- update vendor

master
李光春 2 years ago
parent cd0cd1c79d
commit 16cf68e8af

129
go.mod

@ -1,107 +1,106 @@
module github.com/dtapps/go-library
go 1.18
go 1.19
require (
github.com/aliyun/aliyun-oss-go-sdk v2.2.4+incompatible
github.com/allegro/bigcache/v3 v3.0.2
github.com/aws/aws-sdk-go v1.44.61
github.com/baidubce/bce-sdk-go v0.9.130
github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible
github.com/allegro/bigcache/v3 v3.1.0
github.com/aws/aws-sdk-go v1.44.161
github.com/baidubce/bce-sdk-go v0.9.138
github.com/basgys/goxml2json v1.1.0
github.com/docker/docker v20.10.17+incompatible
github.com/docker/docker v20.10.21+incompatible
github.com/drone/drone-go v1.7.1
github.com/gin-gonic/gin v1.8.1
github.com/go-playground/locales v0.14.0
github.com/go-playground/universal-translator v0.18.0
github.com/go-playground/validator/v10 v10.11.0
github.com/go-redis/redis/v9 v9.0.0-beta.1
github.com/go-sql-driver/mysql v1.6.0
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible
github.com/go-playground/validator/v10 v10.11.1
github.com/go-redis/redis/v9 v9.0.0-rc.2
github.com/go-sql-driver/mysql v1.7.0
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.22.11+incompatible
github.com/jasonlvhit/gocron v0.0.1
github.com/kamva/mgm/v3 v3.4.1
github.com/ks3sdklib/aws-sdk-go v1.1.6
github.com/lib/pq v1.10.6
github.com/kamva/mgm/v3 v3.5.0
github.com/ks3sdklib/aws-sdk-go v1.1.8
github.com/lib/pq v1.10.7
github.com/mitchellh/mapstructure v1.5.0
github.com/mvdan/xurls v1.1.0
github.com/natefinch/lumberjack v2.0.0+incompatible
github.com/nilorg/sdk v0.0.0-20220617065147-3001fb840741
github.com/nilorg/sdk v0.0.0-20221104025912-4b6ccb7004d8
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/qiniu/go-sdk/v7 v7.13.0
github.com/qiniu/qmgo v1.1.1
github.com/qiniu/go-sdk/v7 v7.14.0
github.com/qiniu/qmgo v1.1.4
github.com/robfig/cron/v3 v3.0.1
github.com/rs/xid v1.4.0
github.com/saracen/go7z v0.0.0-20191010121135-9c09b6bd7fda
github.com/shopspring/decimal v1.3.1
github.com/sirupsen/logrus v1.9.0
github.com/tencentyun/cos-go-sdk-v5 v0.7.35
github.com/tencentyun/cos-go-sdk-v5 v0.7.40
github.com/upper/db/v4 v4.6.0
github.com/uptrace/bun v1.1.6
github.com/uptrace/bun/dialect/mysqldialect v1.1.6
github.com/uptrace/bun/dialect/pgdialect v1.1.6
github.com/uptrace/bun/driver/pgdriver v1.1.6
github.com/upyun/go-sdk/v3 v3.0.2
go.mongodb.org/mongo-driver v1.10.0
go.uber.org/zap v1.21.0
golang.org/x/crypto v0.3.0
golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c
github.com/uptrace/bun v1.1.9
github.com/uptrace/bun/dialect/mysqldialect v1.1.9
github.com/uptrace/bun/dialect/pgdialect v1.1.9
github.com/uptrace/bun/driver/pgdriver v1.1.9
github.com/upyun/go-sdk/v3 v3.0.3
go.mongodb.org/mongo-driver v1.11.1
go.uber.org/zap v1.24.0
golang.org/x/crypto v0.4.0
golang.org/x/oauth2 v0.3.0
golang.org/x/text v0.5.0
google.golang.org/grpc v1.48.0
google.golang.org/protobuf v1.28.0
google.golang.org/grpc v1.51.0
google.golang.org/protobuf v1.28.1
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
gorm.io/datatypes v1.0.7
gorm.io/driver/mysql v1.3.5
gorm.io/driver/postgres v1.3.8
gorm.io/gorm v1.23.8
k8s.io/client-go v0.24.3
xorm.io/xorm v1.3.1
gorm.io/datatypes v1.1.0
gorm.io/driver/mysql v1.4.4
gorm.io/driver/postgres v1.4.5
gorm.io/gorm v1.24.2
k8s.io/client-go v0.26.0
xorm.io/xorm v1.3.2
)
require (
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
github.com/Microsoft/go-winio v0.6.0 // indirect
github.com/bitly/go-simplejson v0.5.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/clbanning/mxj v1.8.4 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/deckarep/golang-set v1.8.0 // indirect
github.com/denisenkom/go-mssqldb v0.12.2 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/docker/distribution v2.8.1+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/emicklei/go-restful/v3 v3.10.1 // indirect
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
github.com/go-openapi/swag v0.21.1 // indirect
github.com/goccy/go-json v0.9.10 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/goccy/go-json v0.10.0 // indirect
github.com/gofrs/uuid v4.2.0+incompatible // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/gnostic v0.6.9 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgconn v1.12.1 // indirect
github.com/jackc/pgconn v1.13.0 // indirect
github.com/jackc/pgio v1.0.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgproto3/v2 v2.3.0 // indirect
github.com/jackc/pgproto3/v2 v2.3.1 // indirect
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
github.com/jackc/pgtype v1.11.0 // indirect
github.com/jackc/pgx/v4 v4.16.1 // indirect
github.com/jackc/pgtype v1.13.0 // indirect
github.com/jackc/pgx/v4 v4.17.2 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.15.9 // indirect
github.com/klauspost/compress v1.15.13 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@ -124,46 +123,44 @@ require (
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
github.com/ugorji/go/codec v1.2.7 // indirect
github.com/ulikunitz/xz v0.5.10 // indirect
github.com/ulikunitz/xz v0.5.11 // indirect
github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.1.1 // indirect
github.com/xdg-go/stringprep v1.0.3 // indirect
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
golang.org/x/mod v0.7.0 // indirect
golang.org/x/net v0.3.0 // indirect
golang.org/x/net v0.4.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.3.0 // indirect
golang.org/x/term v0.3.0 // indirect
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.4.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252 // indirect
google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37 // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
gorm.io/driver/sqlite v1.3.6 // indirect
gorm.io/driver/sqlserver v1.3.2 // indirect
gotest.tools/v3 v3.3.0 // indirect
k8s.io/api v0.24.3 // indirect
k8s.io/apimachinery v0.24.3 // indirect
k8s.io/klog/v2 v2.70.1 // indirect
k8s.io/kube-openapi v0.0.0-20220627174259-011e075b9cb8 // indirect
k8s.io/utils v0.0.0-20220713171938-56c0de1e6f5e // indirect
k8s.io/api v0.26.0 // indirect
k8s.io/apimachinery v0.26.0 // indirect
k8s.io/klog/v2 v2.80.1 // indirect
k8s.io/kube-openapi v0.0.0-20221207184640-f3cff1453715 // indirect
k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 // indirect
lukechampine.com/uint128 v1.2.0 // indirect
mellium.im/sasl v0.2.1 // indirect
mellium.im/sasl v0.3.0 // indirect
modernc.org/ccgo/v3 v3.16.8 // indirect
modernc.org/libc v1.16.17 // indirect
modernc.org/opt v0.1.3 // indirect
modernc.org/sqlite v1.17.3 // indirect
modernc.org/strutil v1.1.2 // indirect
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
xorm.io/builder v0.3.12 // indirect
)

729
go.sum

File diff suppressed because it is too large Load Diff

@ -0,0 +1 @@
* text=auto eol=lf

@ -1 +1,10 @@
.vscode/
*.exe
# testing
testdata
# go workspaces
go.work
go.work.sum

@ -0,0 +1,144 @@
run:
skip-dirs:
- pkg/etw/sample
linters:
enable:
# style
- containedctx # struct contains a context
- dupl # duplicate code
- errname # erorrs are named correctly
- goconst # strings that should be constants
- godot # comments end in a period
- misspell
- nolintlint # "//nolint" directives are properly explained
- revive # golint replacement
- stylecheck # golint replacement, less configurable than revive
- unconvert # unnecessary conversions
- wastedassign
# bugs, performance, unused, etc ...
- contextcheck # function uses a non-inherited context
- errorlint # errors not wrapped for 1.13
- exhaustive # check exhaustiveness of enum switch statements
- gofmt # files are gofmt'ed
- gosec # security
- nestif # deeply nested ifs
- nilerr # returns nil even with non-nil error
- prealloc # slices that can be pre-allocated
- structcheck # unused struct fields
- unparam # unused function params
issues:
exclude-rules:
# err is very often shadowed in nested scopes
- linters:
- govet
text: '^shadow: declaration of "err" shadows declaration'
# ignore long lines for skip autogen directives
- linters:
- revive
text: "^line-length-limit: "
source: "^//(go:generate|sys) "
# allow unjustified ignores of error checks in defer statements
- linters:
- nolintlint
text: "^directive `//nolint:errcheck` should provide explanation"
source: '^\s*defer '
# allow unjustified ignores of error lints for io.EOF
- linters:
- nolintlint
text: "^directive `//nolint:errorlint` should provide explanation"
source: '[=|!]= io.EOF'
linters-settings:
govet:
enable-all: true
disable:
# struct order is often for Win32 compat
# also, ignore pointer bytes/GC issues for now until performance becomes an issue
- fieldalignment
check-shadowing: true
nolintlint:
allow-leading-space: false
require-explanation: true
require-specific: true
revive:
# revive is more configurable than static check, so likely the preferred alternative to static-check
# (once the perf issue is solved: https://github.com/golangci/golangci-lint/issues/2997)
enable-all-rules:
true
# https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md
rules:
# rules with required arguments
- name: argument-limit
disabled: true
- name: banned-characters
disabled: true
- name: cognitive-complexity
disabled: true
- name: cyclomatic
disabled: true
- name: file-header
disabled: true
- name: function-length
disabled: true
- name: function-result-limit
disabled: true
- name: max-public-structs
disabled: true
# geneally annoying rules
- name: add-constant # complains about any and all strings and integers
disabled: true
- name: confusing-naming # we frequently use "Foo()" and "foo()" together
disabled: true
- name: flag-parameter # excessive, and a common idiom we use
disabled: true
# general config
- name: line-length-limit
arguments:
- 140
- name: var-naming
arguments:
- []
- - CID
- CRI
- CTRD
- DACL
- DLL
- DOS
- ETW
- FSCTL
- GCS
- GMSA
- HCS
- HV
- IO
- LCOW
- LDAP
- LPAC
- LTSC
- MMIO
- NT
- OCI
- PMEM
- PWSH
- RX
- SACl
- SID
- SMB
- TX
- VHD
- VHDX
- VMID
- VPCI
- WCOW
- WIM
stylecheck:
checks:
- "all"
- "-ST1003" # use revive's var naming

@ -13,16 +13,60 @@ Please see the LICENSE file for licensing information.
## Contributing
This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA)
declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.microsoft.com.
This project welcomes contributions and suggestions.
Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that
you have the right to, and actually do, grant us the rights to use your contribution.
For details, visit [Microsoft CLA](https://cla.microsoft.com).
When you submit a pull request, a CLA-bot will automatically determine whether you need to provide a CLA and decorate the PR
appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA.
When you submit a pull request, a CLA-bot will automatically determine whether you need to
provide a CLA and decorate the PR appropriately (e.g., label, comment).
Simply follow the instructions provided by the bot.
You will only need to do this once across all repos using our CLA.
We also require that contributors sign their commits using git commit -s or git commit --signoff to certify they either authored the work themselves
or otherwise have permission to use it in this project. Please see https://developercertificate.org/ for more info, as well as to make sure that you can
attest to the rules listed. Our CI uses the DCO Github app to ensure that all commits in a given PR are signed-off.
Additionally, the pull request pipeline requires the following steps to be performed before
mergining.
### Code Sign-Off
We require that contributors sign their commits using [`git commit --signoff`][git-commit-s]
to certify they either authored the work themselves or otherwise have permission to use it in this project.
A range of commits can be signed off using [`git rebase --signoff`][git-rebase-s].
Please see [the developer certificate](https://developercertificate.org) for more info,
as well as to make sure that you can attest to the rules listed.
Our CI uses the DCO Github app to ensure that all commits in a given PR are signed-off.
### Linting
Code must pass a linting stage, which uses [`golangci-lint`][lint].
The linting settings are stored in [`.golangci.yaml`](./.golangci.yaml), and can be run
automatically with VSCode by adding the following to your workspace or folder settings:
```json
"go.lintTool": "golangci-lint",
"go.lintOnSave": "package",
```
Additional editor [integrations options are also available][lint-ide].
Alternatively, `golangci-lint` can be [installed locally][lint-install] and run from the repo root:
```shell
# use . or specify a path to only lint a package
# to show all lint errors, use flags "--max-issues-per-linter=0 --max-same-issues=0"
> golangci-lint run ./...
```
### Go Generate
The pipeline checks that auto-generated code, via `go generate`, are up to date.
This can be done for the entire repo:
```shell
> go generate ./...
```
## Code of Conduct
@ -30,8 +74,16 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
## Special Thanks
Thanks to [natefinch][natefinch] for the inspiration for this library.
See [npipe](https://github.com/natefinch/npipe) for another named pipe implementation.
## Special Thanks
Thanks to natefinch for the inspiration for this library. See https://github.com/natefinch/npipe
for another named pipe implementation.
[lint]: https://golangci-lint.run/
[lint-ide]: https://golangci-lint.run/usage/integrations/#editor-integration
[lint-install]: https://golangci-lint.run/usage/install/#local-installation
[git-commit-s]: https://git-scm.com/docs/git-commit#Documentation/git-commit.txt--s
[git-rebase-s]: https://git-scm.com/docs/git-rebase#Documentation/git-rebase.txt---signoff
[natefinch]: https://github.com/natefinch

@ -0,0 +1,41 @@
<!-- BEGIN MICROSOFT SECURITY.MD V0.0.7 BLOCK -->
## Security
Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below.
## Reporting Security Issues
**Please do not report security vulnerabilities through public GitHub issues.**
Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report).
If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey).
You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc).
Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
* Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
* Full paths of source file(s) related to the manifestation of the issue
* The location of the affected source code (tag/branch/commit or direct URL)
* Any special configuration required to reproduce the issue
* Step-by-step instructions to reproduce the issue
* Proof-of-concept or exploit code (if possible)
* Impact of the issue, including how an attacker might exploit the issue
This information will help us triage your report more quickly.
If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs.
## Preferred Languages
We prefer all communications to be in English.
## Policy
Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd).
<!-- END MICROSOFT SECURITY.MD BLOCK -->

@ -1,3 +1,4 @@
//go:build windows
// +build windows
package winio
@ -7,11 +8,12 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"runtime"
"syscall"
"unicode/utf16"
"golang.org/x/sys/windows"
)
//sys backupRead(h syscall.Handle, b []byte, bytesRead *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupRead
@ -24,7 +26,7 @@ const (
BackupAlternateData
BackupLink
BackupPropertyData
BackupObjectId
BackupObjectId //revive:disable-line:var-naming ID, not Id
BackupReparseData
BackupSparseBlock
BackupTxfsData
@ -34,14 +36,16 @@ const (
StreamSparseAttributes = uint32(8)
)
//nolint:revive // var-naming: ALL_CAPS
const (
WRITE_DAC = 0x40000
WRITE_OWNER = 0x80000
ACCESS_SYSTEM_SECURITY = 0x1000000
WRITE_DAC = windows.WRITE_DAC
WRITE_OWNER = windows.WRITE_OWNER
ACCESS_SYSTEM_SECURITY = windows.ACCESS_SYSTEM_SECURITY
)
// BackupHeader represents a backup stream of a file.
type BackupHeader struct {
//revive:disable-next-line:var-naming ID, not Id
Id uint32 // The backup stream ID
Attributes uint32 // Stream attributes
Size int64 // The size of the stream in bytes
@ -49,8 +53,8 @@ type BackupHeader struct {
Offset int64 // The offset of the stream in the file (for BackupSparseBlock only).
}
type win32StreamId struct {
StreamId uint32
type win32StreamID struct {
StreamID uint32
Attributes uint32
Size uint64
NameSize uint32
@ -71,7 +75,7 @@ func NewBackupStreamReader(r io.Reader) *BackupStreamReader {
// Next returns the next backup stream and prepares for calls to Read(). It skips the remainder of the current stream if
// it was not completely read.
func (r *BackupStreamReader) Next() (*BackupHeader, error) {
if r.bytesLeft > 0 {
if r.bytesLeft > 0 { //nolint:nestif // todo: flatten this
if s, ok := r.r.(io.Seeker); ok {
// Make sure Seek on io.SeekCurrent sometimes succeeds
// before trying the actual seek.
@ -82,16 +86,16 @@ func (r *BackupStreamReader) Next() (*BackupHeader, error) {
r.bytesLeft = 0
}
}
if _, err := io.Copy(ioutil.Discard, r); err != nil {
if _, err := io.Copy(io.Discard, r); err != nil {
return nil, err
}
}
var wsi win32StreamId
var wsi win32StreamID
if err := binary.Read(r.r, binary.LittleEndian, &wsi); err != nil {
return nil, err
}
hdr := &BackupHeader{
Id: wsi.StreamId,
Id: wsi.StreamID,
Attributes: wsi.Attributes,
Size: int64(wsi.Size),
}
@ -102,7 +106,7 @@ func (r *BackupStreamReader) Next() (*BackupHeader, error) {
}
hdr.Name = syscall.UTF16ToString(name)
}
if wsi.StreamId == BackupSparseBlock {
if wsi.StreamID == BackupSparseBlock {
if err := binary.Read(r.r, binary.LittleEndian, &hdr.Offset); err != nil {
return nil, err
}
@ -147,8 +151,8 @@ func (w *BackupStreamWriter) WriteHeader(hdr *BackupHeader) error {
return fmt.Errorf("missing %d bytes", w.bytesLeft)
}
name := utf16.Encode([]rune(hdr.Name))
wsi := win32StreamId{
StreamId: hdr.Id,
wsi := win32StreamID{
StreamID: hdr.Id,
Attributes: hdr.Attributes,
Size: uint64(hdr.Size),
NameSize: uint32(len(name) * 2),
@ -203,7 +207,7 @@ func (r *BackupFileReader) Read(b []byte) (int, error) {
var bytesRead uint32
err := backupRead(syscall.Handle(r.f.Fd()), b, &bytesRead, false, r.includeSecurity, &r.ctx)
if err != nil {
return 0, &os.PathError{"BackupRead", r.f.Name(), err}
return 0, &os.PathError{Op: "BackupRead", Path: r.f.Name(), Err: err}
}
runtime.KeepAlive(r.f)
if bytesRead == 0 {
@ -216,7 +220,7 @@ func (r *BackupFileReader) Read(b []byte) (int, error) {
// the underlying file.
func (r *BackupFileReader) Close() error {
if r.ctx != 0 {
backupRead(syscall.Handle(r.f.Fd()), nil, nil, true, false, &r.ctx)
_ = backupRead(syscall.Handle(r.f.Fd()), nil, nil, true, false, &r.ctx)
runtime.KeepAlive(r.f)
r.ctx = 0
}
@ -242,7 +246,7 @@ func (w *BackupFileWriter) Write(b []byte) (int, error) {
var bytesWritten uint32
err := backupWrite(syscall.Handle(w.f.Fd()), b, &bytesWritten, false, w.includeSecurity, &w.ctx)
if err != nil {
return 0, &os.PathError{"BackupWrite", w.f.Name(), err}
return 0, &os.PathError{Op: "BackupWrite", Path: w.f.Name(), Err: err}
}
runtime.KeepAlive(w.f)
if int(bytesWritten) != len(b) {
@ -255,7 +259,7 @@ func (w *BackupFileWriter) Write(b []byte) (int, error) {
// close the underlying file.
func (w *BackupFileWriter) Close() error {
if w.ctx != 0 {
backupWrite(syscall.Handle(w.f.Fd()), nil, nil, true, false, &w.ctx)
_ = backupWrite(syscall.Handle(w.f.Fd()), nil, nil, true, false, &w.ctx)
runtime.KeepAlive(w.f)
w.ctx = 0
}
@ -271,7 +275,13 @@ func OpenForBackup(path string, access uint32, share uint32, createmode uint32)
if err != nil {
return nil, err
}
h, err := syscall.CreateFile(&winPath[0], access, share, nil, createmode, syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OPEN_REPARSE_POINT, 0)
h, err := syscall.CreateFile(&winPath[0],
access,
share,
nil,
createmode,
syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OPEN_REPARSE_POINT,
0)
if err != nil {
err = &os.PathError{Op: "open", Path: path, Err: err}
return nil, err

@ -0,0 +1,22 @@
// This package provides utilities for efficiently performing Win32 IO operations in Go.
// Currently, this package is provides support for genreal IO and management of
// - named pipes
// - files
// - [Hyper-V sockets]
//
// This code is similar to Go's [net] package, and uses IO completion ports to avoid
// blocking IO on system threads, allowing Go to reuse the thread to schedule other goroutines.
//
// This limits support to Windows Vista and newer operating systems.
//
// Additionally, this package provides support for:
// - creating and managing GUIDs
// - writing to [ETW]
// - opening and manageing VHDs
// - parsing [Windows Image files]
// - auto-generating Win32 API code
//
// [Hyper-V sockets]: https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/user-guide/make-integration-service
// [ETW]: https://docs.microsoft.com/en-us/windows-hardware/drivers/devtest/event-tracing-for-windows--etw-
// [Windows Image files]: https://docs.microsoft.com/en-us/windows-hardware/manufacture/desktop/work-with-windows-images
package winio

@ -33,7 +33,7 @@ func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) {
err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info)
if err != nil {
err = errInvalidEaBuffer
return
return ea, nb, err
}
nameOffset := fileFullEaInformationSize
@ -43,7 +43,7 @@ func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) {
nextOffset := int(info.NextEntryOffset)
if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) {
err = errInvalidEaBuffer
return
return ea, nb, err
}
ea.Name = string(b[nameOffset : nameOffset+nameLen])
@ -52,7 +52,7 @@ func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) {
if info.NextEntryOffset != 0 {
nb = b[info.NextEntryOffset:]
}
return
return ea, nb, err
}
// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION
@ -67,7 +67,7 @@ func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) {
eas = append(eas, ea)
b = nb
}
return
return eas, err
}
func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error {

@ -11,6 +11,8 @@ import (
"sync/atomic"
"syscall"
"time"
"golang.org/x/sys/windows"
)
//sys cancelIoEx(file syscall.Handle, o *syscall.Overlapped) (err error) = CancelIoEx
@ -24,6 +26,8 @@ type atomicBool int32
func (b *atomicBool) isSet() bool { return atomic.LoadInt32((*int32)(b)) != 0 }
func (b *atomicBool) setFalse() { atomic.StoreInt32((*int32)(b), 0) }
func (b *atomicBool) setTrue() { atomic.StoreInt32((*int32)(b), 1) }
//revive:disable-next-line:predeclared Keep "new" to maintain consistency with "atomic" pkg
func (b *atomicBool) swap(new bool) bool {
var newInt int32
if new {
@ -32,11 +36,6 @@ func (b *atomicBool) swap(new bool) bool {
return atomic.SwapInt32((*int32)(b), newInt) == 1
}
const (
cFILE_SKIP_COMPLETION_PORT_ON_SUCCESS = 1
cFILE_SKIP_SET_EVENT_ON_HANDLE = 2
)
var (
ErrFileClosed = errors.New("file has already been closed")
ErrTimeout = &timeoutError{}
@ -44,28 +43,28 @@ var (
type timeoutError struct{}
func (e *timeoutError) Error() string { return "i/o timeout" }
func (e *timeoutError) Timeout() bool { return true }
func (e *timeoutError) Temporary() bool { return true }
func (*timeoutError) Error() string { return "i/o timeout" }
func (*timeoutError) Timeout() bool { return true }
func (*timeoutError) Temporary() bool { return true }
type timeoutChan chan struct{}
var ioInitOnce sync.Once
var ioCompletionPort syscall.Handle
// ioResult contains the result of an asynchronous IO operation
// ioResult contains the result of an asynchronous IO operation.
type ioResult struct {
bytes uint32
err error
}
// ioOperation represents an outstanding asynchronous Win32 IO
// ioOperation represents an outstanding asynchronous Win32 IO.
type ioOperation struct {
o syscall.Overlapped
ch chan ioResult
}
func initIo() {
func initIO() {
h, err := createIoCompletionPort(syscall.InvalidHandle, 0, 0, 0xffffffff)
if err != nil {
panic(err)
@ -94,15 +93,15 @@ type deadlineHandler struct {
timedout atomicBool
}
// makeWin32File makes a new win32File from an existing file handle
// makeWin32File makes a new win32File from an existing file handle.
func makeWin32File(h syscall.Handle) (*win32File, error) {
f := &win32File{handle: h}
ioInitOnce.Do(initIo)
ioInitOnce.Do(initIO)
_, err := createIoCompletionPort(h, ioCompletionPort, 0, 0xffffffff)
if err != nil {
return nil, err
}
err = setFileCompletionNotificationModes(h, cFILE_SKIP_COMPLETION_PORT_ON_SUCCESS|cFILE_SKIP_SET_EVENT_ON_HANDLE)
err = setFileCompletionNotificationModes(h, windows.FILE_SKIP_COMPLETION_PORT_ON_SUCCESS|windows.FILE_SKIP_SET_EVENT_ON_HANDLE)
if err != nil {
return nil, err
}
@ -121,14 +120,14 @@ func MakeOpenFile(h syscall.Handle) (io.ReadWriteCloser, error) {
return f, nil
}
// closeHandle closes the resources associated with a Win32 handle
// closeHandle closes the resources associated with a Win32 handle.
func (f *win32File) closeHandle() {
f.wgLock.Lock()
// Atomically set that we are closing, releasing the resources only once.
if !f.closing.swap(true) {
f.wgLock.Unlock()
// cancel all IO and wait for it to complete
cancelIoEx(f.handle, nil)
_ = cancelIoEx(f.handle, nil)
f.wg.Wait()
// at this point, no new IO can start
syscall.Close(f.handle)
@ -144,14 +143,14 @@ func (f *win32File) Close() error {
return nil
}
// IsClosed checks if the file has been closed
// IsClosed checks if the file has been closed.
func (f *win32File) IsClosed() bool {
return f.closing.isSet()
}
// prepareIo prepares for a new IO operation.
// prepareIO prepares for a new IO operation.
// The caller must call f.wg.Done() when the IO is finished, prior to Close() returning.
func (f *win32File) prepareIo() (*ioOperation, error) {
func (f *win32File) prepareIO() (*ioOperation, error) {
f.wgLock.RLock()
if f.closing.isSet() {
f.wgLock.RUnlock()
@ -164,7 +163,7 @@ func (f *win32File) prepareIo() (*ioOperation, error) {
return c, nil
}
// ioCompletionProcessor processes completed async IOs forever
// ioCompletionProcessor processes completed async IOs forever.
func ioCompletionProcessor(h syscall.Handle) {
for {
var bytes uint32
@ -178,15 +177,17 @@ func ioCompletionProcessor(h syscall.Handle) {
}
}
// asyncIo processes the return value from ReadFile or WriteFile, blocking until
// todo: helsaawy - create an asyncIO version that takes a context
// asyncIO processes the return value from ReadFile or WriteFile, blocking until
// the operation has actually completed.
func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, err error) (int, error) {
if err != syscall.ERROR_IO_PENDING {
func (f *win32File) asyncIO(c *ioOperation, d *deadlineHandler, bytes uint32, err error) (int, error) {
if err != syscall.ERROR_IO_PENDING { //nolint:errorlint // err is Errno
return int(bytes), err
}
if f.closing.isSet() {
cancelIoEx(f.handle, &c.o)
_ = cancelIoEx(f.handle, &c.o)
}
var timeout timeoutChan
@ -200,7 +201,7 @@ func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, er
select {
case r = <-c.ch:
err = r.err
if err == syscall.ERROR_OPERATION_ABORTED {
if err == syscall.ERROR_OPERATION_ABORTED { //nolint:errorlint // err is Errno
if f.closing.isSet() {
err = ErrFileClosed
}
@ -210,10 +211,10 @@ func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, er
err = wsaGetOverlappedResult(f.handle, &c.o, &bytes, false, &flags)
}
case <-timeout:
cancelIoEx(f.handle, &c.o)
_ = cancelIoEx(f.handle, &c.o)
r = <-c.ch
err = r.err
if err == syscall.ERROR_OPERATION_ABORTED {
if err == syscall.ERROR_OPERATION_ABORTED { //nolint:errorlint // err is Errno
err = ErrTimeout
}
}
@ -221,13 +222,14 @@ func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, er
// runtime.KeepAlive is needed, as c is passed via native
// code to ioCompletionProcessor, c must remain alive
// until the channel read is complete.
// todo: (de)allocate *ioOperation via win32 heap functions, instead of needing to KeepAlive?
runtime.KeepAlive(c)
return int(r.bytes), err
}
// Read reads from a file handle.
func (f *win32File) Read(b []byte) (int, error) {
c, err := f.prepareIo()
c, err := f.prepareIO()
if err != nil {
return 0, err
}
@ -239,13 +241,13 @@ func (f *win32File) Read(b []byte) (int, error) {
var bytes uint32
err = syscall.ReadFile(f.handle, b, &bytes, &c.o)
n, err := f.asyncIo(c, &f.readDeadline, bytes, err)
n, err := f.asyncIO(c, &f.readDeadline, bytes, err)
runtime.KeepAlive(b)
// Handle EOF conditions.
if err == nil && n == 0 && len(b) != 0 {
return 0, io.EOF
} else if err == syscall.ERROR_BROKEN_PIPE {
} else if err == syscall.ERROR_BROKEN_PIPE { //nolint:errorlint // err is Errno
return 0, io.EOF
} else {
return n, err
@ -254,7 +256,7 @@ func (f *win32File) Read(b []byte) (int, error) {
// Write writes to a file handle.
func (f *win32File) Write(b []byte) (int, error) {
c, err := f.prepareIo()
c, err := f.prepareIO()
if err != nil {
return 0, err
}
@ -266,7 +268,7 @@ func (f *win32File) Write(b []byte) (int, error) {
var bytes uint32
err = syscall.WriteFile(f.handle, b, &bytes, &c.o)
n, err := f.asyncIo(c, &f.writeDeadline, bytes, err)
n, err := f.asyncIO(c, &f.writeDeadline, bytes, err)
runtime.KeepAlive(b)
return n, err
}

@ -1,3 +1,4 @@
//go:build windows
// +build windows
package winio
@ -14,13 +15,18 @@ import (
type FileBasicInfo struct {
CreationTime, LastAccessTime, LastWriteTime, ChangeTime windows.Filetime
FileAttributes uint32
pad uint32 // padding
_ uint32 // padding
}
// GetFileBasicInfo retrieves times and attributes for a file.
func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) {
bi := &FileBasicInfo{}
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
if err := windows.GetFileInformationByHandleEx(
windows.Handle(f.Fd()),
windows.FileBasicInfo,
(*byte)(unsafe.Pointer(bi)),
uint32(unsafe.Sizeof(*bi)),
); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
@ -29,7 +35,12 @@ func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) {
// SetFileBasicInfo sets times and attributes for a file.
func SetFileBasicInfo(f *os.File, bi *FileBasicInfo) error {
if err := windows.SetFileInformationByHandle(windows.Handle(f.Fd()), windows.FileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
if err := windows.SetFileInformationByHandle(
windows.Handle(f.Fd()),
windows.FileBasicInfo,
(*byte)(unsafe.Pointer(bi)),
uint32(unsafe.Sizeof(*bi)),
); err != nil {
return &os.PathError{Op: "SetFileInformationByHandle", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
@ -48,7 +59,10 @@ type FileStandardInfo struct {
// GetFileStandardInfo retrieves ended information for the file.
func GetFileStandardInfo(f *os.File) (*FileStandardInfo, error) {
si := &FileStandardInfo{}
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileStandardInfo, (*byte)(unsafe.Pointer(si)), uint32(unsafe.Sizeof(*si))); err != nil {
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()),
windows.FileStandardInfo,
(*byte)(unsafe.Pointer(si)),
uint32(unsafe.Sizeof(*si))); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
@ -65,7 +79,12 @@ type FileIDInfo struct {
// GetFileID retrieves the unique (volume, file ID) pair for a file.
func GetFileID(f *os.File) (*FileIDInfo, error) {
fileID := &FileIDInfo{}
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileIdInfo, (*byte)(unsafe.Pointer(fileID)), uint32(unsafe.Sizeof(*fileID))); err != nil {
if err := windows.GetFileInformationByHandleEx(
windows.Handle(f.Fd()),
windows.FileIdInfo,
(*byte)(unsafe.Pointer(fileID)),
uint32(unsafe.Sizeof(*fileID)),
); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)

@ -4,6 +4,8 @@
package winio
import (
"context"
"errors"
"fmt"
"io"
"net"
@ -12,16 +14,87 @@ import (
"time"
"unsafe"
"golang.org/x/sys/windows"
"github.com/Microsoft/go-winio/internal/socket"
"github.com/Microsoft/go-winio/pkg/guid"
)
//sys bind(s syscall.Handle, name unsafe.Pointer, namelen int32) (err error) [failretval==socketError] = ws2_32.bind
const afHVSock = 34 // AF_HYPERV
const (
afHvSock = 34 // AF_HYPERV
// Well known Service and VM IDs
//https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/user-guide/make-integration-service#vmid-wildcards
socketError = ^uintptr(0)
)
// HvsockGUIDWildcard is the wildcard VmId for accepting connections from all partitions.
func HvsockGUIDWildcard() guid.GUID { // 00000000-0000-0000-0000-000000000000
return guid.GUID{}
}
// HvsockGUIDBroadcast is the wildcard VmId for broadcasting sends to all partitions.
func HvsockGUIDBroadcast() guid.GUID { //ffffffff-ffff-ffff-ffff-ffffffffffff
return guid.GUID{
Data1: 0xffffffff,
Data2: 0xffff,
Data3: 0xffff,
Data4: [8]uint8{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
}
}
// HvsockGUIDLoopback is the Loopback VmId for accepting connections to the same partition as the connector.
func HvsockGUIDLoopback() guid.GUID { // e0e16197-dd56-4a10-9195-5ee7a155a838
return guid.GUID{
Data1: 0xe0e16197,
Data2: 0xdd56,
Data3: 0x4a10,
Data4: [8]uint8{0x91, 0x95, 0x5e, 0xe7, 0xa1, 0x55, 0xa8, 0x38},
}
}
// HvsockGUIDSiloHost is the address of a silo's host partition:
// - The silo host of a hosted silo is the utility VM.
// - The silo host of a silo on a physical host is the physical host.
func HvsockGUIDSiloHost() guid.GUID { // 36bd0c5c-7276-4223-88ba-7d03b654c568
return guid.GUID{
Data1: 0x36bd0c5c,
Data2: 0x7276,
Data3: 0x4223,
Data4: [8]byte{0x88, 0xba, 0x7d, 0x03, 0xb6, 0x54, 0xc5, 0x68},
}
}
// HvsockGUIDChildren is the wildcard VmId for accepting connections from the connector's child partitions.
func HvsockGUIDChildren() guid.GUID { // 90db8b89-0d35-4f79-8ce9-49ea0ac8b7cd
return guid.GUID{
Data1: 0x90db8b89,
Data2: 0xd35,
Data3: 0x4f79,
Data4: [8]uint8{0x8c, 0xe9, 0x49, 0xea, 0xa, 0xc8, 0xb7, 0xcd},
}
}
// HvsockGUIDParent is the wildcard VmId for accepting connections from the connector's parent partition.
// Listening on this VmId accepts connection from:
// - Inside silos: silo host partition.
// - Inside hosted silo: host of the VM.
// - Inside VM: VM host.
// - Physical host: Not supported.
func HvsockGUIDParent() guid.GUID { // a42e7cda-d03f-480c-9cc2-a4de20abb878
return guid.GUID{
Data1: 0xa42e7cda,
Data2: 0xd03f,
Data3: 0x480c,
Data4: [8]uint8{0x9c, 0xc2, 0xa4, 0xde, 0x20, 0xab, 0xb8, 0x78},
}
}
// hvsockVsockServiceTemplate is the Service GUID used for the VSOCK protocol.
func hvsockVsockServiceTemplate() guid.GUID { // 00000000-facb-11e6-bd58-64006a7986d3
return guid.GUID{
Data2: 0xfacb,
Data3: 0x11e6,
Data4: [8]uint8{0xbd, 0x58, 0x64, 0x00, 0x6a, 0x79, 0x86, 0xd3},
}
}
// An HvsockAddr is an address for a AF_HYPERV socket.
type HvsockAddr struct {
@ -36,8 +109,10 @@ type rawHvsockAddr struct {
ServiceID guid.GUID
}
var _ socket.RawSockaddr = &rawHvsockAddr{}
// Network returns the address's network name, "hvsock".
func (addr *HvsockAddr) Network() string {
func (*HvsockAddr) Network() string {
return "hvsock"
}
@ -47,14 +122,14 @@ func (addr *HvsockAddr) String() string {
// VsockServiceID returns an hvsock service ID corresponding to the specified AF_VSOCK port.
func VsockServiceID(port uint32) guid.GUID {
g, _ := guid.FromString("00000000-facb-11e6-bd58-64006a7986d3")
g := hvsockVsockServiceTemplate() // make a copy
g.Data1 = port
return g
}
func (addr *HvsockAddr) raw() rawHvsockAddr {
return rawHvsockAddr{
Family: afHvSock,
Family: afHVSock,
VMID: addr.VMID,
ServiceID: addr.ServiceID,
}
@ -65,20 +140,48 @@ func (addr *HvsockAddr) fromRaw(raw *rawHvsockAddr) {
addr.ServiceID = raw.ServiceID
}
// Sockaddr returns a pointer to and the size of this struct.
//
// Implements the [socket.RawSockaddr] interface, and allows use in
// [socket.Bind] and [socket.ConnectEx].
func (r *rawHvsockAddr) Sockaddr() (unsafe.Pointer, int32, error) {
return unsafe.Pointer(r), int32(unsafe.Sizeof(rawHvsockAddr{})), nil
}
// Sockaddr interface allows use with `sockets.Bind()` and `.ConnectEx()`.
func (r *rawHvsockAddr) FromBytes(b []byte) error {
n := int(unsafe.Sizeof(rawHvsockAddr{}))
if len(b) < n {
return fmt.Errorf("got %d, want %d: %w", len(b), n, socket.ErrBufferSize)
}
copy(unsafe.Slice((*byte)(unsafe.Pointer(r)), n), b[:n])
if r.Family != afHVSock {
return fmt.Errorf("got %d, want %d: %w", r.Family, afHVSock, socket.ErrAddrFamily)
}
return nil
}
// HvsockListener is a socket listener for the AF_HYPERV address family.
type HvsockListener struct {
sock *win32File
addr HvsockAddr
}
var _ net.Listener = &HvsockListener{}
// HvsockConn is a connected socket of the AF_HYPERV address family.
type HvsockConn struct {
sock *win32File
local, remote HvsockAddr
}
func newHvSocket() (*win32File, error) {
fd, err := syscall.Socket(afHvSock, syscall.SOCK_STREAM, 1)
var _ net.Conn = &HvsockConn{}
func newHVSocket() (*win32File, error) {
fd, err := syscall.Socket(afHVSock, syscall.SOCK_STREAM, 1)
if err != nil {
return nil, os.NewSyscallError("socket", err)
}
@ -94,12 +197,12 @@ func newHvSocket() (*win32File, error) {
// ListenHvsock listens for connections on the specified hvsock address.
func ListenHvsock(addr *HvsockAddr) (_ *HvsockListener, err error) {
l := &HvsockListener{addr: *addr}
sock, err := newHvSocket()
sock, err := newHVSocket()
if err != nil {
return nil, l.opErr("listen", err)
}
sa := addr.raw()
err = bind(sock.handle, unsafe.Pointer(&sa), int32(unsafe.Sizeof(sa)))
err = socket.Bind(windows.Handle(sock.handle), &sa)
if err != nil {
return nil, l.opErr("listen", os.NewSyscallError("socket", err))
}
@ -121,7 +224,7 @@ func (l *HvsockListener) Addr() net.Addr {
// Accept waits for the next connection and returns it.
func (l *HvsockListener) Accept() (_ net.Conn, err error) {
sock, err := newHvSocket()
sock, err := newHVSocket()
if err != nil {
return nil, l.opErr("accept", err)
}
@ -130,27 +233,42 @@ func (l *HvsockListener) Accept() (_ net.Conn, err error) {
sock.Close()
}
}()
c, err := l.sock.prepareIo()
c, err := l.sock.prepareIO()
if err != nil {
return nil, l.opErr("accept", err)
}
defer l.sock.wg.Done()
// AcceptEx, per documentation, requires an extra 16 bytes per address.
//
// https://docs.microsoft.com/en-us/windows/win32/api/mswsock/nf-mswsock-acceptex
const addrlen = uint32(16 + unsafe.Sizeof(rawHvsockAddr{}))
var addrbuf [addrlen * 2]byte
var bytes uint32
err = syscall.AcceptEx(l.sock.handle, sock.handle, &addrbuf[0], 0, addrlen, addrlen, &bytes, &c.o)
_, err = l.sock.asyncIo(c, nil, bytes, err)
if err != nil {
err = syscall.AcceptEx(l.sock.handle, sock.handle, &addrbuf[0], 0 /*rxdatalen*/, addrlen, addrlen, &bytes, &c.o)
if _, err = l.sock.asyncIO(c, nil, bytes, err); err != nil {
return nil, l.opErr("accept", os.NewSyscallError("acceptex", err))
}
conn := &HvsockConn{
sock: sock,
}
// The local address returned in the AcceptEx buffer is the same as the Listener socket's
// address. However, the service GUID reported by GetSockName is different from the Listeners
// socket, and is sometimes the same as the local address of the socket that dialed the
// address, with the service GUID.Data1 incremented, but othertimes is different.
// todo: does the local address matter? is the listener's address or the actual address appropriate?
conn.local.fromRaw((*rawHvsockAddr)(unsafe.Pointer(&addrbuf[0])))
conn.remote.fromRaw((*rawHvsockAddr)(unsafe.Pointer(&addrbuf[addrlen])))
// initialize the accepted socket and update its properties with those of the listening socket
if err = windows.Setsockopt(windows.Handle(sock.handle),
windows.SOL_SOCKET, windows.SO_UPDATE_ACCEPT_CONTEXT,
(*byte)(unsafe.Pointer(&l.sock.handle)), int32(unsafe.Sizeof(l.sock.handle))); err != nil {
return nil, conn.opErr("accept", os.NewSyscallError("setsockopt", err))
}
sock = nil
return conn, nil
}
@ -160,43 +278,171 @@ func (l *HvsockListener) Close() error {
return l.sock.Close()
}
/* Need to finish ConnectEx handling
func DialHvsock(ctx context.Context, addr *HvsockAddr) (*HvsockConn, error) {
sock, err := newHvSocket()
// HvsockDialer configures and dials a Hyper-V Socket (ie, [HvsockConn]).
type HvsockDialer struct {
// Deadline is the time the Dial operation must connect before erroring.
Deadline time.Time
// Retries is the number of additional connects to try if the connection times out, is refused,
// or the host is unreachable
Retries uint
// RetryWait is the time to wait after a connection error to retry
RetryWait time.Duration
rt *time.Timer // redial wait timer
}
// Dial the Hyper-V socket at addr.
//
// See [HvsockDialer.Dial] for more information.
func Dial(ctx context.Context, addr *HvsockAddr) (conn *HvsockConn, err error) {
return (&HvsockDialer{}).Dial(ctx, addr)
}
// Dial attempts to connect to the Hyper-V socket at addr, and returns a connection if successful.
// Will attempt (HvsockDialer).Retries if dialing fails, waiting (HvsockDialer).RetryWait between
// retries.
//
// Dialing can be cancelled either by providing (HvsockDialer).Deadline, or cancelling ctx.
func (d *HvsockDialer) Dial(ctx context.Context, addr *HvsockAddr) (conn *HvsockConn, err error) {
op := "dial"
// create the conn early to use opErr()
conn = &HvsockConn{
remote: *addr,
}
if !d.Deadline.IsZero() {
var cancel context.CancelFunc
ctx, cancel = context.WithDeadline(ctx, d.Deadline)
defer cancel()
}
// preemptive timeout/cancellation check
if err = ctx.Err(); err != nil {
return nil, conn.opErr(op, err)
}
sock, err := newHVSocket()
if err != nil {
return nil, err
return nil, conn.opErr(op, err)
}
defer func() {
if sock != nil {
sock.Close()
}
}()
c, err := sock.prepareIo()
sa := addr.raw()
err = socket.Bind(windows.Handle(sock.handle), &sa)
if err != nil {
return nil, err
return nil, conn.opErr(op, os.NewSyscallError("bind", err))
}
c, err := sock.prepareIO()
if err != nil {
return nil, conn.opErr(op, err)
}
defer sock.wg.Done()
var bytes uint32
err = windows.ConnectEx(windows.Handle(sock.handle), sa, nil, 0, &bytes, &c.o)
_, err = sock.asyncIo(ctx, c, nil, bytes, err)
for i := uint(0); i <= d.Retries; i++ {
err = socket.ConnectEx(
windows.Handle(sock.handle),
&sa,
nil, // sendBuf
0, // sendDataLen
&bytes,
(*windows.Overlapped)(unsafe.Pointer(&c.o)))
_, err = sock.asyncIO(c, nil, bytes, err)
if i < d.Retries && canRedial(err) {
if err = d.redialWait(ctx); err == nil {
continue
}
}
break
}
if err != nil {
return nil, err
return nil, conn.opErr(op, os.NewSyscallError("connectex", err))
}
conn := &HvsockConn{
sock: sock,
remote: *addr,
// update the connection properties, so shutdown can be used
if err = windows.Setsockopt(
windows.Handle(sock.handle),
windows.SOL_SOCKET,
windows.SO_UPDATE_CONNECT_CONTEXT,
nil, // optvalue
0, // optlen
); err != nil {
return nil, conn.opErr(op, os.NewSyscallError("setsockopt", err))
}
// get the local name
var sal rawHvsockAddr
err = socket.GetSockName(windows.Handle(sock.handle), &sal)
if err != nil {
return nil, conn.opErr(op, os.NewSyscallError("getsockname", err))
}
conn.local.fromRaw(&sal)
// one last check for timeout, since asyncIO doesn't check the context
if err = ctx.Err(); err != nil {
return nil, conn.opErr(op, err)
}
conn.sock = sock
sock = nil
return conn, nil
}
*/
// redialWait waits before attempting to redial, resetting the timer as appropriate.
func (d *HvsockDialer) redialWait(ctx context.Context) (err error) {
if d.RetryWait == 0 {
return nil
}
if d.rt == nil {
d.rt = time.NewTimer(d.RetryWait)
} else {
// should already be stopped and drained
d.rt.Reset(d.RetryWait)
}
select {
case <-ctx.Done():
case <-d.rt.C:
return nil
}
// stop and drain the timer
if !d.rt.Stop() {
<-d.rt.C
}
return ctx.Err()
}
// assumes error is a plain, unwrapped syscall.Errno provided by direct syscall.
func canRedial(err error) bool {
//nolint:errorlint // guaranteed to be an Errno
switch err {
case windows.WSAECONNREFUSED, windows.WSAENETUNREACH, windows.WSAETIMEDOUT,
windows.ERROR_CONNECTION_REFUSED, windows.ERROR_CONNECTION_UNAVAIL:
return true
default:
return false
}
}
func (conn *HvsockConn) opErr(op string, err error) error {
// translate from "file closed" to "socket closed"
if errors.Is(err, ErrFileClosed) {
err = socket.ErrSocketClosed
}
return &net.OpError{Op: op, Net: "hvsock", Source: &conn.local, Addr: &conn.remote, Err: err}
}
func (conn *HvsockConn) Read(b []byte) (int, error) {
c, err := conn.sock.prepareIo()
c, err := conn.sock.prepareIO()
if err != nil {
return 0, conn.opErr("read", err)
}
@ -204,10 +450,11 @@ func (conn *HvsockConn) Read(b []byte) (int, error) {
buf := syscall.WSABuf{Buf: &b[0], Len: uint32(len(b))}
var flags, bytes uint32
err = syscall.WSARecv(conn.sock.handle, &buf, 1, &bytes, &flags, &c.o, nil)
n, err := conn.sock.asyncIo(c, &conn.sock.readDeadline, bytes, err)
n, err := conn.sock.asyncIO(c, &conn.sock.readDeadline, bytes, err)
if err != nil {
if _, ok := err.(syscall.Errno); ok {
err = os.NewSyscallError("wsarecv", err)
var eno windows.Errno
if errors.As(err, &eno) {
err = os.NewSyscallError("wsarecv", eno)
}
return 0, conn.opErr("read", err)
} else if n == 0 {
@ -230,7 +477,7 @@ func (conn *HvsockConn) Write(b []byte) (int, error) {
}
func (conn *HvsockConn) write(b []byte) (int, error) {
c, err := conn.sock.prepareIo()
c, err := conn.sock.prepareIO()
if err != nil {
return 0, conn.opErr("write", err)
}
@ -238,10 +485,11 @@ func (conn *HvsockConn) write(b []byte) (int, error) {
buf := syscall.WSABuf{Buf: &b[0], Len: uint32(len(b))}
var bytes uint32
err = syscall.WSASend(conn.sock.handle, &buf, 1, &bytes, 0, &c.o, nil)
n, err := conn.sock.asyncIo(c, &conn.sock.writeDeadline, bytes, err)
n, err := conn.sock.asyncIO(c, &conn.sock.writeDeadline, bytes, err)
if err != nil {
if _, ok := err.(syscall.Errno); ok {
err = os.NewSyscallError("wsasend", err)
var eno windows.Errno
if errors.As(err, &eno) {
err = os.NewSyscallError("wsasend", eno)
}
return 0, conn.opErr("write", err)
}
@ -257,13 +505,19 @@ func (conn *HvsockConn) IsClosed() bool {
return conn.sock.IsClosed()
}
// shutdown disables sending or receiving on a socket.
func (conn *HvsockConn) shutdown(how int) error {
if conn.IsClosed() {
return ErrFileClosed
return socket.ErrSocketClosed
}
err := syscall.Shutdown(conn.sock.handle, how)
if err != nil {
// If the connection was closed, shutdowns fail with "not connected"
if errors.Is(err, windows.WSAENOTCONN) ||
errors.Is(err, windows.WSAESHUTDOWN) {
err = socket.ErrSocketClosed
}
return os.NewSyscallError("shutdown", err)
}
return nil
@ -273,7 +527,7 @@ func (conn *HvsockConn) shutdown(how int) error {
func (conn *HvsockConn) CloseRead() error {
err := conn.shutdown(syscall.SHUT_RD)
if err != nil {
return conn.opErr("close", err)
return conn.opErr("closeread", err)
}
return nil
}
@ -283,7 +537,7 @@ func (conn *HvsockConn) CloseRead() error {
func (conn *HvsockConn) CloseWrite() error {
err := conn.shutdown(syscall.SHUT_WR)
if err != nil {
return conn.opErr("close", err)
return conn.opErr("closewrite", err)
}
return nil
}
@ -300,8 +554,13 @@ func (conn *HvsockConn) RemoteAddr() net.Addr {
// SetDeadline implements the net.Conn SetDeadline method.
func (conn *HvsockConn) SetDeadline(t time.Time) error {
conn.SetReadDeadline(t)
conn.SetWriteDeadline(t)
// todo: implement `SetDeadline` for `win32File`
if err := conn.SetReadDeadline(t); err != nil {
return fmt.Errorf("set read deadline: %w", err)
}
if err := conn.SetWriteDeadline(t); err != nil {
return fmt.Errorf("set write deadline: %w", err)
}
return nil
}

@ -0,0 +1,20 @@
package socket
import (
"unsafe"
)
// RawSockaddr allows structs to be used with [Bind] and [ConnectEx]. The
// struct must meet the Win32 sockaddr requirements specified here:
// https://docs.microsoft.com/en-us/windows/win32/winsock/sockaddr-2
//
// Specifically, the struct size must be least larger than an int16 (unsigned short)
// for the address family.
type RawSockaddr interface {
// Sockaddr returns a pointer to the RawSockaddr and its struct size, allowing
// for the RawSockaddr's data to be overwritten by syscalls (if necessary).
//
// It is the callers responsibility to validate that the values are valid; invalid
// pointers or size can cause a panic.
Sockaddr() (unsafe.Pointer, int32, error)
}

@ -0,0 +1,179 @@
//go:build windows
package socket
import (
"errors"
"fmt"
"net"
"sync"
"syscall"
"unsafe"
"github.com/Microsoft/go-winio/pkg/guid"
"golang.org/x/sys/windows"
)
//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zsyscall_windows.go socket.go
//sys getsockname(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) [failretval==socketError] = ws2_32.getsockname
//sys getpeername(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) [failretval==socketError] = ws2_32.getpeername
//sys bind(s windows.Handle, name unsafe.Pointer, namelen int32) (err error) [failretval==socketError] = ws2_32.bind
const socketError = uintptr(^uint32(0))
var (
// todo(helsaawy): create custom error types to store the desired vs actual size and addr family?
ErrBufferSize = errors.New("buffer size")
ErrAddrFamily = errors.New("address family")
ErrInvalidPointer = errors.New("invalid pointer")
ErrSocketClosed = fmt.Errorf("socket closed: %w", net.ErrClosed)
)
// todo(helsaawy): replace these with generics, ie: GetSockName[S RawSockaddr](s windows.Handle) (S, error)
// GetSockName writes the local address of socket s to the [RawSockaddr] rsa.
// If rsa is not large enough, the [windows.WSAEFAULT] is returned.
func GetSockName(s windows.Handle, rsa RawSockaddr) error {
ptr, l, err := rsa.Sockaddr()
if err != nil {
return fmt.Errorf("could not retrieve socket pointer and size: %w", err)
}
// although getsockname returns WSAEFAULT if the buffer is too small, it does not set
// &l to the correct size, so--apart from doubling the buffer repeatedly--there is no remedy
return getsockname(s, ptr, &l)
}
// GetPeerName returns the remote address the socket is connected to.
//
// See [GetSockName] for more information.
func GetPeerName(s windows.Handle, rsa RawSockaddr) error {
ptr, l, err := rsa.Sockaddr()
if err != nil {
return fmt.Errorf("could not retrieve socket pointer and size: %w", err)
}
return getpeername(s, ptr, &l)
}
func Bind(s windows.Handle, rsa RawSockaddr) (err error) {
ptr, l, err := rsa.Sockaddr()
if err != nil {
return fmt.Errorf("could not retrieve socket pointer and size: %w", err)
}
return bind(s, ptr, l)
}
// "golang.org/x/sys/windows".ConnectEx and .Bind only accept internal implementations of the
// their sockaddr interface, so they cannot be used with HvsockAddr
// Replicate functionality here from
// https://cs.opensource.google/go/x/sys/+/master:windows/syscall_windows.go
// The function pointers to `AcceptEx`, `ConnectEx` and `GetAcceptExSockaddrs` must be loaded at
// runtime via a WSAIoctl call:
// https://docs.microsoft.com/en-us/windows/win32/api/Mswsock/nc-mswsock-lpfn_connectex#remarks
type runtimeFunc struct {
id guid.GUID
once sync.Once
addr uintptr
err error
}
func (f *runtimeFunc) Load() error {
f.once.Do(func() {
var s windows.Handle
s, f.err = windows.Socket(windows.AF_INET, windows.SOCK_STREAM, windows.IPPROTO_TCP)
if f.err != nil {
return
}
defer windows.CloseHandle(s) //nolint:errcheck
var n uint32
f.err = windows.WSAIoctl(s,
windows.SIO_GET_EXTENSION_FUNCTION_POINTER,
(*byte)(unsafe.Pointer(&f.id)),
uint32(unsafe.Sizeof(f.id)),
(*byte)(unsafe.Pointer(&f.addr)),
uint32(unsafe.Sizeof(f.addr)),
&n,
nil, //overlapped
0, //completionRoutine
)
})
return f.err
}
var (
// todo: add `AcceptEx` and `GetAcceptExSockaddrs`
WSAID_CONNECTEX = guid.GUID{ //revive:disable-line:var-naming ALL_CAPS
Data1: 0x25a207b9,
Data2: 0xddf3,
Data3: 0x4660,
Data4: [8]byte{0x8e, 0xe9, 0x76, 0xe5, 0x8c, 0x74, 0x06, 0x3e},
}
connectExFunc = runtimeFunc{id: WSAID_CONNECTEX}
)
func ConnectEx(
fd windows.Handle,
rsa RawSockaddr,
sendBuf *byte,
sendDataLen uint32,
bytesSent *uint32,
overlapped *windows.Overlapped,
) error {
if err := connectExFunc.Load(); err != nil {
return fmt.Errorf("failed to load ConnectEx function pointer: %w", err)
}
ptr, n, err := rsa.Sockaddr()
if err != nil {
return err
}
return connectEx(fd, ptr, n, sendBuf, sendDataLen, bytesSent, overlapped)
}
// BOOL LpfnConnectex(
// [in] SOCKET s,
// [in] const sockaddr *name,
// [in] int namelen,
// [in, optional] PVOID lpSendBuffer,
// [in] DWORD dwSendDataLength,
// [out] LPDWORD lpdwBytesSent,
// [in] LPOVERLAPPED lpOverlapped
// )
func connectEx(
s windows.Handle,
name unsafe.Pointer,
namelen int32,
sendBuf *byte,
sendDataLen uint32,
bytesSent *uint32,
overlapped *windows.Overlapped,
) (err error) {
// todo: after upgrading to 1.18, switch from syscall.Syscall9 to syscall.SyscallN
r1, _, e1 := syscall.Syscall9(connectExFunc.addr,
7,
uintptr(s),
uintptr(name),
uintptr(namelen),
uintptr(unsafe.Pointer(sendBuf)),
uintptr(sendDataLen),
uintptr(unsafe.Pointer(bytesSent)),
uintptr(unsafe.Pointer(overlapped)),
0,
0)
if r1 == 0 {
if e1 != 0 {
err = error(e1)
} else {
err = syscall.EINVAL
}
}
return err
}

@ -0,0 +1,72 @@
//go:build windows
// Code generated by 'go generate' using "github.com/Microsoft/go-winio/tools/mkwinsyscall"; DO NOT EDIT.
package socket
import (
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
var _ unsafe.Pointer
// Do the interface allocations only once for common
// Errno values.
const (
errnoERROR_IO_PENDING = 997
)
var (
errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
errERROR_EINVAL error = syscall.EINVAL
)
// errnoErr returns common boxed Errno values, to prevent
// allocations at runtime.
func errnoErr(e syscall.Errno) error {
switch e {
case 0:
return errERROR_EINVAL
case errnoERROR_IO_PENDING:
return errERROR_IO_PENDING
}
// TODO: add more here, after collecting data on the common
// error values see on Windows. (perhaps when running
// all.bat?)
return e
}
var (
modws2_32 = windows.NewLazySystemDLL("ws2_32.dll")
procbind = modws2_32.NewProc("bind")
procgetpeername = modws2_32.NewProc("getpeername")
procgetsockname = modws2_32.NewProc("getsockname")
)
func bind(s windows.Handle, name unsafe.Pointer, namelen int32) (err error) {
r1, _, e1 := syscall.Syscall(procbind.Addr(), 3, uintptr(s), uintptr(name), uintptr(namelen))
if r1 == socketError {
err = errnoErr(e1)
}
return
}
func getpeername(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) {
r1, _, e1 := syscall.Syscall(procgetpeername.Addr(), 3, uintptr(s), uintptr(name), uintptr(unsafe.Pointer(namelen)))
if r1 == socketError {
err = errnoErr(e1)
}
return
}
func getsockname(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) {
r1, _, e1 := syscall.Syscall(procgetsockname.Addr(), 3, uintptr(s), uintptr(name), uintptr(unsafe.Pointer(namelen)))
if r1 == socketError {
err = errnoErr(e1)
}
return
}

@ -1,3 +1,4 @@
//go:build windows
// +build windows
package winio
@ -13,6 +14,8 @@ import (
"syscall"
"time"
"unsafe"
"golang.org/x/sys/windows"
)
//sys connectNamedPipe(pipe syscall.Handle, o *syscall.Overlapped) (err error) = ConnectNamedPipe
@ -21,10 +24,10 @@ import (
//sys getNamedPipeInfo(pipe syscall.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) = GetNamedPipeInfo
//sys getNamedPipeHandleState(pipe syscall.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) = GetNamedPipeHandleStateW
//sys localAlloc(uFlags uint32, length uint32) (ptr uintptr) = LocalAlloc
//sys ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntstatus) = ntdll.NtCreateNamedPipeFile
//sys rtlNtStatusToDosError(status ntstatus) (winerr error) = ntdll.RtlNtStatusToDosErrorNoTeb
//sys rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntstatus) = ntdll.RtlDosPathNameToNtPathName_U
//sys rtlDefaultNpAcl(dacl *uintptr) (status ntstatus) = ntdll.RtlDefaultNpAcl
//sys ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntStatus) = ntdll.NtCreateNamedPipeFile
//sys rtlNtStatusToDosError(status ntStatus) (winerr error) = ntdll.RtlNtStatusToDosErrorNoTeb
//sys rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntStatus) = ntdll.RtlDosPathNameToNtPathName_U
//sys rtlDefaultNpAcl(dacl *uintptr) (status ntStatus) = ntdll.RtlDefaultNpAcl
type ioStatusBlock struct {
Status, Information uintptr
@ -51,45 +54,22 @@ type securityDescriptor struct {
Control uint16
Owner uintptr
Group uintptr
Sacl uintptr
Dacl uintptr
Sacl uintptr //revive:disable-line:var-naming SACL, not Sacl
Dacl uintptr //revive:disable-line:var-naming DACL, not Dacl
}
type ntstatus int32
type ntStatus int32
func (status ntstatus) Err() error {
func (status ntStatus) Err() error {
if status >= 0 {
return nil
}
return rtlNtStatusToDosError(status)
}
const (
cERROR_PIPE_BUSY = syscall.Errno(231)
cERROR_NO_DATA = syscall.Errno(232)
cERROR_PIPE_CONNECTED = syscall.Errno(535)
cERROR_SEM_TIMEOUT = syscall.Errno(121)
cSECURITY_SQOS_PRESENT = 0x100000
cSECURITY_ANONYMOUS = 0
cPIPE_TYPE_MESSAGE = 4
cPIPE_READMODE_MESSAGE = 2
cFILE_OPEN = 1
cFILE_CREATE = 2
cFILE_PIPE_MESSAGE_TYPE = 1
cFILE_PIPE_REJECT_REMOTE_CLIENTS = 2
cSE_DACL_PRESENT = 4
)
var (
// ErrPipeListenerClosed is returned for pipe operations on listeners that have been closed.
// This error should match net.errClosing since docker takes a dependency on its text.
ErrPipeListenerClosed = errors.New("use of closed network connection")
ErrPipeListenerClosed = net.ErrClosed
errPipeWriteClosed = errors.New("pipe has been closed for write")
)
@ -116,9 +96,10 @@ func (f *win32Pipe) RemoteAddr() net.Addr {
}
func (f *win32Pipe) SetDeadline(t time.Time) error {
f.SetReadDeadline(t)
f.SetWriteDeadline(t)
return nil
if err := f.SetReadDeadline(t); err != nil {
return err
}
return f.SetWriteDeadline(t)
}
// CloseWrite closes the write side of a message pipe in byte mode.
@ -157,14 +138,14 @@ func (f *win32MessageBytePipe) Read(b []byte) (int, error) {
return 0, io.EOF
}
n, err := f.win32File.Read(b)
if err == io.EOF {
if err == io.EOF { //nolint:errorlint
// If this was the result of a zero-byte read, then
// it is possible that the read was due to a zero-size
// message. Since we are simulating CloseWrite with a
// zero-byte message, ensure that all future Read() calls
// also return EOF.
f.readEOF = true
} else if err == syscall.ERROR_MORE_DATA {
} else if err == syscall.ERROR_MORE_DATA { //nolint:errorlint // err is Errno
// ERROR_MORE_DATA indicates that the pipe's read mode is message mode
// and the message still has more bytes. Treat this as a success, since
// this package presents all named pipes as byte streams.
@ -173,7 +154,7 @@ func (f *win32MessageBytePipe) Read(b []byte) (int, error) {
return n, err
}
func (s pipeAddress) Network() string {
func (pipeAddress) Network() string {
return "pipe"
}
@ -184,16 +165,21 @@ func (s pipeAddress) String() string {
// tryDialPipe attempts to dial the pipe at `path` until `ctx` cancellation or timeout.
func tryDialPipe(ctx context.Context, path *string, access uint32) (syscall.Handle, error) {
for {
select {
case <-ctx.Done():
return syscall.Handle(0), ctx.Err()
default:
h, err := createFile(*path, access, 0, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_OVERLAPPED|cSECURITY_SQOS_PRESENT|cSECURITY_ANONYMOUS, 0)
h, err := createFile(*path,
access,
0,
nil,
syscall.OPEN_EXISTING,
windows.FILE_FLAG_OVERLAPPED|windows.SECURITY_SQOS_PRESENT|windows.SECURITY_ANONYMOUS,
0)
if err == nil {
return h, nil
}
if err != cERROR_PIPE_BUSY {
if err != windows.ERROR_PIPE_BUSY { //nolint:errorlint // err is Errno
return h, &os.PathError{Err: err, Op: "open", Path: *path}
}
// Wait 10 msec and try again. This is a rather simplistic
@ -213,9 +199,10 @@ func DialPipe(path string, timeout *time.Duration) (net.Conn, error) {
} else {
absTimeout = time.Now().Add(2 * time.Second)
}
ctx, _ := context.WithDeadline(context.Background(), absTimeout)
ctx, cancel := context.WithDeadline(context.Background(), absTimeout)
defer cancel()
conn, err := DialPipeContext(ctx, path)
if err == context.DeadlineExceeded {
if errors.Is(err, context.DeadlineExceeded) {
return nil, ErrTimeout
}
return conn, err
@ -251,7 +238,7 @@ func DialPipeAccess(ctx context.Context, path string, access uint32) (net.Conn,
// If the pipe is in message mode, return a message byte pipe, which
// supports CloseWrite().
if flags&cPIPE_TYPE_MESSAGE != 0 {
if flags&windows.PIPE_TYPE_MESSAGE != 0 {
return &win32MessageBytePipe{
win32Pipe: win32Pipe{win32File: f, path: path},
}, nil
@ -283,7 +270,11 @@ func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (sy
oa.Length = unsafe.Sizeof(oa)
var ntPath unicodeString
if err := rtlDosPathNameToNtPathName(&path16[0], &ntPath, 0, 0).Err(); err != nil {
if err := rtlDosPathNameToNtPathName(&path16[0],
&ntPath,
0,
0,
).Err(); err != nil {
return 0, &os.PathError{Op: "open", Path: path, Err: err}
}
defer localFree(ntPath.Buffer)
@ -292,8 +283,8 @@ func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (sy
// The security descriptor is only needed for the first pipe.
if first {
if sd != nil {
len := uint32(len(sd))
sdb := localAlloc(0, len)
l := uint32(len(sd))
sdb := localAlloc(0, l)
defer localFree(sdb)
copy((*[0xffff]byte)(unsafe.Pointer(sdb))[:], sd)
oa.SecurityDescriptor = (*securityDescriptor)(unsafe.Pointer(sdb))
@ -301,28 +292,28 @@ func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (sy
// Construct the default named pipe security descriptor.
var dacl uintptr
if err := rtlDefaultNpAcl(&dacl).Err(); err != nil {
return 0, fmt.Errorf("getting default named pipe ACL: %s", err)
return 0, fmt.Errorf("getting default named pipe ACL: %w", err)
}
defer localFree(dacl)
sdb := &securityDescriptor{
Revision: 1,
Control: cSE_DACL_PRESENT,
Control: windows.SE_DACL_PRESENT,
Dacl: dacl,
}
oa.SecurityDescriptor = sdb
}
}
typ := uint32(cFILE_PIPE_REJECT_REMOTE_CLIENTS)
typ := uint32(windows.FILE_PIPE_REJECT_REMOTE_CLIENTS)
if c.MessageMode {
typ |= cFILE_PIPE_MESSAGE_TYPE
typ |= windows.FILE_PIPE_MESSAGE_TYPE
}
disposition := uint32(cFILE_OPEN)
disposition := uint32(windows.FILE_OPEN)
access := uint32(syscall.GENERIC_READ | syscall.GENERIC_WRITE | syscall.SYNCHRONIZE)
if first {
disposition = cFILE_CREATE
disposition = windows.FILE_CREATE
// By not asking for read or write access, the named pipe file system
// will put this pipe into an initially disconnected state, blocking
// client connections until the next call with first == false.
@ -335,7 +326,20 @@ func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (sy
h syscall.Handle
iosb ioStatusBlock
)
err = ntCreateNamedPipeFile(&h, access, &oa, &iosb, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE, disposition, 0, typ, 0, 0, 0xffffffff, uint32(c.InputBufferSize), uint32(c.OutputBufferSize), &timeout).Err()
err = ntCreateNamedPipeFile(&h,
access,
&oa,
&iosb,
syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE,
disposition,
0,
typ,
0,
0,
0xffffffff,
uint32(c.InputBufferSize),
uint32(c.OutputBufferSize),
&timeout).Err()
if err != nil {
return 0, &os.PathError{Op: "open", Path: path, Err: err}
}
@ -380,7 +384,7 @@ func (l *win32PipeListener) makeConnectedServerPipe() (*win32File, error) {
p.Close()
p = nil
err = <-ch
if err == nil || err == ErrFileClosed {
if err == nil || err == ErrFileClosed { //nolint:errorlint // err is Errno
err = ErrPipeListenerClosed
}
}
@ -402,12 +406,12 @@ func (l *win32PipeListener) listenerRoutine() {
p, err = l.makeConnectedServerPipe()
// If the connection was immediately closed by the client, try
// again.
if err != cERROR_NO_DATA {
if err != windows.ERROR_NO_DATA { //nolint:errorlint // err is Errno
break
}
}
responseCh <- acceptResponse{p, err}
closed = err == ErrPipeListenerClosed
closed = err == ErrPipeListenerClosed //nolint:errorlint // err is Errno
}
}
syscall.Close(l.firstHandle)
@ -469,15 +473,15 @@ func ListenPipe(path string, c *PipeConfig) (net.Listener, error) {
}
func connectPipe(p *win32File) error {
c, err := p.prepareIo()
c, err := p.prepareIO()
if err != nil {
return err
}
defer p.wg.Done()
err = connectNamedPipe(p.handle, &c.o)
_, err = p.asyncIo(c, nil, 0, err)
if err != nil && err != cERROR_PIPE_CONNECTED {
_, err = p.asyncIO(c, nil, 0, err)
if err != nil && err != windows.ERROR_PIPE_CONNECTED { //nolint:errorlint // err is Errno
return err
}
return nil

@ -1,5 +1,3 @@
// +build windows
// Package guid provides a GUID type. The backing structure for a GUID is
// identical to that used by the golang.org/x/sys/windows GUID type.
// There are two main binary encodings used for a GUID, the big-endian encoding,
@ -9,24 +7,26 @@ package guid
import (
"crypto/rand"
"crypto/sha1"
"crypto/sha1" //nolint:gosec // not used for secure application
"encoding"
"encoding/binary"
"fmt"
"strconv"
)
//go:generate go run golang.org/x/tools/cmd/stringer -type=Variant -trimprefix=Variant -linecomment
// Variant specifies which GUID variant (or "type") of the GUID. It determines
// how the entirety of the rest of the GUID is interpreted.
type Variant uint8
// The variants specified by RFC 4122.
// The variants specified by RFC 4122 section 4.1.1.
const (
// VariantUnknown specifies a GUID variant which does not conform to one of
// the variant encodings specified in RFC 4122.
VariantUnknown Variant = iota
VariantNCS
VariantRFC4122
VariantRFC4122 // RFC 4122
VariantMicrosoft
VariantFuture
)
@ -36,6 +36,10 @@ const (
// hash of an input string.
type Version uint8
func (v Version) String() string {
return strconv.FormatUint(uint64(v), 10)
}
var _ = (encoding.TextMarshaler)(GUID{})
var _ = (encoding.TextUnmarshaler)(&GUID{})
@ -61,7 +65,7 @@ func NewV4() (GUID, error) {
// big-endian UTF16 stream of bytes. If that is desired, the string can be
// encoded as such before being passed to this function.
func NewV5(namespace GUID, name []byte) (GUID, error) {
b := sha1.New()
b := sha1.New() //nolint:gosec // not used for secure application
namespaceBytes := namespace.ToArray()
b.Write(namespaceBytes[:])
b.Write(name)

@ -1,3 +1,4 @@
//go:build !windows
// +build !windows
package guid

@ -1,3 +1,6 @@
//go:build windows
// +build windows
package guid
import "golang.org/x/sys/windows"

@ -0,0 +1,27 @@
// Code generated by "stringer -type=Variant -trimprefix=Variant -linecomment"; DO NOT EDIT.
package guid
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[VariantUnknown-0]
_ = x[VariantNCS-1]
_ = x[VariantRFC4122-2]
_ = x[VariantMicrosoft-3]
_ = x[VariantFuture-4]
}
const _Variant_name = "UnknownNCSRFC 4122MicrosoftFuture"
var _Variant_index = [...]uint8{0, 7, 10, 18, 27, 33}
func (i Variant) String() string {
if i >= Variant(len(_Variant_index)-1) {
return "Variant(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _Variant_name[_Variant_index[i]:_Variant_index[i+1]]
}

@ -1,3 +1,4 @@
//go:build windows
// +build windows
package winio
@ -24,22 +25,17 @@ import (
//sys lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) = advapi32.LookupPrivilegeDisplayNameW
const (
SE_PRIVILEGE_ENABLED = 2
//revive:disable-next-line:var-naming ALL_CAPS
SE_PRIVILEGE_ENABLED = windows.SE_PRIVILEGE_ENABLED
ERROR_NOT_ALL_ASSIGNED syscall.Errno = 1300
//revive:disable-next-line:var-naming ALL_CAPS
ERROR_NOT_ALL_ASSIGNED syscall.Errno = windows.ERROR_NOT_ALL_ASSIGNED
SeBackupPrivilege = "SeBackupPrivilege"
SeRestorePrivilege = "SeRestorePrivilege"
SeSecurityPrivilege = "SeSecurityPrivilege"
)
const (
securityAnonymous = iota
securityIdentification
securityImpersonation
securityDelegation
)
var (
privNames = make(map[string]uint64)
privNameMutex sync.Mutex
@ -51,11 +47,9 @@ type PrivilegeError struct {
}
func (e *PrivilegeError) Error() string {
s := ""
s := "Could not enable privilege "
if len(e.privileges) > 1 {
s = "Could not enable privileges "
} else {
s = "Could not enable privilege "
}
for i, p := range e.privileges {
if i != 0 {
@ -94,7 +88,7 @@ func RunWithPrivileges(names []string, fn func() error) error {
}
func mapPrivileges(names []string) ([]uint64, error) {
var privileges []uint64
privileges := make([]uint64, 0, len(names))
privNameMutex.Lock()
defer privNameMutex.Unlock()
for _, name := range names {
@ -127,7 +121,7 @@ func enableDisableProcessPrivilege(names []string, action uint32) error {
return err
}
p, _ := windows.GetCurrentProcess()
p := windows.CurrentProcess()
var token windows.Token
err = windows.OpenProcessToken(p, windows.TOKEN_ADJUST_PRIVILEGES|windows.TOKEN_QUERY, &token)
if err != nil {
@ -140,10 +134,10 @@ func enableDisableProcessPrivilege(names []string, action uint32) error {
func adjustPrivileges(token windows.Token, privileges []uint64, action uint32) error {
var b bytes.Buffer
binary.Write(&b, binary.LittleEndian, uint32(len(privileges)))
_ = binary.Write(&b, binary.LittleEndian, uint32(len(privileges)))
for _, p := range privileges {
binary.Write(&b, binary.LittleEndian, p)
binary.Write(&b, binary.LittleEndian, action)
_ = binary.Write(&b, binary.LittleEndian, p)
_ = binary.Write(&b, binary.LittleEndian, action)
}
prevState := make([]byte, b.Len())
reqSize := uint32(0)
@ -151,7 +145,7 @@ func adjustPrivileges(token windows.Token, privileges []uint64, action uint32) e
if !success {
return err
}
if err == ERROR_NOT_ALL_ASSIGNED {
if err == ERROR_NOT_ALL_ASSIGNED { //nolint:errorlint // err is Errno
return &PrivilegeError{privileges}
}
return nil
@ -177,7 +171,7 @@ func getPrivilegeName(luid uint64) string {
}
func newThreadToken() (windows.Token, error) {
err := impersonateSelf(securityImpersonation)
err := impersonateSelf(windows.SecurityImpersonation)
if err != nil {
return 0, err
}

@ -1,3 +1,6 @@
//go:build windows
// +build windows
package winio
import (
@ -113,16 +116,16 @@ func EncodeReparsePoint(rp *ReparsePoint) []byte {
}
var b bytes.Buffer
binary.Write(&b, binary.LittleEndian, &data)
_ = binary.Write(&b, binary.LittleEndian, &data)
if !rp.IsMountPoint {
flags := uint32(0)
if relative {
flags |= 1
}
binary.Write(&b, binary.LittleEndian, flags)
_ = binary.Write(&b, binary.LittleEndian, flags)
}
binary.Write(&b, binary.LittleEndian, ntTarget16)
binary.Write(&b, binary.LittleEndian, target16)
_ = binary.Write(&b, binary.LittleEndian, ntTarget16)
_ = binary.Write(&b, binary.LittleEndian, target16)
return b.Bytes()
}

@ -1,23 +1,25 @@
//go:build windows
// +build windows
package winio
import (
"errors"
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
//sys lookupAccountName(systemName *uint16, accountName string, sid *byte, sidSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) = advapi32.LookupAccountNameW
//sys lookupAccountSid(systemName *uint16, sid *byte, name *uint16, nameSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) = advapi32.LookupAccountSidW
//sys convertSidToStringSid(sid *byte, str **uint16) (err error) = advapi32.ConvertSidToStringSidW
//sys convertStringSidToSid(str *uint16, sid **byte) (err error) = advapi32.ConvertStringSidToSidW
//sys convertStringSecurityDescriptorToSecurityDescriptor(str string, revision uint32, sd *uintptr, size *uint32) (err error) = advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW
//sys convertSecurityDescriptorToStringSecurityDescriptor(sd *byte, revision uint32, secInfo uint32, sddl **uint16, sddlSize *uint32) (err error) = advapi32.ConvertSecurityDescriptorToStringSecurityDescriptorW
//sys localFree(mem uintptr) = LocalFree
//sys getSecurityDescriptorLength(sd uintptr) (len uint32) = advapi32.GetSecurityDescriptorLength
const (
cERROR_NONE_MAPPED = syscall.Errno(1332)
)
type AccountLookupError struct {
Name string
Err error
@ -28,8 +30,10 @@ func (e *AccountLookupError) Error() string {
return "lookup account: empty account name specified"
}
var s string
switch e.Err {
case cERROR_NONE_MAPPED:
switch {
case errors.Is(e.Err, windows.ERROR_INVALID_SID):
s = "the security ID structure is invalid"
case errors.Is(e.Err, windows.ERROR_NONE_MAPPED):
s = "not found"
default:
s = e.Err.Error()
@ -37,6 +41,8 @@ func (e *AccountLookupError) Error() string {
return "lookup account " + e.Name + ": " + s
}
func (e *AccountLookupError) Unwrap() error { return e.Err }
type SddlConversionError struct {
Sddl string
Err error
@ -46,15 +52,19 @@ func (e *SddlConversionError) Error() string {
return "convert " + e.Sddl + ": " + e.Err.Error()
}
func (e *SddlConversionError) Unwrap() error { return e.Err }
// LookupSidByName looks up the SID of an account by name
//
//revive:disable-next-line:var-naming SID, not Sid
func LookupSidByName(name string) (sid string, err error) {
if name == "" {
return "", &AccountLookupError{name, cERROR_NONE_MAPPED}
return "", &AccountLookupError{name, windows.ERROR_NONE_MAPPED}
}
var sidSize, sidNameUse, refDomainSize uint32
err = lookupAccountName(nil, name, nil, &sidSize, nil, &refDomainSize, &sidNameUse)
if err != nil && err != syscall.ERROR_INSUFFICIENT_BUFFER {
if err != nil && err != syscall.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // err is Errno
return "", &AccountLookupError{name, err}
}
sidBuffer := make([]byte, sidSize)
@ -73,6 +83,42 @@ func LookupSidByName(name string) (sid string, err error) {
return sid, nil
}
// LookupNameBySid looks up the name of an account by SID
//
//revive:disable-next-line:var-naming SID, not Sid
func LookupNameBySid(sid string) (name string, err error) {
if sid == "" {
return "", &AccountLookupError{sid, windows.ERROR_NONE_MAPPED}
}
sidBuffer, err := windows.UTF16PtrFromString(sid)
if err != nil {
return "", &AccountLookupError{sid, err}
}
var sidPtr *byte
if err = convertStringSidToSid(sidBuffer, &sidPtr); err != nil {
return "", &AccountLookupError{sid, err}
}
defer localFree(uintptr(unsafe.Pointer(sidPtr)))
var nameSize, refDomainSize, sidNameUse uint32
err = lookupAccountSid(nil, sidPtr, nil, &nameSize, nil, &refDomainSize, &sidNameUse)
if err != nil && err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // err is Errno
return "", &AccountLookupError{sid, err}
}
nameBuffer := make([]uint16, nameSize)
refDomainBuffer := make([]uint16, refDomainSize)
err = lookupAccountSid(nil, sidPtr, &nameBuffer[0], &nameSize, &refDomainBuffer[0], &refDomainSize, &sidNameUse)
if err != nil {
return "", &AccountLookupError{sid, err}
}
name = windows.UTF16ToString(nameBuffer)
return name, nil
}
func SddlToSecurityDescriptor(sddl string) ([]byte, error) {
var sdBuffer uintptr
err := convertStringSecurityDescriptorToSecurityDescriptor(sddl, 1, &sdBuffer, nil)
@ -87,7 +133,7 @@ func SddlToSecurityDescriptor(sddl string) ([]byte, error) {
func SecurityDescriptorToSddl(sd []byte) (string, error) {
var sddl *uint16
// The returned string length seems to including an aribtrary number of terminating NULs.
// The returned string length seems to include an arbitrary number of terminating NULs.
// Don't use it.
err := convertSecurityDescriptorToStringSecurityDescriptor(&sd[0], 1, 0xff, &sddl, nil)
if err != nil {

@ -1,3 +1,5 @@
//go:build windows
package winio
//go:generate go run golang.org/x/sys/windows/mkwinsyscall -output zsyscall_windows.go file.go pipe.go sd.go fileinfo.go privilege.go backup.go hvsock.go
//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zsyscall_windows.go ./*.go

@ -0,0 +1,5 @@
//go:build tools
package winio
import _ "golang.org/x/tools/cmd/stringer"

@ -1,4 +1,6 @@
// Code generated by 'go generate'; DO NOT EDIT.
//go:build windows
// Code generated by 'go generate' using "github.com/Microsoft/go-winio/tools/mkwinsyscall"; DO NOT EDIT.
package winio
@ -47,9 +49,11 @@ var (
procConvertSecurityDescriptorToStringSecurityDescriptorW = modadvapi32.NewProc("ConvertSecurityDescriptorToStringSecurityDescriptorW")
procConvertSidToStringSidW = modadvapi32.NewProc("ConvertSidToStringSidW")
procConvertStringSecurityDescriptorToSecurityDescriptorW = modadvapi32.NewProc("ConvertStringSecurityDescriptorToSecurityDescriptorW")
procConvertStringSidToSidW = modadvapi32.NewProc("ConvertStringSidToSidW")
procGetSecurityDescriptorLength = modadvapi32.NewProc("GetSecurityDescriptorLength")
procImpersonateSelf = modadvapi32.NewProc("ImpersonateSelf")
procLookupAccountNameW = modadvapi32.NewProc("LookupAccountNameW")
procLookupAccountSidW = modadvapi32.NewProc("LookupAccountSidW")
procLookupPrivilegeDisplayNameW = modadvapi32.NewProc("LookupPrivilegeDisplayNameW")
procLookupPrivilegeNameW = modadvapi32.NewProc("LookupPrivilegeNameW")
procLookupPrivilegeValueW = modadvapi32.NewProc("LookupPrivilegeValueW")
@ -74,7 +78,6 @@ var (
procRtlDosPathNameToNtPathName_U = modntdll.NewProc("RtlDosPathNameToNtPathName_U")
procRtlNtStatusToDosErrorNoTeb = modntdll.NewProc("RtlNtStatusToDosErrorNoTeb")
procWSAGetOverlappedResult = modws2_32.NewProc("WSAGetOverlappedResult")
procbind = modws2_32.NewProc("bind")
)
func adjustTokenPrivileges(token windows.Token, releaseAll bool, input *byte, outputSize uint32, output *byte, requiredSize *uint32) (success bool, err error) {
@ -123,6 +126,14 @@ func _convertStringSecurityDescriptorToSecurityDescriptor(str *uint16, revision
return
}
func convertStringSidToSid(str *uint16, sid **byte) (err error) {
r1, _, e1 := syscall.Syscall(procConvertStringSidToSidW.Addr(), 2, uintptr(unsafe.Pointer(str)), uintptr(unsafe.Pointer(sid)), 0)
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func getSecurityDescriptorLength(sd uintptr) (len uint32) {
r0, _, _ := syscall.Syscall(procGetSecurityDescriptorLength.Addr(), 1, uintptr(sd), 0, 0)
len = uint32(r0)
@ -154,6 +165,14 @@ func _lookupAccountName(systemName *uint16, accountName *uint16, sid *byte, sidS
return
}
func lookupAccountSid(systemName *uint16, sid *byte, name *uint16, nameSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) {
r1, _, e1 := syscall.Syscall9(procLookupAccountSidW.Addr(), 7, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(nameSize)), uintptr(unsafe.Pointer(refDomain)), uintptr(unsafe.Pointer(refDomainSize)), uintptr(unsafe.Pointer(sidNameUse)), 0, 0)
if r1 == 0 {
err = errnoErr(e1)
}
return
}
func lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) {
var _p0 *uint16
_p0, err = syscall.UTF16PtrFromString(systemName)
@ -380,25 +399,25 @@ func setFileCompletionNotificationModes(h syscall.Handle, flags uint8) (err erro
return
}
func ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntstatus) {
func ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntStatus) {
r0, _, _ := syscall.Syscall15(procNtCreateNamedPipeFile.Addr(), 14, uintptr(unsafe.Pointer(pipe)), uintptr(access), uintptr(unsafe.Pointer(oa)), uintptr(unsafe.Pointer(iosb)), uintptr(share), uintptr(disposition), uintptr(options), uintptr(typ), uintptr(readMode), uintptr(completionMode), uintptr(maxInstances), uintptr(inboundQuota), uintptr(outputQuota), uintptr(unsafe.Pointer(timeout)), 0)
status = ntstatus(r0)
status = ntStatus(r0)
return
}
func rtlDefaultNpAcl(dacl *uintptr) (status ntstatus) {
func rtlDefaultNpAcl(dacl *uintptr) (status ntStatus) {
r0, _, _ := syscall.Syscall(procRtlDefaultNpAcl.Addr(), 1, uintptr(unsafe.Pointer(dacl)), 0, 0)
status = ntstatus(r0)
status = ntStatus(r0)
return
}
func rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntstatus) {
func rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntStatus) {
r0, _, _ := syscall.Syscall6(procRtlDosPathNameToNtPathName_U.Addr(), 4, uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(ntName)), uintptr(filePart), uintptr(reserved), 0, 0)
status = ntstatus(r0)
status = ntStatus(r0)
return
}
func rtlNtStatusToDosError(status ntstatus) (winerr error) {
func rtlNtStatusToDosError(status ntStatus) (winerr error) {
r0, _, _ := syscall.Syscall(procRtlNtStatusToDosErrorNoTeb.Addr(), 1, uintptr(status), 0, 0)
if r0 != 0 {
winerr = syscall.Errno(r0)
@ -417,11 +436,3 @@ func wsaGetOverlappedResult(h syscall.Handle, o *syscall.Overlapped, bytes *uint
}
return
}
func bind(s syscall.Handle, name unsafe.Pointer, namelen int32) (err error) {
r1, _, e1 := syscall.Syscall(procbind.Addr(), 3, uintptr(s), uintptr(name), uintptr(namelen))
if r1 == socketError {
err = errnoErr(e1)
}
return
}

@ -9,6 +9,7 @@ import (
"hash"
"hash/crc64"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
@ -449,46 +450,24 @@ func (bucket Bucket) DeleteObjects(objectKeys []string, options ...Option) (Dele
for _, key := range objectKeys {
dxml.Objects = append(dxml.Objects, DeleteObject{Key: key})
}
isQuiet, _ := FindOption(options, deleteObjectsQuiet, false)
dxml.Quiet = isQuiet.(bool)
bs, err := xml.Marshal(dxml)
if err != nil {
return out, err
}
buffer := new(bytes.Buffer)
buffer.Write(bs)
contentType := http.DetectContentType(buffer.Bytes())
options = append(options, ContentType(contentType))
sum := md5.Sum(bs)
b64 := base64.StdEncoding.EncodeToString(sum[:])
options = append(options, ContentMD5(b64))
params := map[string]interface{}{}
params["delete"] = nil
params["encoding-type"] = "url"
resp, err := bucket.do("POST", "", params, options, buffer, nil)
xmlData := marshalDeleteObjectToXml(dxml)
body, err := bucket.DeleteMultipleObjectsXml(xmlData, options...)
if err != nil {
return out, err
}
defer resp.Body.Close()
deletedResult := DeleteObjectVersionsResult{}
if !dxml.Quiet {
if err = xmlUnmarshal(resp.Body, &deletedResult); err == nil {
if err = xmlUnmarshal(strings.NewReader(body), &deletedResult); err == nil {
err = decodeDeleteObjectsResult(&deletedResult)
}
}
// Keep compatibility:need convert to struct DeleteObjectsResult
out.XMLName = deletedResult.XMLName
for _, v := range deletedResult.DeletedObjectsDetail {
out.DeletedObjects = append(out.DeletedObjects, v.Key)
}
return out, err
}
@ -505,38 +484,48 @@ func (bucket Bucket) DeleteObjectVersions(objectVersions []DeleteObject, options
out := DeleteObjectVersionsResult{}
dxml := deleteXML{}
dxml.Objects = objectVersions
isQuiet, _ := FindOption(options, deleteObjectsQuiet, false)
dxml.Quiet = isQuiet.(bool)
bs, err := xml.Marshal(dxml)
xmlData := marshalDeleteObjectToXml(dxml)
body, err := bucket.DeleteMultipleObjectsXml(xmlData, options...)
if err != nil {
return out, err
}
if !dxml.Quiet {
if err = xmlUnmarshal(strings.NewReader(body), &out); err == nil {
err = decodeDeleteObjectsResult(&out)
}
}
return out, err
}
// DeleteMultipleObjectsXml deletes multiple object or deletes multiple object versions.
//
// xmlData the object keys and versions to delete as the xml format.
// options the options for deleting objects.
//
// string the result response body.
// error it's nil if no error, otherwise it's an error.
//
func (bucket Bucket) DeleteMultipleObjectsXml(xmlData string, options ...Option) (string, error) {
buffer := new(bytes.Buffer)
bs := []byte(xmlData)
buffer.Write(bs)
contentType := http.DetectContentType(buffer.Bytes())
options = append(options, ContentType(contentType))
options = append(options, ContentType("application/xml"))
sum := md5.Sum(bs)
b64 := base64.StdEncoding.EncodeToString(sum[:])
options = append(options, ContentMD5(b64))
params := map[string]interface{}{}
params["delete"] = nil
params["encoding-type"] = "url"
resp, err := bucket.do("POST", "", params, options, buffer, nil)
resp, err := bucket.doInner("POST", "", params, options, buffer, nil)
if err != nil {
return out, err
return "", err
}
defer resp.Body.Close()
if !dxml.Quiet {
if err = xmlUnmarshal(resp.Body, &out); err == nil {
err = decodeDeleteObjectsResult(&out)
}
}
body, err := ioutil.ReadAll(resp.Body)
out := string(body)
return out, err
}
@ -591,7 +580,7 @@ func (bucket Bucket) ListObjects(options ...Option) (ListObjectsResult, error) {
return out, err
}
resp, err := bucket.do("GET", "", params, options, nil, nil)
resp, err := bucket.doInner("GET", "", params, options, nil, nil)
if err != nil {
return out, err
}
@ -619,7 +608,7 @@ func (bucket Bucket) ListObjectsV2(options ...Option) (ListObjectsResultV2, erro
return out, err
}
resp, err := bucket.do("GET", "", params, options, nil, nil)
resp, err := bucket.doInner("GET", "", params, options, nil, nil)
if err != nil {
return out, err
}
@ -645,7 +634,7 @@ func (bucket Bucket) ListObjectVersions(options ...Option) (ListObjectVersionsRe
}
params["versions"] = nil
resp, err := bucket.do("GET", "", params, options, nil, nil)
resp, err := bucket.doInner("GET", "", params, options, nil, nil)
if err != nil {
return out, err
}
@ -901,6 +890,11 @@ func (bucket Bucket) RestoreObjectXML(objectKey, configXML string, options ...Op
// error it's nil if no error, otherwise it's an error object.
//
func (bucket Bucket) SignURL(objectKey string, method HTTPMethod, expiredInSec int64, options ...Option) (string, error) {
err := CheckObjectName(objectKey)
if err != nil {
return "", err
}
if expiredInSec < 0 {
return "", fmt.Errorf("invalid expires: %d, expires must bigger than 0", expiredInSec)
}
@ -1191,11 +1185,6 @@ func (bucket Bucket) GetObjectTagging(objectKey string, options ...Option) (GetO
func (bucket Bucket) DeleteObjectTagging(objectKey string, options ...Option) error {
params, _ := GetRawParams(options)
params["tagging"] = nil
if objectKey == "" {
return fmt.Errorf("invalid argument: object name is empty")
}
resp, err := bucket.do("DELETE", objectKey, params, options, nil, nil)
if err != nil {
return err
@ -1207,7 +1196,7 @@ func (bucket Bucket) DeleteObjectTagging(objectKey string, options ...Option) er
func (bucket Bucket) OptionsMethod(objectKey string, options ...Option) (http.Header, error) {
var out http.Header
resp, err := bucket.do("OPTIONS", objectKey, nil, options, nil, nil)
resp, err := bucket.doInner("OPTIONS", objectKey, nil, options, nil, nil)
if err != nil {
return out, err
}
@ -1219,11 +1208,11 @@ func (bucket Bucket) OptionsMethod(objectKey string, options ...Option) (http.He
// public
func (bucket Bucket) Do(method, objectName string, params map[string]interface{}, options []Option,
data io.Reader, listener ProgressListener) (*Response, error) {
return bucket.do(method, objectName, params, options, data, listener)
return bucket.doInner(method, objectName, params, options, data, listener)
}
// Private
func (bucket Bucket) do(method, objectName string, params map[string]interface{}, options []Option,
func (bucket Bucket) doInner(method, objectName string, params map[string]interface{}, options []Option,
data io.Reader, listener ProgressListener) (*Response, error) {
headers := make(map[string]string)
err := handleOptions(headers, options)
@ -1235,7 +1224,6 @@ func (bucket Bucket) do(method, objectName string, params map[string]interface{}
if len(bucket.BucketName) > 0 && err != nil {
return nil, err
}
resp, err := bucket.Client.Conn.Do(method, bucket.BucketName, objectName,
params, headers, data, 0, listener)
@ -1251,6 +1239,17 @@ func (bucket Bucket) do(method, objectName string, params map[string]interface{}
return resp, err
}
// Private check object name before bucket.do
func (bucket Bucket) do(method, objectName string, params map[string]interface{}, options []Option,
data io.Reader, listener ProgressListener) (*Response, error) {
err := CheckObjectName(objectName)
if err != nil {
return nil, err
}
resp, err := bucket.doInner(method, objectName, params, options, data, listener)
return resp, err
}
func (bucket Bucket) doURL(method HTTPMethod, signedURL string, params map[string]interface{}, options []Option,
data io.Reader, listener ProgressListener) (*Response, error) {
headers := make(map[string]string)

@ -732,6 +732,112 @@ func (client Client) DeleteBucketWebsite(bucketName string, options ...Option) e
return CheckRespCode(resp.StatusCode, []int{http.StatusNoContent})
}
// OpenMetaQuery Enables the metadata management feature for a bucket.
//
// bucketName the bucket name.
//
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) OpenMetaQuery(bucketName string, options ...Option) error {
params := map[string]interface{}{}
params["metaQuery"] = nil
params["comp"] = "add"
resp, err := client.do("POST", bucketName, params, nil, nil, options...)
if err != nil {
return err
}
defer resp.Body.Close()
return CheckRespCode(resp.StatusCode, []int{http.StatusOK})
}
// GetMetaQueryStatus Queries the information about the metadata index library of a bucket.
//
// bucketName the bucket name
//
// GetMetaQueryStatusResult the result object upon successful request. It's only valid when error is nil.
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) GetMetaQueryStatus(bucketName string, options ...Option) (GetMetaQueryStatusResult, error) {
var out GetMetaQueryStatusResult
params := map[string]interface{}{}
params["metaQuery"] = nil
resp, err := client.do("GET", bucketName, params, nil, nil, options...)
if err != nil {
return out, err
}
defer resp.Body.Close()
err = xmlUnmarshal(resp.Body, &out)
return out, err
}
// DoMetaQuery Queries the objects that meet specified conditions and lists the information about objects based on specified fields and sorting methods.
//
// bucketName the bucket name
//
// metaQuery the option of query
//
// DoMetaQueryResult the result object upon successful request. It's only valid when error is nil.
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) DoMetaQuery(bucketName string, metaQuery MetaQuery, options ...Option) (DoMetaQueryResult, error) {
var out DoMetaQueryResult
bs, err := xml.Marshal(metaQuery)
if err != nil {
return out, err
}
out, err = client.DoMetaQueryXml(bucketName, string(bs), options...)
return out, err
}
// DoMetaQuery Queries the objects that meet specified conditions and lists the information about objects based on specified fields and sorting methods.
//
// bucketName the bucket name
//
// metaQuery the option of query
//
// DoMetaQueryResult the result object upon successful request. It's only valid when error is nil.
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) DoMetaQueryXml(bucketName string, metaQueryXml string, options ...Option) (DoMetaQueryResult, error) {
var out DoMetaQueryResult
buffer := new(bytes.Buffer)
buffer.Write([]byte(metaQueryXml))
contentType := http.DetectContentType(buffer.Bytes())
headers := map[string]string{}
headers[HTTPHeaderContentType] = contentType
params := map[string]interface{}{}
params["metaQuery"] = nil
params["comp"] = "query"
resp, err := client.do("POST", bucketName, params, headers, buffer, options...)
if err != nil {
return out, err
}
defer resp.Body.Close()
err = xmlUnmarshal(resp.Body, &out)
return out, err
}
// CloseMetaQuery Disables the metadata management feature for a bucket.
//
// bucketName the bucket name.
//
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) CloseMetaQuery(bucketName string, options ...Option) error {
params := map[string]interface{}{}
params["metaQuery"] = nil
params["comp"] = "delete"
resp, err := client.do("POST", bucketName, params, nil, nil, options...)
if err != nil {
return err
}
defer resp.Body.Close()
return CheckRespCode(resp.StatusCode, []int{http.StatusOK})
}
// GetBucketWebsite gets the bucket's default page (index page) and the error page.
//
// bucketName the bucket name
@ -1087,8 +1193,14 @@ func (client Client) GetBucketTagging(bucketName string, options ...Option) (Get
// error nil if success, otherwise error
//
func (client Client) DeleteBucketTagging(bucketName string, options ...Option) error {
key, _ := FindOption(options, "tagging", nil)
params := map[string]interface{}{}
params["tagging"] = nil
if key == nil {
params["tagging"] = nil
} else {
params["tagging"] = key.(string)
}
resp, err := client.do("DELETE", bucketName, params, nil, nil, options...)
if err != nil {
return err
@ -1946,6 +2058,67 @@ func (client Client) GetBucketReplicationProgress(bucketName string, ruleId stri
return string(data), err
}
// GetBucketAccessMonitor get bucket's access monitor config
// bucketName the bucket name.
// GetBucketAccessMonitorResult the access monitor configuration result of bucket.
// error it's nil if no error, otherwise it's an error object.
func (client Client) GetBucketAccessMonitor(bucketName string, options ...Option) (GetBucketAccessMonitorResult, error) {
var out GetBucketAccessMonitorResult
body, err := client.GetBucketAccessMonitorXml(bucketName, options...)
err = xmlUnmarshal(strings.NewReader(body), &out)
return out, err
}
// GetBucketAccessMonitor get bucket's access monitor config
// bucketName the bucket name.
// string the access monitor configuration result of bucket xml foramt.
// error it's nil if no error, otherwise it's an error object.
func (client Client) GetBucketAccessMonitorXml(bucketName string, options ...Option) (string, error) {
params := map[string]interface{}{}
params["accessmonitor"] = nil
resp, err := client.do("GET", bucketName, params, nil, nil, options...)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
out := string(body)
return out, err
}
// PutBucketAccessMonitor get bucket's access monitor config
// bucketName the bucket name.
// accessMonitor the access monitor configuration of bucket.
// error it's nil if no error, otherwise it's an error object.
func (client Client) PutBucketAccessMonitor(bucketName string, accessMonitor PutBucketAccessMonitor, options ...Option) error {
bs, err := xml.Marshal(accessMonitor)
if err != nil {
return err
}
err = client.PutBucketAccessMonitorXml(bucketName, string(bs), options...)
return err
}
// PutBucketAccessMonitor get bucket's access monitor config
// bucketName the bucket name.
// xmlData the access monitor configuration in xml foramt
// error it's nil if no error, otherwise it's an error object.
func (client Client) PutBucketAccessMonitorXml(bucketName string, xmlData string, options ...Option) error {
buffer := new(bytes.Buffer)
buffer.Write([]byte(xmlData))
contentType := http.DetectContentType(buffer.Bytes())
headers := map[string]string{}
headers[HTTPHeaderContentType] = contentType
params := map[string]interface{}{}
params["accessmonitor"] = nil
resp, err := client.do("PUT", bucketName, params, nil, buffer, options...)
if err != nil {
return err
}
defer resp.Body.Close()
return CheckRespCode(resp.StatusCode, []int{http.StatusOK})
}
// GetBucketCname get bucket's binding cname
// bucketName the bucket name.
// string the xml configuration of bucket.

@ -48,7 +48,8 @@ var signKeyList = []string{"acl", "uploads", "location", "cors",
"worm", "wormId", "wormExtend", "withHashContext",
"x-oss-enable-md5", "x-oss-enable-sha1", "x-oss-enable-sha256",
"x-oss-hash-ctx", "x-oss-md5-ctx", "transferAcceleration",
"regionList", "cloudboxes",
"regionList", "cloudboxes", "x-oss-ac-source-ip", "x-oss-ac-subnet-mask", "x-oss-ac-vpc-id", "x-oss-ac-forward-allow",
"metaQuery",
}
// init initializes Conn
@ -802,7 +803,7 @@ func (um *urlMaker) Init(endpoint string, isCname bool, isProxy bool) error {
host, _, err := net.SplitHostPort(um.NetLoc)
if err != nil {
host = um.NetLoc
if host[0] == '[' && host[len(host)-1] == ']' {
if len(host) > 0 && host[0] == '[' && host[len(host)-1] == ']' {
host = host[1 : len(host)-1]
}
}

@ -206,6 +206,7 @@ const (
HTTPHeaderAllowSameActionOverLap = "X-Oss-Allow-Same-Action-Overlap"
HttpHeaderOssDate = "X-Oss-Date"
HttpHeaderOssContentSha256 = "X-Oss-Content-Sha256"
HttpHeaderOssNotification = "X-Oss-Notification"
)
// HTTP Param
@ -239,7 +240,7 @@ const (
DefaultContentSha256 = "UNSIGNED-PAYLOAD" // for v4 signature
Version = "v2.2.4" // Go SDK version
Version = "v2.2.6" // Go SDK version
)
// FrameType

@ -1,3 +1,4 @@
//go:build !go1.7
// +build !go1.7
// "golang.org/x/time/rate" is depended on golang context package go1.7 onward

@ -1,3 +1,4 @@
//go:build go1.7
// +build go1.7
package oss

@ -203,7 +203,7 @@ func (bucket Bucket) ListLiveChannel(options ...Option) (ListLiveChannelResult,
params["live"] = nil
resp, err := bucket.do("GET", "", params, nil, nil, nil)
resp, err := bucket.doInner("GET", "", params, nil, nil, nil)
if err != nil {
return out, err
}

@ -567,6 +567,28 @@ func TypeByExtension(filePath string) string {
typ := mime.TypeByExtension(path.Ext(filePath))
if typ == "" {
typ = extToMimeType[strings.ToLower(path.Ext(filePath))]
} else {
if strings.HasPrefix(typ, "text/") && strings.Contains(typ, "charset=") {
typ = removeCharsetInMimeType(typ)
}
}
return typ
}
// Remove charset from mime type
func removeCharsetInMimeType(typ string) (str string) {
temArr := strings.Split(typ, ";")
var builder strings.Builder
for i, s := range temArr {
tmpStr := strings.Trim(s, " ")
if strings.Contains(tmpStr, "charset=") {
continue
}
if i == 0 {
builder.WriteString(s)
} else {
builder.WriteString("; " + s)
}
}
return builder.String()
}

@ -290,7 +290,7 @@ func (bucket Bucket) ListMultipartUploads(options ...Option) (ListMultipartUploa
}
params["uploads"] = nil
resp, err := bucket.do("GET", "", params, options, nil, nil)
resp, err := bucket.doInner("GET", "", params, options, nil, nil)
if err != nil {
return out, err
}

@ -10,7 +10,7 @@ type ProgressEventType int
const (
// TransferStartedEvent transfer started, set TotalBytes
TransferStartedEvent ProgressEventType = 1 + iota
// TransferDataEvent transfer data, set ConsumedBytes anmd TotalBytes
// TransferDataEvent transfer data, set ConsumedBytes and TotalBytes
TransferDataEvent
// TransferCompletedEvent transfer completed
TransferCompletedEvent

@ -1,3 +1,4 @@
//go:build !go1.7
// +build !go1.7
package oss

@ -1,3 +1,4 @@
//go:build go1.7
// +build go1.7
package oss

@ -107,6 +107,7 @@ func (sr *SelectObjectResponse) readFrames(p []byte) (int, error) {
if err != nil || !checkValid {
return nn, fmt.Errorf("%s", err.Error())
}
sr.Frame.OpenLine = false
} else if sr.Frame.FrameType == EndFrameType {
err = sr.analysisEndFrame()
if err != nil {

@ -1,3 +1,4 @@
//go:build !go1.7
// +build !go1.7
package oss

@ -1,3 +1,4 @@
//go:build go1.7
// +build go1.7
package oss

@ -5,6 +5,8 @@ import (
"encoding/xml"
"fmt"
"net/url"
"strconv"
"strings"
"time"
)
@ -78,6 +80,7 @@ type LifecycleRule struct {
// Deprecated: Use NonVersionTransitions instead.
NonVersionTransition *LifecycleVersionTransition `xml:"-"` // NonVersionTransition is not suggested to use
NonVersionTransitions []LifecycleVersionTransition `xml:"NoncurrentVersionTransition,omitempty"`
Filter *LifecycleFilter `xml:Filter,omitempty` //condition parameter container of this exclusion rule
}
// LifecycleExpiration defines the rule's expiration property
@ -91,10 +94,13 @@ type LifecycleExpiration struct {
// LifecycleTransition defines the rule's transition propery
type LifecycleTransition struct {
XMLName xml.Name `xml:"Transition"`
Days int `xml:"Days,omitempty"` // Relative transition time: The transition time in days after the last modified time
CreatedBeforeDate string `xml:"CreatedBeforeDate,omitempty"` // objects created before the date will be expired
StorageClass StorageClassType `xml:"StorageClass,omitempty"` // Specifies the target storage type
XMLName xml.Name `xml:"Transition"`
Days int `xml:"Days,omitempty"` // Relative transition time: The transition time in days after the last modified time
CreatedBeforeDate string `xml:"CreatedBeforeDate,omitempty"` // objects created before the date will be expired
StorageClass StorageClassType `xml:"StorageClass,omitempty"` // Specifies the target storage type
IsAccessTime *bool `xml:"IsAccessTime,omitempty"` // access time
ReturnToStdWhenVisit *bool `xml:"ReturnToStdWhenVisit,omitempty"` // Return To Std When Visit
AllowSmallFile *bool `xml:AllowSmallFile,omitempty`
}
// LifecycleAbortMultipartUpload defines the rule's abort multipart upload propery
@ -112,9 +118,25 @@ type LifecycleVersionExpiration struct {
// LifecycleVersionTransition defines the rule's NoncurrentVersionTransition propery
type LifecycleVersionTransition struct {
XMLName xml.Name `xml:"NoncurrentVersionTransition"`
NoncurrentDays int `xml:"NoncurrentDays,omitempty"` // How many days after the Object becomes a non-current version
StorageClass StorageClassType `xml:"StorageClass,omitempty"`
XMLName xml.Name `xml:"NoncurrentVersionTransition"`
NoncurrentDays int `xml:"NoncurrentDays,omitempty"` // How many days after the Object becomes a non-current version
StorageClass StorageClassType `xml:"StorageClass,omitempty"`
IsAccessTime *bool `xml:"IsAccessTime,omitempty"` // access time
ReturnToStdWhenVisit *bool `xml:"ReturnToStdWhenVisit,omitempty"` // Return To Std When Visit
AllowSmallFile *bool `xml:AllowSmallFile,omitempty`
}
// LifecycleFilter defines the rule's Filter propery
type LifecycleFilter struct {
XMLName xml.Name `xml:"Filter"`
Not []LifecycleFilterNot `xml:"Not,omitempty"`
}
// LifecycleFilterNot defines the rule's Filter Not propery
type LifecycleFilterNot struct {
XMLName xml.Name `xml:"Not"`
Prefix string `xml:"Prefix,omitempty"` //Object prefix applicable to this exclusion rule
Tag *Tag `xml:"Tag,omitempty"` //the tags applicable to this exclusion rule
}
const iso8601DateFormat = "2006-01-02T15:04:05.000Z"
@ -321,6 +343,7 @@ type GetBucketInfoResult struct {
type BucketInfo struct {
XMLName xml.Name `xml:"Bucket"`
Name string `xml:"Name"` // Bucket name
AccessMonitor string `xml:"AccessMonitor"` // Bucket Access Monitor
Location string `xml:"Location"` // Bucket datacenter
CreationDate time.Time `xml:"CreationDate"` // Bucket creation time
ExtranetEndpoint string `xml:"ExtranetEndpoint"` // Bucket external endpoint
@ -358,13 +381,14 @@ type ListObjectsResult struct {
// ObjectProperties defines Objecct properties
type ObjectProperties struct {
XMLName xml.Name `xml:"Contents"`
Key string `xml:"Key"` // Object key
Type string `xml:"Type"` // Object type
Size int64 `xml:"Size"` // Object size
ETag string `xml:"ETag"` // Object ETag
Owner Owner `xml:"Owner"` // Object owner information
LastModified time.Time `xml:"LastModified"` // Object last modified time
StorageClass string `xml:"StorageClass"` // Object storage class (Standard, IA, Archive)
Key string `xml:"Key"` // Object key
Type string `xml:"Type"` // Object type
Size int64 `xml:"Size"` // Object size
ETag string `xml:"ETag"` // Object ETag
Owner Owner `xml:"Owner"` // Object owner information
LastModified time.Time `xml:"LastModified"` // Object last modified time
StorageClass string `xml:"StorageClass"` // Object storage class (Standard, IA, Archive)
RestoreInfo string `xml:"RestoreInfo,omitempty"` // Object restoreInfo
}
// ListObjectsResultV2 defines the result from ListObjectsV2 request
@ -410,15 +434,16 @@ type ObjectDeleteMarkerProperties struct {
type ObjectVersionProperties struct {
XMLName xml.Name `xml:"Version"`
Key string `xml:"Key"` // The Object Key
VersionId string `xml:"VersionId"` // The Object VersionId
IsLatest bool `xml:"IsLatest"` // is latest version or not
LastModified time.Time `xml:"LastModified"` // Object last modified time
Type string `xml:"Type"` // Object type
Size int64 `xml:"Size"` // Object size
ETag string `xml:"ETag"` // Object ETag
StorageClass string `xml:"StorageClass"` // Object storage class (Standard, IA, Archive)
Owner Owner `xml:"Owner"` // bucket owner element
Key string `xml:"Key"` // The Object Key
VersionId string `xml:"VersionId"` // The Object VersionId
IsLatest bool `xml:"IsLatest"` // is latest version or not
LastModified time.Time `xml:"LastModified"` // Object last modified time
Type string `xml:"Type"` // Object type
Size int64 `xml:"Size"` // Object size
ETag string `xml:"ETag"` // Object ETag
StorageClass string `xml:"StorageClass"` // Object storage class (Standard, IA, Archive)
Owner Owner `xml:"Owner"` // bucket owner element
RestoreInfo string `xml:"RestoreInfo,omitempty"` // Object restoreInfo
}
// Owner defines Bucket/Object's owner
@ -766,6 +791,33 @@ func decodeListMultipartUploadResult(result *ListMultipartUploadResult) error {
return nil
}
// marshalDeleteObjectToXml deleteXML struct to xml
func marshalDeleteObjectToXml(dxml deleteXML) string {
var builder strings.Builder
builder.WriteString("<Delete>")
builder.WriteString("<Quiet>")
builder.WriteString(strconv.FormatBool(dxml.Quiet))
builder.WriteString("</Quiet>")
if len(dxml.Objects) > 0 {
for _, object := range dxml.Objects {
builder.WriteString("<Object>")
if object.Key != "" {
builder.WriteString("<Key>")
builder.WriteString(EscapeXml(object.Key))
builder.WriteString("</Key>")
}
if object.VersionId != "" {
builder.WriteString("<VersionId>")
builder.WriteString(object.VersionId)
builder.WriteString("</VersionId>")
}
builder.WriteString("</Object>")
}
}
builder.WriteString("</Delete>")
return builder.String()
}
// createBucketConfiguration defines the configuration for creating a bucket.
type createBucketConfiguration struct {
XMLName xml.Name `xml:"CreateBucketConfiguration"`
@ -914,10 +966,23 @@ type GetBucketEncryptionResult ServerEncryptionRule
type GetBucketTaggingResult Tagging
type BucketStat struct {
XMLName xml.Name `xml:"BucketStat"`
Storage int64 `xml:"Storage"`
ObjectCount int64 `xml:"ObjectCount"`
MultipartUploadCount int64 `xml:"MultipartUploadCount"`
XMLName xml.Name `xml:"BucketStat"`
Storage int64 `xml:"Storage"`
ObjectCount int64 `xml:"ObjectCount"`
MultipartUploadCount int64 `xml:"MultipartUploadCount"`
LiveChannelCount int64 `xml:"LiveChannelCount"`
LastModifiedTime int64 `xml:"LastModifiedTime"`
StandardStorage int64 `xml:"StandardStorage"`
StandardObjectCount int64 `xml:"StandardObjectCount"`
InfrequentAccessStorage int64 `xml:"InfrequentAccessStorage"`
InfrequentAccessRealStorage int64 `xml:"InfrequentAccessRealStorage"`
InfrequentAccessObjectCount int64 `xml:"InfrequentAccessObjectCount"`
ArchiveStorage int64 `xml:"ArchiveStorage"`
ArchiveRealStorage int64 `xml:"ArchiveRealStorage"`
ArchiveObjectCount int64 `xml:"ArchiveObjectCount"`
ColdArchiveStorage int64 `xml:"ColdArchiveStorage"`
ColdArchiveRealStorage int64 `xml:"ColdArchiveRealStorage"`
ColdArchiveObjectCount int64 `xml:"ColdArchiveObjectCount"`
}
type GetBucketStatResult BucketStat
@ -1304,3 +1369,104 @@ type CreateBucketCnameTokenResult CnameTokenXML
// GetBucketCnameTokenResult defines result object for GetBucketCnameToken request
type GetBucketCnameTokenResult CnameTokenXML
// GetMetaQueryStatusResult defines result for GetMetaQueryStatus result
type GetMetaQueryStatusResult GetMetaQueryStatusResultXml
// GetMetaQueryStatusResultXml define get meta query status information
type GetMetaQueryStatusResultXml struct {
XMLName xml.Name `xml:"MetaQueryStatus"`
State string `xml:"State"`
Phase string `xml:"Phase"`
CreateTime string `xml:"CreateTime"`
UpdateTime string `xml:"UpdateTime"`
}
// DoMetaQuery defines meta query struct
type MetaQuery struct {
XMLName xml.Name `xml:"MetaQuery"`
NextToken string `xml:"NextToken,omitempty"`
MaxResults int64 `xml:"MaxResults,omitempty"`
Query string `xml:"Query"`
Sort string `xml:"Sort,omitempty"`
Order string `xml:"Order,omitempty"`
Aggregations []MetaQueryAggregationRequest `xml:"Aggregations>Aggregation,omitempty"`
}
// MetaQueryAggregationRequest defines meta query aggregation request
type MetaQueryAggregationRequest struct {
XMLName xml.Name `xml:"Aggregation"`
Field string `xml:"Field,omitempty"`
Operation string `xml:"Operation,omitempty"`
}
// MetaQueryAggregationResponse defines meta query aggregation response
type MetaQueryAggregationResponse struct {
XMLName xml.Name `xml:"Aggregation"`
Field string `xml:"Field,omitempty"`
Operation string `xml:"Operation,omitempty"`
Value float64 `xml:"Value,omitempty"`
Groups []MetaQueryGroup `xml:"Groups>Group,omitempty"`
}
// DoMetaQueryResult defines result for DoMetaQuery result
type DoMetaQueryResult DoMetaQueryResultXml
// DoMetaQueryResultXml defines do meta query information
type DoMetaQueryResultXml struct {
XMLName xml.Name `xml:"MetaQuery"`
NextToken string `xml:"NextToken,omitempty"` // next token
Files []MetaQueryFile `xml:"Files>File,omitempty"` // file
Aggregations []MetaQueryAggregationResponse `xml:"Aggregations>Aggregation,omitempty"'` // Aggregation
}
// MetaQueryFile defines do meta query result file information
type MetaQueryFile struct {
XMLName xml.Name `xml:"File"`
Filename string `xml:"Filename"` //file name
Size int64 `xml:"Size"` // file size
FileModifiedTime string `xml:"FileModifiedTime"` // file Modified Time
OssObjectType string `xml:"OSSObjectType"` // Oss Object Type
OssStorageClass string `xml:"OSSStorageClass"` // Oss Storage Class
ObjectACL string `xml:"ObjectACL"` // Object Acl
ETag string `xml:"ETag"` // ETag
OssCRC64 string `xml:"OSSCRC64"` // Oss CRC64
OssTaggingCount int64 `xml:"OSSTaggingCount,omitempty"` // Oss Tagging Count
OssTagging []MetaQueryTagging `xml:"OSSTagging>Tagging,omitempty"` // Tagging
OssUserMeta []MetaQueryUserMeta `xml:"OSSUserMeta>UserMeta,omitempty"` // UserMeta
ServerSideEncryption string `xml:"ServerSideEncryption,omitempty"` //Server Side Encryption
ServerSideEncryptionCustomerAlgorithm string `xml:"ServerSideEncryptionCustomerAlgorithm,omitempty"` // Server Side Encryption Customer Algorithm
}
// MetaQueryTagging defines do meta query result tagging information
type MetaQueryTagging struct {
XMLName xml.Name `xml:"Tagging"`
Key string `xml:"Key"`
Value string `xml:"Value"`
}
// MetaQueryUserMeta defines do meta query result user meta information
type MetaQueryUserMeta struct {
XMLName xml.Name `xml:"UserMeta"`
Key string `xml:"Key"`
Value string `xml:"Value"`
}
// MetaQueryGroup defines do meta query result group information
type MetaQueryGroup struct {
XMLName xml.Name `xml:"Group"`
Value string `xml:"Value"`
Count int64 `xml:"Count"`
}
//GetBucketAccessMonitorResult define config for get bucket access monitor
type GetBucketAccessMonitorResult BucketAccessMonitorXml
//BucketAccessMonitor define the xml of bucket access monitor config
type PutBucketAccessMonitor BucketAccessMonitorXml
// GetBucketAccessMonitorXml define get bucket access monitor information
type BucketAccessMonitorXml struct {
XMLName xml.Name `xml:"AccessMonitorConfiguration"`
Status string `xml:"Status"` // access monitor status
}

@ -14,12 +14,25 @@ import (
"strconv"
"strings"
"time"
"unicode/utf8"
)
var sys_name string
var sys_release string
var sys_machine string
var (
escQuot = []byte("&#34;") // shorter than "&quot;"
escApos = []byte("&#39;") // shorter than "&apos;"
escAmp = []byte("&amp;")
escLT = []byte("&lt;")
escGT = []byte("&gt;")
escTab = []byte("&#x9;")
escNL = []byte("&#xA;")
escCR = []byte("&#xD;")
escFFFD = []byte("\uFFFD") // Unicode replacement character
)
func init() {
sys_name = runtime.GOOS
sys_release = "-"
@ -376,6 +389,11 @@ func ChoiceCompletePartOption(options []Option) []Option {
}
}
notification, _ := FindOption(options, HttpHeaderOssNotification, nil)
if notification != nil {
outOption = append(outOption, SetHeader(HttpHeaderOssNotification, notification))
}
return outOption
}
@ -435,6 +453,13 @@ func CheckBucketName(bucketName string) error {
return nil
}
func CheckObjectName(objectName string) error {
if len(objectName) == 0 {
return fmt.Errorf("object name is empty")
}
return nil
}
func GetReaderLen(reader io.Reader) (int64, error) {
var contentLength int64
var err error
@ -532,3 +557,64 @@ func EscapeLFString(str string) string {
}
return log.String()
}
// EscapeString writes to p the properly escaped XML equivalent
// of the plain text data s.
func EscapeXml(s string) string {
var p strings.Builder
var esc []byte
hextable := "0123456789ABCDEF"
escPattern := []byte("&#x00;")
last := 0
for i := 0; i < len(s); {
r, width := utf8.DecodeRuneInString(s[i:])
i += width
switch r {
case '"':
esc = escQuot
case '\'':
esc = escApos
case '&':
esc = escAmp
case '<':
esc = escLT
case '>':
esc = escGT
case '\t':
esc = escTab
case '\n':
esc = escNL
case '\r':
esc = escCR
default:
if !isInCharacterRange(r) || (r == 0xFFFD && width == 1) {
if r >= 0x00 && r < 0x20 {
escPattern[3] = hextable[r>>4]
escPattern[4] = hextable[r&0x0f]
esc = escPattern
} else {
esc = escFFFD
}
break
}
continue
}
p.WriteString(s[last : i-width])
p.Write(esc)
last = i
}
p.WriteString(s[last:])
return p.String()
}
// Decide whether the given rune is in the XML Character Range, per
// the Char production of https://www.xml.com/axml/testaxml.htm,
// Section 2.2 Characters.
func isInCharacterRange(r rune) (inrange bool) {
return r == 0x09 ||
r == 0x0A ||
r == 0x0D ||
r >= 0x20 && r <= 0xD7FF ||
r >= 0xE000 && r <= 0xFFFD ||
r >= 0x10000 && r <= 0x10FFFF
}

@ -1,4 +1,4 @@
# BigCache [![Build Status](https://github.com/allegro/bigcache/workflows/build/badge.svg)](https://github.com/allegro/bigcache/actions?query=workflow%3Abuild)&nbsp;[![Coverage Status](https://coveralls.io/repos/github/allegro/bigcache/badge.svg?branch=master)](https://coveralls.io/github/allegro/bigcache?branch=master)&nbsp;[![GoDoc](https://godoc.org/github.com/allegro/bigcache?status.svg)](https://godoc.org/github.com/allegro/bigcache)&nbsp;[![Go Report Card](https://goreportcard.com/badge/github.com/allegro/bigcache)](https://goreportcard.com/report/github.com/allegro/bigcache)
# BigCache [![Build Status](https://github.com/allegro/bigcache/workflows/build/badge.svg)](https://github.com/allegro/bigcache/actions?query=workflow%3Abuild)&nbsp;[![Coverage Status](https://coveralls.io/repos/github/allegro/bigcache/badge.svg?branch=master)](https://coveralls.io/github/allegro/bigcache?branch=master)&nbsp;[![GoDoc](https://godoc.org/github.com/allegro/bigcache/v3?status.svg)](https://godoc.org/github.com/allegro/bigcache/v3)&nbsp;[![Go Report Card](https://goreportcard.com/badge/github.com/allegro/bigcache/v3)](https://goreportcard.com/report/github.com/allegro/bigcache/v3)
Fast, concurrent, evicting in-memory cache written to keep big number of entries without impact on performance.
BigCache keeps entries on heap but omits GC for them. To achieve that, operations on byte slices take place,
@ -11,9 +11,13 @@ Requires Go 1.12 or newer.
### Simple initialization
```go
import "github.com/allegro/bigcache/v3"
import (
"fmt"
"context"
"github.com/allegro/bigcache/v3"
)
cache, _ := bigcache.NewBigCache(bigcache.DefaultConfig(10 * time.Minute))
cache, _ := bigcache.New(context.Background(), bigcache.DefaultConfig(10 * time.Minute))
cache.Set("my-unique-key", []byte("value"))
@ -71,7 +75,7 @@ config := bigcache.Config {
OnRemoveWithReason: nil,
}
cache, initErr := bigcache.NewBigCache(config)
cache, initErr := bigcache.New(context.Background(), config)
if initErr != nil {
log.Fatal(initErr)
}

@ -1,6 +1,7 @@
package bigcache
import (
"context"
"fmt"
"time"
)
@ -40,12 +41,21 @@ const (
Deleted = RemoveReason(3)
)
// New initialize new instance of BigCache
func New(ctx context.Context, config Config) (*BigCache, error) {
return newBigCache(ctx, config, &systemClock{})
}
// NewBigCache initialize new instance of BigCache
//
// Deprecated: NewBigCache is deprecated, please use New(ctx, config) instead,
// New takes in context and can gracefully
// shutdown with context cancellations
func NewBigCache(config Config) (*BigCache, error) {
return newBigCache(config, &systemClock{})
return newBigCache(context.Background(), config, &systemClock{})
}
func newBigCache(config Config, clock clock) (*BigCache, error) {
func newBigCache(ctx context.Context, config Config, clock clock) (*BigCache, error) {
if !isPowerOfTwo(config.Shards) {
return nil, fmt.Errorf("Shards number must be power of two")
}
@ -94,6 +104,9 @@ func newBigCache(config Config, clock clock) (*BigCache, error) {
defer ticker.Stop()
for {
select {
case <-ctx.Done():
fmt.Println("ctx done, shutting down bigcache cleanup routine")
return
case t := <-ticker.C:
cache.cleanUp(uint64(t.Unix()))
case <-cache.close:
@ -163,6 +176,14 @@ func (c *BigCache) Reset() error {
return nil
}
// ResetStats resets cache stats
func (c *BigCache) ResetStats() error {
for _, shard := range c.shards {
shard.resetStats()
}
return nil
}
// Len computes number of entries in cache
func (c *BigCache) Len() int {
var len int
@ -209,6 +230,9 @@ func (c *BigCache) Iterator() *EntryInfoIterator {
func (c *BigCache) onEvict(oldestEntry []byte, currentTimestamp uint64, evict func(reason RemoveReason) error) bool {
oldestTimestamp := readTimestampFromEntry(oldestEntry)
if currentTimestamp < oldestTimestamp {
return false
}
if currentTimestamp-oldestTimestamp > c.lifeWindow {
evict(Expired)
return true

@ -30,6 +30,7 @@ type cacheShard struct {
hashmapStats map[uint64]uint32
stats Stats
cleanEnabled bool
}
func (s *cacheShard) getWithInfo(key string, hashedKey uint64) (entry []byte, resp Response, err error) {
@ -50,12 +51,11 @@ func (s *cacheShard) getWithInfo(key string, hashedKey uint64) (entry []byte, re
}
entry = readEntry(wrappedEntry)
oldestTimeStamp := readTimestampFromEntry(wrappedEntry)
s.lock.RUnlock()
s.hit(hashedKey)
if currentTime-oldestTimeStamp >= s.lifeWindow {
if s.isExpired(wrappedEntry, currentTime) {
resp.EntryStatus = Expired
}
s.lock.RUnlock()
s.hit(hashedKey)
return entry, resp, nil
}
@ -130,8 +130,10 @@ func (s *cacheShard) set(key string, hashedKey uint64, entry []byte) error {
}
}
if oldestEntry, err := s.entries.Peek(); err == nil {
s.onEvict(oldestEntry, currentTimestamp, s.removeOldestEntry)
if !s.cleanEnabled {
if oldestEntry, err := s.entries.Peek(); err == nil {
s.onEvict(oldestEntry, currentTimestamp, s.removeOldestEntry)
}
}
w := wrapEntry(currentTimestamp, hashedKey, key, entry, &s.entryBuffer)
@ -152,8 +154,10 @@ func (s *cacheShard) set(key string, hashedKey uint64, entry []byte) error {
func (s *cacheShard) addNewWithoutLock(key string, hashedKey uint64, entry []byte) error {
currentTimestamp := uint64(s.clock.Epoch())
if oldestEntry, err := s.entries.Peek(); err == nil {
s.onEvict(oldestEntry, currentTimestamp, s.removeOldestEntry)
if !s.cleanEnabled {
if oldestEntry, err := s.entries.Peek(); err == nil {
s.onEvict(oldestEntry, currentTimestamp, s.removeOldestEntry)
}
}
w := wrapEntry(currentTimestamp, hashedKey, key, entry, &s.entryBuffer)
@ -176,8 +180,10 @@ func (s *cacheShard) setWrappedEntryWithoutLock(currentTimestamp uint64, w []byt
}
}
if oldestEntry, err := s.entries.Peek(); err == nil {
s.onEvict(oldestEntry, currentTimestamp, s.removeOldestEntry)
if !s.cleanEnabled {
if oldestEntry, err := s.entries.Peek(); err == nil {
s.onEvict(oldestEntry, currentTimestamp, s.removeOldestEntry)
}
}
for {
@ -268,14 +274,21 @@ func (s *cacheShard) del(hashedKey uint64) error {
}
func (s *cacheShard) onEvict(oldestEntry []byte, currentTimestamp uint64, evict func(reason RemoveReason) error) bool {
oldestTimestamp := readTimestampFromEntry(oldestEntry)
if currentTimestamp-oldestTimestamp > s.lifeWindow {
if s.isExpired(oldestEntry, currentTimestamp) {
evict(Expired)
return true
}
return false
}
func (s *cacheShard) isExpired(oldestEntry []byte, currentTimestamp uint64) bool {
oldestTimestamp := readTimestampFromEntry(oldestEntry)
if currentTimestamp <= oldestTimestamp { // if currentTimestamp < oldestTimestamp, the result will out of uint64 limits;
return false
}
return currentTimestamp-oldestTimestamp > s.lifeWindow
}
func (s *cacheShard) cleanUp(currentTimestamp uint64) {
s.lock.Lock()
for {
@ -340,6 +353,12 @@ func (s *cacheShard) reset(config Config) {
s.lock.Unlock()
}
func (s *cacheShard) resetStats() {
s.lock.Lock()
s.stats = Stats{}
s.lock.Unlock()
}
func (s *cacheShard) len() int {
s.lock.RLock()
res := len(s.hashmap)
@ -430,5 +449,6 @@ func initNewShard(config Config, callback onRemoveCallback, clock clock) *cacheS
clock: clock,
lifeWindow: uint64(config.LifeWindow.Seconds()),
statsEnabled: config.StatsEnabled,
cleanEnabled: config.CleanWindow > 0,
}
}

@ -31,12 +31,12 @@ func (d *DecodeModelOptions) Set(optFns ...func(*DecodeModelOptions)) {
// allow you to get a list of the partitions in the order the endpoints
// will be resolved in.
//
// resolver, err := endpoints.DecodeModel(reader)
// resolver, err := endpoints.DecodeModel(reader)
//
// partitions := resolver.(endpoints.EnumPartitions).Partitions()
// for _, p := range partitions {
// // ... inspect partitions
// }
// partitions := resolver.(endpoints.EnumPartitions).Partitions()
// for _, p := range partitions {
// // ... inspect partitions
// }
func DecodeModel(r io.Reader, optFns ...func(*DecodeModelOptions)) (Resolver, error) {
var opts DecodeModelOptions
opts.Set(optFns...)

File diff suppressed because it is too large Load Diff

@ -9,7 +9,7 @@
// AWS GovCloud (US) (aws-us-gov).
// .
//
// Enumerating Regions and Endpoint Metadata
// # Enumerating Regions and Endpoint Metadata
//
// Casting the Resolver returned by DefaultResolver to a EnumPartitions interface
// will allow you to get access to the list of underlying Partitions with the
@ -17,22 +17,22 @@
// resolving to a single partition, or enumerate regions, services, and endpoints
// in the partition.
//
// resolver := endpoints.DefaultResolver()
// partitions := resolver.(endpoints.EnumPartitions).Partitions()
// resolver := endpoints.DefaultResolver()
// partitions := resolver.(endpoints.EnumPartitions).Partitions()
//
// for _, p := range partitions {
// fmt.Println("Regions for", p.ID())
// for id, _ := range p.Regions() {
// fmt.Println("*", id)
// }
// for _, p := range partitions {
// fmt.Println("Regions for", p.ID())
// for id, _ := range p.Regions() {
// fmt.Println("*", id)
// }
//
// fmt.Println("Services for", p.ID())
// for id, _ := range p.Services() {
// fmt.Println("*", id)
// }
// }
// fmt.Println("Services for", p.ID())
// for id, _ := range p.Services() {
// fmt.Println("*", id)
// }
// }
//
// Using Custom Endpoints
// # Using Custom Endpoints
//
// The endpoints package also gives you the ability to use your own logic how
// endpoints are resolved. This is a great way to define a custom endpoint
@ -47,20 +47,19 @@
// of Resolver.EndpointFor, converting it to a type that satisfies the
// Resolver interface.
//
// myCustomResolver := func(service, region string, optFns ...func(*endpoints.Options)) (endpoints.ResolvedEndpoint, error) {
// if service == endpoints.S3ServiceID {
// return endpoints.ResolvedEndpoint{
// URL: "s3.custom.endpoint.com",
// SigningRegion: "custom-signing-region",
// }, nil
// }
//
// myCustomResolver := func(service, region string, optFns ...func(*endpoints.Options)) (endpoints.ResolvedEndpoint, error) {
// if service == endpoints.S3ServiceID {
// return endpoints.ResolvedEndpoint{
// URL: "s3.custom.endpoint.com",
// SigningRegion: "custom-signing-region",
// }, nil
// }
// return endpoints.DefaultResolver().EndpointFor(service, region, optFns...)
// }
//
// return endpoints.DefaultResolver().EndpointFor(service, region, optFns...)
// }
//
// sess := session.Must(session.NewSession(&aws.Config{
// Region: aws.String("us-west-2"),
// EndpointResolver: endpoints.ResolverFunc(myCustomResolver),
// }))
// sess := session.Must(session.NewSession(&aws.Config{
// Region: aws.String("us-west-2"),
// EndpointResolver: endpoints.ResolverFunc(myCustomResolver),
// }))
package endpoints

@ -353,10 +353,12 @@ type EnumPartitions interface {
// as the second parameter.
//
// This example shows how to get the regions for DynamoDB in the AWS partition.
// rs, exists := endpoints.RegionsForService(endpoints.DefaultPartitions(), endpoints.AwsPartitionID, endpoints.DynamodbServiceID)
//
// rs, exists := endpoints.RegionsForService(endpoints.DefaultPartitions(), endpoints.AwsPartitionID, endpoints.DynamodbServiceID)
//
// This is equivalent to using the partition directly.
// rs := endpoints.AwsPartition().Services()[endpoints.DynamodbServiceID].Regions()
//
// rs := endpoints.AwsPartition().Services()[endpoints.DynamodbServiceID].Regions()
func RegionsForService(ps []Partition, partitionID, serviceID string) (map[string]Region, bool) {
for _, p := range ps {
if p.ID() != partitionID {
@ -423,8 +425,8 @@ func (p Partition) ID() string { return p.id }
// of new regions and services expansions.
//
// Errors that can be returned.
// * UnknownServiceError
// * UnknownEndpointError
// - UnknownServiceError
// - UnknownEndpointError
func (p Partition) EndpointFor(service, region string, opts ...func(*Options)) (ResolvedEndpoint, error) {
return p.p.EndpointFor(service, region, opts...)
}

@ -5,4 +5,4 @@ package aws
const SDKName = "aws-sdk-go"
// SDKVersion is the version of this SDK
const SDKVersion = "1.44.61"
const SDKVersion = "1.44.161"

@ -1,9 +1,8 @@
package shareddefaults
import (
"os"
"os/user"
"path/filepath"
"runtime"
)
// SharedCredentialsFilename returns the SDK's default file path
@ -31,10 +30,17 @@ func SharedConfigFilename() string {
// UserHomeDir returns the home directory for the user the process is
// running under.
func UserHomeDir() string {
if runtime.GOOS == "windows" { // Windows
return os.Getenv("USERPROFILE")
var home string
home = userHomeDir()
if len(home) > 0 {
return home
}
currUser, _ := user.Current()
if currUser != nil {
home = currUser.HomeDir
}
// *nix
return os.Getenv("HOME")
return home
}

@ -0,0 +1,18 @@
//go:build !go1.12
// +build !go1.12
package shareddefaults
import (
"os"
"runtime"
)
func userHomeDir() string {
if runtime.GOOS == "windows" { // Windows
return os.Getenv("USERPROFILE")
}
// *nix
return os.Getenv("HOME")
}

@ -0,0 +1,13 @@
//go:build go1.12
// +build go1.12
package shareddefaults
import (
"os"
)
func userHomeDir() string {
home, _ := os.UserHomeDir()
return home
}

@ -4,7 +4,6 @@ package jsonutil
import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"math"
"reflect"
@ -16,6 +15,12 @@ import (
"github.com/aws/aws-sdk-go/private/protocol"
)
const (
floatNaN = "NaN"
floatInf = "Infinity"
floatNegInf = "-Infinity"
)
var timeType = reflect.ValueOf(time.Time{}).Type()
var byteSliceType = reflect.ValueOf([]byte{}).Type()
@ -211,10 +216,16 @@ func buildScalar(v reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) erro
buf.Write(strconv.AppendInt(scratch[:0], value.Int(), 10))
case reflect.Float64:
f := value.Float()
if math.IsInf(f, 0) || math.IsNaN(f) {
return &json.UnsupportedValueError{Value: v, Str: strconv.FormatFloat(f, 'f', -1, 64)}
switch {
case math.IsNaN(f):
writeString(floatNaN, buf)
case math.IsInf(f, 1):
writeString(floatInf, buf)
case math.IsInf(f, -1):
writeString(floatNegInf, buf)
default:
buf.Write(strconv.AppendFloat(scratch[:0], f, 'f', -1, 64))
}
buf.Write(strconv.AppendFloat(scratch[:0], f, 'f', -1, 64))
default:
switch converted := value.Interface().(type) {
case time.Time:

@ -6,6 +6,7 @@ import (
"encoding/json"
"fmt"
"io"
"math"
"math/big"
"reflect"
"strings"
@ -258,6 +259,18 @@ func (u unmarshaler) unmarshalScalar(value reflect.Value, data interface{}, tag
return err
}
value.Set(reflect.ValueOf(v))
case *float64:
// These are regular strings when parsed by encoding/json's unmarshaler.
switch {
case strings.EqualFold(d, floatNaN):
value.Set(reflect.ValueOf(aws.Float64(math.NaN())))
case strings.EqualFold(d, floatInf):
value.Set(reflect.ValueOf(aws.Float64(math.Inf(1))))
case strings.EqualFold(d, floatNegInf):
value.Set(reflect.ValueOf(aws.Float64(math.Inf(-1))))
default:
return fmt.Errorf("unknown JSON number value: %s", d)
}
default:
return fmt.Errorf("unsupported value: %v (%s)", value.Interface(), value.Type())
}

@ -3,6 +3,7 @@ package queryutil
import (
"encoding/base64"
"fmt"
"math"
"net/url"
"reflect"
"sort"
@ -13,6 +14,12 @@ import (
"github.com/aws/aws-sdk-go/private/protocol"
)
const (
floatNaN = "NaN"
floatInf = "Infinity"
floatNegInf = "-Infinity"
)
// Parse parses an object i and fills a url.Values object. The isEC2 flag
// indicates if this is the EC2 Query sub-protocol.
func Parse(body url.Values, i interface{}, isEC2 bool) error {
@ -228,9 +235,32 @@ func (q *queryParser) parseScalar(v url.Values, r reflect.Value, name string, ta
case int:
v.Set(name, strconv.Itoa(value))
case float64:
v.Set(name, strconv.FormatFloat(value, 'f', -1, 64))
var str string
switch {
case math.IsNaN(value):
str = floatNaN
case math.IsInf(value, 1):
str = floatInf
case math.IsInf(value, -1):
str = floatNegInf
default:
str = strconv.FormatFloat(value, 'f', -1, 64)
}
v.Set(name, str)
case float32:
v.Set(name, strconv.FormatFloat(float64(value), 'f', -1, 32))
asFloat64 := float64(value)
var str string
switch {
case math.IsNaN(asFloat64):
str = floatNaN
case math.IsInf(asFloat64, 1):
str = floatInf
case math.IsInf(asFloat64, -1):
str = floatNegInf
default:
str = strconv.FormatFloat(asFloat64, 'f', -1, 32)
}
v.Set(name, str)
case time.Time:
const ISO8601UTC = "2006-01-02T15:04:05Z"
format := tag.Get("timestampFormat")

@ -3,6 +3,7 @@ package query
import (
"encoding/xml"
"fmt"
"strings"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
@ -62,7 +63,7 @@ func UnmarshalError(r *request.Request) {
}
r.Error = awserr.NewRequestFailure(
awserr.New(respErr.Code, respErr.Message, nil),
awserr.New(strings.TrimSpace(respErr.Code), strings.TrimSpace(respErr.Message), nil),
r.HTTPResponse.StatusCode,
reqID,
)

@ -6,6 +6,7 @@ import (
"encoding/base64"
"fmt"
"io"
"math"
"net/http"
"net/url"
"path"
@ -20,6 +21,12 @@ import (
"github.com/aws/aws-sdk-go/private/protocol"
)
const (
floatNaN = "NaN"
floatInf = "Infinity"
floatNegInf = "-Infinity"
)
// Whether the byte value can be sent without escaping in AWS URLs
var noEscape [256]bool
@ -302,7 +309,16 @@ func convertType(v reflect.Value, tag reflect.StructTag) (str string, err error)
case int64:
str = strconv.FormatInt(value, 10)
case float64:
str = strconv.FormatFloat(value, 'f', -1, 64)
switch {
case math.IsNaN(value):
str = floatNaN
case math.IsInf(value, 1):
str = floatInf
case math.IsInf(value, -1):
str = floatNegInf
default:
str = strconv.FormatFloat(value, 'f', -1, 64)
}
case time.Time:
format := tag.Get("timestampFormat")
if len(format) == 0 {

@ -6,6 +6,7 @@ import (
"fmt"
"io"
"io/ioutil"
"math"
"net/http"
"reflect"
"strconv"
@ -231,9 +232,20 @@ func unmarshalHeader(v reflect.Value, header string, tag reflect.StructTag) erro
}
v.Set(reflect.ValueOf(&i))
case *float64:
f, err := strconv.ParseFloat(header, 64)
if err != nil {
return err
var f float64
switch {
case strings.EqualFold(header, floatNaN):
f = math.NaN()
case strings.EqualFold(header, floatInf):
f = math.Inf(1)
case strings.EqualFold(header, floatNegInf):
f = math.Inf(-1)
default:
var err error
f, err = strconv.ParseFloat(header, 64)
if err != nil {
return err
}
}
v.Set(reflect.ValueOf(&f))
case *time.Time:

@ -5,6 +5,7 @@ import (
"encoding/base64"
"encoding/xml"
"fmt"
"math"
"reflect"
"sort"
"strconv"
@ -14,6 +15,12 @@ import (
"github.com/aws/aws-sdk-go/private/protocol"
)
const (
floatNaN = "NaN"
floatInf = "Infinity"
floatNegInf = "-Infinity"
)
// BuildXML will serialize params into an xml.Encoder. Error will be returned
// if the serialization of any of the params or nested values fails.
func BuildXML(params interface{}, e *xml.Encoder) error {
@ -275,6 +282,7 @@ func (b *xmlBuilder) buildMap(value reflect.Value, current *XMLNode, tag reflect
// Error will be returned if the value type is unsupported.
func (b *xmlBuilder) buildScalar(value reflect.Value, current *XMLNode, tag reflect.StructTag) error {
var str string
switch converted := value.Interface().(type) {
case string:
str = converted
@ -289,9 +297,29 @@ func (b *xmlBuilder) buildScalar(value reflect.Value, current *XMLNode, tag refl
case int:
str = strconv.Itoa(converted)
case float64:
str = strconv.FormatFloat(converted, 'f', -1, 64)
switch {
case math.IsNaN(converted):
str = floatNaN
case math.IsInf(converted, 1):
str = floatInf
case math.IsInf(converted, -1):
str = floatNegInf
default:
str = strconv.FormatFloat(converted, 'f', -1, 64)
}
case float32:
str = strconv.FormatFloat(float64(converted), 'f', -1, 32)
// The SDK doesn't render float32 values in types, only float64. This case would never be hit currently.
asFloat64 := float64(converted)
switch {
case math.IsNaN(asFloat64):
str = floatNaN
case math.IsInf(asFloat64, 1):
str = floatInf
case math.IsInf(asFloat64, -1):
str = floatNegInf
default:
str = strconv.FormatFloat(asFloat64, 'f', -1, 32)
}
case time.Time:
format := tag.Get("timestampFormat")
if len(format) == 0 {

@ -6,6 +6,7 @@ import (
"encoding/xml"
"fmt"
"io"
"math"
"reflect"
"strconv"
"strings"
@ -276,9 +277,20 @@ func parseScalar(r reflect.Value, node *XMLNode, tag reflect.StructTag) error {
}
r.Set(reflect.ValueOf(&v))
case *float64:
v, err := strconv.ParseFloat(node.Text, 64)
if err != nil {
return err
var v float64
switch {
case strings.EqualFold(node.Text, floatNaN):
v = math.NaN()
case strings.EqualFold(node.Text, floatInf):
v = math.Inf(1)
case strings.EqualFold(node.Text, floatNegInf):
v = math.Inf(-1)
default:
var err error
v, err = strconv.ParseFloat(node.Text, 64)
if err != nil {
return err
}
}
r.Set(reflect.ValueOf(&v))
case *time.Time:

File diff suppressed because it is too large Load Diff

@ -39,11 +39,11 @@ func NormalizeBucketLocation(loc string) string {
// See http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETlocation.html
// for more information on the values that can be returned.
//
// req, result := svc.GetBucketLocationRequest(&s3.GetBucketLocationInput{
// Bucket: aws.String(bucket),
// })
// req.Handlers.Unmarshal.PushBackNamed(NormalizeBucketLocationHandler)
// err := req.Send()
// req, result := svc.GetBucketLocationRequest(&s3.GetBucketLocationInput{
// Bucket: aws.String(bucket),
// })
// req.Handlers.Unmarshal.PushBackNamed(NormalizeBucketLocationHandler)
// err := req.Send()
var NormalizeBucketLocationHandler = request.NamedHandler{
Name: "awssdk.s3.NormalizeBucketLocation",
Fn: func(req *request.Request) {
@ -65,12 +65,12 @@ var NormalizeBucketLocationHandler = request.NamedHandler{
// See http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETlocation.html
// for more information on the values that can be returned.
//
// result, err := svc.GetBucketLocationWithContext(ctx,
// &s3.GetBucketLocationInput{
// Bucket: aws.String(bucket),
// },
// s3.WithNormalizeBucketLocation,
// )
// result, err := svc.GetBucketLocationWithContext(ctx,
// &s3.GetBucketLocationInput{
// Bucket: aws.String(bucket),
// },
// s3.WithNormalizeBucketLocation,
// )
func WithNormalizeBucketLocation(r *request.Request) {
r.Handlers.Unmarshal.PushBackNamed(NormalizeBucketLocationHandler)
}

@ -8,7 +8,7 @@
// See s3 package documentation for more information.
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/
//
// Using the Client
// # Using the Client
//
// To contact Amazon Simple Storage Service with the SDK use the New function to create
// a new service client. With that client you can make API requests to the service.

@ -6,99 +6,99 @@
// for optimizations if the Body satisfies that type. Once the Uploader instance
// is created you can call Upload concurrently from multiple goroutines safely.
//
// // The session the S3 Uploader will use
// sess := session.Must(session.NewSession())
//
// // Create an uploader with the session and default options
// uploader := s3manager.NewUploader(sess)
//
// f, err := os.Open(filename)
// if err != nil {
// return fmt.Errorf("failed to open file %q, %v", filename, err)
// }
//
// // Upload the file to S3.
// result, err := uploader.Upload(&s3manager.UploadInput{
// Bucket: aws.String(myBucket),
// Key: aws.String(myString),
// Body: f,
// })
// if err != nil {
// return fmt.Errorf("failed to upload file, %v", err)
// }
// fmt.Printf("file uploaded to, %s\n", aws.StringValue(result.Location))
// // The session the S3 Uploader will use
// sess := session.Must(session.NewSession())
//
// // Create an uploader with the session and default options
// uploader := s3manager.NewUploader(sess)
//
// f, err := os.Open(filename)
// if err != nil {
// return fmt.Errorf("failed to open file %q, %v", filename, err)
// }
//
// // Upload the file to S3.
// result, err := uploader.Upload(&s3manager.UploadInput{
// Bucket: aws.String(myBucket),
// Key: aws.String(myString),
// Body: f,
// })
// if err != nil {
// return fmt.Errorf("failed to upload file, %v", err)
// }
// fmt.Printf("file uploaded to, %s\n", aws.StringValue(result.Location))
//
// See the s3manager package's Uploader type documentation for more information.
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3manager/#Uploader
//
// Download Manager
// # Download Manager
//
// The s3manager package's Downloader provides concurrently downloading of Objects
// from S3. The Downloader will write S3 Object content with an io.WriterAt.
// Once the Downloader instance is created you can call Download concurrently from
// multiple goroutines safely.
//
// // The session the S3 Downloader will use
// sess := session.Must(session.NewSession())
// // The session the S3 Downloader will use
// sess := session.Must(session.NewSession())
//
// // Create a downloader with the session and default options
// downloader := s3manager.NewDownloader(sess)
// // Create a downloader with the session and default options
// downloader := s3manager.NewDownloader(sess)
//
// // Create a file to write the S3 Object contents to.
// f, err := os.Create(filename)
// if err != nil {
// return fmt.Errorf("failed to create file %q, %v", filename, err)
// }
// // Create a file to write the S3 Object contents to.
// f, err := os.Create(filename)
// if err != nil {
// return fmt.Errorf("failed to create file %q, %v", filename, err)
// }
//
// // Write the contents of S3 Object to the file
// n, err := downloader.Download(f, &s3.GetObjectInput{
// Bucket: aws.String(myBucket),
// Key: aws.String(myString),
// })
// if err != nil {
// return fmt.Errorf("failed to download file, %v", err)
// }
// fmt.Printf("file downloaded, %d bytes\n", n)
// // Write the contents of S3 Object to the file
// n, err := downloader.Download(f, &s3.GetObjectInput{
// Bucket: aws.String(myBucket),
// Key: aws.String(myString),
// })
// if err != nil {
// return fmt.Errorf("failed to download file, %v", err)
// }
// fmt.Printf("file downloaded, %d bytes\n", n)
//
// See the s3manager package's Downloader type documentation for more information.
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3manager/#Downloader
//
// Automatic URI cleaning
// # Automatic URI cleaning
//
// Interacting with objects whose keys contain adjacent slashes (e.g. bucketname/foo//bar/objectname)
// requires setting DisableRestProtocolURICleaning to true in the aws.Config struct
// used by the service client.
//
// svc := s3.New(sess, &aws.Config{
// DisableRestProtocolURICleaning: aws.Bool(true),
// })
// out, err := svc.GetObject(&s3.GetObjectInput {
// Bucket: aws.String("bucketname"),
// Key: aws.String("//foo//bar//moo"),
// })
// svc := s3.New(sess, &aws.Config{
// DisableRestProtocolURICleaning: aws.Bool(true),
// })
// out, err := svc.GetObject(&s3.GetObjectInput {
// Bucket: aws.String("bucketname"),
// Key: aws.String("//foo//bar//moo"),
// })
//
// Get Bucket Region
// # Get Bucket Region
//
// GetBucketRegion will attempt to get the region for a bucket using a region
// hint to determine which AWS partition to perform the query on. Use this utility
// to determine the region a bucket is in.
//
// sess := session.Must(session.NewSession())
// sess := session.Must(session.NewSession())
//
// bucket := "my-bucket"
// region, err := s3manager.GetBucketRegion(ctx, sess, bucket, "us-west-2")
// if err != nil {
// if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" {
// fmt.Fprintf(os.Stderr, "unable to find bucket %s's region not found\n", bucket)
// }
// return err
// }
// fmt.Printf("Bucket %s is in %s region\n", bucket, region)
// bucket := "my-bucket"
// region, err := s3manager.GetBucketRegion(ctx, sess, bucket, "us-west-2")
// if err != nil {
// if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" {
// fmt.Fprintf(os.Stderr, "unable to find bucket %s's region not found\n", bucket)
// }
// return err
// }
// fmt.Printf("Bucket %s is in %s region\n", bucket, region)
//
// See the s3manager package's GetBucketRegion function documentation for more information
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3manager/#GetBucketRegion
//
// S3 Crypto Client
// # S3 Crypto Client
//
// The s3crypto package provides the tools to upload and download encrypted
// content from S3. The Encryption and Decryption clients can be used concurrently
@ -106,5 +106,4 @@
//
// See the s3crypto package documentation for more information.
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3crypto/
//
package s3

@ -52,9 +52,8 @@ func accessPointResourceParser(a awsarn.ARN) (arn.Resource, error) {
// outpost access-point resource.
//
// Supported Outpost AccessPoint ARN format:
// - ARN format: arn:{partition}:s3-outposts:{region}:{accountId}:outpost/{outpostId}/accesspoint/{accesspointName}
// - example: arn:aws:s3-outposts:us-west-2:012345678901:outpost/op-1234567890123456/accesspoint/myaccesspoint
//
// - ARN format: arn:{partition}:s3-outposts:{region}:{accountId}:outpost/{outpostId}/accesspoint/{accesspointName}
// - example: arn:aws:s3-outposts:us-west-2:012345678901:outpost/op-1234567890123456/accesspoint/myaccesspoint
func parseOutpostAccessPointResource(a awsarn.ARN, resParts []string) (arn.OutpostAccessPointARN, error) {
// outpost accesspoint arn is only valid if service is s3-outposts
if a.Service != "s3-outposts" {

@ -37,7 +37,6 @@ type accessPointEndpointBuilder arn.AccessPointARN
// - example : myaccesspoint-012345678901.s3-accesspoint.us-west-2.amazonaws.com
//
// Access Point Endpoint requests are signed using "s3" as signing name.
//
func (a accessPointEndpointBuilder) build(req *request.Request) error {
resolveService := arn.AccessPointARN(a).Service
resolveRegion := arn.AccessPointARN(a).Region
@ -92,7 +91,6 @@ type s3ObjectLambdaAccessPointEndpointBuilder arn.S3ObjectLambdaAccessPointARN
// - example : myaccesspoint-012345678901.s3-object-lambda.us-west-2.amazonaws.com
//
// Access Point Endpoint requests are signed using "s3-object-lambda" as signing name.
//
func (a s3ObjectLambdaAccessPointEndpointBuilder) build(req *request.Request) error {
resolveRegion := arn.S3ObjectLambdaAccessPointARN(a).Region
@ -147,7 +145,6 @@ type outpostAccessPointEndpointBuilder arn.OutpostAccessPointARN
// - example : myaccesspoint-012345678901.op-01234567890123456.s3-outposts.us-west-2.amazonaws.com
//
// Outpost AccessPoint Endpoint request are signed using "s3-outposts" as signing name.
//
func (o outpostAccessPointEndpointBuilder) build(req *request.Request) error {
resolveRegion := o.Region
resolveService := o.Service

@ -39,13 +39,14 @@ const (
// aws.Config parameter to add your extra config.
//
// Example:
// mySession := session.Must(session.NewSession())
//
// // Create a S3 client from just a session.
// svc := s3.New(mySession)
// mySession := session.Must(session.NewSession())
//
// // Create a S3 client with additional configuration
// svc := s3.New(mySession, aws.NewConfig().WithRegion("us-west-2"))
// // Create a S3 client from just a session.
// svc := s3.New(mySession)
//
// // Create a S3 client with additional configuration
// svc := s3.New(mySession, aws.NewConfig().WithRegion("us-west-2"))
func New(p client.ConfigProvider, cfgs ...*aws.Config) *S3 {
c := p.ClientConfig(EndpointsID, cfgs...)
if c.SigningNameDerived || len(c.SigningName) == 0 {

@ -29,14 +29,13 @@ const opGetRoleCredentials = "GetRoleCredentials"
// This method is useful when you want to inject custom logic or configuration
// into the SDK's request lifecycle. Such as custom headers, or retry logic.
//
// // Example sending a request using the GetRoleCredentialsRequest method.
// req, resp := client.GetRoleCredentialsRequest(params)
//
// // Example sending a request using the GetRoleCredentialsRequest method.
// req, resp := client.GetRoleCredentialsRequest(params)
//
// err := req.Send()
// if err == nil { // resp is now filled
// fmt.Println(resp)
// }
// err := req.Send()
// if err == nil { // resp is now filled
// fmt.Println(resp)
// }
//
// See also, https://docs.aws.amazon.com/goto/WebAPI/sso-2019-06-10/GetRoleCredentials
func (c *SSO) GetRoleCredentialsRequest(input *GetRoleCredentialsInput) (req *request.Request, output *GetRoleCredentialsOutput) {
@ -69,20 +68,21 @@ func (c *SSO) GetRoleCredentialsRequest(input *GetRoleCredentialsInput) (req *re
// API operation GetRoleCredentials for usage and error information.
//
// Returned Error Types:
// * InvalidRequestException
// Indicates that a problem occurred with the input to the request. For example,
// a required parameter might be missing or out of range.
//
// * UnauthorizedException
// Indicates that the request is not authorized. This can happen due to an invalid
// access token in the request.
// - InvalidRequestException
// Indicates that a problem occurred with the input to the request. For example,
// a required parameter might be missing or out of range.
//
// - UnauthorizedException
// Indicates that the request is not authorized. This can happen due to an invalid
// access token in the request.
//
// * TooManyRequestsException
// Indicates that the request is being made too frequently and is more than
// what the server can handle.
// - TooManyRequestsException
// Indicates that the request is being made too frequently and is more than
// what the server can handle.
//
// * ResourceNotFoundException
// The specified resource doesn't exist.
// - ResourceNotFoundException
// The specified resource doesn't exist.
//
// See also, https://docs.aws.amazon.com/goto/WebAPI/sso-2019-06-10/GetRoleCredentials
func (c *SSO) GetRoleCredentials(input *GetRoleCredentialsInput) (*GetRoleCredentialsOutput, error) {
@ -122,14 +122,13 @@ const opListAccountRoles = "ListAccountRoles"
// This method is useful when you want to inject custom logic or configuration
// into the SDK's request lifecycle. Such as custom headers, or retry logic.
//
// // Example sending a request using the ListAccountRolesRequest method.
// req, resp := client.ListAccountRolesRequest(params)
//
// // Example sending a request using the ListAccountRolesRequest method.
// req, resp := client.ListAccountRolesRequest(params)
//
// err := req.Send()
// if err == nil { // resp is now filled
// fmt.Println(resp)
// }
// err := req.Send()
// if err == nil { // resp is now filled
// fmt.Println(resp)
// }
//
// See also, https://docs.aws.amazon.com/goto/WebAPI/sso-2019-06-10/ListAccountRoles
func (c *SSO) ListAccountRolesRequest(input *ListAccountRolesInput) (req *request.Request, output *ListAccountRolesOutput) {
@ -167,20 +166,21 @@ func (c *SSO) ListAccountRolesRequest(input *ListAccountRolesInput) (req *reques
// API operation ListAccountRoles for usage and error information.
//
// Returned Error Types:
// * InvalidRequestException
// Indicates that a problem occurred with the input to the request. For example,
// a required parameter might be missing or out of range.
//
// * UnauthorizedException
// Indicates that the request is not authorized. This can happen due to an invalid
// access token in the request.
// - InvalidRequestException
// Indicates that a problem occurred with the input to the request. For example,
// a required parameter might be missing or out of range.
//
// * TooManyRequestsException
// Indicates that the request is being made too frequently and is more than
// what the server can handle.
// - UnauthorizedException
// Indicates that the request is not authorized. This can happen due to an invalid
// access token in the request.
//
// * ResourceNotFoundException
// The specified resource doesn't exist.
// - TooManyRequestsException
// Indicates that the request is being made too frequently and is more than
// what the server can handle.
//
// - ResourceNotFoundException
// The specified resource doesn't exist.
//
// See also, https://docs.aws.amazon.com/goto/WebAPI/sso-2019-06-10/ListAccountRoles
func (c *SSO) ListAccountRoles(input *ListAccountRolesInput) (*ListAccountRolesOutput, error) {
@ -212,15 +212,14 @@ func (c *SSO) ListAccountRolesWithContext(ctx aws.Context, input *ListAccountRol
//
// Note: This operation can generate multiple requests to a service.
//
// // Example iterating over at most 3 pages of a ListAccountRoles operation.
// pageNum := 0
// err := client.ListAccountRolesPages(params,
// func(page *sso.ListAccountRolesOutput, lastPage bool) bool {
// pageNum++
// fmt.Println(page)
// return pageNum <= 3
// })
//
// // Example iterating over at most 3 pages of a ListAccountRoles operation.
// pageNum := 0
// err := client.ListAccountRolesPages(params,
// func(page *sso.ListAccountRolesOutput, lastPage bool) bool {
// pageNum++
// fmt.Println(page)
// return pageNum <= 3
// })
func (c *SSO) ListAccountRolesPages(input *ListAccountRolesInput, fn func(*ListAccountRolesOutput, bool) bool) error {
return c.ListAccountRolesPagesWithContext(aws.BackgroundContext(), input, fn)
}
@ -272,14 +271,13 @@ const opListAccounts = "ListAccounts"
// This method is useful when you want to inject custom logic or configuration
// into the SDK's request lifecycle. Such as custom headers, or retry logic.
//
// // Example sending a request using the ListAccountsRequest method.
// req, resp := client.ListAccountsRequest(params)
//
// // Example sending a request using the ListAccountsRequest method.
// req, resp := client.ListAccountsRequest(params)
//
// err := req.Send()
// if err == nil { // resp is now filled
// fmt.Println(resp)
// }
// err := req.Send()
// if err == nil { // resp is now filled
// fmt.Println(resp)
// }
//
// See also, https://docs.aws.amazon.com/goto/WebAPI/sso-2019-06-10/ListAccounts
func (c *SSO) ListAccountsRequest(input *ListAccountsInput) (req *request.Request, output *ListAccountsOutput) {
@ -310,7 +308,8 @@ func (c *SSO) ListAccountsRequest(input *ListAccountsInput) (req *request.Reques
// Lists all AWS accounts assigned to the user. These AWS accounts are assigned
// by the administrator of the account. For more information, see Assign User
// Access (https://docs.aws.amazon.com/singlesignon/latest/userguide/useraccess.html#assignusers)
// in the AWS SSO User Guide. This operation returns a paginated response.
// in the IAM Identity Center User Guide. This operation returns a paginated
// response.
//
// Returns awserr.Error for service API and SDK errors. Use runtime type assertions
// with awserr.Error's Code and Message methods to get detailed information about
@ -320,20 +319,21 @@ func (c *SSO) ListAccountsRequest(input *ListAccountsInput) (req *request.Reques
// API operation ListAccounts for usage and error information.
//
// Returned Error Types:
// * InvalidRequestException
// Indicates that a problem occurred with the input to the request. For example,
// a required parameter might be missing or out of range.
//
// * UnauthorizedException
// Indicates that the request is not authorized. This can happen due to an invalid
// access token in the request.
// - InvalidRequestException
// Indicates that a problem occurred with the input to the request. For example,
// a required parameter might be missing or out of range.
//
// * TooManyRequestsException
// Indicates that the request is being made too frequently and is more than
// what the server can handle.
// - UnauthorizedException
// Indicates that the request is not authorized. This can happen due to an invalid
// access token in the request.
//
// * ResourceNotFoundException
// The specified resource doesn't exist.
// - TooManyRequestsException
// Indicates that the request is being made too frequently and is more than
// what the server can handle.
//
// - ResourceNotFoundException
// The specified resource doesn't exist.
//
// See also, https://docs.aws.amazon.com/goto/WebAPI/sso-2019-06-10/ListAccounts
func (c *SSO) ListAccounts(input *ListAccountsInput) (*ListAccountsOutput, error) {
@ -365,15 +365,14 @@ func (c *SSO) ListAccountsWithContext(ctx aws.Context, input *ListAccountsInput,
//
// Note: This operation can generate multiple requests to a service.
//
// // Example iterating over at most 3 pages of a ListAccounts operation.
// pageNum := 0
// err := client.ListAccountsPages(params,
// func(page *sso.ListAccountsOutput, lastPage bool) bool {
// pageNum++
// fmt.Println(page)
// return pageNum <= 3
// })
//
// // Example iterating over at most 3 pages of a ListAccounts operation.
// pageNum := 0
// err := client.ListAccountsPages(params,
// func(page *sso.ListAccountsOutput, lastPage bool) bool {
// pageNum++
// fmt.Println(page)
// return pageNum <= 3
// })
func (c *SSO) ListAccountsPages(input *ListAccountsInput, fn func(*ListAccountsOutput, bool) bool) error {
return c.ListAccountsPagesWithContext(aws.BackgroundContext(), input, fn)
}
@ -425,14 +424,13 @@ const opLogout = "Logout"
// This method is useful when you want to inject custom logic or configuration
// into the SDK's request lifecycle. Such as custom headers, or retry logic.
//
// // Example sending a request using the LogoutRequest method.
// req, resp := client.LogoutRequest(params)
//
// // Example sending a request using the LogoutRequest method.
// req, resp := client.LogoutRequest(params)
//
// err := req.Send()
// if err == nil { // resp is now filled
// fmt.Println(resp)
// }
// err := req.Send()
// if err == nil { // resp is now filled
// fmt.Println(resp)
// }
//
// See also, https://docs.aws.amazon.com/goto/WebAPI/sso-2019-06-10/Logout
func (c *SSO) LogoutRequest(input *LogoutInput) (req *request.Request, output *LogoutOutput) {
@ -455,7 +453,21 @@ func (c *SSO) LogoutRequest(input *LogoutInput) (req *request.Request, output *L
// Logout API operation for AWS Single Sign-On.
//
// Removes the client- and server-side session that is associated with the user.
// Removes the locally stored SSO tokens from the client-side cache and sends
// an API call to the IAM Identity Center service to invalidate the corresponding
// server-side IAM Identity Center sign in session.
//
// If a user uses IAM Identity Center to access the AWS CLI, the users IAM
// Identity Center sign in session is used to obtain an IAM session, as specified
// in the corresponding IAM Identity Center permission set. More specifically,
// IAM Identity Center assumes an IAM role in the target account on behalf of
// the user, and the corresponding temporary AWS credentials are returned to
// the client.
//
// After user logout, any existing IAM role sessions that were created by using
// IAM Identity Center permission sets continue based on the duration configured
// in the permission set. For more information, see User authentications (https://docs.aws.amazon.com/singlesignon/latest/userguide/authconcept.html)
// in the IAM Identity Center User Guide.
//
// Returns awserr.Error for service API and SDK errors. Use runtime type assertions
// with awserr.Error's Code and Message methods to get detailed information about
@ -465,17 +477,18 @@ func (c *SSO) LogoutRequest(input *LogoutInput) (req *request.Request, output *L
// API operation Logout for usage and error information.
//
// Returned Error Types:
// * InvalidRequestException
// Indicates that a problem occurred with the input to the request. For example,
// a required parameter might be missing or out of range.
//
// * UnauthorizedException
// Indicates that the request is not authorized. This can happen due to an invalid
// access token in the request.
// - InvalidRequestException
// Indicates that a problem occurred with the input to the request. For example,
// a required parameter might be missing or out of range.
//
// - UnauthorizedException
// Indicates that the request is not authorized. This can happen due to an invalid
// access token in the request.
//
// * TooManyRequestsException
// Indicates that the request is being made too frequently and is more than
// what the server can handle.
// - TooManyRequestsException
// Indicates that the request is being made too frequently and is more than
// what the server can handle.
//
// See also, https://docs.aws.amazon.com/goto/WebAPI/sso-2019-06-10/Logout
func (c *SSO) Logout(input *LogoutInput) (*LogoutOutput, error) {
@ -554,7 +567,7 @@ type GetRoleCredentialsInput struct {
// The token issued by the CreateToken API call. For more information, see CreateToken
// (https://docs.aws.amazon.com/singlesignon/latest/OIDCAPIReference/API_CreateToken.html)
// in the AWS SSO OIDC API Reference Guide.
// in the IAM Identity Center OIDC API Reference Guide.
//
// AccessToken is a sensitive parameter and its value will be
// replaced with "sensitive" in string returned by GetRoleCredentialsInput's
@ -730,7 +743,7 @@ type ListAccountRolesInput struct {
// The token issued by the CreateToken API call. For more information, see CreateToken
// (https://docs.aws.amazon.com/singlesignon/latest/OIDCAPIReference/API_CreateToken.html)
// in the AWS SSO OIDC API Reference Guide.
// in the IAM Identity Center OIDC API Reference Guide.
//
// AccessToken is a sensitive parameter and its value will be
// replaced with "sensitive" in string returned by ListAccountRolesInput's
@ -859,7 +872,7 @@ type ListAccountsInput struct {
// The token issued by the CreateToken API call. For more information, see CreateToken
// (https://docs.aws.amazon.com/singlesignon/latest/OIDCAPIReference/API_CreateToken.html)
// in the AWS SSO OIDC API Reference Guide.
// in the IAM Identity Center OIDC API Reference Guide.
//
// AccessToken is a sensitive parameter and its value will be
// replaced with "sensitive" in string returned by ListAccountsInput's
@ -974,7 +987,7 @@ type LogoutInput struct {
// The token issued by the CreateToken API call. For more information, see CreateToken
// (https://docs.aws.amazon.com/singlesignon/latest/OIDCAPIReference/API_CreateToken.html)
// in the AWS SSO OIDC API Reference Guide.
// in the IAM Identity Center OIDC API Reference Guide.
//
// AccessToken is a sensitive parameter and its value will be
// replaced with "sensitive" in string returned by LogoutInput's

@ -3,30 +3,31 @@
// Package sso provides the client and types for making API
// requests to AWS Single Sign-On.
//
// AWS Single Sign-On Portal is a web service that makes it easy for you to
// assign user access to AWS SSO resources such as the user portal. Users can
// get AWS account applications and roles assigned to them and get federated
// into the application.
// AWS IAM Identity Center (successor to AWS Single Sign-On) Portal is a web
// service that makes it easy for you to assign user access to IAM Identity
// Center resources such as the AWS access portal. Users can get AWS account
// applications and roles assigned to them and get federated into the application.
//
// For general information about AWS SSO, see What is AWS Single Sign-On? (https://docs.aws.amazon.com/singlesignon/latest/userguide/what-is.html)
// in the AWS SSO User Guide.
// Although AWS Single Sign-On was renamed, the sso and identitystore API namespaces
// will continue to retain their original name for backward compatibility purposes.
// For more information, see IAM Identity Center rename (https://docs.aws.amazon.com/singlesignon/latest/userguide/what-is.html#renamed).
//
// This API reference guide describes the AWS SSO Portal operations that you
// can call programatically and includes detailed information on data types
// and errors.
// This reference guide describes the IAM Identity Center Portal operations
// that you can call programatically and includes detailed information on data
// types and errors.
//
// AWS provides SDKs that consist of libraries and sample code for various programming
// languages and platforms, such as Java, Ruby, .Net, iOS, or Android. The SDKs
// provide a convenient way to create programmatic access to AWS SSO and other
// AWS services. For more information about the AWS SDKs, including how to download
// and install them, see Tools for Amazon Web Services (http://aws.amazon.com/tools/).
// provide a convenient way to create programmatic access to IAM Identity Center
// and other AWS services. For more information about the AWS SDKs, including
// how to download and install them, see Tools for Amazon Web Services (http://aws.amazon.com/tools/).
//
// See https://docs.aws.amazon.com/goto/WebAPI/sso-2019-06-10 for more information on this service.
//
// See sso package documentation for more information.
// https://docs.aws.amazon.com/sdk-for-go/api/service/sso/
//
// Using the Client
// # Using the Client
//
// To contact AWS Single Sign-On with the SDK use the New function to create
// a new service client. With that client you can make API requests to the service.

@ -40,13 +40,14 @@ const (
// aws.Config parameter to add your extra config.
//
// Example:
// mySession := session.Must(session.NewSession())
//
// // Create a SSO client from just a session.
// svc := sso.New(mySession)
// mySession := session.Must(session.NewSession())
//
// // Create a SSO client with additional configuration
// svc := sso.New(mySession, aws.NewConfig().WithRegion("us-west-2"))
// // Create a SSO client from just a session.
// svc := sso.New(mySession)
//
// // Create a SSO client with additional configuration
// svc := sso.New(mySession, aws.NewConfig().WithRegion("us-west-2"))
func New(p client.ConfigProvider, cfgs ...*aws.Config) *SSO {
c := p.ClientConfig(EndpointsID, cfgs...)
if c.SigningNameDerived || len(c.SigningName) == 0 {

@ -23,37 +23,37 @@ import (
// can be stubbed out for unit testing your code with the SDK without needing
// to inject custom request handlers into the SDK's request pipeline.
//
// // myFunc uses an SDK service client to make a request to
// // AWS Single Sign-On.
// func myFunc(svc ssoiface.SSOAPI) bool {
// // Make svc.GetRoleCredentials request
// }
// // myFunc uses an SDK service client to make a request to
// // AWS Single Sign-On.
// func myFunc(svc ssoiface.SSOAPI) bool {
// // Make svc.GetRoleCredentials request
// }
//
// func main() {
// sess := session.New()
// svc := sso.New(sess)
// func main() {
// sess := session.New()
// svc := sso.New(sess)
//
// myFunc(svc)
// }
// myFunc(svc)
// }
//
// In your _test.go file:
//
// // Define a mock struct to be used in your unit tests of myFunc.
// type mockSSOClient struct {
// ssoiface.SSOAPI
// }
// func (m *mockSSOClient) GetRoleCredentials(input *sso.GetRoleCredentialsInput) (*sso.GetRoleCredentialsOutput, error) {
// // mock response/functionality
// }
// // Define a mock struct to be used in your unit tests of myFunc.
// type mockSSOClient struct {
// ssoiface.SSOAPI
// }
// func (m *mockSSOClient) GetRoleCredentials(input *sso.GetRoleCredentialsInput) (*sso.GetRoleCredentialsOutput, error) {
// // mock response/functionality
// }
//
// func TestMyFunc(t *testing.T) {
// // Setup Test
// mockSvc := &mockSSOClient{}
// func TestMyFunc(t *testing.T) {
// // Setup Test
// mockSvc := &mockSSOClient{}
//
// myfunc(mockSvc)
// myfunc(mockSvc)
//
// // Verify myFunc's functionality
// }
// // Verify myFunc's functionality
// }
//
// It is important to note that this interface will have breaking changes
// when the service model is updated and adds new API operations, paginators,

File diff suppressed because it is too large Load Diff

@ -14,7 +14,7 @@
// See sts package documentation for more information.
// https://docs.aws.amazon.com/sdk-for-go/api/service/sts/
//
// Using the Client
// # Using the Client
//
// To contact AWS Security Token Service with the SDK use the New function to create
// a new service client. With that client you can make API requests to the service.

@ -39,13 +39,14 @@ const (
// aws.Config parameter to add your extra config.
//
// Example:
// mySession := session.Must(session.NewSession())
//
// // Create a STS client from just a session.
// svc := sts.New(mySession)
// mySession := session.Must(session.NewSession())
//
// // Create a STS client with additional configuration
// svc := sts.New(mySession, aws.NewConfig().WithRegion("us-west-2"))
// // Create a STS client from just a session.
// svc := sts.New(mySession)
//
// // Create a STS client with additional configuration
// svc := sts.New(mySession, aws.NewConfig().WithRegion("us-west-2"))
func New(p client.ConfigProvider, cfgs ...*aws.Config) *STS {
c := p.ClientConfig(EndpointsID, cfgs...)
if c.SigningNameDerived || len(c.SigningName) == 0 {

@ -23,37 +23,37 @@ import (
// can be stubbed out for unit testing your code with the SDK without needing
// to inject custom request handlers into the SDK's request pipeline.
//
// // myFunc uses an SDK service client to make a request to
// // AWS Security Token Service.
// func myFunc(svc stsiface.STSAPI) bool {
// // Make svc.AssumeRole request
// }
// // myFunc uses an SDK service client to make a request to
// // AWS Security Token Service.
// func myFunc(svc stsiface.STSAPI) bool {
// // Make svc.AssumeRole request
// }
//
// func main() {
// sess := session.New()
// svc := sts.New(sess)
// func main() {
// sess := session.New()
// svc := sts.New(sess)
//
// myFunc(svc)
// }
// myFunc(svc)
// }
//
// In your _test.go file:
//
// // Define a mock struct to be used in your unit tests of myFunc.
// type mockSTSClient struct {
// stsiface.STSAPI
// }
// func (m *mockSTSClient) AssumeRole(input *sts.AssumeRoleInput) (*sts.AssumeRoleOutput, error) {
// // mock response/functionality
// }
// // Define a mock struct to be used in your unit tests of myFunc.
// type mockSTSClient struct {
// stsiface.STSAPI
// }
// func (m *mockSTSClient) AssumeRole(input *sts.AssumeRoleInput) (*sts.AssumeRoleOutput, error) {
// // mock response/functionality
// }
//
// func TestMyFunc(t *testing.T) {
// // Setup Test
// mockSvc := &mockSTSClient{}
// func TestMyFunc(t *testing.T) {
// // Setup Test
// mockSvc := &mockSTSClient{}
//
// myfunc(mockSvc)
// myfunc(mockSvc)
//
// // Verify myFunc's functionality
// }
// // Verify myFunc's functionality
// }
//
// It is important to note that this interface will have breaking changes
// when the service model is updated and adds new API operations, paginators,

@ -88,8 +88,8 @@ func (b *BceV1Signer) Sign(req *http.Request, cred *BceCredentials, opt *SignOpt
signDate = util.FormatISO8601Date(opt.Timestamp)
}
// Set security token if using session credentials
if len(cred.SessionToken) != 0 {
// Set security token if using session credentials and session token not in param
if len(cred.SessionToken) != 0 && req.Param(http.BCE_SECURITY_TOKEN) == "" {
req.SetHeader(http.BCE_SECURITY_TOKEN, cred.SessionToken)
}

@ -26,7 +26,7 @@ import (
// Constants and default values for the package bce
const (
SDK_VERSION = "0.9.130"
SDK_VERSION = "0.9.138"
URI_PREFIX = "/" // now support uri without prefix "v1" so just set root path
DEFAULT_DOMAIN = "baidubce.com"
DEFAULT_PROTOCOL = "http"

@ -80,4 +80,5 @@ const (
BCE_RESTORE = "x-bce-restore"
BCE_FORBID_OVERWRITE = "x-bce-forbid-overwrite"
BCE_SYMLINK_TARGET = "x-bce-symlink-target"
BCE_TRAFFIC_LIMIT = "x-bce-traffic-limit"
)

@ -95,6 +95,7 @@ func ListObjects(cli bce.Client, bucket string,
if err := resp.ParseJsonBody(result); err != nil {
return nil, err
}
defer func() { resp.Body().Close() }()
return result, nil
}
@ -189,6 +190,7 @@ func GetBucketLocation(cli bce.Client, bucket string) (string, error) {
if err := resp.ParseJsonBody(result); err != nil {
return "", err
}
defer func() { resp.Body().Close() }()
return result.LocationConstraint, nil
}
@ -255,6 +257,7 @@ func GetBucketAcl(cli bce.Client, bucket string) (*GetBucketAclResult, error) {
if err := resp.ParseJsonBody(result); err != nil {
return nil, err
}
defer func() { resp.Body().Close() }()
return result, nil
}

@ -126,7 +126,7 @@ type PutBucketLoggingArgs struct {
type GetBucketLoggingResult struct {
Status string `json:"status"`
TargetBucket string `json:"targetBucket,omitempty"`
TargetPrefix string `json:"targetPrefix, omitempty"`
TargetPrefix string `json:"targetPrefix,omitempty"`
}
// LifecycleConditionTimeType defines the structure of time condition
@ -259,6 +259,7 @@ type PutObjectArgs struct {
ContentCrc32 string
StorageClass string
Process string
TrafficLimit int64
}
// CopyObjectArgs defines the optional args structure for the copy object api.
@ -269,6 +270,7 @@ type CopyObjectArgs struct {
IfNoneMatch string
IfModifiedSince string
IfUnmodifiedSince string
TrafficLimit int64
}
type MultiCopyObjectArgs struct {
@ -321,8 +323,8 @@ type SelectObjectResult struct {
// selectObject request args
type SelectObjectArgs struct {
SelectType string `json: "-"`
SelectRequest *SelectObjectRequest `json: "selectRequest"`
SelectType string `json:"-"`
SelectRequest *SelectObjectRequest `json:"selectRequest"`
}
type SelectObjectRequest struct {
@ -397,6 +399,7 @@ type AppendObjectArgs struct {
ContentSha256 string
ContentCrc32 string
StorageClass string
TrafficLimit int64
}
// AppendObjectResult defines the result data structure for appending object.
@ -449,6 +452,7 @@ type UploadPartArgs struct {
ContentMD5 string
ContentSha256 string
ContentCrc32 string
TrafficLimit int64
}
// UploadPartCopyArgs defines the optional arguments of UploadPartCopy.
@ -458,6 +462,7 @@ type UploadPartCopyArgs struct {
IfNoneMatch string
IfModifiedSince string
IfUnmodifiedSince string
TrafficLimit int64
}
type PutSymlinkArgs struct {

@ -118,6 +118,14 @@ func UploadPart(cli bce.Client, bucket, object, uploadId string, partNumber int,
http.BCE_CONTENT_SHA256: args.ContentSha256,
http.BCE_CONTENT_CRC32: args.ContentCrc32,
})
//set traffic-limit
if args.TrafficLimit > 0 {
if args.TrafficLimit > TRAFFIC_LIMIT_MAX || args.TrafficLimit < TRAFFIC_LIMIT_MIN {
return "", bce.NewBceClientError(fmt.Sprintf("TrafficLimit must between %d ~ %d, current value:%d", TRAFFIC_LIMIT_MIN, TRAFFIC_LIMIT_MAX, args.TrafficLimit))
}
req.SetHeader(http.BCE_TRAFFIC_LIMIT, fmt.Sprintf("%d", args.TrafficLimit))
}
}
// Send request and get the result
@ -227,6 +235,13 @@ func UploadPartCopy(cli bce.Client, bucket, object, source, uploadId string, par
http.BCE_COPY_SOURCE_IF_MODIFIED_SINCE: args.IfModifiedSince,
http.BCE_COPY_SOURCE_IF_UNMODIFIED_SINCE: args.IfUnmodifiedSince,
})
//set traffic-limit
if args.TrafficLimit > 0 {
if args.TrafficLimit > TRAFFIC_LIMIT_MAX || args.TrafficLimit < TRAFFIC_LIMIT_MIN {
return nil, bce.NewBceClientError(fmt.Sprintf("TrafficLimit must between %d ~ %d, current value:%d", TRAFFIC_LIMIT_MIN, TRAFFIC_LIMIT_MAX, args.TrafficLimit))
}
req.SetHeader(http.BCE_TRAFFIC_LIMIT, fmt.Sprintf("%d", args.TrafficLimit))
}
}
// Send request and get the result

@ -78,6 +78,14 @@ func PutObject(cli bce.Client, bucket, object string, body *bce.Body,
req.SetBody(body) // re-assign body
}
//set traffic-limit
if args.TrafficLimit > 0 {
if args.TrafficLimit > TRAFFIC_LIMIT_MAX || args.TrafficLimit < TRAFFIC_LIMIT_MIN {
return "", bce.NewBceClientError(fmt.Sprintf("TrafficLimit must between %d ~ %d, current value:%d", TRAFFIC_LIMIT_MIN, TRAFFIC_LIMIT_MAX, args.TrafficLimit))
}
req.SetHeader(http.BCE_TRAFFIC_LIMIT, fmt.Sprintf("%d", args.TrafficLimit))
}
// Reset the contentMD5 if set by user
if len(args.ContentMD5) != 0 {
req.SetHeader(http.CONTENT_MD5, args.ContentMD5)
@ -173,6 +181,15 @@ func CopyObject(cli bce.Client, bucket, object, source string,
args.StorageClass)
}
}
//set traffic-limit
if args.TrafficLimit > 0 {
if args.TrafficLimit > TRAFFIC_LIMIT_MAX || args.TrafficLimit < TRAFFIC_LIMIT_MIN {
return nil, bce.NewBceClientError(fmt.Sprintf("TrafficLimit must between %d ~ %d, current value:%d", TRAFFIC_LIMIT_MIN, TRAFFIC_LIMIT_MAX, args.TrafficLimit))
}
req.SetHeader(http.BCE_TRAFFIC_LIMIT, fmt.Sprintf("%d", args.TrafficLimit))
}
if err := setUserMetadata(req, args.UserMeta); err != nil {
return nil, err
}
@ -549,6 +566,13 @@ func AppendObject(cli bce.Client, bucket, object string, content *bce.Body,
if err := setUserMetadata(req, args.UserMeta); err != nil {
return nil, err
}
//set traffic-limit
if args.TrafficLimit > 0 {
if args.TrafficLimit > TRAFFIC_LIMIT_MAX || args.TrafficLimit < TRAFFIC_LIMIT_MIN {
return nil, bce.NewBceClientError(fmt.Sprintf("TrafficLimit must between %d ~ %d, current value:%d", TRAFFIC_LIMIT_MIN, TRAFFIC_LIMIT_MAX, args.TrafficLimit))
}
req.SetHeader(http.BCE_TRAFFIC_LIMIT, fmt.Sprintf("%d", args.TrafficLimit))
}
}
// Send request and get the result
@ -719,6 +743,11 @@ func GeneratePresignedUrlInternal(conf *bce.BceClientConfiguration, signer auth.
if expire != 0 {
option.ExpireSeconds = expire
}
if conf.Credentials.SessionToken != "" {
req.SetParam(http.BCE_SECURITY_TOKEN, conf.Credentials.SessionToken)
}
// Generate the authorization string and return the signed url.
signer.Sign(&req.Request, conf.Credentials, &option)
req.SetParam("authorization", req.Header(http.AUTHORIZATION))

@ -45,6 +45,9 @@ const (
THRESHOLD_100_CONTINUE = 1 << 20 // add 100 continue header if body size bigger than 1MB
TRAFFIC_LIMIT_MAX = 8 * (100 << 20) // 100M bit = 838860800
TRAFFIC_LIMIT_MIN = 8 * (100 << 10) // 100K bit = 819200
STATUS_ENABLED = "enabled"
STATUS_DISABLED = "disabled"

@ -29,6 +29,7 @@ import (
"github.com/baidubce/bce-sdk-go/auth"
"github.com/baidubce/bce-sdk-go/bce"
"github.com/baidubce/bce-sdk-go/services/bos/api"
"github.com/baidubce/bce-sdk-go/services/sts"
"github.com/baidubce/bce-sdk-go/util/log"
)
@ -71,6 +72,37 @@ func NewClient(ak, sk, endpoint string) (*Client, error) {
RedirectDisabled: false,
})
}
// NewStsClient make the BOS service client with STS configuration, it will first apply stsAK,stsSK, sessionToken, then return bosClient using temporary sts Credential
func NewStsClient(ak, sk, endpoint string, expiration int) (*Client, error) {
stsClient, err := sts.NewClient(ak, sk)
if err != nil {
fmt.Println("create sts client object :", err)
return nil, err
}
sts, err := stsClient.GetSessionToken(expiration, "")
if err != nil {
fmt.Println("get session token failed:", err)
return nil, err
}
bosClient, err := NewClient(sts.AccessKeyId, sts.SecretAccessKey, endpoint)
if err != nil {
fmt.Println("create bos client failed:", err)
return nil, err
}
stsCredential, err := auth.NewSessionBceCredentials(
sts.AccessKeyId,
sts.SecretAccessKey,
sts.SessionToken)
if err != nil {
fmt.Println("create sts credential object failed:", err)
return nil, err
}
bosClient.Config.Credentials = stsCredential
return bosClient, nil
}
func NewClientWithConfig(config *BosClientConfiguration) (*Client, error) {
var credentials *auth.BceCredentials
var err error

@ -0,0 +1,30 @@
package api
import "time"
type GetSessionTokenResult struct {
AccessKeyId string
SecretAccessKey string
SessionToken string
CreateTime string
Expiration string
UserId string
}
type AssumeRoleArgs struct {
DurationSeconds int
AccountId string
UserId string
RoleName string
Acl string
}
type Credential struct {
AccessKeyId string
SecretAccessKey string
SessionToken string
CreateTime time.Time
Expiration time.Time
UserId string
RoleId string
}

@ -0,0 +1,140 @@
/*
* Copyright 2017 Baidu, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
* except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
// sts.go - define the response for STS service
// Package api defines all APIs supported by the STS service of BCE.
package api
import (
"fmt"
"strconv"
"github.com/baidubce/bce-sdk-go/bce"
"github.com/baidubce/bce-sdk-go/http"
)
const (
DEFAULT_DURATION_SECONDS = 43200 // default duration is 12 hours
URI_PREFIX = bce.URI_PREFIX + "v1" // sts service not support uri without prefix "v1"
REQUEST_URI = "/sessionToken"
DEFAULT_ASSUMEROLE_DURATION_SECONDS = 7200 // default duration is 2 hours
REQUEST_ASSUMEROLE_URI = "/credential"
)
// GetSessionToken - get the session token from the STS service
//
// PARAMS:
// - cli: the client object which can perform sending request
// - durationSec: the duration seconds of the token
// - acl: the acl string
//
// RETURNS:
// - *GetSessionTokenResult: result of this api
// - error: nil if ok otherwise the specific error
func GetSessionToken(cli bce.Client, durationSec int, acl string) (*GetSessionTokenResult, error) {
// If the duration seconds is not a positive, use the default value
if durationSec <= 0 {
durationSec = DEFAULT_DURATION_SECONDS
}
// Build the request
req := &bce.BceRequest{}
req.SetUri(URI_PREFIX + REQUEST_URI)
req.SetMethod(http.POST)
req.SetParam("durationSeconds", strconv.Itoa(durationSec))
if len(acl) > 0 {
req.SetHeader(http.CONTENT_TYPE, bce.DEFAULT_CONTENT_TYPE)
body, err := bce.NewBodyFromString(acl)
if err != nil {
return nil, err
}
req.SetBody(body)
}
// Send requet and get response
resp := &bce.BceResponse{}
if err := cli.SendRequest(req, resp); err != nil {
return nil, err
}
if resp.IsFail() {
return nil, resp.ServiceError()
}
jsonBody := &GetSessionTokenResult{}
if err := resp.ParseJsonBody(jsonBody); err != nil {
return nil, err
}
return jsonBody, nil
}
// AssumeRole - get the credential for the assume role from the STS service
//
// PARAMS:
// - cli: the client object which can perform sending request
// - args: the args for assumeRole
// RETURNS:
// - *Credential: result of this api
// - error: nil if ok otherwise the specific error
func AssumeRole(cli bce.Client, args *AssumeRoleArgs) (*Credential, error) {
// If the duration seconds is not a positive, use the default value
if args.DurationSeconds <= 0 {
args.DurationSeconds = DEFAULT_ASSUMEROLE_DURATION_SECONDS
}
if args.AccountId == "" {
return nil, fmt.Errorf("please set accountId")
}
if args.RoleName == "" {
return nil, fmt.Errorf("please set roleName")
}
// Build the request
req := &bce.BceRequest{}
req.SetUri(URI_PREFIX + REQUEST_ASSUMEROLE_URI)
req.SetMethod(http.POST)
req.SetParam("assumeRole", "")
req.SetParam("durationSeconds", strconv.Itoa(args.DurationSeconds))
req.SetParam("accountId", args.AccountId)
req.SetParam("roleName", args.RoleName)
if args.UserId != "" {
req.SetParam("userId", args.UserId)
}
if len(args.Acl) > 0 {
req.SetHeader(http.CONTENT_TYPE, bce.DEFAULT_CONTENT_TYPE)
body, err := bce.NewBodyFromString(args.Acl)
if err != nil {
return nil, err
}
req.SetBody(body)
}
// Send requet and get response
resp := &bce.BceResponse{}
if err := cli.SendRequest(req, resp); err != nil {
return nil, err
}
if resp.IsFail() {
return nil, resp.ServiceError()
}
jsonBody := &Credential{}
if err := resp.ParseJsonBody(jsonBody); err != nil {
return nil, err
}
return jsonBody, nil
}

@ -0,0 +1,75 @@
/*
* Copyright 2017 Baidu, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
* except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
// client.go - define the client for STS service which is derived from BceClient
// Package sts defines the STS service of BCE.
// It contains the model sub package to implement the concrete request and response of the
// GetSessionToken API.
package sts
import (
"github.com/baidubce/bce-sdk-go/auth"
"github.com/baidubce/bce-sdk-go/bce"
"github.com/baidubce/bce-sdk-go/services/sts/api"
"github.com/baidubce/bce-sdk-go/util"
)
const DEFAULT_SERVICE_DOMAIN = "sts." + bce.DEFAULT_REGION + "." + bce.DEFAULT_DOMAIN
// Client of STS service is a kind of BceClient, so it derived from the BceClient and it only
// supports the GetSessionToken API. There is no other fields needed.
type Client struct {
*bce.BceClient
}
func (c *Client) GetSessionToken(duration int, acl string) (*api.GetSessionTokenResult, error) {
return api.GetSessionToken(c, duration, acl)
}
func (c *Client) AssumeRole(args *api.AssumeRoleArgs) (*api.Credential, error) {
return api.AssumeRole(c, args)
}
// NewClient make the STS service client with default configuration.
// Use `cli.Config.xxx` to access the config or change it to non-default value.
func NewClient(ak, sk string) (*Client, error) {
return NewStsClient(ak, sk, DEFAULT_SERVICE_DOMAIN)
}
func NewStsClient(ak, sk, endpoint string) (*Client, error) {
credentials, err := auth.NewBceCredentials(ak, sk)
if err != nil {
return nil, err
}
if len(endpoint) == 0 {
endpoint = DEFAULT_SERVICE_DOMAIN
}
defaultSignOptions := &auth.SignOptions{
HeadersToSign: auth.DEFAULT_HEADERS_TO_SIGN,
Timestamp: util.NowUTCSeconds(),
ExpireSeconds: auth.DEFAULT_EXPIRE_SECONDS}
defaultConf := &bce.BceClientConfiguration{
Endpoint: endpoint,
Region: bce.DEFAULT_REGION,
UserAgent: bce.DEFAULT_USER_AGENT,
Credentials: credentials,
SignOption: defaultSignOptions,
Retry: bce.DEFAULT_RETRY_POLICY,
ConnectionTimeoutInMillis: bce.DEFAULT_CONNECTION_TIMEOUT_IN_MILLIS}
v1Signer := &auth.BceV1Signer{}
client := &Client{bce.NewBceClient(defaultConf, v1Signer)}
return client, nil
}

@ -3,8 +3,7 @@
[![Go Reference](https://pkg.go.dev/badge/github.com/cespare/xxhash/v2.svg)](https://pkg.go.dev/github.com/cespare/xxhash/v2)
[![Test](https://github.com/cespare/xxhash/actions/workflows/test.yml/badge.svg)](https://github.com/cespare/xxhash/actions/workflows/test.yml)
xxhash is a Go implementation of the 64-bit
[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a
xxhash is a Go implementation of the 64-bit [xxHash] algorithm, XXH64. This is a
high-quality hashing algorithm that is much faster than anything in the Go
standard library.
@ -25,8 +24,11 @@ func (*Digest) WriteString(string) (int, error)
func (*Digest) Sum64() uint64
```
This implementation provides a fast pure-Go implementation and an even faster
assembly implementation for amd64.
The package is written with optimized pure Go and also contains even faster
assembly implementations for amd64 and arm64. If desired, the `purego` build tag
opts into using the Go code even on those architectures.
[xxHash]: http://cyan4973.github.io/xxHash/
## Compatibility
@ -45,19 +47,20 @@ I recommend using the latest release of Go.
Here are some quick benchmarks comparing the pure-Go and assembly
implementations of Sum64.
| input size | purego | asm |
| --- | --- | --- |
| 5 B | 979.66 MB/s | 1291.17 MB/s |
| 100 B | 7475.26 MB/s | 7973.40 MB/s |
| 4 KB | 17573.46 MB/s | 17602.65 MB/s |
| 10 MB | 17131.46 MB/s | 17142.16 MB/s |
| input size | purego | asm |
| ---------- | --------- | --------- |
| 4 B | 1.3 GB/s | 1.2 GB/s |
| 16 B | 2.9 GB/s | 3.5 GB/s |
| 100 B | 6.9 GB/s | 8.1 GB/s |
| 4 KB | 11.7 GB/s | 16.7 GB/s |
| 10 MB | 12.0 GB/s | 17.3 GB/s |
These numbers were generated on Ubuntu 18.04 with an Intel i7-8700K CPU using
the following commands under Go 1.11.2:
These numbers were generated on Ubuntu 20.04 with an Intel Xeon Platinum 8252C
CPU using the following commands under Go 1.19.2:
```
$ go test -tags purego -benchtime 10s -bench '/xxhash,direct,bytes'
$ go test -benchtime 10s -bench '/xxhash,direct,bytes'
benchstat <(go test -tags purego -benchtime 500ms -count 15 -bench 'Sum64$')
benchstat <(go test -benchtime 500ms -count 15 -bench 'Sum64$')
```
## Projects using this package

@ -16,19 +16,11 @@ const (
prime5 uint64 = 2870177450012600261
)
// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where
// possible in the Go code is worth a small (but measurable) performance boost
// by avoiding some MOVQs. Vars are needed for the asm and also are useful for
// convenience in the Go code in a few places where we need to intentionally
// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the
// result overflows a uint64).
var (
prime1v = prime1
prime2v = prime2
prime3v = prime3
prime4v = prime4
prime5v = prime5
)
// Store the primes in an array as well.
//
// The consts are used when possible in Go code to avoid MOVs but we need a
// contiguous array of the assembly code.
var primes = [...]uint64{prime1, prime2, prime3, prime4, prime5}
// Digest implements hash.Hash64.
type Digest struct {
@ -50,10 +42,10 @@ func New() *Digest {
// Reset clears the Digest's state so that it can be reused.
func (d *Digest) Reset() {
d.v1 = prime1v + prime2
d.v1 = primes[0] + prime2
d.v2 = prime2
d.v3 = 0
d.v4 = -prime1v
d.v4 = -primes[0]
d.total = 0
d.n = 0
}
@ -69,21 +61,23 @@ func (d *Digest) Write(b []byte) (n int, err error) {
n = len(b)
d.total += uint64(n)
memleft := d.mem[d.n&(len(d.mem)-1):]
if d.n+n < 32 {
// This new data doesn't even fill the current block.
copy(d.mem[d.n:], b)
copy(memleft, b)
d.n += n
return
}
if d.n > 0 {
// Finish off the partial block.
copy(d.mem[d.n:], b)
c := copy(memleft, b)
d.v1 = round(d.v1, u64(d.mem[0:8]))
d.v2 = round(d.v2, u64(d.mem[8:16]))
d.v3 = round(d.v3, u64(d.mem[16:24]))
d.v4 = round(d.v4, u64(d.mem[24:32]))
b = b[32-d.n:]
b = b[c:]
d.n = 0
}
@ -133,21 +127,20 @@ func (d *Digest) Sum64() uint64 {
h += d.total
i, end := 0, d.n
for ; i+8 <= end; i += 8 {
k1 := round(0, u64(d.mem[i:i+8]))
b := d.mem[:d.n&(len(d.mem)-1)]
for ; len(b) >= 8; b = b[8:] {
k1 := round(0, u64(b[:8]))
h ^= k1
h = rol27(h)*prime1 + prime4
}
if i+4 <= end {
h ^= uint64(u32(d.mem[i:i+4])) * prime1
if len(b) >= 4 {
h ^= uint64(u32(b[:4])) * prime1
h = rol23(h)*prime2 + prime3
i += 4
b = b[4:]
}
for i < end {
h ^= uint64(d.mem[i]) * prime5
for ; len(b) > 0; b = b[1:] {
h ^= uint64(b[0]) * prime5
h = rol11(h) * prime1
i++
}
h ^= h >> 33

@ -1,215 +1,209 @@
//go:build !appengine && gc && !purego
// +build !appengine
// +build gc
// +build !purego
#include "textflag.h"
// Register allocation:
// AX h
// SI pointer to advance through b
// DX n
// BX loop end
// R8 v1, k1
// R9 v2
// R10 v3
// R11 v4
// R12 tmp
// R13 prime1v
// R14 prime2v
// DI prime4v
// round reads from and advances the buffer pointer in SI.
// It assumes that R13 has prime1v and R14 has prime2v.
#define round(r) \
MOVQ (SI), R12 \
ADDQ $8, SI \
IMULQ R14, R12 \
ADDQ R12, r \
ROLQ $31, r \
IMULQ R13, r
// mergeRound applies a merge round on the two registers acc and val.
// It assumes that R13 has prime1v, R14 has prime2v, and DI has prime4v.
#define mergeRound(acc, val) \
IMULQ R14, val \
ROLQ $31, val \
IMULQ R13, val \
XORQ val, acc \
IMULQ R13, acc \
ADDQ DI, acc
// Registers:
#define h AX
#define d AX
#define p SI // pointer to advance through b
#define n DX
#define end BX // loop end
#define v1 R8
#define v2 R9
#define v3 R10
#define v4 R11
#define x R12
#define prime1 R13
#define prime2 R14
#define prime4 DI
#define round(acc, x) \
IMULQ prime2, x \
ADDQ x, acc \
ROLQ $31, acc \
IMULQ prime1, acc
// round0 performs the operation x = round(0, x).
#define round0(x) \
IMULQ prime2, x \
ROLQ $31, x \
IMULQ prime1, x
// mergeRound applies a merge round on the two registers acc and x.
// It assumes that prime1, prime2, and prime4 have been loaded.
#define mergeRound(acc, x) \
round0(x) \
XORQ x, acc \
IMULQ prime1, acc \
ADDQ prime4, acc
// blockLoop processes as many 32-byte blocks as possible,
// updating v1, v2, v3, and v4. It assumes that there is at least one block
// to process.
#define blockLoop() \
loop: \
MOVQ +0(p), x \
round(v1, x) \
MOVQ +8(p), x \
round(v2, x) \
MOVQ +16(p), x \
round(v3, x) \
MOVQ +24(p), x \
round(v4, x) \
ADDQ $32, p \
CMPQ p, end \
JLE loop
// func Sum64(b []byte) uint64
TEXT ·Sum64(SB), NOSPLIT, $0-32
TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32
// Load fixed primes.
MOVQ ·prime1v(SB), R13
MOVQ ·prime2v(SB), R14
MOVQ ·prime4v(SB), DI
MOVQ ·primes+0(SB), prime1
MOVQ ·primes+8(SB), prime2
MOVQ ·primes+24(SB), prime4
// Load slice.
MOVQ b_base+0(FP), SI
MOVQ b_len+8(FP), DX
LEAQ (SI)(DX*1), BX
MOVQ b_base+0(FP), p
MOVQ b_len+8(FP), n
LEAQ (p)(n*1), end
// The first loop limit will be len(b)-32.
SUBQ $32, BX
SUBQ $32, end
// Check whether we have at least one block.
CMPQ DX, $32
CMPQ n, $32
JLT noBlocks
// Set up initial state (v1, v2, v3, v4).
MOVQ R13, R8
ADDQ R14, R8
MOVQ R14, R9
XORQ R10, R10
XORQ R11, R11
SUBQ R13, R11
// Loop until SI > BX.
blockLoop:
round(R8)
round(R9)
round(R10)
round(R11)
CMPQ SI, BX
JLE blockLoop
MOVQ R8, AX
ROLQ $1, AX
MOVQ R9, R12
ROLQ $7, R12
ADDQ R12, AX
MOVQ R10, R12
ROLQ $12, R12
ADDQ R12, AX
MOVQ R11, R12
ROLQ $18, R12
ADDQ R12, AX
mergeRound(AX, R8)
mergeRound(AX, R9)
mergeRound(AX, R10)
mergeRound(AX, R11)
MOVQ prime1, v1
ADDQ prime2, v1
MOVQ prime2, v2
XORQ v3, v3
XORQ v4, v4
SUBQ prime1, v4
blockLoop()
MOVQ v1, h
ROLQ $1, h
MOVQ v2, x
ROLQ $7, x
ADDQ x, h
MOVQ v3, x
ROLQ $12, x
ADDQ x, h
MOVQ v4, x
ROLQ $18, x
ADDQ x, h
mergeRound(h, v1)
mergeRound(h, v2)
mergeRound(h, v3)
mergeRound(h, v4)
JMP afterBlocks
noBlocks:
MOVQ ·prime5v(SB), AX
MOVQ ·primes+32(SB), h
afterBlocks:
ADDQ DX, AX
// Right now BX has len(b)-32, and we want to loop until SI > len(b)-8.
ADDQ $24, BX
CMPQ SI, BX
JG fourByte
wordLoop:
// Calculate k1.
MOVQ (SI), R8
ADDQ $8, SI
IMULQ R14, R8
ROLQ $31, R8
IMULQ R13, R8
XORQ R8, AX
ROLQ $27, AX
IMULQ R13, AX
ADDQ DI, AX
CMPQ SI, BX
JLE wordLoop
fourByte:
ADDQ $4, BX
CMPQ SI, BX
JG singles
MOVL (SI), R8
ADDQ $4, SI
IMULQ R13, R8
XORQ R8, AX
ROLQ $23, AX
IMULQ R14, AX
ADDQ ·prime3v(SB), AX
singles:
ADDQ $4, BX
CMPQ SI, BX
ADDQ n, h
ADDQ $24, end
CMPQ p, end
JG try4
loop8:
MOVQ (p), x
ADDQ $8, p
round0(x)
XORQ x, h
ROLQ $27, h
IMULQ prime1, h
ADDQ prime4, h
CMPQ p, end
JLE loop8
try4:
ADDQ $4, end
CMPQ p, end
JG try1
MOVL (p), x
ADDQ $4, p
IMULQ prime1, x
XORQ x, h
ROLQ $23, h
IMULQ prime2, h
ADDQ ·primes+16(SB), h
try1:
ADDQ $4, end
CMPQ p, end
JGE finalize
singlesLoop:
MOVBQZX (SI), R12
ADDQ $1, SI
IMULQ ·prime5v(SB), R12
XORQ R12, AX
loop1:
MOVBQZX (p), x
ADDQ $1, p
IMULQ ·primes+32(SB), x
XORQ x, h
ROLQ $11, h
IMULQ prime1, h
ROLQ $11, AX
IMULQ R13, AX
CMPQ SI, BX
JL singlesLoop
CMPQ p, end
JL loop1
finalize:
MOVQ AX, R12
SHRQ $33, R12
XORQ R12, AX
IMULQ R14, AX
MOVQ AX, R12
SHRQ $29, R12
XORQ R12, AX
IMULQ ·prime3v(SB), AX
MOVQ AX, R12
SHRQ $32, R12
XORQ R12, AX
MOVQ AX, ret+24(FP)
MOVQ h, x
SHRQ $33, x
XORQ x, h
IMULQ prime2, h
MOVQ h, x
SHRQ $29, x
XORQ x, h
IMULQ ·primes+16(SB), h
MOVQ h, x
SHRQ $32, x
XORQ x, h
MOVQ h, ret+24(FP)
RET
// writeBlocks uses the same registers as above except that it uses AX to store
// the d pointer.
// func writeBlocks(d *Digest, b []byte) int
TEXT ·writeBlocks(SB), NOSPLIT, $0-40
TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40
// Load fixed primes needed for round.
MOVQ ·prime1v(SB), R13
MOVQ ·prime2v(SB), R14
MOVQ ·primes+0(SB), prime1
MOVQ ·primes+8(SB), prime2
// Load slice.
MOVQ b_base+8(FP), SI
MOVQ b_len+16(FP), DX
LEAQ (SI)(DX*1), BX
SUBQ $32, BX
MOVQ b_base+8(FP), p
MOVQ b_len+16(FP), n
LEAQ (p)(n*1), end
SUBQ $32, end
// Load vN from d.
MOVQ d+0(FP), AX
MOVQ 0(AX), R8 // v1
MOVQ 8(AX), R9 // v2
MOVQ 16(AX), R10 // v3
MOVQ 24(AX), R11 // v4
MOVQ s+0(FP), d
MOVQ 0(d), v1
MOVQ 8(d), v2
MOVQ 16(d), v3
MOVQ 24(d), v4
// We don't need to check the loop condition here; this function is
// always called with at least one block of data to process.
blockLoop:
round(R8)
round(R9)
round(R10)
round(R11)
CMPQ SI, BX
JLE blockLoop
blockLoop()
// Copy vN back to d.
MOVQ R8, 0(AX)
MOVQ R9, 8(AX)
MOVQ R10, 16(AX)
MOVQ R11, 24(AX)
// The number of bytes written is SI minus the old base pointer.
SUBQ b_base+8(FP), SI
MOVQ SI, ret+32(FP)
MOVQ v1, 0(d)
MOVQ v2, 8(d)
MOVQ v3, 16(d)
MOVQ v4, 24(d)
// The number of bytes written is p minus the old base pointer.
SUBQ b_base+8(FP), p
MOVQ p, ret+32(FP)
RET

@ -0,0 +1,183 @@
//go:build !appengine && gc && !purego
// +build !appengine
// +build gc
// +build !purego
#include "textflag.h"
// Registers:
#define digest R1
#define h R2 // return value
#define p R3 // input pointer
#define n R4 // input length
#define nblocks R5 // n / 32
#define prime1 R7
#define prime2 R8
#define prime3 R9
#define prime4 R10
#define prime5 R11
#define v1 R12
#define v2 R13
#define v3 R14
#define v4 R15
#define x1 R20
#define x2 R21
#define x3 R22
#define x4 R23
#define round(acc, x) \
MADD prime2, acc, x, acc \
ROR $64-31, acc \
MUL prime1, acc
// round0 performs the operation x = round(0, x).
#define round0(x) \
MUL prime2, x \
ROR $64-31, x \
MUL prime1, x
#define mergeRound(acc, x) \
round0(x) \
EOR x, acc \
MADD acc, prime4, prime1, acc
// blockLoop processes as many 32-byte blocks as possible,
// updating v1, v2, v3, and v4. It assumes that n >= 32.
#define blockLoop() \
LSR $5, n, nblocks \
PCALIGN $16 \
loop: \
LDP.P 16(p), (x1, x2) \
LDP.P 16(p), (x3, x4) \
round(v1, x1) \
round(v2, x2) \
round(v3, x3) \
round(v4, x4) \
SUB $1, nblocks \
CBNZ nblocks, loop
// func Sum64(b []byte) uint64
TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32
LDP b_base+0(FP), (p, n)
LDP ·primes+0(SB), (prime1, prime2)
LDP ·primes+16(SB), (prime3, prime4)
MOVD ·primes+32(SB), prime5
CMP $32, n
CSEL LT, prime5, ZR, h // if n < 32 { h = prime5 } else { h = 0 }
BLT afterLoop
ADD prime1, prime2, v1
MOVD prime2, v2
MOVD $0, v3
NEG prime1, v4
blockLoop()
ROR $64-1, v1, x1
ROR $64-7, v2, x2
ADD x1, x2
ROR $64-12, v3, x3
ROR $64-18, v4, x4
ADD x3, x4
ADD x2, x4, h
mergeRound(h, v1)
mergeRound(h, v2)
mergeRound(h, v3)
mergeRound(h, v4)
afterLoop:
ADD n, h
TBZ $4, n, try8
LDP.P 16(p), (x1, x2)
round0(x1)
// NOTE: here and below, sequencing the EOR after the ROR (using a
// rotated register) is worth a small but measurable speedup for small
// inputs.
ROR $64-27, h
EOR x1 @> 64-27, h, h
MADD h, prime4, prime1, h
round0(x2)
ROR $64-27, h
EOR x2 @> 64-27, h, h
MADD h, prime4, prime1, h
try8:
TBZ $3, n, try4
MOVD.P 8(p), x1
round0(x1)
ROR $64-27, h
EOR x1 @> 64-27, h, h
MADD h, prime4, prime1, h
try4:
TBZ $2, n, try2
MOVWU.P 4(p), x2
MUL prime1, x2
ROR $64-23, h
EOR x2 @> 64-23, h, h
MADD h, prime3, prime2, h
try2:
TBZ $1, n, try1
MOVHU.P 2(p), x3
AND $255, x3, x1
LSR $8, x3, x2
MUL prime5, x1
ROR $64-11, h
EOR x1 @> 64-11, h, h
MUL prime1, h
MUL prime5, x2
ROR $64-11, h
EOR x2 @> 64-11, h, h
MUL prime1, h
try1:
TBZ $0, n, finalize
MOVBU (p), x4
MUL prime5, x4
ROR $64-11, h
EOR x4 @> 64-11, h, h
MUL prime1, h
finalize:
EOR h >> 33, h
MUL prime2, h
EOR h >> 29, h
MUL prime3, h
EOR h >> 32, h
MOVD h, ret+24(FP)
RET
// func writeBlocks(d *Digest, b []byte) int
TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40
LDP ·primes+0(SB), (prime1, prime2)
// Load state. Assume v[1-4] are stored contiguously.
MOVD d+0(FP), digest
LDP 0(digest), (v1, v2)
LDP 16(digest), (v3, v4)
LDP b_base+8(FP), (p, n)
blockLoop()
// Store updated state.
STP (v1, v2), 0(digest)
STP (v3, v4), 16(digest)
BIC $31, n
MOVD n, ret+32(FP)
RET

@ -1,3 +1,5 @@
//go:build (amd64 || arm64) && !appengine && gc && !purego
// +build amd64 arm64
// +build !appengine
// +build gc
// +build !purego

@ -1,4 +1,5 @@
// +build !amd64 appengine !gc purego
//go:build (!amd64 && !arm64) || appengine || !gc || purego
// +build !amd64,!arm64 appengine !gc purego
package xxhash
@ -14,10 +15,10 @@ func Sum64(b []byte) uint64 {
var h uint64
if n >= 32 {
v1 := prime1v + prime2
v1 := primes[0] + prime2
v2 := prime2
v3 := uint64(0)
v4 := -prime1v
v4 := -primes[0]
for len(b) >= 32 {
v1 = round(v1, u64(b[0:8:len(b)]))
v2 = round(v2, u64(b[8:16:len(b)]))
@ -36,19 +37,18 @@ func Sum64(b []byte) uint64 {
h += uint64(n)
i, end := 0, len(b)
for ; i+8 <= end; i += 8 {
k1 := round(0, u64(b[i:i+8:len(b)]))
for ; len(b) >= 8; b = b[8:] {
k1 := round(0, u64(b[:8]))
h ^= k1
h = rol27(h)*prime1 + prime4
}
if i+4 <= end {
h ^= uint64(u32(b[i:i+4:len(b)])) * prime1
if len(b) >= 4 {
h ^= uint64(u32(b[:4])) * prime1
h = rol23(h)*prime2 + prime3
i += 4
b = b[4:]
}
for ; i < end; i++ {
h ^= uint64(b[i]) * prime5
for ; len(b) > 0; b = b[1:] {
h ^= uint64(b[0]) * prime5
h = rol11(h) * prime1
}

@ -1,3 +1,4 @@
//go:build appengine
// +build appengine
// This file contains the safe implementations of otherwise unsafe-using code.

@ -1,3 +1,4 @@
//go:build !appengine
// +build !appengine
// This file encapsulates usage of unsafe.
@ -11,7 +12,7 @@ import (
// In the future it's possible that compiler optimizations will make these
// XxxString functions unnecessary by realizing that calls such as
// Sum64([]byte(s)) don't need to copy s. See https://golang.org/issue/2205.
// Sum64([]byte(s)) don't need to copy s. See https://go.dev/issue/2205.
// If that happens, even if we keep these functions they can be replaced with
// the trivial safe code.

@ -24,7 +24,7 @@ info:
title: "Docker Engine API"
version: "1.41"
x-logo:
url: "https://docs.docker.com/images/logo-docker-main.png"
url: "https://docs.docker.com/assets/images/logo-docker-main.png"
description: |
The Engine API is an HTTP API served by Docker Engine. It is the API the
Docker client uses to communicate with the Engine, so everything the Docker
@ -1891,23 +1891,52 @@ definitions:
BuildCache:
type: "object"
description: |
BuildCache contains information about a build cache record.
properties:
ID:
type: "string"
description: |
Unique ID of the build cache record.
example: "ndlpt0hhvkqcdfkputsk4cq9c"
Parent:
description: |
ID of the parent build cache record.
type: "string"
example: "hw53o5aio51xtltp5xjp8v7fx"
Type:
type: "string"
description: |
Cache record type.
example: "regular"
# see https://github.com/moby/buildkit/blob/fce4a32258dc9d9664f71a4831d5de10f0670677/client/diskusage.go#L75-L84
enum:
- "internal"
- "frontend"
- "source.local"
- "source.git.checkout"
- "exec.cachemount"
- "regular"
Description:
type: "string"
description: |
Description of the build-step that produced the build cache.
example: "mount / from exec /bin/sh -c echo 'Binary::apt::APT::Keep-Downloaded-Packages \"true\";' > /etc/apt/apt.conf.d/keep-cache"
InUse:
type: "boolean"
description: |
Indicates if the build cache is in use.
example: false
Shared:
type: "boolean"
description: |
Indicates if the build cache is shared.
example: true
Size:
description: |
Amount of disk space used by the build cache (in bytes).
type: "integer"
example: 51
CreatedAt:
description: |
Date and time at which the build cache was created in
@ -1925,6 +1954,7 @@ definitions:
example: "2017-08-09T07:09:37.632105588Z"
UsageCount:
type: "integer"
example: 26
ImageID:
type: "object"
@ -5415,6 +5445,28 @@ paths:
`/?[a-zA-Z0-9][a-zA-Z0-9_.-]+`.
type: "string"
pattern: "^/?[a-zA-Z0-9][a-zA-Z0-9_.-]+$"
- name: "platform"
in: "query"
description: |
Platform in the format `os[/arch[/variant]]` used for image lookup.
When specified, the daemon checks if the requested image is present
in the local image cache with the given OS and Architecture, and
otherwise returns a `404` status.
If the option is not set, the host's native OS and Architecture are
used to look up the image in the image cache. However, if no platform
is passed and the given image does exist in the local image cache,
but its OS or architecture does not match, the container is created
with the available image, and a warning is added to the `Warnings`
field in the response, for example;
WARNING: The requested image's platform (linux/arm64/v8) does not
match the detected host platform (linux/amd64) and no
specific platform was requested
type: "string"
default: ""
- name: "body"
in: "body"
description: "Container to create"

@ -135,9 +135,6 @@ func NewClientWithOpts(ops ...Opt) (*Client, error) {
}
}
if _, ok := c.client.Transport.(http.RoundTripper); !ok {
return nil, fmt.Errorf("unable to verify TLS configuration, invalid transport %v", c.client.Transport)
}
if c.scheme == "" {
c.scheme = "http"

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save