parent
534f6dea99
commit
c2c1239c54
@ -0,0 +1,6 @@
|
||||
.db
|
||||
*.test
|
||||
*~
|
||||
*.swp
|
||||
.idea
|
||||
.vscode
|
||||
@ -0,0 +1,8 @@
|
||||
Copyright (c) 2011-2013, 'pq' Contributors
|
||||
Portions Copyright (C) 2011 Blake Mizerany
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
@ -0,0 +1,36 @@
|
||||
# pq - A pure Go postgres driver for Go's database/sql package
|
||||
|
||||
[](https://pkg.go.dev/github.com/lib/pq?tab=doc)
|
||||
|
||||
## Install
|
||||
|
||||
go get github.com/lib/pq
|
||||
|
||||
## Features
|
||||
|
||||
* SSL
|
||||
* Handles bad connections for `database/sql`
|
||||
* Scan `time.Time` correctly (i.e. `timestamp[tz]`, `time[tz]`, `date`)
|
||||
* Scan binary blobs correctly (i.e. `bytea`)
|
||||
* Package for `hstore` support
|
||||
* COPY FROM support
|
||||
* pq.ParseURL for converting urls to connection strings for sql.Open.
|
||||
* Many libpq compatible environment variables
|
||||
* Unix socket support
|
||||
* Notifications: `LISTEN`/`NOTIFY`
|
||||
* pgpass support
|
||||
* GSS (Kerberos) auth
|
||||
|
||||
## Tests
|
||||
|
||||
`go test` is used for testing. See [TESTS.md](TESTS.md) for more details.
|
||||
|
||||
## Status
|
||||
|
||||
This package is currently in maintenance mode, which means:
|
||||
1. It generally does not accept new features.
|
||||
2. It does accept bug fixes and version compatability changes provided by the community.
|
||||
3. Maintainers usually do not resolve reported issues.
|
||||
4. Community members are encouraged to help each other with reported issues.
|
||||
|
||||
For users that require new features or reliable resolution of reported bugs, we recommend using [pgx](https://github.com/jackc/pgx) which is under active development.
|
||||
@ -0,0 +1,33 @@
|
||||
# Tests
|
||||
|
||||
## Running Tests
|
||||
|
||||
`go test` is used for testing. A running PostgreSQL
|
||||
server is required, with the ability to log in. The
|
||||
database to connect to test with is "pqgotest," on
|
||||
"localhost" but these can be overridden using [environment
|
||||
variables](https://www.postgresql.org/docs/9.3/static/libpq-envars.html).
|
||||
|
||||
Example:
|
||||
|
||||
PGHOST=/run/postgresql go test
|
||||
|
||||
## Benchmarks
|
||||
|
||||
A benchmark suite can be run as part of the tests:
|
||||
|
||||
go test -bench .
|
||||
|
||||
## Example setup (Docker)
|
||||
|
||||
Run a postgres container:
|
||||
|
||||
```
|
||||
docker run --expose 5432:5432 postgres
|
||||
```
|
||||
|
||||
Run tests:
|
||||
|
||||
```
|
||||
PGHOST=localhost PGPORT=5432 PGUSER=postgres PGSSLMODE=disable PGDATABASE=postgres go test
|
||||
```
|
||||
@ -0,0 +1,895 @@
|
||||
package pq
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var typeByteSlice = reflect.TypeOf([]byte{})
|
||||
var typeDriverValuer = reflect.TypeOf((*driver.Valuer)(nil)).Elem()
|
||||
var typeSQLScanner = reflect.TypeOf((*sql.Scanner)(nil)).Elem()
|
||||
|
||||
// Array returns the optimal driver.Valuer and sql.Scanner for an array or
|
||||
// slice of any dimension.
|
||||
//
|
||||
// For example:
|
||||
// db.Query(`SELECT * FROM t WHERE id = ANY($1)`, pq.Array([]int{235, 401}))
|
||||
//
|
||||
// var x []sql.NullInt64
|
||||
// db.QueryRow(`SELECT ARRAY[235, 401]`).Scan(pq.Array(&x))
|
||||
//
|
||||
// Scanning multi-dimensional arrays is not supported. Arrays where the lower
|
||||
// bound is not one (such as `[0:0]={1}') are not supported.
|
||||
func Array(a interface{}) interface {
|
||||
driver.Valuer
|
||||
sql.Scanner
|
||||
} {
|
||||
switch a := a.(type) {
|
||||
case []bool:
|
||||
return (*BoolArray)(&a)
|
||||
case []float64:
|
||||
return (*Float64Array)(&a)
|
||||
case []float32:
|
||||
return (*Float32Array)(&a)
|
||||
case []int64:
|
||||
return (*Int64Array)(&a)
|
||||
case []int32:
|
||||
return (*Int32Array)(&a)
|
||||
case []string:
|
||||
return (*StringArray)(&a)
|
||||
case [][]byte:
|
||||
return (*ByteaArray)(&a)
|
||||
|
||||
case *[]bool:
|
||||
return (*BoolArray)(a)
|
||||
case *[]float64:
|
||||
return (*Float64Array)(a)
|
||||
case *[]float32:
|
||||
return (*Float32Array)(a)
|
||||
case *[]int64:
|
||||
return (*Int64Array)(a)
|
||||
case *[]int32:
|
||||
return (*Int32Array)(a)
|
||||
case *[]string:
|
||||
return (*StringArray)(a)
|
||||
case *[][]byte:
|
||||
return (*ByteaArray)(a)
|
||||
}
|
||||
|
||||
return GenericArray{a}
|
||||
}
|
||||
|
||||
// ArrayDelimiter may be optionally implemented by driver.Valuer or sql.Scanner
|
||||
// to override the array delimiter used by GenericArray.
|
||||
type ArrayDelimiter interface {
|
||||
// ArrayDelimiter returns the delimiter character(s) for this element's type.
|
||||
ArrayDelimiter() string
|
||||
}
|
||||
|
||||
// BoolArray represents a one-dimensional array of the PostgreSQL boolean type.
|
||||
type BoolArray []bool
|
||||
|
||||
// Scan implements the sql.Scanner interface.
|
||||
func (a *BoolArray) Scan(src interface{}) error {
|
||||
switch src := src.(type) {
|
||||
case []byte:
|
||||
return a.scanBytes(src)
|
||||
case string:
|
||||
return a.scanBytes([]byte(src))
|
||||
case nil:
|
||||
*a = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("pq: cannot convert %T to BoolArray", src)
|
||||
}
|
||||
|
||||
func (a *BoolArray) scanBytes(src []byte) error {
|
||||
elems, err := scanLinearArray(src, []byte{','}, "BoolArray")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *a != nil && len(elems) == 0 {
|
||||
*a = (*a)[:0]
|
||||
} else {
|
||||
b := make(BoolArray, len(elems))
|
||||
for i, v := range elems {
|
||||
if len(v) != 1 {
|
||||
return fmt.Errorf("pq: could not parse boolean array index %d: invalid boolean %q", i, v)
|
||||
}
|
||||
switch v[0] {
|
||||
case 't':
|
||||
b[i] = true
|
||||
case 'f':
|
||||
b[i] = false
|
||||
default:
|
||||
return fmt.Errorf("pq: could not parse boolean array index %d: invalid boolean %q", i, v)
|
||||
}
|
||||
}
|
||||
*a = b
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface.
|
||||
func (a BoolArray) Value() (driver.Value, error) {
|
||||
if a == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if n := len(a); n > 0 {
|
||||
// There will be exactly two curly brackets, N bytes of values,
|
||||
// and N-1 bytes of delimiters.
|
||||
b := make([]byte, 1+2*n)
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
b[2*i] = ','
|
||||
if a[i] {
|
||||
b[1+2*i] = 't'
|
||||
} else {
|
||||
b[1+2*i] = 'f'
|
||||
}
|
||||
}
|
||||
|
||||
b[0] = '{'
|
||||
b[2*n] = '}'
|
||||
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
return "{}", nil
|
||||
}
|
||||
|
||||
// ByteaArray represents a one-dimensional array of the PostgreSQL bytea type.
|
||||
type ByteaArray [][]byte
|
||||
|
||||
// Scan implements the sql.Scanner interface.
|
||||
func (a *ByteaArray) Scan(src interface{}) error {
|
||||
switch src := src.(type) {
|
||||
case []byte:
|
||||
return a.scanBytes(src)
|
||||
case string:
|
||||
return a.scanBytes([]byte(src))
|
||||
case nil:
|
||||
*a = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("pq: cannot convert %T to ByteaArray", src)
|
||||
}
|
||||
|
||||
func (a *ByteaArray) scanBytes(src []byte) error {
|
||||
elems, err := scanLinearArray(src, []byte{','}, "ByteaArray")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *a != nil && len(elems) == 0 {
|
||||
*a = (*a)[:0]
|
||||
} else {
|
||||
b := make(ByteaArray, len(elems))
|
||||
for i, v := range elems {
|
||||
b[i], err = parseBytea(v)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not parse bytea array index %d: %s", i, err.Error())
|
||||
}
|
||||
}
|
||||
*a = b
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface. It uses the "hex" format which
|
||||
// is only supported on PostgreSQL 9.0 or newer.
|
||||
func (a ByteaArray) Value() (driver.Value, error) {
|
||||
if a == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if n := len(a); n > 0 {
|
||||
// There will be at least two curly brackets, 2*N bytes of quotes,
|
||||
// 3*N bytes of hex formatting, and N-1 bytes of delimiters.
|
||||
size := 1 + 6*n
|
||||
for _, x := range a {
|
||||
size += hex.EncodedLen(len(x))
|
||||
}
|
||||
|
||||
b := make([]byte, size)
|
||||
|
||||
for i, s := 0, b; i < n; i++ {
|
||||
o := copy(s, `,"\\x`)
|
||||
o += hex.Encode(s[o:], a[i])
|
||||
s[o] = '"'
|
||||
s = s[o+1:]
|
||||
}
|
||||
|
||||
b[0] = '{'
|
||||
b[size-1] = '}'
|
||||
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
return "{}", nil
|
||||
}
|
||||
|
||||
// Float64Array represents a one-dimensional array of the PostgreSQL double
|
||||
// precision type.
|
||||
type Float64Array []float64
|
||||
|
||||
// Scan implements the sql.Scanner interface.
|
||||
func (a *Float64Array) Scan(src interface{}) error {
|
||||
switch src := src.(type) {
|
||||
case []byte:
|
||||
return a.scanBytes(src)
|
||||
case string:
|
||||
return a.scanBytes([]byte(src))
|
||||
case nil:
|
||||
*a = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("pq: cannot convert %T to Float64Array", src)
|
||||
}
|
||||
|
||||
func (a *Float64Array) scanBytes(src []byte) error {
|
||||
elems, err := scanLinearArray(src, []byte{','}, "Float64Array")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *a != nil && len(elems) == 0 {
|
||||
*a = (*a)[:0]
|
||||
} else {
|
||||
b := make(Float64Array, len(elems))
|
||||
for i, v := range elems {
|
||||
if b[i], err = strconv.ParseFloat(string(v), 64); err != nil {
|
||||
return fmt.Errorf("pq: parsing array element index %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
*a = b
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface.
|
||||
func (a Float64Array) Value() (driver.Value, error) {
|
||||
if a == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if n := len(a); n > 0 {
|
||||
// There will be at least two curly brackets, N bytes of values,
|
||||
// and N-1 bytes of delimiters.
|
||||
b := make([]byte, 1, 1+2*n)
|
||||
b[0] = '{'
|
||||
|
||||
b = strconv.AppendFloat(b, a[0], 'f', -1, 64)
|
||||
for i := 1; i < n; i++ {
|
||||
b = append(b, ',')
|
||||
b = strconv.AppendFloat(b, a[i], 'f', -1, 64)
|
||||
}
|
||||
|
||||
return string(append(b, '}')), nil
|
||||
}
|
||||
|
||||
return "{}", nil
|
||||
}
|
||||
|
||||
// Float32Array represents a one-dimensional array of the PostgreSQL double
|
||||
// precision type.
|
||||
type Float32Array []float32
|
||||
|
||||
// Scan implements the sql.Scanner interface.
|
||||
func (a *Float32Array) Scan(src interface{}) error {
|
||||
switch src := src.(type) {
|
||||
case []byte:
|
||||
return a.scanBytes(src)
|
||||
case string:
|
||||
return a.scanBytes([]byte(src))
|
||||
case nil:
|
||||
*a = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("pq: cannot convert %T to Float32Array", src)
|
||||
}
|
||||
|
||||
func (a *Float32Array) scanBytes(src []byte) error {
|
||||
elems, err := scanLinearArray(src, []byte{','}, "Float32Array")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *a != nil && len(elems) == 0 {
|
||||
*a = (*a)[:0]
|
||||
} else {
|
||||
b := make(Float32Array, len(elems))
|
||||
for i, v := range elems {
|
||||
var x float64
|
||||
if x, err = strconv.ParseFloat(string(v), 32); err != nil {
|
||||
return fmt.Errorf("pq: parsing array element index %d: %v", i, err)
|
||||
}
|
||||
b[i] = float32(x)
|
||||
}
|
||||
*a = b
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface.
|
||||
func (a Float32Array) Value() (driver.Value, error) {
|
||||
if a == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if n := len(a); n > 0 {
|
||||
// There will be at least two curly brackets, N bytes of values,
|
||||
// and N-1 bytes of delimiters.
|
||||
b := make([]byte, 1, 1+2*n)
|
||||
b[0] = '{'
|
||||
|
||||
b = strconv.AppendFloat(b, float64(a[0]), 'f', -1, 32)
|
||||
for i := 1; i < n; i++ {
|
||||
b = append(b, ',')
|
||||
b = strconv.AppendFloat(b, float64(a[i]), 'f', -1, 32)
|
||||
}
|
||||
|
||||
return string(append(b, '}')), nil
|
||||
}
|
||||
|
||||
return "{}", nil
|
||||
}
|
||||
|
||||
// GenericArray implements the driver.Valuer and sql.Scanner interfaces for
|
||||
// an array or slice of any dimension.
|
||||
type GenericArray struct{ A interface{} }
|
||||
|
||||
func (GenericArray) evaluateDestination(rt reflect.Type) (reflect.Type, func([]byte, reflect.Value) error, string) {
|
||||
var assign func([]byte, reflect.Value) error
|
||||
var del = ","
|
||||
|
||||
// TODO calculate the assign function for other types
|
||||
// TODO repeat this section on the element type of arrays or slices (multidimensional)
|
||||
{
|
||||
if reflect.PtrTo(rt).Implements(typeSQLScanner) {
|
||||
// dest is always addressable because it is an element of a slice.
|
||||
assign = func(src []byte, dest reflect.Value) (err error) {
|
||||
ss := dest.Addr().Interface().(sql.Scanner)
|
||||
if src == nil {
|
||||
err = ss.Scan(nil)
|
||||
} else {
|
||||
err = ss.Scan(src)
|
||||
}
|
||||
return
|
||||
}
|
||||
goto FoundType
|
||||
}
|
||||
|
||||
assign = func([]byte, reflect.Value) error {
|
||||
return fmt.Errorf("pq: scanning to %s is not implemented; only sql.Scanner", rt)
|
||||
}
|
||||
}
|
||||
|
||||
FoundType:
|
||||
|
||||
if ad, ok := reflect.Zero(rt).Interface().(ArrayDelimiter); ok {
|
||||
del = ad.ArrayDelimiter()
|
||||
}
|
||||
|
||||
return rt, assign, del
|
||||
}
|
||||
|
||||
// Scan implements the sql.Scanner interface.
|
||||
func (a GenericArray) Scan(src interface{}) error {
|
||||
dpv := reflect.ValueOf(a.A)
|
||||
switch {
|
||||
case dpv.Kind() != reflect.Ptr:
|
||||
return fmt.Errorf("pq: destination %T is not a pointer to array or slice", a.A)
|
||||
case dpv.IsNil():
|
||||
return fmt.Errorf("pq: destination %T is nil", a.A)
|
||||
}
|
||||
|
||||
dv := dpv.Elem()
|
||||
switch dv.Kind() {
|
||||
case reflect.Slice:
|
||||
case reflect.Array:
|
||||
default:
|
||||
return fmt.Errorf("pq: destination %T is not a pointer to array or slice", a.A)
|
||||
}
|
||||
|
||||
switch src := src.(type) {
|
||||
case []byte:
|
||||
return a.scanBytes(src, dv)
|
||||
case string:
|
||||
return a.scanBytes([]byte(src), dv)
|
||||
case nil:
|
||||
if dv.Kind() == reflect.Slice {
|
||||
dv.Set(reflect.Zero(dv.Type()))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("pq: cannot convert %T to %s", src, dv.Type())
|
||||
}
|
||||
|
||||
func (a GenericArray) scanBytes(src []byte, dv reflect.Value) error {
|
||||
dtype, assign, del := a.evaluateDestination(dv.Type().Elem())
|
||||
dims, elems, err := parseArray(src, []byte(del))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO allow multidimensional
|
||||
|
||||
if len(dims) > 1 {
|
||||
return fmt.Errorf("pq: scanning from multidimensional ARRAY%s is not implemented",
|
||||
strings.Replace(fmt.Sprint(dims), " ", "][", -1))
|
||||
}
|
||||
|
||||
// Treat a zero-dimensional array like an array with a single dimension of zero.
|
||||
if len(dims) == 0 {
|
||||
dims = append(dims, 0)
|
||||
}
|
||||
|
||||
for i, rt := 0, dv.Type(); i < len(dims); i, rt = i+1, rt.Elem() {
|
||||
switch rt.Kind() {
|
||||
case reflect.Slice:
|
||||
case reflect.Array:
|
||||
if rt.Len() != dims[i] {
|
||||
return fmt.Errorf("pq: cannot convert ARRAY%s to %s",
|
||||
strings.Replace(fmt.Sprint(dims), " ", "][", -1), dv.Type())
|
||||
}
|
||||
default:
|
||||
// TODO handle multidimensional
|
||||
}
|
||||
}
|
||||
|
||||
values := reflect.MakeSlice(reflect.SliceOf(dtype), len(elems), len(elems))
|
||||
for i, e := range elems {
|
||||
if err := assign(e, values.Index(i)); err != nil {
|
||||
return fmt.Errorf("pq: parsing array element index %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TODO handle multidimensional
|
||||
|
||||
switch dv.Kind() {
|
||||
case reflect.Slice:
|
||||
dv.Set(values.Slice(0, dims[0]))
|
||||
case reflect.Array:
|
||||
for i := 0; i < dims[0]; i++ {
|
||||
dv.Index(i).Set(values.Index(i))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface.
|
||||
func (a GenericArray) Value() (driver.Value, error) {
|
||||
if a.A == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
rv := reflect.ValueOf(a.A)
|
||||
|
||||
switch rv.Kind() {
|
||||
case reflect.Slice:
|
||||
if rv.IsNil() {
|
||||
return nil, nil
|
||||
}
|
||||
case reflect.Array:
|
||||
default:
|
||||
return nil, fmt.Errorf("pq: Unable to convert %T to array", a.A)
|
||||
}
|
||||
|
||||
if n := rv.Len(); n > 0 {
|
||||
// There will be at least two curly brackets, N bytes of values,
|
||||
// and N-1 bytes of delimiters.
|
||||
b := make([]byte, 0, 1+2*n)
|
||||
|
||||
b, _, err := appendArray(b, rv, n)
|
||||
return string(b), err
|
||||
}
|
||||
|
||||
return "{}", nil
|
||||
}
|
||||
|
||||
// Int64Array represents a one-dimensional array of the PostgreSQL integer types.
|
||||
type Int64Array []int64
|
||||
|
||||
// Scan implements the sql.Scanner interface.
|
||||
func (a *Int64Array) Scan(src interface{}) error {
|
||||
switch src := src.(type) {
|
||||
case []byte:
|
||||
return a.scanBytes(src)
|
||||
case string:
|
||||
return a.scanBytes([]byte(src))
|
||||
case nil:
|
||||
*a = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("pq: cannot convert %T to Int64Array", src)
|
||||
}
|
||||
|
||||
func (a *Int64Array) scanBytes(src []byte) error {
|
||||
elems, err := scanLinearArray(src, []byte{','}, "Int64Array")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *a != nil && len(elems) == 0 {
|
||||
*a = (*a)[:0]
|
||||
} else {
|
||||
b := make(Int64Array, len(elems))
|
||||
for i, v := range elems {
|
||||
if b[i], err = strconv.ParseInt(string(v), 10, 64); err != nil {
|
||||
return fmt.Errorf("pq: parsing array element index %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
*a = b
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface.
|
||||
func (a Int64Array) Value() (driver.Value, error) {
|
||||
if a == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if n := len(a); n > 0 {
|
||||
// There will be at least two curly brackets, N bytes of values,
|
||||
// and N-1 bytes of delimiters.
|
||||
b := make([]byte, 1, 1+2*n)
|
||||
b[0] = '{'
|
||||
|
||||
b = strconv.AppendInt(b, a[0], 10)
|
||||
for i := 1; i < n; i++ {
|
||||
b = append(b, ',')
|
||||
b = strconv.AppendInt(b, a[i], 10)
|
||||
}
|
||||
|
||||
return string(append(b, '}')), nil
|
||||
}
|
||||
|
||||
return "{}", nil
|
||||
}
|
||||
|
||||
// Int32Array represents a one-dimensional array of the PostgreSQL integer types.
|
||||
type Int32Array []int32
|
||||
|
||||
// Scan implements the sql.Scanner interface.
|
||||
func (a *Int32Array) Scan(src interface{}) error {
|
||||
switch src := src.(type) {
|
||||
case []byte:
|
||||
return a.scanBytes(src)
|
||||
case string:
|
||||
return a.scanBytes([]byte(src))
|
||||
case nil:
|
||||
*a = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("pq: cannot convert %T to Int32Array", src)
|
||||
}
|
||||
|
||||
func (a *Int32Array) scanBytes(src []byte) error {
|
||||
elems, err := scanLinearArray(src, []byte{','}, "Int32Array")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *a != nil && len(elems) == 0 {
|
||||
*a = (*a)[:0]
|
||||
} else {
|
||||
b := make(Int32Array, len(elems))
|
||||
for i, v := range elems {
|
||||
x, err := strconv.ParseInt(string(v), 10, 32)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pq: parsing array element index %d: %v", i, err)
|
||||
}
|
||||
b[i] = int32(x)
|
||||
}
|
||||
*a = b
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface.
|
||||
func (a Int32Array) Value() (driver.Value, error) {
|
||||
if a == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if n := len(a); n > 0 {
|
||||
// There will be at least two curly brackets, N bytes of values,
|
||||
// and N-1 bytes of delimiters.
|
||||
b := make([]byte, 1, 1+2*n)
|
||||
b[0] = '{'
|
||||
|
||||
b = strconv.AppendInt(b, int64(a[0]), 10)
|
||||
for i := 1; i < n; i++ {
|
||||
b = append(b, ',')
|
||||
b = strconv.AppendInt(b, int64(a[i]), 10)
|
||||
}
|
||||
|
||||
return string(append(b, '}')), nil
|
||||
}
|
||||
|
||||
return "{}", nil
|
||||
}
|
||||
|
||||
// StringArray represents a one-dimensional array of the PostgreSQL character types.
|
||||
type StringArray []string
|
||||
|
||||
// Scan implements the sql.Scanner interface.
|
||||
func (a *StringArray) Scan(src interface{}) error {
|
||||
switch src := src.(type) {
|
||||
case []byte:
|
||||
return a.scanBytes(src)
|
||||
case string:
|
||||
return a.scanBytes([]byte(src))
|
||||
case nil:
|
||||
*a = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("pq: cannot convert %T to StringArray", src)
|
||||
}
|
||||
|
||||
func (a *StringArray) scanBytes(src []byte) error {
|
||||
elems, err := scanLinearArray(src, []byte{','}, "StringArray")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *a != nil && len(elems) == 0 {
|
||||
*a = (*a)[:0]
|
||||
} else {
|
||||
b := make(StringArray, len(elems))
|
||||
for i, v := range elems {
|
||||
if b[i] = string(v); v == nil {
|
||||
return fmt.Errorf("pq: parsing array element index %d: cannot convert nil to string", i)
|
||||
}
|
||||
}
|
||||
*a = b
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface.
|
||||
func (a StringArray) Value() (driver.Value, error) {
|
||||
if a == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if n := len(a); n > 0 {
|
||||
// There will be at least two curly brackets, 2*N bytes of quotes,
|
||||
// and N-1 bytes of delimiters.
|
||||
b := make([]byte, 1, 1+3*n)
|
||||
b[0] = '{'
|
||||
|
||||
b = appendArrayQuotedBytes(b, []byte(a[0]))
|
||||
for i := 1; i < n; i++ {
|
||||
b = append(b, ',')
|
||||
b = appendArrayQuotedBytes(b, []byte(a[i]))
|
||||
}
|
||||
|
||||
return string(append(b, '}')), nil
|
||||
}
|
||||
|
||||
return "{}", nil
|
||||
}
|
||||
|
||||
// appendArray appends rv to the buffer, returning the extended buffer and
|
||||
// the delimiter used between elements.
|
||||
//
|
||||
// It panics when n <= 0 or rv's Kind is not reflect.Array nor reflect.Slice.
|
||||
func appendArray(b []byte, rv reflect.Value, n int) ([]byte, string, error) {
|
||||
var del string
|
||||
var err error
|
||||
|
||||
b = append(b, '{')
|
||||
|
||||
if b, del, err = appendArrayElement(b, rv.Index(0)); err != nil {
|
||||
return b, del, err
|
||||
}
|
||||
|
||||
for i := 1; i < n; i++ {
|
||||
b = append(b, del...)
|
||||
if b, del, err = appendArrayElement(b, rv.Index(i)); err != nil {
|
||||
return b, del, err
|
||||
}
|
||||
}
|
||||
|
||||
return append(b, '}'), del, nil
|
||||
}
|
||||
|
||||
// appendArrayElement appends rv to the buffer, returning the extended buffer
|
||||
// and the delimiter to use before the next element.
|
||||
//
|
||||
// When rv's Kind is neither reflect.Array nor reflect.Slice, it is converted
|
||||
// using driver.DefaultParameterConverter and the resulting []byte or string
|
||||
// is double-quoted.
|
||||
//
|
||||
// See http://www.postgresql.org/docs/current/static/arrays.html#ARRAYS-IO
|
||||
func appendArrayElement(b []byte, rv reflect.Value) ([]byte, string, error) {
|
||||
if k := rv.Kind(); k == reflect.Array || k == reflect.Slice {
|
||||
if t := rv.Type(); t != typeByteSlice && !t.Implements(typeDriverValuer) {
|
||||
if n := rv.Len(); n > 0 {
|
||||
return appendArray(b, rv, n)
|
||||
}
|
||||
|
||||
return b, "", nil
|
||||
}
|
||||
}
|
||||
|
||||
var del = ","
|
||||
var err error
|
||||
var iv interface{} = rv.Interface()
|
||||
|
||||
if ad, ok := iv.(ArrayDelimiter); ok {
|
||||
del = ad.ArrayDelimiter()
|
||||
}
|
||||
|
||||
if iv, err = driver.DefaultParameterConverter.ConvertValue(iv); err != nil {
|
||||
return b, del, err
|
||||
}
|
||||
|
||||
switch v := iv.(type) {
|
||||
case nil:
|
||||
return append(b, "NULL"...), del, nil
|
||||
case []byte:
|
||||
return appendArrayQuotedBytes(b, v), del, nil
|
||||
case string:
|
||||
return appendArrayQuotedBytes(b, []byte(v)), del, nil
|
||||
}
|
||||
|
||||
b, err = appendValue(b, iv)
|
||||
return b, del, err
|
||||
}
|
||||
|
||||
func appendArrayQuotedBytes(b, v []byte) []byte {
|
||||
b = append(b, '"')
|
||||
for {
|
||||
i := bytes.IndexAny(v, `"\`)
|
||||
if i < 0 {
|
||||
b = append(b, v...)
|
||||
break
|
||||
}
|
||||
if i > 0 {
|
||||
b = append(b, v[:i]...)
|
||||
}
|
||||
b = append(b, '\\', v[i])
|
||||
v = v[i+1:]
|
||||
}
|
||||
return append(b, '"')
|
||||
}
|
||||
|
||||
func appendValue(b []byte, v driver.Value) ([]byte, error) {
|
||||
return append(b, encode(nil, v, 0)...), nil
|
||||
}
|
||||
|
||||
// parseArray extracts the dimensions and elements of an array represented in
|
||||
// text format. Only representations emitted by the backend are supported.
|
||||
// Notably, whitespace around brackets and delimiters is significant, and NULL
|
||||
// is case-sensitive.
|
||||
//
|
||||
// See http://www.postgresql.org/docs/current/static/arrays.html#ARRAYS-IO
|
||||
func parseArray(src, del []byte) (dims []int, elems [][]byte, err error) {
|
||||
var depth, i int
|
||||
|
||||
if len(src) < 1 || src[0] != '{' {
|
||||
return nil, nil, fmt.Errorf("pq: unable to parse array; expected %q at offset %d", '{', 0)
|
||||
}
|
||||
|
||||
Open:
|
||||
for i < len(src) {
|
||||
switch src[i] {
|
||||
case '{':
|
||||
depth++
|
||||
i++
|
||||
case '}':
|
||||
elems = make([][]byte, 0)
|
||||
goto Close
|
||||
default:
|
||||
break Open
|
||||
}
|
||||
}
|
||||
dims = make([]int, i)
|
||||
|
||||
Element:
|
||||
for i < len(src) {
|
||||
switch src[i] {
|
||||
case '{':
|
||||
if depth == len(dims) {
|
||||
break Element
|
||||
}
|
||||
depth++
|
||||
dims[depth-1] = 0
|
||||
i++
|
||||
case '"':
|
||||
var elem = []byte{}
|
||||
var escape bool
|
||||
for i++; i < len(src); i++ {
|
||||
if escape {
|
||||
elem = append(elem, src[i])
|
||||
escape = false
|
||||
} else {
|
||||
switch src[i] {
|
||||
default:
|
||||
elem = append(elem, src[i])
|
||||
case '\\':
|
||||
escape = true
|
||||
case '"':
|
||||
elems = append(elems, elem)
|
||||
i++
|
||||
break Element
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
for start := i; i < len(src); i++ {
|
||||
if bytes.HasPrefix(src[i:], del) || src[i] == '}' {
|
||||
elem := src[start:i]
|
||||
if len(elem) == 0 {
|
||||
return nil, nil, fmt.Errorf("pq: unable to parse array; unexpected %q at offset %d", src[i], i)
|
||||
}
|
||||
if bytes.Equal(elem, []byte("NULL")) {
|
||||
elem = nil
|
||||
}
|
||||
elems = append(elems, elem)
|
||||
break Element
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for i < len(src) {
|
||||
if bytes.HasPrefix(src[i:], del) && depth > 0 {
|
||||
dims[depth-1]++
|
||||
i += len(del)
|
||||
goto Element
|
||||
} else if src[i] == '}' && depth > 0 {
|
||||
dims[depth-1]++
|
||||
depth--
|
||||
i++
|
||||
} else {
|
||||
return nil, nil, fmt.Errorf("pq: unable to parse array; unexpected %q at offset %d", src[i], i)
|
||||
}
|
||||
}
|
||||
|
||||
Close:
|
||||
for i < len(src) {
|
||||
if src[i] == '}' && depth > 0 {
|
||||
depth--
|
||||
i++
|
||||
} else {
|
||||
return nil, nil, fmt.Errorf("pq: unable to parse array; unexpected %q at offset %d", src[i], i)
|
||||
}
|
||||
}
|
||||
if depth > 0 {
|
||||
err = fmt.Errorf("pq: unable to parse array; expected %q at offset %d", '}', i)
|
||||
}
|
||||
if err == nil {
|
||||
for _, d := range dims {
|
||||
if (len(elems) % d) != 0 {
|
||||
err = fmt.Errorf("pq: multidimensional arrays must have elements with matching dimensions")
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func scanLinearArray(src, del []byte, typ string) (elems [][]byte, err error) {
|
||||
dims, elems, err := parseArray(src, del)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(dims) > 1 {
|
||||
return nil, fmt.Errorf("pq: cannot convert ARRAY%s to %s", strings.Replace(fmt.Sprint(dims), " ", "][", -1), typ)
|
||||
}
|
||||
return elems, err
|
||||
}
|
||||
@ -0,0 +1,91 @@
|
||||
package pq
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
|
||||
"github.com/lib/pq/oid"
|
||||
)
|
||||
|
||||
type readBuf []byte
|
||||
|
||||
func (b *readBuf) int32() (n int) {
|
||||
n = int(int32(binary.BigEndian.Uint32(*b)))
|
||||
*b = (*b)[4:]
|
||||
return
|
||||
}
|
||||
|
||||
func (b *readBuf) oid() (n oid.Oid) {
|
||||
n = oid.Oid(binary.BigEndian.Uint32(*b))
|
||||
*b = (*b)[4:]
|
||||
return
|
||||
}
|
||||
|
||||
// N.B: this is actually an unsigned 16-bit integer, unlike int32
|
||||
func (b *readBuf) int16() (n int) {
|
||||
n = int(binary.BigEndian.Uint16(*b))
|
||||
*b = (*b)[2:]
|
||||
return
|
||||
}
|
||||
|
||||
func (b *readBuf) string() string {
|
||||
i := bytes.IndexByte(*b, 0)
|
||||
if i < 0 {
|
||||
errorf("invalid message format; expected string terminator")
|
||||
}
|
||||
s := (*b)[:i]
|
||||
*b = (*b)[i+1:]
|
||||
return string(s)
|
||||
}
|
||||
|
||||
func (b *readBuf) next(n int) (v []byte) {
|
||||
v = (*b)[:n]
|
||||
*b = (*b)[n:]
|
||||
return
|
||||
}
|
||||
|
||||
func (b *readBuf) byte() byte {
|
||||
return b.next(1)[0]
|
||||
}
|
||||
|
||||
type writeBuf struct {
|
||||
buf []byte
|
||||
pos int
|
||||
}
|
||||
|
||||
func (b *writeBuf) int32(n int) {
|
||||
x := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(x, uint32(n))
|
||||
b.buf = append(b.buf, x...)
|
||||
}
|
||||
|
||||
func (b *writeBuf) int16(n int) {
|
||||
x := make([]byte, 2)
|
||||
binary.BigEndian.PutUint16(x, uint16(n))
|
||||
b.buf = append(b.buf, x...)
|
||||
}
|
||||
|
||||
func (b *writeBuf) string(s string) {
|
||||
b.buf = append(append(b.buf, s...), '\000')
|
||||
}
|
||||
|
||||
func (b *writeBuf) byte(c byte) {
|
||||
b.buf = append(b.buf, c)
|
||||
}
|
||||
|
||||
func (b *writeBuf) bytes(v []byte) {
|
||||
b.buf = append(b.buf, v...)
|
||||
}
|
||||
|
||||
func (b *writeBuf) wrap() []byte {
|
||||
p := b.buf[b.pos:]
|
||||
binary.BigEndian.PutUint32(p, uint32(len(p)))
|
||||
return b.buf
|
||||
}
|
||||
|
||||
func (b *writeBuf) next(c byte) {
|
||||
p := b.buf[b.pos:]
|
||||
binary.BigEndian.PutUint32(p, uint32(len(p)))
|
||||
b.pos = len(b.buf) + 1
|
||||
b.buf = append(b.buf, c, 0, 0, 0, 0)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,247 @@
|
||||
package pq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
watchCancelDialContextTimeout = time.Second * 10
|
||||
)
|
||||
|
||||
// Implement the "QueryerContext" interface
|
||||
func (cn *conn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
|
||||
list := make([]driver.Value, len(args))
|
||||
for i, nv := range args {
|
||||
list[i] = nv.Value
|
||||
}
|
||||
finish := cn.watchCancel(ctx)
|
||||
r, err := cn.query(query, list)
|
||||
if err != nil {
|
||||
if finish != nil {
|
||||
finish()
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
r.finish = finish
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Implement the "ExecerContext" interface
|
||||
func (cn *conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) {
|
||||
list := make([]driver.Value, len(args))
|
||||
for i, nv := range args {
|
||||
list[i] = nv.Value
|
||||
}
|
||||
|
||||
if finish := cn.watchCancel(ctx); finish != nil {
|
||||
defer finish()
|
||||
}
|
||||
|
||||
return cn.Exec(query, list)
|
||||
}
|
||||
|
||||
// Implement the "ConnPrepareContext" interface
|
||||
func (cn *conn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) {
|
||||
if finish := cn.watchCancel(ctx); finish != nil {
|
||||
defer finish()
|
||||
}
|
||||
return cn.Prepare(query)
|
||||
}
|
||||
|
||||
// Implement the "ConnBeginTx" interface
|
||||
func (cn *conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) {
|
||||
var mode string
|
||||
|
||||
switch sql.IsolationLevel(opts.Isolation) {
|
||||
case sql.LevelDefault:
|
||||
// Don't touch mode: use the server's default
|
||||
case sql.LevelReadUncommitted:
|
||||
mode = " ISOLATION LEVEL READ UNCOMMITTED"
|
||||
case sql.LevelReadCommitted:
|
||||
mode = " ISOLATION LEVEL READ COMMITTED"
|
||||
case sql.LevelRepeatableRead:
|
||||
mode = " ISOLATION LEVEL REPEATABLE READ"
|
||||
case sql.LevelSerializable:
|
||||
mode = " ISOLATION LEVEL SERIALIZABLE"
|
||||
default:
|
||||
return nil, fmt.Errorf("pq: isolation level not supported: %d", opts.Isolation)
|
||||
}
|
||||
|
||||
if opts.ReadOnly {
|
||||
mode += " READ ONLY"
|
||||
} else {
|
||||
mode += " READ WRITE"
|
||||
}
|
||||
|
||||
tx, err := cn.begin(mode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cn.txnFinish = cn.watchCancel(ctx)
|
||||
return tx, nil
|
||||
}
|
||||
|
||||
func (cn *conn) Ping(ctx context.Context) error {
|
||||
if finish := cn.watchCancel(ctx); finish != nil {
|
||||
defer finish()
|
||||
}
|
||||
rows, err := cn.simpleQuery(";")
|
||||
if err != nil {
|
||||
return driver.ErrBadConn // https://golang.org/pkg/database/sql/driver/#Pinger
|
||||
}
|
||||
rows.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cn *conn) watchCancel(ctx context.Context) func() {
|
||||
if done := ctx.Done(); done != nil {
|
||||
finished := make(chan struct{}, 1)
|
||||
go func() {
|
||||
select {
|
||||
case <-done:
|
||||
select {
|
||||
case finished <- struct{}{}:
|
||||
default:
|
||||
// We raced with the finish func, let the next query handle this with the
|
||||
// context.
|
||||
return
|
||||
}
|
||||
|
||||
// Set the connection state to bad so it does not get reused.
|
||||
cn.err.set(ctx.Err())
|
||||
|
||||
// At this point the function level context is canceled,
|
||||
// so it must not be used for the additional network
|
||||
// request to cancel the query.
|
||||
// Create a new context to pass into the dial.
|
||||
ctxCancel, cancel := context.WithTimeout(context.Background(), watchCancelDialContextTimeout)
|
||||
defer cancel()
|
||||
|
||||
_ = cn.cancel(ctxCancel)
|
||||
case <-finished:
|
||||
}
|
||||
}()
|
||||
return func() {
|
||||
select {
|
||||
case <-finished:
|
||||
cn.err.set(ctx.Err())
|
||||
cn.Close()
|
||||
case finished <- struct{}{}:
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cn *conn) cancel(ctx context.Context) error {
|
||||
// Create a new values map (copy). This makes sure the connection created
|
||||
// in this method cannot write to the same underlying data, which could
|
||||
// cause a concurrent map write panic. This is necessary because cancel
|
||||
// is called from a goroutine in watchCancel.
|
||||
o := make(values)
|
||||
for k, v := range cn.opts {
|
||||
o[k] = v
|
||||
}
|
||||
|
||||
c, err := dial(ctx, cn.dialer, o)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer c.Close()
|
||||
|
||||
{
|
||||
can := conn{
|
||||
c: c,
|
||||
}
|
||||
err = can.ssl(o)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w := can.writeBuf(0)
|
||||
w.int32(80877102) // cancel request code
|
||||
w.int32(cn.processID)
|
||||
w.int32(cn.secretKey)
|
||||
|
||||
if err := can.sendStartupPacket(w); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Read until EOF to ensure that the server received the cancel.
|
||||
{
|
||||
_, err := io.Copy(ioutil.Discard, c)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Implement the "StmtQueryContext" interface
|
||||
func (st *stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) {
|
||||
list := make([]driver.Value, len(args))
|
||||
for i, nv := range args {
|
||||
list[i] = nv.Value
|
||||
}
|
||||
finish := st.watchCancel(ctx)
|
||||
r, err := st.query(list)
|
||||
if err != nil {
|
||||
if finish != nil {
|
||||
finish()
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
r.finish = finish
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Implement the "StmtExecContext" interface
|
||||
func (st *stmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) {
|
||||
list := make([]driver.Value, len(args))
|
||||
for i, nv := range args {
|
||||
list[i] = nv.Value
|
||||
}
|
||||
|
||||
if finish := st.watchCancel(ctx); finish != nil {
|
||||
defer finish()
|
||||
}
|
||||
|
||||
return st.Exec(list)
|
||||
}
|
||||
|
||||
// watchCancel is implemented on stmt in order to not mark the parent conn as bad
|
||||
func (st *stmt) watchCancel(ctx context.Context) func() {
|
||||
if done := ctx.Done(); done != nil {
|
||||
finished := make(chan struct{})
|
||||
go func() {
|
||||
select {
|
||||
case <-done:
|
||||
// At this point the function level context is canceled,
|
||||
// so it must not be used for the additional network
|
||||
// request to cancel the query.
|
||||
// Create a new context to pass into the dial.
|
||||
ctxCancel, cancel := context.WithTimeout(context.Background(), watchCancelDialContextTimeout)
|
||||
defer cancel()
|
||||
|
||||
_ = st.cancel(ctxCancel)
|
||||
finished <- struct{}{}
|
||||
case <-finished:
|
||||
}
|
||||
}()
|
||||
return func() {
|
||||
select {
|
||||
case <-finished:
|
||||
case finished <- struct{}{}:
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (st *stmt) cancel(ctx context.Context) error {
|
||||
return st.cn.cancel(ctx)
|
||||
}
|
||||
@ -0,0 +1,120 @@
|
||||
package pq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql/driver"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Connector represents a fixed configuration for the pq driver with a given
|
||||
// name. Connector satisfies the database/sql/driver Connector interface and
|
||||
// can be used to create any number of DB Conn's via the database/sql OpenDB
|
||||
// function.
|
||||
//
|
||||
// See https://golang.org/pkg/database/sql/driver/#Connector.
|
||||
// See https://golang.org/pkg/database/sql/#OpenDB.
|
||||
type Connector struct {
|
||||
opts values
|
||||
dialer Dialer
|
||||
}
|
||||
|
||||
// Connect returns a connection to the database using the fixed configuration
|
||||
// of this Connector. Context is not used.
|
||||
func (c *Connector) Connect(ctx context.Context) (driver.Conn, error) {
|
||||
return c.open(ctx)
|
||||
}
|
||||
|
||||
// Dialer allows change the dialer used to open connections.
|
||||
func (c *Connector) Dialer(dialer Dialer) {
|
||||
c.dialer = dialer
|
||||
}
|
||||
|
||||
// Driver returns the underlying driver of this Connector.
|
||||
func (c *Connector) Driver() driver.Driver {
|
||||
return &Driver{}
|
||||
}
|
||||
|
||||
// NewConnector returns a connector for the pq driver in a fixed configuration
|
||||
// with the given dsn. The returned connector can be used to create any number
|
||||
// of equivalent Conn's. The returned connector is intended to be used with
|
||||
// database/sql.OpenDB.
|
||||
//
|
||||
// See https://golang.org/pkg/database/sql/driver/#Connector.
|
||||
// See https://golang.org/pkg/database/sql/#OpenDB.
|
||||
func NewConnector(dsn string) (*Connector, error) {
|
||||
var err error
|
||||
o := make(values)
|
||||
|
||||
// A number of defaults are applied here, in this order:
|
||||
//
|
||||
// * Very low precedence defaults applied in every situation
|
||||
// * Environment variables
|
||||
// * Explicitly passed connection information
|
||||
o["host"] = "localhost"
|
||||
o["port"] = "5432"
|
||||
// N.B.: Extra float digits should be set to 3, but that breaks
|
||||
// Postgres 8.4 and older, where the max is 2.
|
||||
o["extra_float_digits"] = "2"
|
||||
for k, v := range parseEnviron(os.Environ()) {
|
||||
o[k] = v
|
||||
}
|
||||
|
||||
if strings.HasPrefix(dsn, "postgres://") || strings.HasPrefix(dsn, "postgresql://") {
|
||||
dsn, err = ParseURL(dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := parseOpts(dsn, o); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Use the "fallback" application name if necessary
|
||||
if fallback, ok := o["fallback_application_name"]; ok {
|
||||
if _, ok := o["application_name"]; !ok {
|
||||
o["application_name"] = fallback
|
||||
}
|
||||
}
|
||||
|
||||
// We can't work with any client_encoding other than UTF-8 currently.
|
||||
// However, we have historically allowed the user to set it to UTF-8
|
||||
// explicitly, and there's no reason to break such programs, so allow that.
|
||||
// Note that the "options" setting could also set client_encoding, but
|
||||
// parsing its value is not worth it. Instead, we always explicitly send
|
||||
// client_encoding as a separate run-time parameter, which should override
|
||||
// anything set in options.
|
||||
if enc, ok := o["client_encoding"]; ok && !isUTF8(enc) {
|
||||
return nil, errors.New("client_encoding must be absent or 'UTF8'")
|
||||
}
|
||||
o["client_encoding"] = "UTF8"
|
||||
// DateStyle needs a similar treatment.
|
||||
if datestyle, ok := o["datestyle"]; ok {
|
||||
if datestyle != "ISO, MDY" {
|
||||
return nil, fmt.Errorf("setting datestyle must be absent or %v; got %v", "ISO, MDY", datestyle)
|
||||
}
|
||||
} else {
|
||||
o["datestyle"] = "ISO, MDY"
|
||||
}
|
||||
|
||||
// If a user is not provided by any other means, the last
|
||||
// resort is to use the current operating system provided user
|
||||
// name.
|
||||
if _, ok := o["user"]; !ok {
|
||||
u, err := userCurrent()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o["user"] = u
|
||||
}
|
||||
|
||||
// SSL is not necessary or supported over UNIX domain sockets
|
||||
if network, _ := network(o); network == "unix" {
|
||||
o["sslmode"] = "disable"
|
||||
}
|
||||
|
||||
return &Connector{opts: o, dialer: defaultDialer{}}, nil
|
||||
}
|
||||
@ -0,0 +1,341 @@
|
||||
package pq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql/driver"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var (
|
||||
errCopyInClosed = errors.New("pq: copyin statement has already been closed")
|
||||
errBinaryCopyNotSupported = errors.New("pq: only text format supported for COPY")
|
||||
errCopyToNotSupported = errors.New("pq: COPY TO is not supported")
|
||||
errCopyNotSupportedOutsideTxn = errors.New("pq: COPY is only allowed inside a transaction")
|
||||
errCopyInProgress = errors.New("pq: COPY in progress")
|
||||
)
|
||||
|
||||
// CopyIn creates a COPY FROM statement which can be prepared with
|
||||
// Tx.Prepare(). The target table should be visible in search_path.
|
||||
func CopyIn(table string, columns ...string) string {
|
||||
stmt := "COPY " + QuoteIdentifier(table) + " ("
|
||||
for i, col := range columns {
|
||||
if i != 0 {
|
||||
stmt += ", "
|
||||
}
|
||||
stmt += QuoteIdentifier(col)
|
||||
}
|
||||
stmt += ") FROM STDIN"
|
||||
return stmt
|
||||
}
|
||||
|
||||
// CopyInSchema creates a COPY FROM statement which can be prepared with
|
||||
// Tx.Prepare().
|
||||
func CopyInSchema(schema, table string, columns ...string) string {
|
||||
stmt := "COPY " + QuoteIdentifier(schema) + "." + QuoteIdentifier(table) + " ("
|
||||
for i, col := range columns {
|
||||
if i != 0 {
|
||||
stmt += ", "
|
||||
}
|
||||
stmt += QuoteIdentifier(col)
|
||||
}
|
||||
stmt += ") FROM STDIN"
|
||||
return stmt
|
||||
}
|
||||
|
||||
type copyin struct {
|
||||
cn *conn
|
||||
buffer []byte
|
||||
rowData chan []byte
|
||||
done chan bool
|
||||
|
||||
closed bool
|
||||
|
||||
mu struct {
|
||||
sync.Mutex
|
||||
err error
|
||||
driver.Result
|
||||
}
|
||||
}
|
||||
|
||||
const ciBufferSize = 64 * 1024
|
||||
|
||||
// flush buffer before the buffer is filled up and needs reallocation
|
||||
const ciBufferFlushSize = 63 * 1024
|
||||
|
||||
func (cn *conn) prepareCopyIn(q string) (_ driver.Stmt, err error) {
|
||||
if !cn.isInTransaction() {
|
||||
return nil, errCopyNotSupportedOutsideTxn
|
||||
}
|
||||
|
||||
ci := ©in{
|
||||
cn: cn,
|
||||
buffer: make([]byte, 0, ciBufferSize),
|
||||
rowData: make(chan []byte),
|
||||
done: make(chan bool, 1),
|
||||
}
|
||||
// add CopyData identifier + 4 bytes for message length
|
||||
ci.buffer = append(ci.buffer, 'd', 0, 0, 0, 0)
|
||||
|
||||
b := cn.writeBuf('Q')
|
||||
b.string(q)
|
||||
cn.send(b)
|
||||
|
||||
awaitCopyInResponse:
|
||||
for {
|
||||
t, r := cn.recv1()
|
||||
switch t {
|
||||
case 'G':
|
||||
if r.byte() != 0 {
|
||||
err = errBinaryCopyNotSupported
|
||||
break awaitCopyInResponse
|
||||
}
|
||||
go ci.resploop()
|
||||
return ci, nil
|
||||
case 'H':
|
||||
err = errCopyToNotSupported
|
||||
break awaitCopyInResponse
|
||||
case 'E':
|
||||
err = parseError(r)
|
||||
case 'Z':
|
||||
if err == nil {
|
||||
ci.setBad(driver.ErrBadConn)
|
||||
errorf("unexpected ReadyForQuery in response to COPY")
|
||||
}
|
||||
cn.processReadyForQuery(r)
|
||||
return nil, err
|
||||
default:
|
||||
ci.setBad(driver.ErrBadConn)
|
||||
errorf("unknown response for copy query: %q", t)
|
||||
}
|
||||
}
|
||||
|
||||
// something went wrong, abort COPY before we return
|
||||
b = cn.writeBuf('f')
|
||||
b.string(err.Error())
|
||||
cn.send(b)
|
||||
|
||||
for {
|
||||
t, r := cn.recv1()
|
||||
switch t {
|
||||
case 'c', 'C', 'E':
|
||||
case 'Z':
|
||||
// correctly aborted, we're done
|
||||
cn.processReadyForQuery(r)
|
||||
return nil, err
|
||||
default:
|
||||
ci.setBad(driver.ErrBadConn)
|
||||
errorf("unknown response for CopyFail: %q", t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (ci *copyin) flush(buf []byte) {
|
||||
// set message length (without message identifier)
|
||||
binary.BigEndian.PutUint32(buf[1:], uint32(len(buf)-1))
|
||||
|
||||
_, err := ci.cn.c.Write(buf)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (ci *copyin) resploop() {
|
||||
for {
|
||||
var r readBuf
|
||||
t, err := ci.cn.recvMessage(&r)
|
||||
if err != nil {
|
||||
ci.setBad(driver.ErrBadConn)
|
||||
ci.setError(err)
|
||||
ci.done <- true
|
||||
return
|
||||
}
|
||||
switch t {
|
||||
case 'C':
|
||||
// complete
|
||||
res, _ := ci.cn.parseComplete(r.string())
|
||||
ci.setResult(res)
|
||||
case 'N':
|
||||
if n := ci.cn.noticeHandler; n != nil {
|
||||
n(parseError(&r))
|
||||
}
|
||||
case 'Z':
|
||||
ci.cn.processReadyForQuery(&r)
|
||||
ci.done <- true
|
||||
return
|
||||
case 'E':
|
||||
err := parseError(&r)
|
||||
ci.setError(err)
|
||||
default:
|
||||
ci.setBad(driver.ErrBadConn)
|
||||
ci.setError(fmt.Errorf("unknown response during CopyIn: %q", t))
|
||||
ci.done <- true
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (ci *copyin) setBad(err error) {
|
||||
ci.cn.err.set(err)
|
||||
}
|
||||
|
||||
func (ci *copyin) getBad() error {
|
||||
return ci.cn.err.get()
|
||||
}
|
||||
|
||||
func (ci *copyin) err() error {
|
||||
ci.mu.Lock()
|
||||
err := ci.mu.err
|
||||
ci.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
|
||||
// setError() sets ci.err if one has not been set already. Caller must not be
|
||||
// holding ci.Mutex.
|
||||
func (ci *copyin) setError(err error) {
|
||||
ci.mu.Lock()
|
||||
if ci.mu.err == nil {
|
||||
ci.mu.err = err
|
||||
}
|
||||
ci.mu.Unlock()
|
||||
}
|
||||
|
||||
func (ci *copyin) setResult(result driver.Result) {
|
||||
ci.mu.Lock()
|
||||
ci.mu.Result = result
|
||||
ci.mu.Unlock()
|
||||
}
|
||||
|
||||
func (ci *copyin) getResult() driver.Result {
|
||||
ci.mu.Lock()
|
||||
result := ci.mu.Result
|
||||
ci.mu.Unlock()
|
||||
if result == nil {
|
||||
return driver.RowsAffected(0)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (ci *copyin) NumInput() int {
|
||||
return -1
|
||||
}
|
||||
|
||||
func (ci *copyin) Query(v []driver.Value) (r driver.Rows, err error) {
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
|
||||
// Exec inserts values into the COPY stream. The insert is asynchronous
|
||||
// and Exec can return errors from previous Exec calls to the same
|
||||
// COPY stmt.
|
||||
//
|
||||
// You need to call Exec(nil) to sync the COPY stream and to get any
|
||||
// errors from pending data, since Stmt.Close() doesn't return errors
|
||||
// to the user.
|
||||
func (ci *copyin) Exec(v []driver.Value) (r driver.Result, err error) {
|
||||
if ci.closed {
|
||||
return nil, errCopyInClosed
|
||||
}
|
||||
|
||||
if err := ci.getBad(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer ci.cn.errRecover(&err)
|
||||
|
||||
if err := ci.err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(v) == 0 {
|
||||
if err := ci.Close(); err != nil {
|
||||
return driver.RowsAffected(0), err
|
||||
}
|
||||
|
||||
return ci.getResult(), nil
|
||||
}
|
||||
|
||||
numValues := len(v)
|
||||
for i, value := range v {
|
||||
ci.buffer = appendEncodedText(&ci.cn.parameterStatus, ci.buffer, value)
|
||||
if i < numValues-1 {
|
||||
ci.buffer = append(ci.buffer, '\t')
|
||||
}
|
||||
}
|
||||
|
||||
ci.buffer = append(ci.buffer, '\n')
|
||||
|
||||
if len(ci.buffer) > ciBufferFlushSize {
|
||||
ci.flush(ci.buffer)
|
||||
// reset buffer, keep bytes for message identifier and length
|
||||
ci.buffer = ci.buffer[:5]
|
||||
}
|
||||
|
||||
return driver.RowsAffected(0), nil
|
||||
}
|
||||
|
||||
// CopyData inserts a raw string into the COPY stream. The insert is
|
||||
// asynchronous and CopyData can return errors from previous CopyData calls to
|
||||
// the same COPY stmt.
|
||||
//
|
||||
// You need to call Exec(nil) to sync the COPY stream and to get any
|
||||
// errors from pending data, since Stmt.Close() doesn't return errors
|
||||
// to the user.
|
||||
func (ci *copyin) CopyData(ctx context.Context, line string) (r driver.Result, err error) {
|
||||
if ci.closed {
|
||||
return nil, errCopyInClosed
|
||||
}
|
||||
|
||||
if finish := ci.cn.watchCancel(ctx); finish != nil {
|
||||
defer finish()
|
||||
}
|
||||
|
||||
if err := ci.getBad(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer ci.cn.errRecover(&err)
|
||||
|
||||
if err := ci.err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ci.buffer = append(ci.buffer, []byte(line)...)
|
||||
ci.buffer = append(ci.buffer, '\n')
|
||||
|
||||
if len(ci.buffer) > ciBufferFlushSize {
|
||||
ci.flush(ci.buffer)
|
||||
// reset buffer, keep bytes for message identifier and length
|
||||
ci.buffer = ci.buffer[:5]
|
||||
}
|
||||
|
||||
return driver.RowsAffected(0), nil
|
||||
}
|
||||
|
||||
func (ci *copyin) Close() (err error) {
|
||||
if ci.closed { // Don't do anything, we're already closed
|
||||
return nil
|
||||
}
|
||||
ci.closed = true
|
||||
|
||||
if err := ci.getBad(); err != nil {
|
||||
return err
|
||||
}
|
||||
defer ci.cn.errRecover(&err)
|
||||
|
||||
if len(ci.buffer) > 0 {
|
||||
ci.flush(ci.buffer)
|
||||
}
|
||||
// Avoid touching the scratch buffer as resploop could be using it.
|
||||
err = ci.cn.sendSimpleMessage('c')
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
<-ci.done
|
||||
ci.cn.inCopy = false
|
||||
|
||||
if err := ci.err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,268 @@
|
||||
/*
|
||||
Package pq is a pure Go Postgres driver for the database/sql package.
|
||||
|
||||
In most cases clients will use the database/sql package instead of
|
||||
using this package directly. For example:
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
func main() {
|
||||
connStr := "user=pqgotest dbname=pqgotest sslmode=verify-full"
|
||||
db, err := sql.Open("postgres", connStr)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
age := 21
|
||||
rows, err := db.Query("SELECT name FROM users WHERE age = $1", age)
|
||||
…
|
||||
}
|
||||
|
||||
You can also connect to a database using a URL. For example:
|
||||
|
||||
connStr := "postgres://pqgotest:password@localhost/pqgotest?sslmode=verify-full"
|
||||
db, err := sql.Open("postgres", connStr)
|
||||
|
||||
|
||||
Connection String Parameters
|
||||
|
||||
|
||||
Similarly to libpq, when establishing a connection using pq you are expected to
|
||||
supply a connection string containing zero or more parameters.
|
||||
A subset of the connection parameters supported by libpq are also supported by pq.
|
||||
Additionally, pq also lets you specify run-time parameters (such as search_path or work_mem)
|
||||
directly in the connection string. This is different from libpq, which does not allow
|
||||
run-time parameters in the connection string, instead requiring you to supply
|
||||
them in the options parameter.
|
||||
|
||||
For compatibility with libpq, the following special connection parameters are
|
||||
supported:
|
||||
|
||||
* dbname - The name of the database to connect to
|
||||
* user - The user to sign in as
|
||||
* password - The user's password
|
||||
* host - The host to connect to. Values that start with / are for unix
|
||||
domain sockets. (default is localhost)
|
||||
* port - The port to bind to. (default is 5432)
|
||||
* sslmode - Whether or not to use SSL (default is require, this is not
|
||||
the default for libpq)
|
||||
* fallback_application_name - An application_name to fall back to if one isn't provided.
|
||||
* connect_timeout - Maximum wait for connection, in seconds. Zero or
|
||||
not specified means wait indefinitely.
|
||||
* sslcert - Cert file location. The file must contain PEM encoded data.
|
||||
* sslkey - Key file location. The file must contain PEM encoded data.
|
||||
* sslrootcert - The location of the root certificate file. The file
|
||||
must contain PEM encoded data.
|
||||
|
||||
Valid values for sslmode are:
|
||||
|
||||
* disable - No SSL
|
||||
* require - Always SSL (skip verification)
|
||||
* verify-ca - Always SSL (verify that the certificate presented by the
|
||||
server was signed by a trusted CA)
|
||||
* verify-full - Always SSL (verify that the certification presented by
|
||||
the server was signed by a trusted CA and the server host name
|
||||
matches the one in the certificate)
|
||||
|
||||
See http://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
|
||||
for more information about connection string parameters.
|
||||
|
||||
Use single quotes for values that contain whitespace:
|
||||
|
||||
"user=pqgotest password='with spaces'"
|
||||
|
||||
A backslash will escape the next character in values:
|
||||
|
||||
"user=space\ man password='it\'s valid'"
|
||||
|
||||
Note that the connection parameter client_encoding (which sets the
|
||||
text encoding for the connection) may be set but must be "UTF8",
|
||||
matching with the same rules as Postgres. It is an error to provide
|
||||
any other value.
|
||||
|
||||
In addition to the parameters listed above, any run-time parameter that can be
|
||||
set at backend start time can be set in the connection string. For more
|
||||
information, see
|
||||
http://www.postgresql.org/docs/current/static/runtime-config.html.
|
||||
|
||||
Most environment variables as specified at http://www.postgresql.org/docs/current/static/libpq-envars.html
|
||||
supported by libpq are also supported by pq. If any of the environment
|
||||
variables not supported by pq are set, pq will panic during connection
|
||||
establishment. Environment variables have a lower precedence than explicitly
|
||||
provided connection parameters.
|
||||
|
||||
The pgpass mechanism as described in http://www.postgresql.org/docs/current/static/libpq-pgpass.html
|
||||
is supported, but on Windows PGPASSFILE must be specified explicitly.
|
||||
|
||||
|
||||
Queries
|
||||
|
||||
|
||||
database/sql does not dictate any specific format for parameter
|
||||
markers in query strings, and pq uses the Postgres-native ordinal markers,
|
||||
as shown above. The same marker can be reused for the same parameter:
|
||||
|
||||
rows, err := db.Query(`SELECT name FROM users WHERE favorite_fruit = $1
|
||||
OR age BETWEEN $2 AND $2 + 3`, "orange", 64)
|
||||
|
||||
pq does not support the LastInsertId() method of the Result type in database/sql.
|
||||
To return the identifier of an INSERT (or UPDATE or DELETE), use the Postgres
|
||||
RETURNING clause with a standard Query or QueryRow call:
|
||||
|
||||
var userid int
|
||||
err := db.QueryRow(`INSERT INTO users(name, favorite_fruit, age)
|
||||
VALUES('beatrice', 'starfruit', 93) RETURNING id`).Scan(&userid)
|
||||
|
||||
For more details on RETURNING, see the Postgres documentation:
|
||||
|
||||
http://www.postgresql.org/docs/current/static/sql-insert.html
|
||||
http://www.postgresql.org/docs/current/static/sql-update.html
|
||||
http://www.postgresql.org/docs/current/static/sql-delete.html
|
||||
|
||||
For additional instructions on querying see the documentation for the database/sql package.
|
||||
|
||||
|
||||
Data Types
|
||||
|
||||
|
||||
Parameters pass through driver.DefaultParameterConverter before they are handled
|
||||
by this package. When the binary_parameters connection option is enabled,
|
||||
[]byte values are sent directly to the backend as data in binary format.
|
||||
|
||||
This package returns the following types for values from the PostgreSQL backend:
|
||||
|
||||
- integer types smallint, integer, and bigint are returned as int64
|
||||
- floating-point types real and double precision are returned as float64
|
||||
- character types char, varchar, and text are returned as string
|
||||
- temporal types date, time, timetz, timestamp, and timestamptz are
|
||||
returned as time.Time
|
||||
- the boolean type is returned as bool
|
||||
- the bytea type is returned as []byte
|
||||
|
||||
All other types are returned directly from the backend as []byte values in text format.
|
||||
|
||||
|
||||
Errors
|
||||
|
||||
|
||||
pq may return errors of type *pq.Error which can be interrogated for error details:
|
||||
|
||||
if err, ok := err.(*pq.Error); ok {
|
||||
fmt.Println("pq error:", err.Code.Name())
|
||||
}
|
||||
|
||||
See the pq.Error type for details.
|
||||
|
||||
|
||||
Bulk imports
|
||||
|
||||
You can perform bulk imports by preparing a statement returned by pq.CopyIn (or
|
||||
pq.CopyInSchema) in an explicit transaction (sql.Tx). The returned statement
|
||||
handle can then be repeatedly "executed" to copy data into the target table.
|
||||
After all data has been processed you should call Exec() once with no arguments
|
||||
to flush all buffered data. Any call to Exec() might return an error which
|
||||
should be handled appropriately, but because of the internal buffering an error
|
||||
returned by Exec() might not be related to the data passed in the call that
|
||||
failed.
|
||||
|
||||
CopyIn uses COPY FROM internally. It is not possible to COPY outside of an
|
||||
explicit transaction in pq.
|
||||
|
||||
Usage example:
|
||||
|
||||
txn, err := db.Begin()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
stmt, err := txn.Prepare(pq.CopyIn("users", "name", "age"))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
for _, user := range users {
|
||||
_, err = stmt.Exec(user.Name, int64(user.Age))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
_, err = stmt.Exec()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
err = stmt.Close()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
err = txn.Commit()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
|
||||
Notifications
|
||||
|
||||
|
||||
PostgreSQL supports a simple publish/subscribe model over database
|
||||
connections. See http://www.postgresql.org/docs/current/static/sql-notify.html
|
||||
for more information about the general mechanism.
|
||||
|
||||
To start listening for notifications, you first have to open a new connection
|
||||
to the database by calling NewListener. This connection can not be used for
|
||||
anything other than LISTEN / NOTIFY. Calling Listen will open a "notification
|
||||
channel"; once a notification channel is open, a notification generated on that
|
||||
channel will effect a send on the Listener.Notify channel. A notification
|
||||
channel will remain open until Unlisten is called, though connection loss might
|
||||
result in some notifications being lost. To solve this problem, Listener sends
|
||||
a nil pointer over the Notify channel any time the connection is re-established
|
||||
following a connection loss. The application can get information about the
|
||||
state of the underlying connection by setting an event callback in the call to
|
||||
NewListener.
|
||||
|
||||
A single Listener can safely be used from concurrent goroutines, which means
|
||||
that there is often no need to create more than one Listener in your
|
||||
application. However, a Listener is always connected to a single database, so
|
||||
you will need to create a new Listener instance for every database you want to
|
||||
receive notifications in.
|
||||
|
||||
The channel name in both Listen and Unlisten is case sensitive, and can contain
|
||||
any characters legal in an identifier (see
|
||||
http://www.postgresql.org/docs/current/static/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
|
||||
for more information). Note that the channel name will be truncated to 63
|
||||
bytes by the PostgreSQL server.
|
||||
|
||||
You can find a complete, working example of Listener usage at
|
||||
https://godoc.org/github.com/lib/pq/example/listen.
|
||||
|
||||
|
||||
Kerberos Support
|
||||
|
||||
|
||||
If you need support for Kerberos authentication, add the following to your main
|
||||
package:
|
||||
|
||||
import "github.com/lib/pq/auth/kerberos"
|
||||
|
||||
func init() {
|
||||
pq.RegisterGSSProvider(func() (pq.Gss, error) { return kerberos.NewGSS() })
|
||||
}
|
||||
|
||||
This package is in a separate module so that users who don't need Kerberos
|
||||
don't have to download unnecessary dependencies.
|
||||
|
||||
When imported, additional connection string parameters are supported:
|
||||
|
||||
* krbsrvname - GSS (Kerberos) service name when constructing the
|
||||
SPN (default is `postgres`). This will be combined with the host
|
||||
to form the full SPN: `krbsrvname/host`.
|
||||
* krbspn - GSS (Kerberos) SPN. This takes priority over
|
||||
`krbsrvname` if present.
|
||||
*/
|
||||
package pq
|
||||
@ -0,0 +1,632 @@
|
||||
package pq
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql/driver"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq/oid"
|
||||
)
|
||||
|
||||
var time2400Regex = regexp.MustCompile(`^(24:00(?::00(?:\.0+)?)?)(?:[Z+-].*)?$`)
|
||||
|
||||
func binaryEncode(parameterStatus *parameterStatus, x interface{}) []byte {
|
||||
switch v := x.(type) {
|
||||
case []byte:
|
||||
return v
|
||||
default:
|
||||
return encode(parameterStatus, x, oid.T_unknown)
|
||||
}
|
||||
}
|
||||
|
||||
func encode(parameterStatus *parameterStatus, x interface{}, pgtypOid oid.Oid) []byte {
|
||||
switch v := x.(type) {
|
||||
case int64:
|
||||
return strconv.AppendInt(nil, v, 10)
|
||||
case float64:
|
||||
return strconv.AppendFloat(nil, v, 'f', -1, 64)
|
||||
case []byte:
|
||||
if pgtypOid == oid.T_bytea {
|
||||
return encodeBytea(parameterStatus.serverVersion, v)
|
||||
}
|
||||
|
||||
return v
|
||||
case string:
|
||||
if pgtypOid == oid.T_bytea {
|
||||
return encodeBytea(parameterStatus.serverVersion, []byte(v))
|
||||
}
|
||||
|
||||
return []byte(v)
|
||||
case bool:
|
||||
return strconv.AppendBool(nil, v)
|
||||
case time.Time:
|
||||
return formatTs(v)
|
||||
|
||||
default:
|
||||
errorf("encode: unknown type for %T", v)
|
||||
}
|
||||
|
||||
panic("not reached")
|
||||
}
|
||||
|
||||
func decode(parameterStatus *parameterStatus, s []byte, typ oid.Oid, f format) interface{} {
|
||||
switch f {
|
||||
case formatBinary:
|
||||
return binaryDecode(parameterStatus, s, typ)
|
||||
case formatText:
|
||||
return textDecode(parameterStatus, s, typ)
|
||||
default:
|
||||
panic("not reached")
|
||||
}
|
||||
}
|
||||
|
||||
func binaryDecode(parameterStatus *parameterStatus, s []byte, typ oid.Oid) interface{} {
|
||||
switch typ {
|
||||
case oid.T_bytea:
|
||||
return s
|
||||
case oid.T_int8:
|
||||
return int64(binary.BigEndian.Uint64(s))
|
||||
case oid.T_int4:
|
||||
return int64(int32(binary.BigEndian.Uint32(s)))
|
||||
case oid.T_int2:
|
||||
return int64(int16(binary.BigEndian.Uint16(s)))
|
||||
case oid.T_uuid:
|
||||
b, err := decodeUUIDBinary(s)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return b
|
||||
|
||||
default:
|
||||
errorf("don't know how to decode binary parameter of type %d", uint32(typ))
|
||||
}
|
||||
|
||||
panic("not reached")
|
||||
}
|
||||
|
||||
func textDecode(parameterStatus *parameterStatus, s []byte, typ oid.Oid) interface{} {
|
||||
switch typ {
|
||||
case oid.T_char, oid.T_varchar, oid.T_text:
|
||||
return string(s)
|
||||
case oid.T_bytea:
|
||||
b, err := parseBytea(s)
|
||||
if err != nil {
|
||||
errorf("%s", err)
|
||||
}
|
||||
return b
|
||||
case oid.T_timestamptz:
|
||||
return parseTs(parameterStatus.currentLocation, string(s))
|
||||
case oid.T_timestamp, oid.T_date:
|
||||
return parseTs(nil, string(s))
|
||||
case oid.T_time:
|
||||
return mustParse("15:04:05", typ, s)
|
||||
case oid.T_timetz:
|
||||
return mustParse("15:04:05-07", typ, s)
|
||||
case oid.T_bool:
|
||||
return s[0] == 't'
|
||||
case oid.T_int8, oid.T_int4, oid.T_int2:
|
||||
i, err := strconv.ParseInt(string(s), 10, 64)
|
||||
if err != nil {
|
||||
errorf("%s", err)
|
||||
}
|
||||
return i
|
||||
case oid.T_float4, oid.T_float8:
|
||||
// We always use 64 bit parsing, regardless of whether the input text is for
|
||||
// a float4 or float8, because clients expect float64s for all float datatypes
|
||||
// and returning a 32-bit parsed float64 produces lossy results.
|
||||
f, err := strconv.ParseFloat(string(s), 64)
|
||||
if err != nil {
|
||||
errorf("%s", err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// appendEncodedText encodes item in text format as required by COPY
|
||||
// and appends to buf
|
||||
func appendEncodedText(parameterStatus *parameterStatus, buf []byte, x interface{}) []byte {
|
||||
switch v := x.(type) {
|
||||
case int64:
|
||||
return strconv.AppendInt(buf, v, 10)
|
||||
case float64:
|
||||
return strconv.AppendFloat(buf, v, 'f', -1, 64)
|
||||
case []byte:
|
||||
encodedBytea := encodeBytea(parameterStatus.serverVersion, v)
|
||||
return appendEscapedText(buf, string(encodedBytea))
|
||||
case string:
|
||||
return appendEscapedText(buf, v)
|
||||
case bool:
|
||||
return strconv.AppendBool(buf, v)
|
||||
case time.Time:
|
||||
return append(buf, formatTs(v)...)
|
||||
case nil:
|
||||
return append(buf, "\\N"...)
|
||||
default:
|
||||
errorf("encode: unknown type for %T", v)
|
||||
}
|
||||
|
||||
panic("not reached")
|
||||
}
|
||||
|
||||
func appendEscapedText(buf []byte, text string) []byte {
|
||||
escapeNeeded := false
|
||||
startPos := 0
|
||||
var c byte
|
||||
|
||||
// check if we need to escape
|
||||
for i := 0; i < len(text); i++ {
|
||||
c = text[i]
|
||||
if c == '\\' || c == '\n' || c == '\r' || c == '\t' {
|
||||
escapeNeeded = true
|
||||
startPos = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if !escapeNeeded {
|
||||
return append(buf, text...)
|
||||
}
|
||||
|
||||
// copy till first char to escape, iterate the rest
|
||||
result := append(buf, text[:startPos]...)
|
||||
for i := startPos; i < len(text); i++ {
|
||||
c = text[i]
|
||||
switch c {
|
||||
case '\\':
|
||||
result = append(result, '\\', '\\')
|
||||
case '\n':
|
||||
result = append(result, '\\', 'n')
|
||||
case '\r':
|
||||
result = append(result, '\\', 'r')
|
||||
case '\t':
|
||||
result = append(result, '\\', 't')
|
||||
default:
|
||||
result = append(result, c)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func mustParse(f string, typ oid.Oid, s []byte) time.Time {
|
||||
str := string(s)
|
||||
|
||||
// Check for a minute and second offset in the timezone.
|
||||
if typ == oid.T_timestamptz || typ == oid.T_timetz {
|
||||
for i := 3; i <= 6; i += 3 {
|
||||
if str[len(str)-i] == ':' {
|
||||
f += ":00"
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Special case for 24:00 time.
|
||||
// Unfortunately, golang does not parse 24:00 as a proper time.
|
||||
// In this case, we want to try "round to the next day", to differentiate.
|
||||
// As such, we find if the 24:00 time matches at the beginning; if so,
|
||||
// we default it back to 00:00 but add a day later.
|
||||
var is2400Time bool
|
||||
switch typ {
|
||||
case oid.T_timetz, oid.T_time:
|
||||
if matches := time2400Regex.FindStringSubmatch(str); matches != nil {
|
||||
// Concatenate timezone information at the back.
|
||||
str = "00:00:00" + str[len(matches[1]):]
|
||||
is2400Time = true
|
||||
}
|
||||
}
|
||||
t, err := time.Parse(f, str)
|
||||
if err != nil {
|
||||
errorf("decode: %s", err)
|
||||
}
|
||||
if is2400Time {
|
||||
t = t.Add(24 * time.Hour)
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
var errInvalidTimestamp = errors.New("invalid timestamp")
|
||||
|
||||
type timestampParser struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (p *timestampParser) expect(str string, char byte, pos int) {
|
||||
if p.err != nil {
|
||||
return
|
||||
}
|
||||
if pos+1 > len(str) {
|
||||
p.err = errInvalidTimestamp
|
||||
return
|
||||
}
|
||||
if c := str[pos]; c != char && p.err == nil {
|
||||
p.err = fmt.Errorf("expected '%v' at position %v; got '%v'", char, pos, c)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *timestampParser) mustAtoi(str string, begin int, end int) int {
|
||||
if p.err != nil {
|
||||
return 0
|
||||
}
|
||||
if begin < 0 || end < 0 || begin > end || end > len(str) {
|
||||
p.err = errInvalidTimestamp
|
||||
return 0
|
||||
}
|
||||
result, err := strconv.Atoi(str[begin:end])
|
||||
if err != nil {
|
||||
if p.err == nil {
|
||||
p.err = fmt.Errorf("expected number; got '%v'", str)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// The location cache caches the time zones typically used by the client.
|
||||
type locationCache struct {
|
||||
cache map[int]*time.Location
|
||||
lock sync.Mutex
|
||||
}
|
||||
|
||||
// All connections share the same list of timezones. Benchmarking shows that
|
||||
// about 5% speed could be gained by putting the cache in the connection and
|
||||
// losing the mutex, at the cost of a small amount of memory and a somewhat
|
||||
// significant increase in code complexity.
|
||||
var globalLocationCache = newLocationCache()
|
||||
|
||||
func newLocationCache() *locationCache {
|
||||
return &locationCache{cache: make(map[int]*time.Location)}
|
||||
}
|
||||
|
||||
// Returns the cached timezone for the specified offset, creating and caching
|
||||
// it if necessary.
|
||||
func (c *locationCache) getLocation(offset int) *time.Location {
|
||||
c.lock.Lock()
|
||||
defer c.lock.Unlock()
|
||||
|
||||
location, ok := c.cache[offset]
|
||||
if !ok {
|
||||
location = time.FixedZone("", offset)
|
||||
c.cache[offset] = location
|
||||
}
|
||||
|
||||
return location
|
||||
}
|
||||
|
||||
var infinityTsEnabled = false
|
||||
var infinityTsNegative time.Time
|
||||
var infinityTsPositive time.Time
|
||||
|
||||
const (
|
||||
infinityTsEnabledAlready = "pq: infinity timestamp enabled already"
|
||||
infinityTsNegativeMustBeSmaller = "pq: infinity timestamp: negative value must be smaller (before) than positive"
|
||||
)
|
||||
|
||||
// EnableInfinityTs controls the handling of Postgres' "-infinity" and
|
||||
// "infinity" "timestamp"s.
|
||||
//
|
||||
// If EnableInfinityTs is not called, "-infinity" and "infinity" will return
|
||||
// []byte("-infinity") and []byte("infinity") respectively, and potentially
|
||||
// cause error "sql: Scan error on column index 0: unsupported driver -> Scan
|
||||
// pair: []uint8 -> *time.Time", when scanning into a time.Time value.
|
||||
//
|
||||
// Once EnableInfinityTs has been called, all connections created using this
|
||||
// driver will decode Postgres' "-infinity" and "infinity" for "timestamp",
|
||||
// "timestamp with time zone" and "date" types to the predefined minimum and
|
||||
// maximum times, respectively. When encoding time.Time values, any time which
|
||||
// equals or precedes the predefined minimum time will be encoded to
|
||||
// "-infinity". Any values at or past the maximum time will similarly be
|
||||
// encoded to "infinity".
|
||||
//
|
||||
// If EnableInfinityTs is called with negative >= positive, it will panic.
|
||||
// Calling EnableInfinityTs after a connection has been established results in
|
||||
// undefined behavior. If EnableInfinityTs is called more than once, it will
|
||||
// panic.
|
||||
func EnableInfinityTs(negative time.Time, positive time.Time) {
|
||||
if infinityTsEnabled {
|
||||
panic(infinityTsEnabledAlready)
|
||||
}
|
||||
if !negative.Before(positive) {
|
||||
panic(infinityTsNegativeMustBeSmaller)
|
||||
}
|
||||
infinityTsEnabled = true
|
||||
infinityTsNegative = negative
|
||||
infinityTsPositive = positive
|
||||
}
|
||||
|
||||
/*
|
||||
* Testing might want to toggle infinityTsEnabled
|
||||
*/
|
||||
func disableInfinityTs() {
|
||||
infinityTsEnabled = false
|
||||
}
|
||||
|
||||
// This is a time function specific to the Postgres default DateStyle
|
||||
// setting ("ISO, MDY"), the only one we currently support. This
|
||||
// accounts for the discrepancies between the parsing available with
|
||||
// time.Parse and the Postgres date formatting quirks.
|
||||
func parseTs(currentLocation *time.Location, str string) interface{} {
|
||||
switch str {
|
||||
case "-infinity":
|
||||
if infinityTsEnabled {
|
||||
return infinityTsNegative
|
||||
}
|
||||
return []byte(str)
|
||||
case "infinity":
|
||||
if infinityTsEnabled {
|
||||
return infinityTsPositive
|
||||
}
|
||||
return []byte(str)
|
||||
}
|
||||
t, err := ParseTimestamp(currentLocation, str)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// ParseTimestamp parses Postgres' text format. It returns a time.Time in
|
||||
// currentLocation iff that time's offset agrees with the offset sent from the
|
||||
// Postgres server. Otherwise, ParseTimestamp returns a time.Time with the
|
||||
// fixed offset offset provided by the Postgres server.
|
||||
func ParseTimestamp(currentLocation *time.Location, str string) (time.Time, error) {
|
||||
p := timestampParser{}
|
||||
|
||||
monSep := strings.IndexRune(str, '-')
|
||||
// this is Gregorian year, not ISO Year
|
||||
// In Gregorian system, the year 1 BC is followed by AD 1
|
||||
year := p.mustAtoi(str, 0, monSep)
|
||||
daySep := monSep + 3
|
||||
month := p.mustAtoi(str, monSep+1, daySep)
|
||||
p.expect(str, '-', daySep)
|
||||
timeSep := daySep + 3
|
||||
day := p.mustAtoi(str, daySep+1, timeSep)
|
||||
|
||||
minLen := monSep + len("01-01") + 1
|
||||
|
||||
isBC := strings.HasSuffix(str, " BC")
|
||||
if isBC {
|
||||
minLen += 3
|
||||
}
|
||||
|
||||
var hour, minute, second int
|
||||
if len(str) > minLen {
|
||||
p.expect(str, ' ', timeSep)
|
||||
minSep := timeSep + 3
|
||||
p.expect(str, ':', minSep)
|
||||
hour = p.mustAtoi(str, timeSep+1, minSep)
|
||||
secSep := minSep + 3
|
||||
p.expect(str, ':', secSep)
|
||||
minute = p.mustAtoi(str, minSep+1, secSep)
|
||||
secEnd := secSep + 3
|
||||
second = p.mustAtoi(str, secSep+1, secEnd)
|
||||
}
|
||||
remainderIdx := monSep + len("01-01 00:00:00") + 1
|
||||
// Three optional (but ordered) sections follow: the
|
||||
// fractional seconds, the time zone offset, and the BC
|
||||
// designation. We set them up here and adjust the other
|
||||
// offsets if the preceding sections exist.
|
||||
|
||||
nanoSec := 0
|
||||
tzOff := 0
|
||||
|
||||
if remainderIdx < len(str) && str[remainderIdx] == '.' {
|
||||
fracStart := remainderIdx + 1
|
||||
fracOff := strings.IndexAny(str[fracStart:], "-+Z ")
|
||||
if fracOff < 0 {
|
||||
fracOff = len(str) - fracStart
|
||||
}
|
||||
fracSec := p.mustAtoi(str, fracStart, fracStart+fracOff)
|
||||
nanoSec = fracSec * (1000000000 / int(math.Pow(10, float64(fracOff))))
|
||||
|
||||
remainderIdx += fracOff + 1
|
||||
}
|
||||
if tzStart := remainderIdx; tzStart < len(str) && (str[tzStart] == '-' || str[tzStart] == '+') {
|
||||
// time zone separator is always '-' or '+' or 'Z' (UTC is +00)
|
||||
var tzSign int
|
||||
switch c := str[tzStart]; c {
|
||||
case '-':
|
||||
tzSign = -1
|
||||
case '+':
|
||||
tzSign = +1
|
||||
default:
|
||||
return time.Time{}, fmt.Errorf("expected '-' or '+' at position %v; got %v", tzStart, c)
|
||||
}
|
||||
tzHours := p.mustAtoi(str, tzStart+1, tzStart+3)
|
||||
remainderIdx += 3
|
||||
var tzMin, tzSec int
|
||||
if remainderIdx < len(str) && str[remainderIdx] == ':' {
|
||||
tzMin = p.mustAtoi(str, remainderIdx+1, remainderIdx+3)
|
||||
remainderIdx += 3
|
||||
}
|
||||
if remainderIdx < len(str) && str[remainderIdx] == ':' {
|
||||
tzSec = p.mustAtoi(str, remainderIdx+1, remainderIdx+3)
|
||||
remainderIdx += 3
|
||||
}
|
||||
tzOff = tzSign * ((tzHours * 60 * 60) + (tzMin * 60) + tzSec)
|
||||
} else if tzStart < len(str) && str[tzStart] == 'Z' {
|
||||
// time zone Z separator indicates UTC is +00
|
||||
remainderIdx += 1
|
||||
}
|
||||
|
||||
var isoYear int
|
||||
|
||||
if isBC {
|
||||
isoYear = 1 - year
|
||||
remainderIdx += 3
|
||||
} else {
|
||||
isoYear = year
|
||||
}
|
||||
if remainderIdx < len(str) {
|
||||
return time.Time{}, fmt.Errorf("expected end of input, got %v", str[remainderIdx:])
|
||||
}
|
||||
t := time.Date(isoYear, time.Month(month), day,
|
||||
hour, minute, second, nanoSec,
|
||||
globalLocationCache.getLocation(tzOff))
|
||||
|
||||
if currentLocation != nil {
|
||||
// Set the location of the returned Time based on the session's
|
||||
// TimeZone value, but only if the local time zone database agrees with
|
||||
// the remote database on the offset.
|
||||
lt := t.In(currentLocation)
|
||||
_, newOff := lt.Zone()
|
||||
if newOff == tzOff {
|
||||
t = lt
|
||||
}
|
||||
}
|
||||
|
||||
return t, p.err
|
||||
}
|
||||
|
||||
// formatTs formats t into a format postgres understands.
|
||||
func formatTs(t time.Time) []byte {
|
||||
if infinityTsEnabled {
|
||||
// t <= -infinity : ! (t > -infinity)
|
||||
if !t.After(infinityTsNegative) {
|
||||
return []byte("-infinity")
|
||||
}
|
||||
// t >= infinity : ! (!t < infinity)
|
||||
if !t.Before(infinityTsPositive) {
|
||||
return []byte("infinity")
|
||||
}
|
||||
}
|
||||
return FormatTimestamp(t)
|
||||
}
|
||||
|
||||
// FormatTimestamp formats t into Postgres' text format for timestamps.
|
||||
func FormatTimestamp(t time.Time) []byte {
|
||||
// Need to send dates before 0001 A.D. with " BC" suffix, instead of the
|
||||
// minus sign preferred by Go.
|
||||
// Beware, "0000" in ISO is "1 BC", "-0001" is "2 BC" and so on
|
||||
bc := false
|
||||
if t.Year() <= 0 {
|
||||
// flip year sign, and add 1, e.g: "0" will be "1", and "-10" will be "11"
|
||||
t = t.AddDate((-t.Year())*2+1, 0, 0)
|
||||
bc = true
|
||||
}
|
||||
b := []byte(t.Format("2006-01-02 15:04:05.999999999Z07:00"))
|
||||
|
||||
_, offset := t.Zone()
|
||||
offset %= 60
|
||||
if offset != 0 {
|
||||
// RFC3339Nano already printed the minus sign
|
||||
if offset < 0 {
|
||||
offset = -offset
|
||||
}
|
||||
|
||||
b = append(b, ':')
|
||||
if offset < 10 {
|
||||
b = append(b, '0')
|
||||
}
|
||||
b = strconv.AppendInt(b, int64(offset), 10)
|
||||
}
|
||||
|
||||
if bc {
|
||||
b = append(b, " BC"...)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// Parse a bytea value received from the server. Both "hex" and the legacy
|
||||
// "escape" format are supported.
|
||||
func parseBytea(s []byte) (result []byte, err error) {
|
||||
if len(s) >= 2 && bytes.Equal(s[:2], []byte("\\x")) {
|
||||
// bytea_output = hex
|
||||
s = s[2:] // trim off leading "\\x"
|
||||
result = make([]byte, hex.DecodedLen(len(s)))
|
||||
_, err := hex.Decode(result, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
// bytea_output = escape
|
||||
for len(s) > 0 {
|
||||
if s[0] == '\\' {
|
||||
// escaped '\\'
|
||||
if len(s) >= 2 && s[1] == '\\' {
|
||||
result = append(result, '\\')
|
||||
s = s[2:]
|
||||
continue
|
||||
}
|
||||
|
||||
// '\\' followed by an octal number
|
||||
if len(s) < 4 {
|
||||
return nil, fmt.Errorf("invalid bytea sequence %v", s)
|
||||
}
|
||||
r, err := strconv.ParseUint(string(s[1:4]), 8, 8)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not parse bytea value: %s", err.Error())
|
||||
}
|
||||
result = append(result, byte(r))
|
||||
s = s[4:]
|
||||
} else {
|
||||
// We hit an unescaped, raw byte. Try to read in as many as
|
||||
// possible in one go.
|
||||
i := bytes.IndexByte(s, '\\')
|
||||
if i == -1 {
|
||||
result = append(result, s...)
|
||||
break
|
||||
}
|
||||
result = append(result, s[:i]...)
|
||||
s = s[i:]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func encodeBytea(serverVersion int, v []byte) (result []byte) {
|
||||
if serverVersion >= 90000 {
|
||||
// Use the hex format if we know that the server supports it
|
||||
result = make([]byte, 2+hex.EncodedLen(len(v)))
|
||||
result[0] = '\\'
|
||||
result[1] = 'x'
|
||||
hex.Encode(result[2:], v)
|
||||
} else {
|
||||
// .. or resort to "escape"
|
||||
for _, b := range v {
|
||||
if b == '\\' {
|
||||
result = append(result, '\\', '\\')
|
||||
} else if b < 0x20 || b > 0x7e {
|
||||
result = append(result, []byte(fmt.Sprintf("\\%03o", b))...)
|
||||
} else {
|
||||
result = append(result, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// NullTime represents a time.Time that may be null. NullTime implements the
|
||||
// sql.Scanner interface so it can be used as a scan destination, similar to
|
||||
// sql.NullString.
|
||||
type NullTime struct {
|
||||
Time time.Time
|
||||
Valid bool // Valid is true if Time is not NULL
|
||||
}
|
||||
|
||||
// Scan implements the Scanner interface.
|
||||
func (nt *NullTime) Scan(value interface{}) error {
|
||||
nt.Time, nt.Valid = value.(time.Time)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Value implements the driver Valuer interface.
|
||||
func (nt NullTime) Value() (driver.Value, error) {
|
||||
if !nt.Valid {
|
||||
return nil, nil
|
||||
}
|
||||
return nt.Time, nil
|
||||
}
|
||||
@ -0,0 +1,523 @@
|
||||
package pq
|
||||
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
// Error severities
|
||||
const (
|
||||
Efatal = "FATAL"
|
||||
Epanic = "PANIC"
|
||||
Ewarning = "WARNING"
|
||||
Enotice = "NOTICE"
|
||||
Edebug = "DEBUG"
|
||||
Einfo = "INFO"
|
||||
Elog = "LOG"
|
||||
)
|
||||
|
||||
// Error represents an error communicating with the server.
|
||||
//
|
||||
// See http://www.postgresql.org/docs/current/static/protocol-error-fields.html for details of the fields
|
||||
type Error struct {
|
||||
Severity string
|
||||
Code ErrorCode
|
||||
Message string
|
||||
Detail string
|
||||
Hint string
|
||||
Position string
|
||||
InternalPosition string
|
||||
InternalQuery string
|
||||
Where string
|
||||
Schema string
|
||||
Table string
|
||||
Column string
|
||||
DataTypeName string
|
||||
Constraint string
|
||||
File string
|
||||
Line string
|
||||
Routine string
|
||||
}
|
||||
|
||||
// ErrorCode is a five-character error code.
|
||||
type ErrorCode string
|
||||
|
||||
// Name returns a more human friendly rendering of the error code, namely the
|
||||
// "condition name".
|
||||
//
|
||||
// See http://www.postgresql.org/docs/9.3/static/errcodes-appendix.html for
|
||||
// details.
|
||||
func (ec ErrorCode) Name() string {
|
||||
return errorCodeNames[ec]
|
||||
}
|
||||
|
||||
// ErrorClass is only the class part of an error code.
|
||||
type ErrorClass string
|
||||
|
||||
// Name returns the condition name of an error class. It is equivalent to the
|
||||
// condition name of the "standard" error code (i.e. the one having the last
|
||||
// three characters "000").
|
||||
func (ec ErrorClass) Name() string {
|
||||
return errorCodeNames[ErrorCode(ec+"000")]
|
||||
}
|
||||
|
||||
// Class returns the error class, e.g. "28".
|
||||
//
|
||||
// See http://www.postgresql.org/docs/9.3/static/errcodes-appendix.html for
|
||||
// details.
|
||||
func (ec ErrorCode) Class() ErrorClass {
|
||||
return ErrorClass(ec[0:2])
|
||||
}
|
||||
|
||||
// errorCodeNames is a mapping between the five-character error codes and the
|
||||
// human readable "condition names". It is derived from the list at
|
||||
// http://www.postgresql.org/docs/9.3/static/errcodes-appendix.html
|
||||
var errorCodeNames = map[ErrorCode]string{
|
||||
// Class 00 - Successful Completion
|
||||
"00000": "successful_completion",
|
||||
// Class 01 - Warning
|
||||
"01000": "warning",
|
||||
"0100C": "dynamic_result_sets_returned",
|
||||
"01008": "implicit_zero_bit_padding",
|
||||
"01003": "null_value_eliminated_in_set_function",
|
||||
"01007": "privilege_not_granted",
|
||||
"01006": "privilege_not_revoked",
|
||||
"01004": "string_data_right_truncation",
|
||||
"01P01": "deprecated_feature",
|
||||
// Class 02 - No Data (this is also a warning class per the SQL standard)
|
||||
"02000": "no_data",
|
||||
"02001": "no_additional_dynamic_result_sets_returned",
|
||||
// Class 03 - SQL Statement Not Yet Complete
|
||||
"03000": "sql_statement_not_yet_complete",
|
||||
// Class 08 - Connection Exception
|
||||
"08000": "connection_exception",
|
||||
"08003": "connection_does_not_exist",
|
||||
"08006": "connection_failure",
|
||||
"08001": "sqlclient_unable_to_establish_sqlconnection",
|
||||
"08004": "sqlserver_rejected_establishment_of_sqlconnection",
|
||||
"08007": "transaction_resolution_unknown",
|
||||
"08P01": "protocol_violation",
|
||||
// Class 09 - Triggered Action Exception
|
||||
"09000": "triggered_action_exception",
|
||||
// Class 0A - Feature Not Supported
|
||||
"0A000": "feature_not_supported",
|
||||
// Class 0B - Invalid Transaction Initiation
|
||||
"0B000": "invalid_transaction_initiation",
|
||||
// Class 0F - Locator Exception
|
||||
"0F000": "locator_exception",
|
||||
"0F001": "invalid_locator_specification",
|
||||
// Class 0L - Invalid Grantor
|
||||
"0L000": "invalid_grantor",
|
||||
"0LP01": "invalid_grant_operation",
|
||||
// Class 0P - Invalid Role Specification
|
||||
"0P000": "invalid_role_specification",
|
||||
// Class 0Z - Diagnostics Exception
|
||||
"0Z000": "diagnostics_exception",
|
||||
"0Z002": "stacked_diagnostics_accessed_without_active_handler",
|
||||
// Class 20 - Case Not Found
|
||||
"20000": "case_not_found",
|
||||
// Class 21 - Cardinality Violation
|
||||
"21000": "cardinality_violation",
|
||||
// Class 22 - Data Exception
|
||||
"22000": "data_exception",
|
||||
"2202E": "array_subscript_error",
|
||||
"22021": "character_not_in_repertoire",
|
||||
"22008": "datetime_field_overflow",
|
||||
"22012": "division_by_zero",
|
||||
"22005": "error_in_assignment",
|
||||
"2200B": "escape_character_conflict",
|
||||
"22022": "indicator_overflow",
|
||||
"22015": "interval_field_overflow",
|
||||
"2201E": "invalid_argument_for_logarithm",
|
||||
"22014": "invalid_argument_for_ntile_function",
|
||||
"22016": "invalid_argument_for_nth_value_function",
|
||||
"2201F": "invalid_argument_for_power_function",
|
||||
"2201G": "invalid_argument_for_width_bucket_function",
|
||||
"22018": "invalid_character_value_for_cast",
|
||||
"22007": "invalid_datetime_format",
|
||||
"22019": "invalid_escape_character",
|
||||
"2200D": "invalid_escape_octet",
|
||||
"22025": "invalid_escape_sequence",
|
||||
"22P06": "nonstandard_use_of_escape_character",
|
||||
"22010": "invalid_indicator_parameter_value",
|
||||
"22023": "invalid_parameter_value",
|
||||
"2201B": "invalid_regular_expression",
|
||||
"2201W": "invalid_row_count_in_limit_clause",
|
||||
"2201X": "invalid_row_count_in_result_offset_clause",
|
||||
"22009": "invalid_time_zone_displacement_value",
|
||||
"2200C": "invalid_use_of_escape_character",
|
||||
"2200G": "most_specific_type_mismatch",
|
||||
"22004": "null_value_not_allowed",
|
||||
"22002": "null_value_no_indicator_parameter",
|
||||
"22003": "numeric_value_out_of_range",
|
||||
"2200H": "sequence_generator_limit_exceeded",
|
||||
"22026": "string_data_length_mismatch",
|
||||
"22001": "string_data_right_truncation",
|
||||
"22011": "substring_error",
|
||||
"22027": "trim_error",
|
||||
"22024": "unterminated_c_string",
|
||||
"2200F": "zero_length_character_string",
|
||||
"22P01": "floating_point_exception",
|
||||
"22P02": "invalid_text_representation",
|
||||
"22P03": "invalid_binary_representation",
|
||||
"22P04": "bad_copy_file_format",
|
||||
"22P05": "untranslatable_character",
|
||||
"2200L": "not_an_xml_document",
|
||||
"2200M": "invalid_xml_document",
|
||||
"2200N": "invalid_xml_content",
|
||||
"2200S": "invalid_xml_comment",
|
||||
"2200T": "invalid_xml_processing_instruction",
|
||||
// Class 23 - Integrity Constraint Violation
|
||||
"23000": "integrity_constraint_violation",
|
||||
"23001": "restrict_violation",
|
||||
"23502": "not_null_violation",
|
||||
"23503": "foreign_key_violation",
|
||||
"23505": "unique_violation",
|
||||
"23514": "check_violation",
|
||||
"23P01": "exclusion_violation",
|
||||
// Class 24 - Invalid Cursor State
|
||||
"24000": "invalid_cursor_state",
|
||||
// Class 25 - Invalid Transaction State
|
||||
"25000": "invalid_transaction_state",
|
||||
"25001": "active_sql_transaction",
|
||||
"25002": "branch_transaction_already_active",
|
||||
"25008": "held_cursor_requires_same_isolation_level",
|
||||
"25003": "inappropriate_access_mode_for_branch_transaction",
|
||||
"25004": "inappropriate_isolation_level_for_branch_transaction",
|
||||
"25005": "no_active_sql_transaction_for_branch_transaction",
|
||||
"25006": "read_only_sql_transaction",
|
||||
"25007": "schema_and_data_statement_mixing_not_supported",
|
||||
"25P01": "no_active_sql_transaction",
|
||||
"25P02": "in_failed_sql_transaction",
|
||||
// Class 26 - Invalid SQL Statement Name
|
||||
"26000": "invalid_sql_statement_name",
|
||||
// Class 27 - Triggered Data Change Violation
|
||||
"27000": "triggered_data_change_violation",
|
||||
// Class 28 - Invalid Authorization Specification
|
||||
"28000": "invalid_authorization_specification",
|
||||
"28P01": "invalid_password",
|
||||
// Class 2B - Dependent Privilege Descriptors Still Exist
|
||||
"2B000": "dependent_privilege_descriptors_still_exist",
|
||||
"2BP01": "dependent_objects_still_exist",
|
||||
// Class 2D - Invalid Transaction Termination
|
||||
"2D000": "invalid_transaction_termination",
|
||||
// Class 2F - SQL Routine Exception
|
||||
"2F000": "sql_routine_exception",
|
||||
"2F005": "function_executed_no_return_statement",
|
||||
"2F002": "modifying_sql_data_not_permitted",
|
||||
"2F003": "prohibited_sql_statement_attempted",
|
||||
"2F004": "reading_sql_data_not_permitted",
|
||||
// Class 34 - Invalid Cursor Name
|
||||
"34000": "invalid_cursor_name",
|
||||
// Class 38 - External Routine Exception
|
||||
"38000": "external_routine_exception",
|
||||
"38001": "containing_sql_not_permitted",
|
||||
"38002": "modifying_sql_data_not_permitted",
|
||||
"38003": "prohibited_sql_statement_attempted",
|
||||
"38004": "reading_sql_data_not_permitted",
|
||||
// Class 39 - External Routine Invocation Exception
|
||||
"39000": "external_routine_invocation_exception",
|
||||
"39001": "invalid_sqlstate_returned",
|
||||
"39004": "null_value_not_allowed",
|
||||
"39P01": "trigger_protocol_violated",
|
||||
"39P02": "srf_protocol_violated",
|
||||
// Class 3B - Savepoint Exception
|
||||
"3B000": "savepoint_exception",
|
||||
"3B001": "invalid_savepoint_specification",
|
||||
// Class 3D - Invalid Catalog Name
|
||||
"3D000": "invalid_catalog_name",
|
||||
// Class 3F - Invalid Schema Name
|
||||
"3F000": "invalid_schema_name",
|
||||
// Class 40 - Transaction Rollback
|
||||
"40000": "transaction_rollback",
|
||||
"40002": "transaction_integrity_constraint_violation",
|
||||
"40001": "serialization_failure",
|
||||
"40003": "statement_completion_unknown",
|
||||
"40P01": "deadlock_detected",
|
||||
// Class 42 - Syntax Error or Access Rule Violation
|
||||
"42000": "syntax_error_or_access_rule_violation",
|
||||
"42601": "syntax_error",
|
||||
"42501": "insufficient_privilege",
|
||||
"42846": "cannot_coerce",
|
||||
"42803": "grouping_error",
|
||||
"42P20": "windowing_error",
|
||||
"42P19": "invalid_recursion",
|
||||
"42830": "invalid_foreign_key",
|
||||
"42602": "invalid_name",
|
||||
"42622": "name_too_long",
|
||||
"42939": "reserved_name",
|
||||
"42804": "datatype_mismatch",
|
||||
"42P18": "indeterminate_datatype",
|
||||
"42P21": "collation_mismatch",
|
||||
"42P22": "indeterminate_collation",
|
||||
"42809": "wrong_object_type",
|
||||
"42703": "undefined_column",
|
||||
"42883": "undefined_function",
|
||||
"42P01": "undefined_table",
|
||||
"42P02": "undefined_parameter",
|
||||
"42704": "undefined_object",
|
||||
"42701": "duplicate_column",
|
||||
"42P03": "duplicate_cursor",
|
||||
"42P04": "duplicate_database",
|
||||
"42723": "duplicate_function",
|
||||
"42P05": "duplicate_prepared_statement",
|
||||
"42P06": "duplicate_schema",
|
||||
"42P07": "duplicate_table",
|
||||
"42712": "duplicate_alias",
|
||||
"42710": "duplicate_object",
|
||||
"42702": "ambiguous_column",
|
||||
"42725": "ambiguous_function",
|
||||
"42P08": "ambiguous_parameter",
|
||||
"42P09": "ambiguous_alias",
|
||||
"42P10": "invalid_column_reference",
|
||||
"42611": "invalid_column_definition",
|
||||
"42P11": "invalid_cursor_definition",
|
||||
"42P12": "invalid_database_definition",
|
||||
"42P13": "invalid_function_definition",
|
||||
"42P14": "invalid_prepared_statement_definition",
|
||||
"42P15": "invalid_schema_definition",
|
||||
"42P16": "invalid_table_definition",
|
||||
"42P17": "invalid_object_definition",
|
||||
// Class 44 - WITH CHECK OPTION Violation
|
||||
"44000": "with_check_option_violation",
|
||||
// Class 53 - Insufficient Resources
|
||||
"53000": "insufficient_resources",
|
||||
"53100": "disk_full",
|
||||
"53200": "out_of_memory",
|
||||
"53300": "too_many_connections",
|
||||
"53400": "configuration_limit_exceeded",
|
||||
// Class 54 - Program Limit Exceeded
|
||||
"54000": "program_limit_exceeded",
|
||||
"54001": "statement_too_complex",
|
||||
"54011": "too_many_columns",
|
||||
"54023": "too_many_arguments",
|
||||
// Class 55 - Object Not In Prerequisite State
|
||||
"55000": "object_not_in_prerequisite_state",
|
||||
"55006": "object_in_use",
|
||||
"55P02": "cant_change_runtime_param",
|
||||
"55P03": "lock_not_available",
|
||||
// Class 57 - Operator Intervention
|
||||
"57000": "operator_intervention",
|
||||
"57014": "query_canceled",
|
||||
"57P01": "admin_shutdown",
|
||||
"57P02": "crash_shutdown",
|
||||
"57P03": "cannot_connect_now",
|
||||
"57P04": "database_dropped",
|
||||
// Class 58 - System Error (errors external to PostgreSQL itself)
|
||||
"58000": "system_error",
|
||||
"58030": "io_error",
|
||||
"58P01": "undefined_file",
|
||||
"58P02": "duplicate_file",
|
||||
// Class F0 - Configuration File Error
|
||||
"F0000": "config_file_error",
|
||||
"F0001": "lock_file_exists",
|
||||
// Class HV - Foreign Data Wrapper Error (SQL/MED)
|
||||
"HV000": "fdw_error",
|
||||
"HV005": "fdw_column_name_not_found",
|
||||
"HV002": "fdw_dynamic_parameter_value_needed",
|
||||
"HV010": "fdw_function_sequence_error",
|
||||
"HV021": "fdw_inconsistent_descriptor_information",
|
||||
"HV024": "fdw_invalid_attribute_value",
|
||||
"HV007": "fdw_invalid_column_name",
|
||||
"HV008": "fdw_invalid_column_number",
|
||||
"HV004": "fdw_invalid_data_type",
|
||||
"HV006": "fdw_invalid_data_type_descriptors",
|
||||
"HV091": "fdw_invalid_descriptor_field_identifier",
|
||||
"HV00B": "fdw_invalid_handle",
|
||||
"HV00C": "fdw_invalid_option_index",
|
||||
"HV00D": "fdw_invalid_option_name",
|
||||
"HV090": "fdw_invalid_string_length_or_buffer_length",
|
||||
"HV00A": "fdw_invalid_string_format",
|
||||
"HV009": "fdw_invalid_use_of_null_pointer",
|
||||
"HV014": "fdw_too_many_handles",
|
||||
"HV001": "fdw_out_of_memory",
|
||||
"HV00P": "fdw_no_schemas",
|
||||
"HV00J": "fdw_option_name_not_found",
|
||||
"HV00K": "fdw_reply_handle",
|
||||
"HV00Q": "fdw_schema_not_found",
|
||||
"HV00R": "fdw_table_not_found",
|
||||
"HV00L": "fdw_unable_to_create_execution",
|
||||
"HV00M": "fdw_unable_to_create_reply",
|
||||
"HV00N": "fdw_unable_to_establish_connection",
|
||||
// Class P0 - PL/pgSQL Error
|
||||
"P0000": "plpgsql_error",
|
||||
"P0001": "raise_exception",
|
||||
"P0002": "no_data_found",
|
||||
"P0003": "too_many_rows",
|
||||
// Class XX - Internal Error
|
||||
"XX000": "internal_error",
|
||||
"XX001": "data_corrupted",
|
||||
"XX002": "index_corrupted",
|
||||
}
|
||||
|
||||
func parseError(r *readBuf) *Error {
|
||||
err := new(Error)
|
||||
for t := r.byte(); t != 0; t = r.byte() {
|
||||
msg := r.string()
|
||||
switch t {
|
||||
case 'S':
|
||||
err.Severity = msg
|
||||
case 'C':
|
||||
err.Code = ErrorCode(msg)
|
||||
case 'M':
|
||||
err.Message = msg
|
||||
case 'D':
|
||||
err.Detail = msg
|
||||
case 'H':
|
||||
err.Hint = msg
|
||||
case 'P':
|
||||
err.Position = msg
|
||||
case 'p':
|
||||
err.InternalPosition = msg
|
||||
case 'q':
|
||||
err.InternalQuery = msg
|
||||
case 'W':
|
||||
err.Where = msg
|
||||
case 's':
|
||||
err.Schema = msg
|
||||
case 't':
|
||||
err.Table = msg
|
||||
case 'c':
|
||||
err.Column = msg
|
||||
case 'd':
|
||||
err.DataTypeName = msg
|
||||
case 'n':
|
||||
err.Constraint = msg
|
||||
case 'F':
|
||||
err.File = msg
|
||||
case 'L':
|
||||
err.Line = msg
|
||||
case 'R':
|
||||
err.Routine = msg
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Fatal returns true if the Error Severity is fatal.
|
||||
func (err *Error) Fatal() bool {
|
||||
return err.Severity == Efatal
|
||||
}
|
||||
|
||||
// SQLState returns the SQLState of the error.
|
||||
func (err *Error) SQLState() string {
|
||||
return string(err.Code)
|
||||
}
|
||||
|
||||
// Get implements the legacy PGError interface. New code should use the fields
|
||||
// of the Error struct directly.
|
||||
func (err *Error) Get(k byte) (v string) {
|
||||
switch k {
|
||||
case 'S':
|
||||
return err.Severity
|
||||
case 'C':
|
||||
return string(err.Code)
|
||||
case 'M':
|
||||
return err.Message
|
||||
case 'D':
|
||||
return err.Detail
|
||||
case 'H':
|
||||
return err.Hint
|
||||
case 'P':
|
||||
return err.Position
|
||||
case 'p':
|
||||
return err.InternalPosition
|
||||
case 'q':
|
||||
return err.InternalQuery
|
||||
case 'W':
|
||||
return err.Where
|
||||
case 's':
|
||||
return err.Schema
|
||||
case 't':
|
||||
return err.Table
|
||||
case 'c':
|
||||
return err.Column
|
||||
case 'd':
|
||||
return err.DataTypeName
|
||||
case 'n':
|
||||
return err.Constraint
|
||||
case 'F':
|
||||
return err.File
|
||||
case 'L':
|
||||
return err.Line
|
||||
case 'R':
|
||||
return err.Routine
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (err *Error) Error() string {
|
||||
return "pq: " + err.Message
|
||||
}
|
||||
|
||||
// PGError is an interface used by previous versions of pq. It is provided
|
||||
// only to support legacy code. New code should use the Error type.
|
||||
type PGError interface {
|
||||
Error() string
|
||||
Fatal() bool
|
||||
Get(k byte) (v string)
|
||||
}
|
||||
|
||||
func errorf(s string, args ...interface{}) {
|
||||
panic(fmt.Errorf("pq: %s", fmt.Sprintf(s, args...)))
|
||||
}
|
||||
|
||||
// TODO(ainar-g) Rename to errorf after removing panics.
|
||||
func fmterrorf(s string, args ...interface{}) error {
|
||||
return fmt.Errorf("pq: %s", fmt.Sprintf(s, args...))
|
||||
}
|
||||
|
||||
func errRecoverNoErrBadConn(err *error) {
|
||||
e := recover()
|
||||
if e == nil {
|
||||
// Do nothing
|
||||
return
|
||||
}
|
||||
var ok bool
|
||||
*err, ok = e.(error)
|
||||
if !ok {
|
||||
*err = fmt.Errorf("pq: unexpected error: %#v", e)
|
||||
}
|
||||
}
|
||||
|
||||
func (cn *conn) errRecover(err *error) {
|
||||
e := recover()
|
||||
switch v := e.(type) {
|
||||
case nil:
|
||||
// Do nothing
|
||||
case runtime.Error:
|
||||
cn.err.set(driver.ErrBadConn)
|
||||
panic(v)
|
||||
case *Error:
|
||||
if v.Fatal() {
|
||||
*err = driver.ErrBadConn
|
||||
} else {
|
||||
*err = v
|
||||
}
|
||||
case *net.OpError:
|
||||
cn.err.set(driver.ErrBadConn)
|
||||
*err = v
|
||||
case *safeRetryError:
|
||||
cn.err.set(driver.ErrBadConn)
|
||||
*err = driver.ErrBadConn
|
||||
case error:
|
||||
if v == io.EOF || v.Error() == "remote error: handshake failure" {
|
||||
*err = driver.ErrBadConn
|
||||
} else {
|
||||
*err = v
|
||||
}
|
||||
|
||||
default:
|
||||
cn.err.set(driver.ErrBadConn)
|
||||
panic(fmt.Sprintf("unknown error: %#v", e))
|
||||
}
|
||||
|
||||
// Any time we return ErrBadConn, we need to remember it since *Tx doesn't
|
||||
// mark the connection bad in database/sql.
|
||||
if *err == driver.ErrBadConn {
|
||||
cn.err.set(driver.ErrBadConn)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,27 @@
|
||||
package pq
|
||||
|
||||
// NewGSSFunc creates a GSS authentication provider, for use with
|
||||
// RegisterGSSProvider.
|
||||
type NewGSSFunc func() (GSS, error)
|
||||
|
||||
var newGss NewGSSFunc
|
||||
|
||||
// RegisterGSSProvider registers a GSS authentication provider. For example, if
|
||||
// you need to use Kerberos to authenticate with your server, add this to your
|
||||
// main package:
|
||||
//
|
||||
// import "github.com/lib/pq/auth/kerberos"
|
||||
//
|
||||
// func init() {
|
||||
// pq.RegisterGSSProvider(func() (pq.GSS, error) { return kerberos.NewGSS() })
|
||||
// }
|
||||
func RegisterGSSProvider(newGssArg NewGSSFunc) {
|
||||
newGss = newGssArg
|
||||
}
|
||||
|
||||
// GSS provides GSSAPI authentication (e.g., Kerberos).
|
||||
type GSS interface {
|
||||
GetInitToken(host string, service string) ([]byte, error)
|
||||
GetInitTokenFromSpn(spn string) ([]byte, error)
|
||||
Continue(inToken []byte) (done bool, outToken []byte, err error)
|
||||
}
|
||||
@ -0,0 +1,72 @@
|
||||
//go:build go1.10
|
||||
// +build go1.10
|
||||
|
||||
package pq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql/driver"
|
||||
)
|
||||
|
||||
// NoticeHandler returns the notice handler on the given connection, if any. A
|
||||
// runtime panic occurs if c is not a pq connection. This is rarely used
|
||||
// directly, use ConnectorNoticeHandler and ConnectorWithNoticeHandler instead.
|
||||
func NoticeHandler(c driver.Conn) func(*Error) {
|
||||
return c.(*conn).noticeHandler
|
||||
}
|
||||
|
||||
// SetNoticeHandler sets the given notice handler on the given connection. A
|
||||
// runtime panic occurs if c is not a pq connection. A nil handler may be used
|
||||
// to unset it. This is rarely used directly, use ConnectorNoticeHandler and
|
||||
// ConnectorWithNoticeHandler instead.
|
||||
//
|
||||
// Note: Notice handlers are executed synchronously by pq meaning commands
|
||||
// won't continue to be processed until the handler returns.
|
||||
func SetNoticeHandler(c driver.Conn, handler func(*Error)) {
|
||||
c.(*conn).noticeHandler = handler
|
||||
}
|
||||
|
||||
// NoticeHandlerConnector wraps a regular connector and sets a notice handler
|
||||
// on it.
|
||||
type NoticeHandlerConnector struct {
|
||||
driver.Connector
|
||||
noticeHandler func(*Error)
|
||||
}
|
||||
|
||||
// Connect calls the underlying connector's connect method and then sets the
|
||||
// notice handler.
|
||||
func (n *NoticeHandlerConnector) Connect(ctx context.Context) (driver.Conn, error) {
|
||||
c, err := n.Connector.Connect(ctx)
|
||||
if err == nil {
|
||||
SetNoticeHandler(c, n.noticeHandler)
|
||||
}
|
||||
return c, err
|
||||
}
|
||||
|
||||
// ConnectorNoticeHandler returns the currently set notice handler, if any. If
|
||||
// the given connector is not a result of ConnectorWithNoticeHandler, nil is
|
||||
// returned.
|
||||
func ConnectorNoticeHandler(c driver.Connector) func(*Error) {
|
||||
if c, ok := c.(*NoticeHandlerConnector); ok {
|
||||
return c.noticeHandler
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ConnectorWithNoticeHandler creates or sets the given handler for the given
|
||||
// connector. If the given connector is a result of calling this function
|
||||
// previously, it is simply set on the given connector and returned. Otherwise,
|
||||
// this returns a new connector wrapping the given one and setting the notice
|
||||
// handler. A nil notice handler may be used to unset it.
|
||||
//
|
||||
// The returned connector is intended to be used with database/sql.OpenDB.
|
||||
//
|
||||
// Note: Notice handlers are executed synchronously by pq meaning commands
|
||||
// won't continue to be processed until the handler returns.
|
||||
func ConnectorWithNoticeHandler(c driver.Connector, handler func(*Error)) *NoticeHandlerConnector {
|
||||
if c, ok := c.(*NoticeHandlerConnector); ok {
|
||||
c.noticeHandler = handler
|
||||
return c
|
||||
}
|
||||
return &NoticeHandlerConnector{Connector: c, noticeHandler: handler}
|
||||
}
|
||||
@ -0,0 +1,858 @@
|
||||
package pq
|
||||
|
||||
// Package pq is a pure Go Postgres driver for the database/sql package.
|
||||
// This module contains support for Postgres LISTEN/NOTIFY.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql/driver"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Notification represents a single notification from the database.
|
||||
type Notification struct {
|
||||
// Process ID (PID) of the notifying postgres backend.
|
||||
BePid int
|
||||
// Name of the channel the notification was sent on.
|
||||
Channel string
|
||||
// Payload, or the empty string if unspecified.
|
||||
Extra string
|
||||
}
|
||||
|
||||
func recvNotification(r *readBuf) *Notification {
|
||||
bePid := r.int32()
|
||||
channel := r.string()
|
||||
extra := r.string()
|
||||
|
||||
return &Notification{bePid, channel, extra}
|
||||
}
|
||||
|
||||
// SetNotificationHandler sets the given notification handler on the given
|
||||
// connection. A runtime panic occurs if c is not a pq connection. A nil handler
|
||||
// may be used to unset it.
|
||||
//
|
||||
// Note: Notification handlers are executed synchronously by pq meaning commands
|
||||
// won't continue to be processed until the handler returns.
|
||||
func SetNotificationHandler(c driver.Conn, handler func(*Notification)) {
|
||||
c.(*conn).notificationHandler = handler
|
||||
}
|
||||
|
||||
// NotificationHandlerConnector wraps a regular connector and sets a notification handler
|
||||
// on it.
|
||||
type NotificationHandlerConnector struct {
|
||||
driver.Connector
|
||||
notificationHandler func(*Notification)
|
||||
}
|
||||
|
||||
// Connect calls the underlying connector's connect method and then sets the
|
||||
// notification handler.
|
||||
func (n *NotificationHandlerConnector) Connect(ctx context.Context) (driver.Conn, error) {
|
||||
c, err := n.Connector.Connect(ctx)
|
||||
if err == nil {
|
||||
SetNotificationHandler(c, n.notificationHandler)
|
||||
}
|
||||
return c, err
|
||||
}
|
||||
|
||||
// ConnectorNotificationHandler returns the currently set notification handler, if any. If
|
||||
// the given connector is not a result of ConnectorWithNotificationHandler, nil is
|
||||
// returned.
|
||||
func ConnectorNotificationHandler(c driver.Connector) func(*Notification) {
|
||||
if c, ok := c.(*NotificationHandlerConnector); ok {
|
||||
return c.notificationHandler
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ConnectorWithNotificationHandler creates or sets the given handler for the given
|
||||
// connector. If the given connector is a result of calling this function
|
||||
// previously, it is simply set on the given connector and returned. Otherwise,
|
||||
// this returns a new connector wrapping the given one and setting the notification
|
||||
// handler. A nil notification handler may be used to unset it.
|
||||
//
|
||||
// The returned connector is intended to be used with database/sql.OpenDB.
|
||||
//
|
||||
// Note: Notification handlers are executed synchronously by pq meaning commands
|
||||
// won't continue to be processed until the handler returns.
|
||||
func ConnectorWithNotificationHandler(c driver.Connector, handler func(*Notification)) *NotificationHandlerConnector {
|
||||
if c, ok := c.(*NotificationHandlerConnector); ok {
|
||||
c.notificationHandler = handler
|
||||
return c
|
||||
}
|
||||
return &NotificationHandlerConnector{Connector: c, notificationHandler: handler}
|
||||
}
|
||||
|
||||
const (
|
||||
connStateIdle int32 = iota
|
||||
connStateExpectResponse
|
||||
connStateExpectReadyForQuery
|
||||
)
|
||||
|
||||
type message struct {
|
||||
typ byte
|
||||
err error
|
||||
}
|
||||
|
||||
var errListenerConnClosed = errors.New("pq: ListenerConn has been closed")
|
||||
|
||||
// ListenerConn is a low-level interface for waiting for notifications. You
|
||||
// should use Listener instead.
|
||||
type ListenerConn struct {
|
||||
// guards cn and err
|
||||
connectionLock sync.Mutex
|
||||
cn *conn
|
||||
err error
|
||||
|
||||
connState int32
|
||||
|
||||
// the sending goroutine will be holding this lock
|
||||
senderLock sync.Mutex
|
||||
|
||||
notificationChan chan<- *Notification
|
||||
|
||||
replyChan chan message
|
||||
}
|
||||
|
||||
// NewListenerConn creates a new ListenerConn. Use NewListener instead.
|
||||
func NewListenerConn(name string, notificationChan chan<- *Notification) (*ListenerConn, error) {
|
||||
return newDialListenerConn(defaultDialer{}, name, notificationChan)
|
||||
}
|
||||
|
||||
func newDialListenerConn(d Dialer, name string, c chan<- *Notification) (*ListenerConn, error) {
|
||||
cn, err := DialOpen(d, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
l := &ListenerConn{
|
||||
cn: cn.(*conn),
|
||||
notificationChan: c,
|
||||
connState: connStateIdle,
|
||||
replyChan: make(chan message, 2),
|
||||
}
|
||||
|
||||
go l.listenerConnMain()
|
||||
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// We can only allow one goroutine at a time to be running a query on the
|
||||
// connection for various reasons, so the goroutine sending on the connection
|
||||
// must be holding senderLock.
|
||||
//
|
||||
// Returns an error if an unrecoverable error has occurred and the ListenerConn
|
||||
// should be abandoned.
|
||||
func (l *ListenerConn) acquireSenderLock() error {
|
||||
// we must acquire senderLock first to avoid deadlocks; see ExecSimpleQuery
|
||||
l.senderLock.Lock()
|
||||
|
||||
l.connectionLock.Lock()
|
||||
err := l.err
|
||||
l.connectionLock.Unlock()
|
||||
if err != nil {
|
||||
l.senderLock.Unlock()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *ListenerConn) releaseSenderLock() {
|
||||
l.senderLock.Unlock()
|
||||
}
|
||||
|
||||
// setState advances the protocol state to newState. Returns false if moving
|
||||
// to that state from the current state is not allowed.
|
||||
func (l *ListenerConn) setState(newState int32) bool {
|
||||
var expectedState int32
|
||||
|
||||
switch newState {
|
||||
case connStateIdle:
|
||||
expectedState = connStateExpectReadyForQuery
|
||||
case connStateExpectResponse:
|
||||
expectedState = connStateIdle
|
||||
case connStateExpectReadyForQuery:
|
||||
expectedState = connStateExpectResponse
|
||||
default:
|
||||
panic(fmt.Sprintf("unexpected listenerConnState %d", newState))
|
||||
}
|
||||
|
||||
return atomic.CompareAndSwapInt32(&l.connState, expectedState, newState)
|
||||
}
|
||||
|
||||
// Main logic is here: receive messages from the postgres backend, forward
|
||||
// notifications and query replies and keep the internal state in sync with the
|
||||
// protocol state. Returns when the connection has been lost, is about to go
|
||||
// away or should be discarded because we couldn't agree on the state with the
|
||||
// server backend.
|
||||
func (l *ListenerConn) listenerConnLoop() (err error) {
|
||||
defer errRecoverNoErrBadConn(&err)
|
||||
|
||||
r := &readBuf{}
|
||||
for {
|
||||
t, err := l.cn.recvMessage(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
switch t {
|
||||
case 'A':
|
||||
// recvNotification copies all the data so we don't need to worry
|
||||
// about the scratch buffer being overwritten.
|
||||
l.notificationChan <- recvNotification(r)
|
||||
|
||||
case 'T', 'D':
|
||||
// only used by tests; ignore
|
||||
|
||||
case 'E':
|
||||
// We might receive an ErrorResponse even when not in a query; it
|
||||
// is expected that the server will close the connection after
|
||||
// that, but we should make sure that the error we display is the
|
||||
// one from the stray ErrorResponse, not io.ErrUnexpectedEOF.
|
||||
if !l.setState(connStateExpectReadyForQuery) {
|
||||
return parseError(r)
|
||||
}
|
||||
l.replyChan <- message{t, parseError(r)}
|
||||
|
||||
case 'C', 'I':
|
||||
if !l.setState(connStateExpectReadyForQuery) {
|
||||
// protocol out of sync
|
||||
return fmt.Errorf("unexpected CommandComplete")
|
||||
}
|
||||
// ExecSimpleQuery doesn't need to know about this message
|
||||
|
||||
case 'Z':
|
||||
if !l.setState(connStateIdle) {
|
||||
// protocol out of sync
|
||||
return fmt.Errorf("unexpected ReadyForQuery")
|
||||
}
|
||||
l.replyChan <- message{t, nil}
|
||||
|
||||
case 'S':
|
||||
// ignore
|
||||
case 'N':
|
||||
if n := l.cn.noticeHandler; n != nil {
|
||||
n(parseError(r))
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unexpected message %q from server in listenerConnLoop", t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This is the main routine for the goroutine receiving on the database
|
||||
// connection. Most of the main logic is in listenerConnLoop.
|
||||
func (l *ListenerConn) listenerConnMain() {
|
||||
err := l.listenerConnLoop()
|
||||
|
||||
// listenerConnLoop terminated; we're done, but we still have to clean up.
|
||||
// Make sure nobody tries to start any new queries by making sure the err
|
||||
// pointer is set. It is important that we do not overwrite its value; a
|
||||
// connection could be closed by either this goroutine or one sending on
|
||||
// the connection -- whoever closes the connection is assumed to have the
|
||||
// more meaningful error message (as the other one will probably get
|
||||
// net.errClosed), so that goroutine sets the error we expose while the
|
||||
// other error is discarded. If the connection is lost while two
|
||||
// goroutines are operating on the socket, it probably doesn't matter which
|
||||
// error we expose so we don't try to do anything more complex.
|
||||
l.connectionLock.Lock()
|
||||
if l.err == nil {
|
||||
l.err = err
|
||||
}
|
||||
l.cn.Close()
|
||||
l.connectionLock.Unlock()
|
||||
|
||||
// There might be a query in-flight; make sure nobody's waiting for a
|
||||
// response to it, since there's not going to be one.
|
||||
close(l.replyChan)
|
||||
|
||||
// let the listener know we're done
|
||||
close(l.notificationChan)
|
||||
|
||||
// this ListenerConn is done
|
||||
}
|
||||
|
||||
// Listen sends a LISTEN query to the server. See ExecSimpleQuery.
|
||||
func (l *ListenerConn) Listen(channel string) (bool, error) {
|
||||
return l.ExecSimpleQuery("LISTEN " + QuoteIdentifier(channel))
|
||||
}
|
||||
|
||||
// Unlisten sends an UNLISTEN query to the server. See ExecSimpleQuery.
|
||||
func (l *ListenerConn) Unlisten(channel string) (bool, error) {
|
||||
return l.ExecSimpleQuery("UNLISTEN " + QuoteIdentifier(channel))
|
||||
}
|
||||
|
||||
// UnlistenAll sends an `UNLISTEN *` query to the server. See ExecSimpleQuery.
|
||||
func (l *ListenerConn) UnlistenAll() (bool, error) {
|
||||
return l.ExecSimpleQuery("UNLISTEN *")
|
||||
}
|
||||
|
||||
// Ping the remote server to make sure it's alive. Non-nil error means the
|
||||
// connection has failed and should be abandoned.
|
||||
func (l *ListenerConn) Ping() error {
|
||||
sent, err := l.ExecSimpleQuery("")
|
||||
if !sent {
|
||||
return err
|
||||
}
|
||||
if err != nil {
|
||||
// shouldn't happen
|
||||
panic(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Attempt to send a query on the connection. Returns an error if sending the
|
||||
// query failed, and the caller should initiate closure of this connection.
|
||||
// The caller must be holding senderLock (see acquireSenderLock and
|
||||
// releaseSenderLock).
|
||||
func (l *ListenerConn) sendSimpleQuery(q string) (err error) {
|
||||
defer errRecoverNoErrBadConn(&err)
|
||||
|
||||
// must set connection state before sending the query
|
||||
if !l.setState(connStateExpectResponse) {
|
||||
panic("two queries running at the same time")
|
||||
}
|
||||
|
||||
// Can't use l.cn.writeBuf here because it uses the scratch buffer which
|
||||
// might get overwritten by listenerConnLoop.
|
||||
b := &writeBuf{
|
||||
buf: []byte("Q\x00\x00\x00\x00"),
|
||||
pos: 1,
|
||||
}
|
||||
b.string(q)
|
||||
l.cn.send(b)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ExecSimpleQuery executes a "simple query" (i.e. one with no bindable
|
||||
// parameters) on the connection. The possible return values are:
|
||||
// 1) "executed" is true; the query was executed to completion on the
|
||||
// database server. If the query failed, err will be set to the error
|
||||
// returned by the database, otherwise err will be nil.
|
||||
// 2) If "executed" is false, the query could not be executed on the remote
|
||||
// server. err will be non-nil.
|
||||
//
|
||||
// After a call to ExecSimpleQuery has returned an executed=false value, the
|
||||
// connection has either been closed or will be closed shortly thereafter, and
|
||||
// all subsequently executed queries will return an error.
|
||||
func (l *ListenerConn) ExecSimpleQuery(q string) (executed bool, err error) {
|
||||
if err = l.acquireSenderLock(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer l.releaseSenderLock()
|
||||
|
||||
err = l.sendSimpleQuery(q)
|
||||
if err != nil {
|
||||
// We can't know what state the protocol is in, so we need to abandon
|
||||
// this connection.
|
||||
l.connectionLock.Lock()
|
||||
// Set the error pointer if it hasn't been set already; see
|
||||
// listenerConnMain.
|
||||
if l.err == nil {
|
||||
l.err = err
|
||||
}
|
||||
l.connectionLock.Unlock()
|
||||
l.cn.c.Close()
|
||||
return false, err
|
||||
}
|
||||
|
||||
// now we just wait for a reply..
|
||||
for {
|
||||
m, ok := <-l.replyChan
|
||||
if !ok {
|
||||
// We lost the connection to server, don't bother waiting for a
|
||||
// a response. err should have been set already.
|
||||
l.connectionLock.Lock()
|
||||
err := l.err
|
||||
l.connectionLock.Unlock()
|
||||
return false, err
|
||||
}
|
||||
switch m.typ {
|
||||
case 'Z':
|
||||
// sanity check
|
||||
if m.err != nil {
|
||||
panic("m.err != nil")
|
||||
}
|
||||
// done; err might or might not be set
|
||||
return true, err
|
||||
|
||||
case 'E':
|
||||
// sanity check
|
||||
if m.err == nil {
|
||||
panic("m.err == nil")
|
||||
}
|
||||
// server responded with an error; ReadyForQuery to follow
|
||||
err = m.err
|
||||
|
||||
default:
|
||||
return false, fmt.Errorf("unknown response for simple query: %q", m.typ)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Close closes the connection.
|
||||
func (l *ListenerConn) Close() error {
|
||||
l.connectionLock.Lock()
|
||||
if l.err != nil {
|
||||
l.connectionLock.Unlock()
|
||||
return errListenerConnClosed
|
||||
}
|
||||
l.err = errListenerConnClosed
|
||||
l.connectionLock.Unlock()
|
||||
// We can't send anything on the connection without holding senderLock.
|
||||
// Simply close the net.Conn to wake up everyone operating on it.
|
||||
return l.cn.c.Close()
|
||||
}
|
||||
|
||||
// Err returns the reason the connection was closed. It is not safe to call
|
||||
// this function until l.Notify has been closed.
|
||||
func (l *ListenerConn) Err() error {
|
||||
return l.err
|
||||
}
|
||||
|
||||
var errListenerClosed = errors.New("pq: Listener has been closed")
|
||||
|
||||
// ErrChannelAlreadyOpen is returned from Listen when a channel is already
|
||||
// open.
|
||||
var ErrChannelAlreadyOpen = errors.New("pq: channel is already open")
|
||||
|
||||
// ErrChannelNotOpen is returned from Unlisten when a channel is not open.
|
||||
var ErrChannelNotOpen = errors.New("pq: channel is not open")
|
||||
|
||||
// ListenerEventType is an enumeration of listener event types.
|
||||
type ListenerEventType int
|
||||
|
||||
const (
|
||||
// ListenerEventConnected is emitted only when the database connection
|
||||
// has been initially initialized. The err argument of the callback
|
||||
// will always be nil.
|
||||
ListenerEventConnected ListenerEventType = iota
|
||||
|
||||
// ListenerEventDisconnected is emitted after a database connection has
|
||||
// been lost, either because of an error or because Close has been
|
||||
// called. The err argument will be set to the reason the database
|
||||
// connection was lost.
|
||||
ListenerEventDisconnected
|
||||
|
||||
// ListenerEventReconnected is emitted after a database connection has
|
||||
// been re-established after connection loss. The err argument of the
|
||||
// callback will always be nil. After this event has been emitted, a
|
||||
// nil pq.Notification is sent on the Listener.Notify channel.
|
||||
ListenerEventReconnected
|
||||
|
||||
// ListenerEventConnectionAttemptFailed is emitted after a connection
|
||||
// to the database was attempted, but failed. The err argument will be
|
||||
// set to an error describing why the connection attempt did not
|
||||
// succeed.
|
||||
ListenerEventConnectionAttemptFailed
|
||||
)
|
||||
|
||||
// EventCallbackType is the event callback type. See also ListenerEventType
|
||||
// constants' documentation.
|
||||
type EventCallbackType func(event ListenerEventType, err error)
|
||||
|
||||
// Listener provides an interface for listening to notifications from a
|
||||
// PostgreSQL database. For general usage information, see section
|
||||
// "Notifications".
|
||||
//
|
||||
// Listener can safely be used from concurrently running goroutines.
|
||||
type Listener struct {
|
||||
// Channel for receiving notifications from the database. In some cases a
|
||||
// nil value will be sent. See section "Notifications" above.
|
||||
Notify chan *Notification
|
||||
|
||||
name string
|
||||
minReconnectInterval time.Duration
|
||||
maxReconnectInterval time.Duration
|
||||
dialer Dialer
|
||||
eventCallback EventCallbackType
|
||||
|
||||
lock sync.Mutex
|
||||
isClosed bool
|
||||
reconnectCond *sync.Cond
|
||||
cn *ListenerConn
|
||||
connNotificationChan <-chan *Notification
|
||||
channels map[string]struct{}
|
||||
}
|
||||
|
||||
// NewListener creates a new database connection dedicated to LISTEN / NOTIFY.
|
||||
//
|
||||
// name should be set to a connection string to be used to establish the
|
||||
// database connection (see section "Connection String Parameters" above).
|
||||
//
|
||||
// minReconnectInterval controls the duration to wait before trying to
|
||||
// re-establish the database connection after connection loss. After each
|
||||
// consecutive failure this interval is doubled, until maxReconnectInterval is
|
||||
// reached. Successfully completing the connection establishment procedure
|
||||
// resets the interval back to minReconnectInterval.
|
||||
//
|
||||
// The last parameter eventCallback can be set to a function which will be
|
||||
// called by the Listener when the state of the underlying database connection
|
||||
// changes. This callback will be called by the goroutine which dispatches the
|
||||
// notifications over the Notify channel, so you should try to avoid doing
|
||||
// potentially time-consuming operations from the callback.
|
||||
func NewListener(name string,
|
||||
minReconnectInterval time.Duration,
|
||||
maxReconnectInterval time.Duration,
|
||||
eventCallback EventCallbackType) *Listener {
|
||||
return NewDialListener(defaultDialer{}, name, minReconnectInterval, maxReconnectInterval, eventCallback)
|
||||
}
|
||||
|
||||
// NewDialListener is like NewListener but it takes a Dialer.
|
||||
func NewDialListener(d Dialer,
|
||||
name string,
|
||||
minReconnectInterval time.Duration,
|
||||
maxReconnectInterval time.Duration,
|
||||
eventCallback EventCallbackType) *Listener {
|
||||
|
||||
l := &Listener{
|
||||
name: name,
|
||||
minReconnectInterval: minReconnectInterval,
|
||||
maxReconnectInterval: maxReconnectInterval,
|
||||
dialer: d,
|
||||
eventCallback: eventCallback,
|
||||
|
||||
channels: make(map[string]struct{}),
|
||||
|
||||
Notify: make(chan *Notification, 32),
|
||||
}
|
||||
l.reconnectCond = sync.NewCond(&l.lock)
|
||||
|
||||
go l.listenerMain()
|
||||
|
||||
return l
|
||||
}
|
||||
|
||||
// NotificationChannel returns the notification channel for this listener.
|
||||
// This is the same channel as Notify, and will not be recreated during the
|
||||
// life time of the Listener.
|
||||
func (l *Listener) NotificationChannel() <-chan *Notification {
|
||||
return l.Notify
|
||||
}
|
||||
|
||||
// Listen starts listening for notifications on a channel. Calls to this
|
||||
// function will block until an acknowledgement has been received from the
|
||||
// server. Note that Listener automatically re-establishes the connection
|
||||
// after connection loss, so this function may block indefinitely if the
|
||||
// connection can not be re-established.
|
||||
//
|
||||
// Listen will only fail in three conditions:
|
||||
// 1) The channel is already open. The returned error will be
|
||||
// ErrChannelAlreadyOpen.
|
||||
// 2) The query was executed on the remote server, but PostgreSQL returned an
|
||||
// error message in response to the query. The returned error will be a
|
||||
// pq.Error containing the information the server supplied.
|
||||
// 3) Close is called on the Listener before the request could be completed.
|
||||
//
|
||||
// The channel name is case-sensitive.
|
||||
func (l *Listener) Listen(channel string) error {
|
||||
l.lock.Lock()
|
||||
defer l.lock.Unlock()
|
||||
|
||||
if l.isClosed {
|
||||
return errListenerClosed
|
||||
}
|
||||
|
||||
// The server allows you to issue a LISTEN on a channel which is already
|
||||
// open, but it seems useful to be able to detect this case to spot for
|
||||
// mistakes in application logic. If the application genuinely does't
|
||||
// care, it can check the exported error and ignore it.
|
||||
_, exists := l.channels[channel]
|
||||
if exists {
|
||||
return ErrChannelAlreadyOpen
|
||||
}
|
||||
|
||||
if l.cn != nil {
|
||||
// If gotResponse is true but error is set, the query was executed on
|
||||
// the remote server, but resulted in an error. This should be
|
||||
// relatively rare, so it's fine if we just pass the error to our
|
||||
// caller. However, if gotResponse is false, we could not complete the
|
||||
// query on the remote server and our underlying connection is about
|
||||
// to go away, so we only add relname to l.channels, and wait for
|
||||
// resync() to take care of the rest.
|
||||
gotResponse, err := l.cn.Listen(channel)
|
||||
if gotResponse && err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
l.channels[channel] = struct{}{}
|
||||
for l.cn == nil {
|
||||
l.reconnectCond.Wait()
|
||||
// we let go of the mutex for a while
|
||||
if l.isClosed {
|
||||
return errListenerClosed
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unlisten removes a channel from the Listener's channel list. Returns
|
||||
// ErrChannelNotOpen if the Listener is not listening on the specified channel.
|
||||
// Returns immediately with no error if there is no connection. Note that you
|
||||
// might still get notifications for this channel even after Unlisten has
|
||||
// returned.
|
||||
//
|
||||
// The channel name is case-sensitive.
|
||||
func (l *Listener) Unlisten(channel string) error {
|
||||
l.lock.Lock()
|
||||
defer l.lock.Unlock()
|
||||
|
||||
if l.isClosed {
|
||||
return errListenerClosed
|
||||
}
|
||||
|
||||
// Similarly to LISTEN, this is not an error in Postgres, but it seems
|
||||
// useful to distinguish from the normal conditions.
|
||||
_, exists := l.channels[channel]
|
||||
if !exists {
|
||||
return ErrChannelNotOpen
|
||||
}
|
||||
|
||||
if l.cn != nil {
|
||||
// Similarly to Listen (see comment in that function), the caller
|
||||
// should only be bothered with an error if it came from the backend as
|
||||
// a response to our query.
|
||||
gotResponse, err := l.cn.Unlisten(channel)
|
||||
if gotResponse && err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Don't bother waiting for resync if there's no connection.
|
||||
delete(l.channels, channel)
|
||||
return nil
|
||||
}
|
||||
|
||||
// UnlistenAll removes all channels from the Listener's channel list. Returns
|
||||
// immediately with no error if there is no connection. Note that you might
|
||||
// still get notifications for any of the deleted channels even after
|
||||
// UnlistenAll has returned.
|
||||
func (l *Listener) UnlistenAll() error {
|
||||
l.lock.Lock()
|
||||
defer l.lock.Unlock()
|
||||
|
||||
if l.isClosed {
|
||||
return errListenerClosed
|
||||
}
|
||||
|
||||
if l.cn != nil {
|
||||
// Similarly to Listen (see comment in that function), the caller
|
||||
// should only be bothered with an error if it came from the backend as
|
||||
// a response to our query.
|
||||
gotResponse, err := l.cn.UnlistenAll()
|
||||
if gotResponse && err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Don't bother waiting for resync if there's no connection.
|
||||
l.channels = make(map[string]struct{})
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ping the remote server to make sure it's alive. Non-nil return value means
|
||||
// that there is no active connection.
|
||||
func (l *Listener) Ping() error {
|
||||
l.lock.Lock()
|
||||
defer l.lock.Unlock()
|
||||
|
||||
if l.isClosed {
|
||||
return errListenerClosed
|
||||
}
|
||||
if l.cn == nil {
|
||||
return errors.New("no connection")
|
||||
}
|
||||
|
||||
return l.cn.Ping()
|
||||
}
|
||||
|
||||
// Clean up after losing the server connection. Returns l.cn.Err(), which
|
||||
// should have the reason the connection was lost.
|
||||
func (l *Listener) disconnectCleanup() error {
|
||||
l.lock.Lock()
|
||||
defer l.lock.Unlock()
|
||||
|
||||
// sanity check; can't look at Err() until the channel has been closed
|
||||
select {
|
||||
case _, ok := <-l.connNotificationChan:
|
||||
if ok {
|
||||
panic("connNotificationChan not closed")
|
||||
}
|
||||
default:
|
||||
panic("connNotificationChan not closed")
|
||||
}
|
||||
|
||||
err := l.cn.Err()
|
||||
l.cn.Close()
|
||||
l.cn = nil
|
||||
return err
|
||||
}
|
||||
|
||||
// Synchronize the list of channels we want to be listening on with the server
|
||||
// after the connection has been established.
|
||||
func (l *Listener) resync(cn *ListenerConn, notificationChan <-chan *Notification) error {
|
||||
doneChan := make(chan error)
|
||||
go func(notificationChan <-chan *Notification) {
|
||||
for channel := range l.channels {
|
||||
// If we got a response, return that error to our caller as it's
|
||||
// going to be more descriptive than cn.Err().
|
||||
gotResponse, err := cn.Listen(channel)
|
||||
if gotResponse && err != nil {
|
||||
doneChan <- err
|
||||
return
|
||||
}
|
||||
|
||||
// If we couldn't reach the server, wait for notificationChan to
|
||||
// close and then return the error message from the connection, as
|
||||
// per ListenerConn's interface.
|
||||
if err != nil {
|
||||
for range notificationChan {
|
||||
}
|
||||
doneChan <- cn.Err()
|
||||
return
|
||||
}
|
||||
}
|
||||
doneChan <- nil
|
||||
}(notificationChan)
|
||||
|
||||
// Ignore notifications while synchronization is going on to avoid
|
||||
// deadlocks. We have to send a nil notification over Notify anyway as
|
||||
// we can't possibly know which notifications (if any) were lost while
|
||||
// the connection was down, so there's no reason to try and process
|
||||
// these messages at all.
|
||||
for {
|
||||
select {
|
||||
case _, ok := <-notificationChan:
|
||||
if !ok {
|
||||
notificationChan = nil
|
||||
}
|
||||
|
||||
case err := <-doneChan:
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// caller should NOT be holding l.lock
|
||||
func (l *Listener) closed() bool {
|
||||
l.lock.Lock()
|
||||
defer l.lock.Unlock()
|
||||
|
||||
return l.isClosed
|
||||
}
|
||||
|
||||
func (l *Listener) connect() error {
|
||||
notificationChan := make(chan *Notification, 32)
|
||||
cn, err := newDialListenerConn(l.dialer, l.name, notificationChan)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
l.lock.Lock()
|
||||
defer l.lock.Unlock()
|
||||
|
||||
err = l.resync(cn, notificationChan)
|
||||
if err != nil {
|
||||
cn.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
l.cn = cn
|
||||
l.connNotificationChan = notificationChan
|
||||
l.reconnectCond.Broadcast()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close disconnects the Listener from the database and shuts it down.
|
||||
// Subsequent calls to its methods will return an error. Close returns an
|
||||
// error if the connection has already been closed.
|
||||
func (l *Listener) Close() error {
|
||||
l.lock.Lock()
|
||||
defer l.lock.Unlock()
|
||||
|
||||
if l.isClosed {
|
||||
return errListenerClosed
|
||||
}
|
||||
|
||||
if l.cn != nil {
|
||||
l.cn.Close()
|
||||
}
|
||||
l.isClosed = true
|
||||
|
||||
// Unblock calls to Listen()
|
||||
l.reconnectCond.Broadcast()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *Listener) emitEvent(event ListenerEventType, err error) {
|
||||
if l.eventCallback != nil {
|
||||
l.eventCallback(event, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Main logic here: maintain a connection to the server when possible, wait
|
||||
// for notifications and emit events.
|
||||
func (l *Listener) listenerConnLoop() {
|
||||
var nextReconnect time.Time
|
||||
|
||||
reconnectInterval := l.minReconnectInterval
|
||||
for {
|
||||
for {
|
||||
err := l.connect()
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
|
||||
if l.closed() {
|
||||
return
|
||||
}
|
||||
l.emitEvent(ListenerEventConnectionAttemptFailed, err)
|
||||
|
||||
time.Sleep(reconnectInterval)
|
||||
reconnectInterval *= 2
|
||||
if reconnectInterval > l.maxReconnectInterval {
|
||||
reconnectInterval = l.maxReconnectInterval
|
||||
}
|
||||
}
|
||||
|
||||
if nextReconnect.IsZero() {
|
||||
l.emitEvent(ListenerEventConnected, nil)
|
||||
} else {
|
||||
l.emitEvent(ListenerEventReconnected, nil)
|
||||
l.Notify <- nil
|
||||
}
|
||||
|
||||
reconnectInterval = l.minReconnectInterval
|
||||
nextReconnect = time.Now().Add(reconnectInterval)
|
||||
|
||||
for {
|
||||
notification, ok := <-l.connNotificationChan
|
||||
if !ok {
|
||||
// lost connection, loop again
|
||||
break
|
||||
}
|
||||
l.Notify <- notification
|
||||
}
|
||||
|
||||
err := l.disconnectCleanup()
|
||||
if l.closed() {
|
||||
return
|
||||
}
|
||||
l.emitEvent(ListenerEventDisconnected, err)
|
||||
|
||||
time.Sleep(time.Until(nextReconnect))
|
||||
}
|
||||
}
|
||||
|
||||
func (l *Listener) listenerMain() {
|
||||
l.listenerConnLoop()
|
||||
close(l.Notify)
|
||||
}
|
||||
@ -0,0 +1,6 @@
|
||||
// Package oid contains OID constants
|
||||
// as defined by the Postgres server.
|
||||
package oid
|
||||
|
||||
// Oid is a Postgres Object ID.
|
||||
type Oid uint32
|
||||
@ -0,0 +1,343 @@
|
||||
// Code generated by gen.go. DO NOT EDIT.
|
||||
|
||||
package oid
|
||||
|
||||
const (
|
||||
T_bool Oid = 16
|
||||
T_bytea Oid = 17
|
||||
T_char Oid = 18
|
||||
T_name Oid = 19
|
||||
T_int8 Oid = 20
|
||||
T_int2 Oid = 21
|
||||
T_int2vector Oid = 22
|
||||
T_int4 Oid = 23
|
||||
T_regproc Oid = 24
|
||||
T_text Oid = 25
|
||||
T_oid Oid = 26
|
||||
T_tid Oid = 27
|
||||
T_xid Oid = 28
|
||||
T_cid Oid = 29
|
||||
T_oidvector Oid = 30
|
||||
T_pg_ddl_command Oid = 32
|
||||
T_pg_type Oid = 71
|
||||
T_pg_attribute Oid = 75
|
||||
T_pg_proc Oid = 81
|
||||
T_pg_class Oid = 83
|
||||
T_json Oid = 114
|
||||
T_xml Oid = 142
|
||||
T__xml Oid = 143
|
||||
T_pg_node_tree Oid = 194
|
||||
T__json Oid = 199
|
||||
T_smgr Oid = 210
|
||||
T_index_am_handler Oid = 325
|
||||
T_point Oid = 600
|
||||
T_lseg Oid = 601
|
||||
T_path Oid = 602
|
||||
T_box Oid = 603
|
||||
T_polygon Oid = 604
|
||||
T_line Oid = 628
|
||||
T__line Oid = 629
|
||||
T_cidr Oid = 650
|
||||
T__cidr Oid = 651
|
||||
T_float4 Oid = 700
|
||||
T_float8 Oid = 701
|
||||
T_abstime Oid = 702
|
||||
T_reltime Oid = 703
|
||||
T_tinterval Oid = 704
|
||||
T_unknown Oid = 705
|
||||
T_circle Oid = 718
|
||||
T__circle Oid = 719
|
||||
T_money Oid = 790
|
||||
T__money Oid = 791
|
||||
T_macaddr Oid = 829
|
||||
T_inet Oid = 869
|
||||
T__bool Oid = 1000
|
||||
T__bytea Oid = 1001
|
||||
T__char Oid = 1002
|
||||
T__name Oid = 1003
|
||||
T__int2 Oid = 1005
|
||||
T__int2vector Oid = 1006
|
||||
T__int4 Oid = 1007
|
||||
T__regproc Oid = 1008
|
||||
T__text Oid = 1009
|
||||
T__tid Oid = 1010
|
||||
T__xid Oid = 1011
|
||||
T__cid Oid = 1012
|
||||
T__oidvector Oid = 1013
|
||||
T__bpchar Oid = 1014
|
||||
T__varchar Oid = 1015
|
||||
T__int8 Oid = 1016
|
||||
T__point Oid = 1017
|
||||
T__lseg Oid = 1018
|
||||
T__path Oid = 1019
|
||||
T__box Oid = 1020
|
||||
T__float4 Oid = 1021
|
||||
T__float8 Oid = 1022
|
||||
T__abstime Oid = 1023
|
||||
T__reltime Oid = 1024
|
||||
T__tinterval Oid = 1025
|
||||
T__polygon Oid = 1027
|
||||
T__oid Oid = 1028
|
||||
T_aclitem Oid = 1033
|
||||
T__aclitem Oid = 1034
|
||||
T__macaddr Oid = 1040
|
||||
T__inet Oid = 1041
|
||||
T_bpchar Oid = 1042
|
||||
T_varchar Oid = 1043
|
||||
T_date Oid = 1082
|
||||
T_time Oid = 1083
|
||||
T_timestamp Oid = 1114
|
||||
T__timestamp Oid = 1115
|
||||
T__date Oid = 1182
|
||||
T__time Oid = 1183
|
||||
T_timestamptz Oid = 1184
|
||||
T__timestamptz Oid = 1185
|
||||
T_interval Oid = 1186
|
||||
T__interval Oid = 1187
|
||||
T__numeric Oid = 1231
|
||||
T_pg_database Oid = 1248
|
||||
T__cstring Oid = 1263
|
||||
T_timetz Oid = 1266
|
||||
T__timetz Oid = 1270
|
||||
T_bit Oid = 1560
|
||||
T__bit Oid = 1561
|
||||
T_varbit Oid = 1562
|
||||
T__varbit Oid = 1563
|
||||
T_numeric Oid = 1700
|
||||
T_refcursor Oid = 1790
|
||||
T__refcursor Oid = 2201
|
||||
T_regprocedure Oid = 2202
|
||||
T_regoper Oid = 2203
|
||||
T_regoperator Oid = 2204
|
||||
T_regclass Oid = 2205
|
||||
T_regtype Oid = 2206
|
||||
T__regprocedure Oid = 2207
|
||||
T__regoper Oid = 2208
|
||||
T__regoperator Oid = 2209
|
||||
T__regclass Oid = 2210
|
||||
T__regtype Oid = 2211
|
||||
T_record Oid = 2249
|
||||
T_cstring Oid = 2275
|
||||
T_any Oid = 2276
|
||||
T_anyarray Oid = 2277
|
||||
T_void Oid = 2278
|
||||
T_trigger Oid = 2279
|
||||
T_language_handler Oid = 2280
|
||||
T_internal Oid = 2281
|
||||
T_opaque Oid = 2282
|
||||
T_anyelement Oid = 2283
|
||||
T__record Oid = 2287
|
||||
T_anynonarray Oid = 2776
|
||||
T_pg_authid Oid = 2842
|
||||
T_pg_auth_members Oid = 2843
|
||||
T__txid_snapshot Oid = 2949
|
||||
T_uuid Oid = 2950
|
||||
T__uuid Oid = 2951
|
||||
T_txid_snapshot Oid = 2970
|
||||
T_fdw_handler Oid = 3115
|
||||
T_pg_lsn Oid = 3220
|
||||
T__pg_lsn Oid = 3221
|
||||
T_tsm_handler Oid = 3310
|
||||
T_anyenum Oid = 3500
|
||||
T_tsvector Oid = 3614
|
||||
T_tsquery Oid = 3615
|
||||
T_gtsvector Oid = 3642
|
||||
T__tsvector Oid = 3643
|
||||
T__gtsvector Oid = 3644
|
||||
T__tsquery Oid = 3645
|
||||
T_regconfig Oid = 3734
|
||||
T__regconfig Oid = 3735
|
||||
T_regdictionary Oid = 3769
|
||||
T__regdictionary Oid = 3770
|
||||
T_jsonb Oid = 3802
|
||||
T__jsonb Oid = 3807
|
||||
T_anyrange Oid = 3831
|
||||
T_event_trigger Oid = 3838
|
||||
T_int4range Oid = 3904
|
||||
T__int4range Oid = 3905
|
||||
T_numrange Oid = 3906
|
||||
T__numrange Oid = 3907
|
||||
T_tsrange Oid = 3908
|
||||
T__tsrange Oid = 3909
|
||||
T_tstzrange Oid = 3910
|
||||
T__tstzrange Oid = 3911
|
||||
T_daterange Oid = 3912
|
||||
T__daterange Oid = 3913
|
||||
T_int8range Oid = 3926
|
||||
T__int8range Oid = 3927
|
||||
T_pg_shseclabel Oid = 4066
|
||||
T_regnamespace Oid = 4089
|
||||
T__regnamespace Oid = 4090
|
||||
T_regrole Oid = 4096
|
||||
T__regrole Oid = 4097
|
||||
)
|
||||
|
||||
var TypeName = map[Oid]string{
|
||||
T_bool: "BOOL",
|
||||
T_bytea: "BYTEA",
|
||||
T_char: "CHAR",
|
||||
T_name: "NAME",
|
||||
T_int8: "INT8",
|
||||
T_int2: "INT2",
|
||||
T_int2vector: "INT2VECTOR",
|
||||
T_int4: "INT4",
|
||||
T_regproc: "REGPROC",
|
||||
T_text: "TEXT",
|
||||
T_oid: "OID",
|
||||
T_tid: "TID",
|
||||
T_xid: "XID",
|
||||
T_cid: "CID",
|
||||
T_oidvector: "OIDVECTOR",
|
||||
T_pg_ddl_command: "PG_DDL_COMMAND",
|
||||
T_pg_type: "PG_TYPE",
|
||||
T_pg_attribute: "PG_ATTRIBUTE",
|
||||
T_pg_proc: "PG_PROC",
|
||||
T_pg_class: "PG_CLASS",
|
||||
T_json: "JSON",
|
||||
T_xml: "XML",
|
||||
T__xml: "_XML",
|
||||
T_pg_node_tree: "PG_NODE_TREE",
|
||||
T__json: "_JSON",
|
||||
T_smgr: "SMGR",
|
||||
T_index_am_handler: "INDEX_AM_HANDLER",
|
||||
T_point: "POINT",
|
||||
T_lseg: "LSEG",
|
||||
T_path: "PATH",
|
||||
T_box: "BOX",
|
||||
T_polygon: "POLYGON",
|
||||
T_line: "LINE",
|
||||
T__line: "_LINE",
|
||||
T_cidr: "CIDR",
|
||||
T__cidr: "_CIDR",
|
||||
T_float4: "FLOAT4",
|
||||
T_float8: "FLOAT8",
|
||||
T_abstime: "ABSTIME",
|
||||
T_reltime: "RELTIME",
|
||||
T_tinterval: "TINTERVAL",
|
||||
T_unknown: "UNKNOWN",
|
||||
T_circle: "CIRCLE",
|
||||
T__circle: "_CIRCLE",
|
||||
T_money: "MONEY",
|
||||
T__money: "_MONEY",
|
||||
T_macaddr: "MACADDR",
|
||||
T_inet: "INET",
|
||||
T__bool: "_BOOL",
|
||||
T__bytea: "_BYTEA",
|
||||
T__char: "_CHAR",
|
||||
T__name: "_NAME",
|
||||
T__int2: "_INT2",
|
||||
T__int2vector: "_INT2VECTOR",
|
||||
T__int4: "_INT4",
|
||||
T__regproc: "_REGPROC",
|
||||
T__text: "_TEXT",
|
||||
T__tid: "_TID",
|
||||
T__xid: "_XID",
|
||||
T__cid: "_CID",
|
||||
T__oidvector: "_OIDVECTOR",
|
||||
T__bpchar: "_BPCHAR",
|
||||
T__varchar: "_VARCHAR",
|
||||
T__int8: "_INT8",
|
||||
T__point: "_POINT",
|
||||
T__lseg: "_LSEG",
|
||||
T__path: "_PATH",
|
||||
T__box: "_BOX",
|
||||
T__float4: "_FLOAT4",
|
||||
T__float8: "_FLOAT8",
|
||||
T__abstime: "_ABSTIME",
|
||||
T__reltime: "_RELTIME",
|
||||
T__tinterval: "_TINTERVAL",
|
||||
T__polygon: "_POLYGON",
|
||||
T__oid: "_OID",
|
||||
T_aclitem: "ACLITEM",
|
||||
T__aclitem: "_ACLITEM",
|
||||
T__macaddr: "_MACADDR",
|
||||
T__inet: "_INET",
|
||||
T_bpchar: "BPCHAR",
|
||||
T_varchar: "VARCHAR",
|
||||
T_date: "DATE",
|
||||
T_time: "TIME",
|
||||
T_timestamp: "TIMESTAMP",
|
||||
T__timestamp: "_TIMESTAMP",
|
||||
T__date: "_DATE",
|
||||
T__time: "_TIME",
|
||||
T_timestamptz: "TIMESTAMPTZ",
|
||||
T__timestamptz: "_TIMESTAMPTZ",
|
||||
T_interval: "INTERVAL",
|
||||
T__interval: "_INTERVAL",
|
||||
T__numeric: "_NUMERIC",
|
||||
T_pg_database: "PG_DATABASE",
|
||||
T__cstring: "_CSTRING",
|
||||
T_timetz: "TIMETZ",
|
||||
T__timetz: "_TIMETZ",
|
||||
T_bit: "BIT",
|
||||
T__bit: "_BIT",
|
||||
T_varbit: "VARBIT",
|
||||
T__varbit: "_VARBIT",
|
||||
T_numeric: "NUMERIC",
|
||||
T_refcursor: "REFCURSOR",
|
||||
T__refcursor: "_REFCURSOR",
|
||||
T_regprocedure: "REGPROCEDURE",
|
||||
T_regoper: "REGOPER",
|
||||
T_regoperator: "REGOPERATOR",
|
||||
T_regclass: "REGCLASS",
|
||||
T_regtype: "REGTYPE",
|
||||
T__regprocedure: "_REGPROCEDURE",
|
||||
T__regoper: "_REGOPER",
|
||||
T__regoperator: "_REGOPERATOR",
|
||||
T__regclass: "_REGCLASS",
|
||||
T__regtype: "_REGTYPE",
|
||||
T_record: "RECORD",
|
||||
T_cstring: "CSTRING",
|
||||
T_any: "ANY",
|
||||
T_anyarray: "ANYARRAY",
|
||||
T_void: "VOID",
|
||||
T_trigger: "TRIGGER",
|
||||
T_language_handler: "LANGUAGE_HANDLER",
|
||||
T_internal: "INTERNAL",
|
||||
T_opaque: "OPAQUE",
|
||||
T_anyelement: "ANYELEMENT",
|
||||
T__record: "_RECORD",
|
||||
T_anynonarray: "ANYNONARRAY",
|
||||
T_pg_authid: "PG_AUTHID",
|
||||
T_pg_auth_members: "PG_AUTH_MEMBERS",
|
||||
T__txid_snapshot: "_TXID_SNAPSHOT",
|
||||
T_uuid: "UUID",
|
||||
T__uuid: "_UUID",
|
||||
T_txid_snapshot: "TXID_SNAPSHOT",
|
||||
T_fdw_handler: "FDW_HANDLER",
|
||||
T_pg_lsn: "PG_LSN",
|
||||
T__pg_lsn: "_PG_LSN",
|
||||
T_tsm_handler: "TSM_HANDLER",
|
||||
T_anyenum: "ANYENUM",
|
||||
T_tsvector: "TSVECTOR",
|
||||
T_tsquery: "TSQUERY",
|
||||
T_gtsvector: "GTSVECTOR",
|
||||
T__tsvector: "_TSVECTOR",
|
||||
T__gtsvector: "_GTSVECTOR",
|
||||
T__tsquery: "_TSQUERY",
|
||||
T_regconfig: "REGCONFIG",
|
||||
T__regconfig: "_REGCONFIG",
|
||||
T_regdictionary: "REGDICTIONARY",
|
||||
T__regdictionary: "_REGDICTIONARY",
|
||||
T_jsonb: "JSONB",
|
||||
T__jsonb: "_JSONB",
|
||||
T_anyrange: "ANYRANGE",
|
||||
T_event_trigger: "EVENT_TRIGGER",
|
||||
T_int4range: "INT4RANGE",
|
||||
T__int4range: "_INT4RANGE",
|
||||
T_numrange: "NUMRANGE",
|
||||
T__numrange: "_NUMRANGE",
|
||||
T_tsrange: "TSRANGE",
|
||||
T__tsrange: "_TSRANGE",
|
||||
T_tstzrange: "TSTZRANGE",
|
||||
T__tstzrange: "_TSTZRANGE",
|
||||
T_daterange: "DATERANGE",
|
||||
T__daterange: "_DATERANGE",
|
||||
T_int8range: "INT8RANGE",
|
||||
T__int8range: "_INT8RANGE",
|
||||
T_pg_shseclabel: "PG_SHSECLABEL",
|
||||
T_regnamespace: "REGNAMESPACE",
|
||||
T__regnamespace: "_REGNAMESPACE",
|
||||
T_regrole: "REGROLE",
|
||||
T__regrole: "_REGROLE",
|
||||
}
|
||||
@ -0,0 +1,93 @@
|
||||
package pq
|
||||
|
||||
import (
|
||||
"math"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq/oid"
|
||||
)
|
||||
|
||||
const headerSize = 4
|
||||
|
||||
type fieldDesc struct {
|
||||
// The object ID of the data type.
|
||||
OID oid.Oid
|
||||
// The data type size (see pg_type.typlen).
|
||||
// Note that negative values denote variable-width types.
|
||||
Len int
|
||||
// The type modifier (see pg_attribute.atttypmod).
|
||||
// The meaning of the modifier is type-specific.
|
||||
Mod int
|
||||
}
|
||||
|
||||
func (fd fieldDesc) Type() reflect.Type {
|
||||
switch fd.OID {
|
||||
case oid.T_int8:
|
||||
return reflect.TypeOf(int64(0))
|
||||
case oid.T_int4:
|
||||
return reflect.TypeOf(int32(0))
|
||||
case oid.T_int2:
|
||||
return reflect.TypeOf(int16(0))
|
||||
case oid.T_varchar, oid.T_text:
|
||||
return reflect.TypeOf("")
|
||||
case oid.T_bool:
|
||||
return reflect.TypeOf(false)
|
||||
case oid.T_date, oid.T_time, oid.T_timetz, oid.T_timestamp, oid.T_timestamptz:
|
||||
return reflect.TypeOf(time.Time{})
|
||||
case oid.T_bytea:
|
||||
return reflect.TypeOf([]byte(nil))
|
||||
default:
|
||||
return reflect.TypeOf(new(interface{})).Elem()
|
||||
}
|
||||
}
|
||||
|
||||
func (fd fieldDesc) Name() string {
|
||||
return oid.TypeName[fd.OID]
|
||||
}
|
||||
|
||||
func (fd fieldDesc) Length() (length int64, ok bool) {
|
||||
switch fd.OID {
|
||||
case oid.T_text, oid.T_bytea:
|
||||
return math.MaxInt64, true
|
||||
case oid.T_varchar, oid.T_bpchar:
|
||||
return int64(fd.Mod - headerSize), true
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func (fd fieldDesc) PrecisionScale() (precision, scale int64, ok bool) {
|
||||
switch fd.OID {
|
||||
case oid.T_numeric, oid.T__numeric:
|
||||
mod := fd.Mod - headerSize
|
||||
precision = int64((mod >> 16) & 0xffff)
|
||||
scale = int64(mod & 0xffff)
|
||||
return precision, scale, true
|
||||
default:
|
||||
return 0, 0, false
|
||||
}
|
||||
}
|
||||
|
||||
// ColumnTypeScanType returns the value type that can be used to scan types into.
|
||||
func (rs *rows) ColumnTypeScanType(index int) reflect.Type {
|
||||
return rs.colTyps[index].Type()
|
||||
}
|
||||
|
||||
// ColumnTypeDatabaseTypeName return the database system type name.
|
||||
func (rs *rows) ColumnTypeDatabaseTypeName(index int) string {
|
||||
return rs.colTyps[index].Name()
|
||||
}
|
||||
|
||||
// ColumnTypeLength returns the length of the column type if the column is a
|
||||
// variable length type. If the column is not a variable length type ok
|
||||
// should return false.
|
||||
func (rs *rows) ColumnTypeLength(index int) (length int64, ok bool) {
|
||||
return rs.colTyps[index].Length()
|
||||
}
|
||||
|
||||
// ColumnTypePrecisionScale should return the precision and scale for decimal
|
||||
// types. If not applicable, ok should be false.
|
||||
func (rs *rows) ColumnTypePrecisionScale(index int) (precision, scale int64, ok bool) {
|
||||
return rs.colTyps[index].PrecisionScale()
|
||||
}
|
||||
@ -0,0 +1,264 @@
|
||||
// Copyright (c) 2014 - Gustavo Niemeyer <gustavo@niemeyer.net>
|
||||
//
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice, this
|
||||
// list of conditions and the following disclaimer.
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
// Package scram implements a SCRAM-{SHA-1,etc} client per RFC5802.
|
||||
//
|
||||
// http://tools.ietf.org/html/rfc5802
|
||||
//
|
||||
package scram
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/hmac"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"hash"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Client implements a SCRAM-* client (SCRAM-SHA-1, SCRAM-SHA-256, etc).
|
||||
//
|
||||
// A Client may be used within a SASL conversation with logic resembling:
|
||||
//
|
||||
// var in []byte
|
||||
// var client = scram.NewClient(sha1.New, user, pass)
|
||||
// for client.Step(in) {
|
||||
// out := client.Out()
|
||||
// // send out to server
|
||||
// in := serverOut
|
||||
// }
|
||||
// if client.Err() != nil {
|
||||
// // auth failed
|
||||
// }
|
||||
//
|
||||
type Client struct {
|
||||
newHash func() hash.Hash
|
||||
|
||||
user string
|
||||
pass string
|
||||
step int
|
||||
out bytes.Buffer
|
||||
err error
|
||||
|
||||
clientNonce []byte
|
||||
serverNonce []byte
|
||||
saltedPass []byte
|
||||
authMsg bytes.Buffer
|
||||
}
|
||||
|
||||
// NewClient returns a new SCRAM-* client with the provided hash algorithm.
|
||||
//
|
||||
// For SCRAM-SHA-256, for example, use:
|
||||
//
|
||||
// client := scram.NewClient(sha256.New, user, pass)
|
||||
//
|
||||
func NewClient(newHash func() hash.Hash, user, pass string) *Client {
|
||||
c := &Client{
|
||||
newHash: newHash,
|
||||
user: user,
|
||||
pass: pass,
|
||||
}
|
||||
c.out.Grow(256)
|
||||
c.authMsg.Grow(256)
|
||||
return c
|
||||
}
|
||||
|
||||
// Out returns the data to be sent to the server in the current step.
|
||||
func (c *Client) Out() []byte {
|
||||
if c.out.Len() == 0 {
|
||||
return nil
|
||||
}
|
||||
return c.out.Bytes()
|
||||
}
|
||||
|
||||
// Err returns the error that occurred, or nil if there were no errors.
|
||||
func (c *Client) Err() error {
|
||||
return c.err
|
||||
}
|
||||
|
||||
// SetNonce sets the client nonce to the provided value.
|
||||
// If not set, the nonce is generated automatically out of crypto/rand on the first step.
|
||||
func (c *Client) SetNonce(nonce []byte) {
|
||||
c.clientNonce = nonce
|
||||
}
|
||||
|
||||
var escaper = strings.NewReplacer("=", "=3D", ",", "=2C")
|
||||
|
||||
// Step processes the incoming data from the server and makes the
|
||||
// next round of data for the server available via Client.Out.
|
||||
// Step returns false if there are no errors and more data is
|
||||
// still expected.
|
||||
func (c *Client) Step(in []byte) bool {
|
||||
c.out.Reset()
|
||||
if c.step > 2 || c.err != nil {
|
||||
return false
|
||||
}
|
||||
c.step++
|
||||
switch c.step {
|
||||
case 1:
|
||||
c.err = c.step1(in)
|
||||
case 2:
|
||||
c.err = c.step2(in)
|
||||
case 3:
|
||||
c.err = c.step3(in)
|
||||
}
|
||||
return c.step > 2 || c.err != nil
|
||||
}
|
||||
|
||||
func (c *Client) step1(in []byte) error {
|
||||
if len(c.clientNonce) == 0 {
|
||||
const nonceLen = 16
|
||||
buf := make([]byte, nonceLen+b64.EncodedLen(nonceLen))
|
||||
if _, err := rand.Read(buf[:nonceLen]); err != nil {
|
||||
return fmt.Errorf("cannot read random SCRAM-SHA-256 nonce from operating system: %v", err)
|
||||
}
|
||||
c.clientNonce = buf[nonceLen:]
|
||||
b64.Encode(c.clientNonce, buf[:nonceLen])
|
||||
}
|
||||
c.authMsg.WriteString("n=")
|
||||
escaper.WriteString(&c.authMsg, c.user)
|
||||
c.authMsg.WriteString(",r=")
|
||||
c.authMsg.Write(c.clientNonce)
|
||||
|
||||
c.out.WriteString("n,,")
|
||||
c.out.Write(c.authMsg.Bytes())
|
||||
return nil
|
||||
}
|
||||
|
||||
var b64 = base64.StdEncoding
|
||||
|
||||
func (c *Client) step2(in []byte) error {
|
||||
c.authMsg.WriteByte(',')
|
||||
c.authMsg.Write(in)
|
||||
|
||||
fields := bytes.Split(in, []byte(","))
|
||||
if len(fields) != 3 {
|
||||
return fmt.Errorf("expected 3 fields in first SCRAM-SHA-256 server message, got %d: %q", len(fields), in)
|
||||
}
|
||||
if !bytes.HasPrefix(fields[0], []byte("r=")) || len(fields[0]) < 2 {
|
||||
return fmt.Errorf("server sent an invalid SCRAM-SHA-256 nonce: %q", fields[0])
|
||||
}
|
||||
if !bytes.HasPrefix(fields[1], []byte("s=")) || len(fields[1]) < 6 {
|
||||
return fmt.Errorf("server sent an invalid SCRAM-SHA-256 salt: %q", fields[1])
|
||||
}
|
||||
if !bytes.HasPrefix(fields[2], []byte("i=")) || len(fields[2]) < 6 {
|
||||
return fmt.Errorf("server sent an invalid SCRAM-SHA-256 iteration count: %q", fields[2])
|
||||
}
|
||||
|
||||
c.serverNonce = fields[0][2:]
|
||||
if !bytes.HasPrefix(c.serverNonce, c.clientNonce) {
|
||||
return fmt.Errorf("server SCRAM-SHA-256 nonce is not prefixed by client nonce: got %q, want %q+\"...\"", c.serverNonce, c.clientNonce)
|
||||
}
|
||||
|
||||
salt := make([]byte, b64.DecodedLen(len(fields[1][2:])))
|
||||
n, err := b64.Decode(salt, fields[1][2:])
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot decode SCRAM-SHA-256 salt sent by server: %q", fields[1])
|
||||
}
|
||||
salt = salt[:n]
|
||||
iterCount, err := strconv.Atoi(string(fields[2][2:]))
|
||||
if err != nil {
|
||||
return fmt.Errorf("server sent an invalid SCRAM-SHA-256 iteration count: %q", fields[2])
|
||||
}
|
||||
c.saltPassword(salt, iterCount)
|
||||
|
||||
c.authMsg.WriteString(",c=biws,r=")
|
||||
c.authMsg.Write(c.serverNonce)
|
||||
|
||||
c.out.WriteString("c=biws,r=")
|
||||
c.out.Write(c.serverNonce)
|
||||
c.out.WriteString(",p=")
|
||||
c.out.Write(c.clientProof())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) step3(in []byte) error {
|
||||
var isv, ise bool
|
||||
var fields = bytes.Split(in, []byte(","))
|
||||
if len(fields) == 1 {
|
||||
isv = bytes.HasPrefix(fields[0], []byte("v="))
|
||||
ise = bytes.HasPrefix(fields[0], []byte("e="))
|
||||
}
|
||||
if ise {
|
||||
return fmt.Errorf("SCRAM-SHA-256 authentication error: %s", fields[0][2:])
|
||||
} else if !isv {
|
||||
return fmt.Errorf("unsupported SCRAM-SHA-256 final message from server: %q", in)
|
||||
}
|
||||
if !bytes.Equal(c.serverSignature(), fields[0][2:]) {
|
||||
return fmt.Errorf("cannot authenticate SCRAM-SHA-256 server signature: %q", fields[0][2:])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) saltPassword(salt []byte, iterCount int) {
|
||||
mac := hmac.New(c.newHash, []byte(c.pass))
|
||||
mac.Write(salt)
|
||||
mac.Write([]byte{0, 0, 0, 1})
|
||||
ui := mac.Sum(nil)
|
||||
hi := make([]byte, len(ui))
|
||||
copy(hi, ui)
|
||||
for i := 1; i < iterCount; i++ {
|
||||
mac.Reset()
|
||||
mac.Write(ui)
|
||||
mac.Sum(ui[:0])
|
||||
for j, b := range ui {
|
||||
hi[j] ^= b
|
||||
}
|
||||
}
|
||||
c.saltedPass = hi
|
||||
}
|
||||
|
||||
func (c *Client) clientProof() []byte {
|
||||
mac := hmac.New(c.newHash, c.saltedPass)
|
||||
mac.Write([]byte("Client Key"))
|
||||
clientKey := mac.Sum(nil)
|
||||
hash := c.newHash()
|
||||
hash.Write(clientKey)
|
||||
storedKey := hash.Sum(nil)
|
||||
mac = hmac.New(c.newHash, storedKey)
|
||||
mac.Write(c.authMsg.Bytes())
|
||||
clientProof := mac.Sum(nil)
|
||||
for i, b := range clientKey {
|
||||
clientProof[i] ^= b
|
||||
}
|
||||
clientProof64 := make([]byte, b64.EncodedLen(len(clientProof)))
|
||||
b64.Encode(clientProof64, clientProof)
|
||||
return clientProof64
|
||||
}
|
||||
|
||||
func (c *Client) serverSignature() []byte {
|
||||
mac := hmac.New(c.newHash, c.saltedPass)
|
||||
mac.Write([]byte("Server Key"))
|
||||
serverKey := mac.Sum(nil)
|
||||
|
||||
mac = hmac.New(c.newHash, serverKey)
|
||||
mac.Write(c.authMsg.Bytes())
|
||||
serverSignature := mac.Sum(nil)
|
||||
|
||||
encoded := make([]byte, b64.EncodedLen(len(serverSignature)))
|
||||
b64.Encode(encoded, serverSignature)
|
||||
return encoded
|
||||
}
|
||||
@ -0,0 +1,204 @@
|
||||
package pq
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"os"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ssl generates a function to upgrade a net.Conn based on the "sslmode" and
|
||||
// related settings. The function is nil when no upgrade should take place.
|
||||
func ssl(o values) (func(net.Conn) (net.Conn, error), error) {
|
||||
verifyCaOnly := false
|
||||
tlsConf := tls.Config{}
|
||||
switch mode := o["sslmode"]; mode {
|
||||
// "require" is the default.
|
||||
case "", "require":
|
||||
// We must skip TLS's own verification since it requires full
|
||||
// verification since Go 1.3.
|
||||
tlsConf.InsecureSkipVerify = true
|
||||
|
||||
// From http://www.postgresql.org/docs/current/static/libpq-ssl.html:
|
||||
//
|
||||
// Note: For backwards compatibility with earlier versions of
|
||||
// PostgreSQL, if a root CA file exists, the behavior of
|
||||
// sslmode=require will be the same as that of verify-ca, meaning the
|
||||
// server certificate is validated against the CA. Relying on this
|
||||
// behavior is discouraged, and applications that need certificate
|
||||
// validation should always use verify-ca or verify-full.
|
||||
if sslrootcert, ok := o["sslrootcert"]; ok {
|
||||
if _, err := os.Stat(sslrootcert); err == nil {
|
||||
verifyCaOnly = true
|
||||
} else {
|
||||
delete(o, "sslrootcert")
|
||||
}
|
||||
}
|
||||
case "verify-ca":
|
||||
// We must skip TLS's own verification since it requires full
|
||||
// verification since Go 1.3.
|
||||
tlsConf.InsecureSkipVerify = true
|
||||
verifyCaOnly = true
|
||||
case "verify-full":
|
||||
tlsConf.ServerName = o["host"]
|
||||
case "disable":
|
||||
return nil, nil
|
||||
default:
|
||||
return nil, fmterrorf(`unsupported sslmode %q; only "require" (default), "verify-full", "verify-ca", and "disable" supported`, mode)
|
||||
}
|
||||
|
||||
// Set Server Name Indication (SNI), if enabled by connection parameters.
|
||||
// By default SNI is on, any value which is not starting with "1" disables
|
||||
// SNI -- that is the same check vanilla libpq uses.
|
||||
if sslsni := o["sslsni"]; sslsni == "" || strings.HasPrefix(sslsni, "1") {
|
||||
// RFC 6066 asks to not set SNI if the host is a literal IP address (IPv4
|
||||
// or IPv6). This check is coded already crypto.tls.hostnameInSNI, so
|
||||
// just always set ServerName here and let crypto/tls do the filtering.
|
||||
tlsConf.ServerName = o["host"]
|
||||
}
|
||||
|
||||
err := sslClientCertificates(&tlsConf, o)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = sslCertificateAuthority(&tlsConf, o)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Accept renegotiation requests initiated by the backend.
|
||||
//
|
||||
// Renegotiation was deprecated then removed from PostgreSQL 9.5, but
|
||||
// the default configuration of older versions has it enabled. Redshift
|
||||
// also initiates renegotiations and cannot be reconfigured.
|
||||
tlsConf.Renegotiation = tls.RenegotiateFreelyAsClient
|
||||
|
||||
return func(conn net.Conn) (net.Conn, error) {
|
||||
client := tls.Client(conn, &tlsConf)
|
||||
if verifyCaOnly {
|
||||
err := sslVerifyCertificateAuthority(client, &tlsConf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return client, nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
// sslClientCertificates adds the certificate specified in the "sslcert" and
|
||||
// "sslkey" settings, or if they aren't set, from the .postgresql directory
|
||||
// in the user's home directory. The configured files must exist and have
|
||||
// the correct permissions.
|
||||
func sslClientCertificates(tlsConf *tls.Config, o values) error {
|
||||
sslinline := o["sslinline"]
|
||||
if sslinline == "true" {
|
||||
cert, err := tls.X509KeyPair([]byte(o["sslcert"]), []byte(o["sslkey"]))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tlsConf.Certificates = []tls.Certificate{cert}
|
||||
return nil
|
||||
}
|
||||
|
||||
// user.Current() might fail when cross-compiling. We have to ignore the
|
||||
// error and continue without home directory defaults, since we wouldn't
|
||||
// know from where to load them.
|
||||
user, _ := user.Current()
|
||||
|
||||
// In libpq, the client certificate is only loaded if the setting is not blank.
|
||||
//
|
||||
// https://github.com/postgres/postgres/blob/REL9_6_2/src/interfaces/libpq/fe-secure-openssl.c#L1036-L1037
|
||||
sslcert := o["sslcert"]
|
||||
if len(sslcert) == 0 && user != nil {
|
||||
sslcert = filepath.Join(user.HomeDir, ".postgresql", "postgresql.crt")
|
||||
}
|
||||
// https://github.com/postgres/postgres/blob/REL9_6_2/src/interfaces/libpq/fe-secure-openssl.c#L1045
|
||||
if len(sslcert) == 0 {
|
||||
return nil
|
||||
}
|
||||
// https://github.com/postgres/postgres/blob/REL9_6_2/src/interfaces/libpq/fe-secure-openssl.c#L1050:L1054
|
||||
if _, err := os.Stat(sslcert); os.IsNotExist(err) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// In libpq, the ssl key is only loaded if the setting is not blank.
|
||||
//
|
||||
// https://github.com/postgres/postgres/blob/REL9_6_2/src/interfaces/libpq/fe-secure-openssl.c#L1123-L1222
|
||||
sslkey := o["sslkey"]
|
||||
if len(sslkey) == 0 && user != nil {
|
||||
sslkey = filepath.Join(user.HomeDir, ".postgresql", "postgresql.key")
|
||||
}
|
||||
|
||||
if len(sslkey) > 0 {
|
||||
if err := sslKeyPermissions(sslkey); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
cert, err := tls.LoadX509KeyPair(sslcert, sslkey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tlsConf.Certificates = []tls.Certificate{cert}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sslCertificateAuthority adds the RootCA specified in the "sslrootcert" setting.
|
||||
func sslCertificateAuthority(tlsConf *tls.Config, o values) error {
|
||||
// In libpq, the root certificate is only loaded if the setting is not blank.
|
||||
//
|
||||
// https://github.com/postgres/postgres/blob/REL9_6_2/src/interfaces/libpq/fe-secure-openssl.c#L950-L951
|
||||
if sslrootcert := o["sslrootcert"]; len(sslrootcert) > 0 {
|
||||
tlsConf.RootCAs = x509.NewCertPool()
|
||||
|
||||
sslinline := o["sslinline"]
|
||||
|
||||
var cert []byte
|
||||
if sslinline == "true" {
|
||||
cert = []byte(sslrootcert)
|
||||
} else {
|
||||
var err error
|
||||
cert, err = ioutil.ReadFile(sslrootcert)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if !tlsConf.RootCAs.AppendCertsFromPEM(cert) {
|
||||
return fmterrorf("couldn't parse pem in sslrootcert")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// sslVerifyCertificateAuthority carries out a TLS handshake to the server and
|
||||
// verifies the presented certificate against the CA, i.e. the one specified in
|
||||
// sslrootcert or the system CA if sslrootcert was not specified.
|
||||
func sslVerifyCertificateAuthority(client *tls.Conn, tlsConf *tls.Config) error {
|
||||
err := client.Handshake()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
certs := client.ConnectionState().PeerCertificates
|
||||
opts := x509.VerifyOptions{
|
||||
DNSName: client.ConnectionState().ServerName,
|
||||
Intermediates: x509.NewCertPool(),
|
||||
Roots: tlsConf.RootCAs,
|
||||
}
|
||||
for i, cert := range certs {
|
||||
if i == 0 {
|
||||
continue
|
||||
}
|
||||
opts.Intermediates.AddCert(cert)
|
||||
}
|
||||
_, err = certs[0].Verify(opts)
|
||||
return err
|
||||
}
|
||||
@ -0,0 +1,93 @@
|
||||
//go:build !windows
|
||||
// +build !windows
|
||||
|
||||
package pq
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
const (
|
||||
rootUserID = uint32(0)
|
||||
|
||||
// The maximum permissions that a private key file owned by a regular user
|
||||
// is allowed to have. This translates to u=rw.
|
||||
maxUserOwnedKeyPermissions os.FileMode = 0600
|
||||
|
||||
// The maximum permissions that a private key file owned by root is allowed
|
||||
// to have. This translates to u=rw,g=r.
|
||||
maxRootOwnedKeyPermissions os.FileMode = 0640
|
||||
)
|
||||
|
||||
var (
|
||||
errSSLKeyHasUnacceptableUserPermissions = errors.New("permissions for files not owned by root should be u=rw (0600) or less")
|
||||
errSSLKeyHasUnacceptableRootPermissions = errors.New("permissions for root owned files should be u=rw,g=r (0640) or less")
|
||||
)
|
||||
|
||||
// sslKeyPermissions checks the permissions on user-supplied ssl key files.
|
||||
// The key file should have very little access.
|
||||
//
|
||||
// libpq does not check key file permissions on Windows.
|
||||
func sslKeyPermissions(sslkey string) error {
|
||||
info, err := os.Stat(sslkey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = hasCorrectPermissions(info)
|
||||
|
||||
// return ErrSSLKeyHasWorldPermissions for backwards compatability with
|
||||
// existing code.
|
||||
if err == errSSLKeyHasUnacceptableUserPermissions || err == errSSLKeyHasUnacceptableRootPermissions {
|
||||
err = ErrSSLKeyHasWorldPermissions
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// hasCorrectPermissions checks the file info (and the unix-specific stat_t
|
||||
// output) to verify that the permissions on the file are correct.
|
||||
//
|
||||
// If the file is owned by the same user the process is running as,
|
||||
// the file should only have 0600 (u=rw). If the file is owned by root,
|
||||
// and the group matches the group that the process is running in, the
|
||||
// permissions cannot be more than 0640 (u=rw,g=r). The file should
|
||||
// never have world permissions.
|
||||
//
|
||||
// Returns an error when the permission check fails.
|
||||
func hasCorrectPermissions(info os.FileInfo) error {
|
||||
// if file's permission matches 0600, allow access.
|
||||
userPermissionMask := (os.FileMode(0777) ^ maxUserOwnedKeyPermissions)
|
||||
|
||||
// regardless of if we're running as root or not, 0600 is acceptable,
|
||||
// so we return if we match the regular user permission mask.
|
||||
if info.Mode().Perm()&userPermissionMask == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// We need to pull the Unix file information to get the file's owner.
|
||||
// If we can't access it, there's some sort of operating system level error
|
||||
// and we should fail rather than attempting to use faulty information.
|
||||
sysInfo := info.Sys()
|
||||
if sysInfo == nil {
|
||||
return ErrSSLKeyUnknownOwnership
|
||||
}
|
||||
|
||||
unixStat, ok := sysInfo.(*syscall.Stat_t)
|
||||
if !ok {
|
||||
return ErrSSLKeyUnknownOwnership
|
||||
}
|
||||
|
||||
// if the file is owned by root, we allow 0640 (u=rw,g=r) to match what
|
||||
// Postgres does.
|
||||
if unixStat.Uid == rootUserID {
|
||||
rootPermissionMask := (os.FileMode(0777) ^ maxRootOwnedKeyPermissions)
|
||||
if info.Mode().Perm()&rootPermissionMask != 0 {
|
||||
return errSSLKeyHasUnacceptableRootPermissions
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return errSSLKeyHasUnacceptableUserPermissions
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
//go:build windows
|
||||
// +build windows
|
||||
|
||||
package pq
|
||||
|
||||
// sslKeyPermissions checks the permissions on user-supplied ssl key files.
|
||||
// The key file should have very little access.
|
||||
//
|
||||
// libpq does not check key file permissions on Windows.
|
||||
func sslKeyPermissions(string) error { return nil }
|
||||
@ -0,0 +1,76 @@
|
||||
package pq
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
nurl "net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ParseURL no longer needs to be used by clients of this library since supplying a URL as a
|
||||
// connection string to sql.Open() is now supported:
|
||||
//
|
||||
// sql.Open("postgres", "postgres://bob:secret@1.2.3.4:5432/mydb?sslmode=verify-full")
|
||||
//
|
||||
// It remains exported here for backwards-compatibility.
|
||||
//
|
||||
// ParseURL converts a url to a connection string for driver.Open.
|
||||
// Example:
|
||||
//
|
||||
// "postgres://bob:secret@1.2.3.4:5432/mydb?sslmode=verify-full"
|
||||
//
|
||||
// converts to:
|
||||
//
|
||||
// "user=bob password=secret host=1.2.3.4 port=5432 dbname=mydb sslmode=verify-full"
|
||||
//
|
||||
// A minimal example:
|
||||
//
|
||||
// "postgres://"
|
||||
//
|
||||
// This will be blank, causing driver.Open to use all of the defaults
|
||||
func ParseURL(url string) (string, error) {
|
||||
u, err := nurl.Parse(url)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if u.Scheme != "postgres" && u.Scheme != "postgresql" {
|
||||
return "", fmt.Errorf("invalid connection protocol: %s", u.Scheme)
|
||||
}
|
||||
|
||||
var kvs []string
|
||||
escaper := strings.NewReplacer(`'`, `\'`, `\`, `\\`)
|
||||
accrue := func(k, v string) {
|
||||
if v != "" {
|
||||
kvs = append(kvs, k+"='"+escaper.Replace(v)+"'")
|
||||
}
|
||||
}
|
||||
|
||||
if u.User != nil {
|
||||
v := u.User.Username()
|
||||
accrue("user", v)
|
||||
|
||||
v, _ = u.User.Password()
|
||||
accrue("password", v)
|
||||
}
|
||||
|
||||
if host, port, err := net.SplitHostPort(u.Host); err != nil {
|
||||
accrue("host", u.Host)
|
||||
} else {
|
||||
accrue("host", host)
|
||||
accrue("port", port)
|
||||
}
|
||||
|
||||
if u.Path != "" {
|
||||
accrue("dbname", u.Path[1:])
|
||||
}
|
||||
|
||||
q := u.Query()
|
||||
for k := range q {
|
||||
accrue(k, q.Get(k))
|
||||
}
|
||||
|
||||
sort.Strings(kvs) // Makes testing easier (not a performance concern)
|
||||
return strings.Join(kvs, " "), nil
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
// Package pq is a pure Go Postgres driver for the database/sql package.
|
||||
|
||||
//go:build js || android || hurd || zos
|
||||
// +build js android hurd zos
|
||||
|
||||
package pq
|
||||
|
||||
func userCurrent() (string, error) {
|
||||
return "", ErrCouldNotDetectUsername
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
// Package pq is a pure Go Postgres driver for the database/sql package.
|
||||
|
||||
//go:build aix || darwin || dragonfly || freebsd || (linux && !android) || nacl || netbsd || openbsd || plan9 || solaris || rumprun || illumos
|
||||
// +build aix darwin dragonfly freebsd linux,!android nacl netbsd openbsd plan9 solaris rumprun illumos
|
||||
|
||||
package pq
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/user"
|
||||
)
|
||||
|
||||
func userCurrent() (string, error) {
|
||||
u, err := user.Current()
|
||||
if err == nil {
|
||||
return u.Username, nil
|
||||
}
|
||||
|
||||
name := os.Getenv("USER")
|
||||
if name != "" {
|
||||
return name, nil
|
||||
}
|
||||
|
||||
return "", ErrCouldNotDetectUsername
|
||||
}
|
||||
@ -0,0 +1,27 @@
|
||||
// Package pq is a pure Go Postgres driver for the database/sql package.
|
||||
package pq
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// Perform Windows user name lookup identically to libpq.
|
||||
//
|
||||
// The PostgreSQL code makes use of the legacy Win32 function
|
||||
// GetUserName, and that function has not been imported into stock Go.
|
||||
// GetUserNameEx is available though, the difference being that a
|
||||
// wider range of names are available. To get the output to be the
|
||||
// same as GetUserName, only the base (or last) component of the
|
||||
// result is returned.
|
||||
func userCurrent() (string, error) {
|
||||
pw_name := make([]uint16, 128)
|
||||
pwname_size := uint32(len(pw_name)) - 1
|
||||
err := syscall.GetUserNameEx(syscall.NameSamCompatible, &pw_name[0], &pwname_size)
|
||||
if err != nil {
|
||||
return "", ErrCouldNotDetectUsername
|
||||
}
|
||||
s := syscall.UTF16ToString(pw_name)
|
||||
u := filepath.Base(s)
|
||||
return u, nil
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
package pq
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// decodeUUIDBinary interprets the binary format of a uuid, returning it in text format.
|
||||
func decodeUUIDBinary(src []byte) ([]byte, error) {
|
||||
if len(src) != 16 {
|
||||
return nil, fmt.Errorf("pq: unable to decode uuid; bad length: %d", len(src))
|
||||
}
|
||||
|
||||
dst := make([]byte, 36)
|
||||
dst[8], dst[13], dst[18], dst[23] = '-', '-', '-', '-'
|
||||
hex.Encode(dst[0:], src[0:4])
|
||||
hex.Encode(dst[9:], src[4:6])
|
||||
hex.Encode(dst[14:], src[6:8])
|
||||
hex.Encode(dst[19:], src[8:10])
|
||||
hex.Encode(dst[24:], src[10:16])
|
||||
|
||||
return dst, nil
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
Copyright 2012 Suryandaru Triandana <syndtr@gmail.com>
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
@ -0,0 +1,349 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/errors"
|
||||
"github.com/syndtr/goleveldb/leveldb/memdb"
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
)
|
||||
|
||||
// ErrBatchCorrupted records reason of batch corruption. This error will be
|
||||
// wrapped with errors.ErrCorrupted.
|
||||
type ErrBatchCorrupted struct {
|
||||
Reason string
|
||||
}
|
||||
|
||||
func (e *ErrBatchCorrupted) Error() string {
|
||||
return fmt.Sprintf("leveldb: batch corrupted: %s", e.Reason)
|
||||
}
|
||||
|
||||
func newErrBatchCorrupted(reason string) error {
|
||||
return errors.NewErrCorrupted(storage.FileDesc{}, &ErrBatchCorrupted{reason})
|
||||
}
|
||||
|
||||
const (
|
||||
batchHeaderLen = 8 + 4
|
||||
batchGrowRec = 3000
|
||||
batchBufioSize = 16
|
||||
)
|
||||
|
||||
// BatchReplay wraps basic batch operations.
|
||||
type BatchReplay interface {
|
||||
Put(key, value []byte)
|
||||
Delete(key []byte)
|
||||
}
|
||||
|
||||
type batchIndex struct {
|
||||
keyType keyType
|
||||
keyPos, keyLen int
|
||||
valuePos, valueLen int
|
||||
}
|
||||
|
||||
func (index batchIndex) k(data []byte) []byte {
|
||||
return data[index.keyPos : index.keyPos+index.keyLen]
|
||||
}
|
||||
|
||||
func (index batchIndex) v(data []byte) []byte {
|
||||
if index.valueLen != 0 {
|
||||
return data[index.valuePos : index.valuePos+index.valueLen]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (index batchIndex) kv(data []byte) (key, value []byte) {
|
||||
return index.k(data), index.v(data)
|
||||
}
|
||||
|
||||
// Batch is a write batch.
|
||||
type Batch struct {
|
||||
data []byte
|
||||
index []batchIndex
|
||||
|
||||
// internalLen is sums of key/value pair length plus 8-bytes internal key.
|
||||
internalLen int
|
||||
}
|
||||
|
||||
func (b *Batch) grow(n int) {
|
||||
o := len(b.data)
|
||||
if cap(b.data)-o < n {
|
||||
div := 1
|
||||
if len(b.index) > batchGrowRec {
|
||||
div = len(b.index) / batchGrowRec
|
||||
}
|
||||
ndata := make([]byte, o, o+n+o/div)
|
||||
copy(ndata, b.data)
|
||||
b.data = ndata
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Batch) appendRec(kt keyType, key, value []byte) {
|
||||
n := 1 + binary.MaxVarintLen32 + len(key)
|
||||
if kt == keyTypeVal {
|
||||
n += binary.MaxVarintLen32 + len(value)
|
||||
}
|
||||
b.grow(n)
|
||||
index := batchIndex{keyType: kt}
|
||||
o := len(b.data)
|
||||
data := b.data[:o+n]
|
||||
data[o] = byte(kt)
|
||||
o++
|
||||
o += binary.PutUvarint(data[o:], uint64(len(key)))
|
||||
index.keyPos = o
|
||||
index.keyLen = len(key)
|
||||
o += copy(data[o:], key)
|
||||
if kt == keyTypeVal {
|
||||
o += binary.PutUvarint(data[o:], uint64(len(value)))
|
||||
index.valuePos = o
|
||||
index.valueLen = len(value)
|
||||
o += copy(data[o:], value)
|
||||
}
|
||||
b.data = data[:o]
|
||||
b.index = append(b.index, index)
|
||||
b.internalLen += index.keyLen + index.valueLen + 8
|
||||
}
|
||||
|
||||
// Put appends 'put operation' of the given key/value pair to the batch.
|
||||
// It is safe to modify the contents of the argument after Put returns but not
|
||||
// before.
|
||||
func (b *Batch) Put(key, value []byte) {
|
||||
b.appendRec(keyTypeVal, key, value)
|
||||
}
|
||||
|
||||
// Delete appends 'delete operation' of the given key to the batch.
|
||||
// It is safe to modify the contents of the argument after Delete returns but
|
||||
// not before.
|
||||
func (b *Batch) Delete(key []byte) {
|
||||
b.appendRec(keyTypeDel, key, nil)
|
||||
}
|
||||
|
||||
// Dump dumps batch contents. The returned slice can be loaded into the
|
||||
// batch using Load method.
|
||||
// The returned slice is not its own copy, so the contents should not be
|
||||
// modified.
|
||||
func (b *Batch) Dump() []byte {
|
||||
return b.data
|
||||
}
|
||||
|
||||
// Load loads given slice into the batch. Previous contents of the batch
|
||||
// will be discarded.
|
||||
// The given slice will not be copied and will be used as batch buffer, so
|
||||
// it is not safe to modify the contents of the slice.
|
||||
func (b *Batch) Load(data []byte) error {
|
||||
return b.decode(data, -1)
|
||||
}
|
||||
|
||||
// Replay replays batch contents.
|
||||
func (b *Batch) Replay(r BatchReplay) error {
|
||||
for _, index := range b.index {
|
||||
switch index.keyType {
|
||||
case keyTypeVal:
|
||||
r.Put(index.k(b.data), index.v(b.data))
|
||||
case keyTypeDel:
|
||||
r.Delete(index.k(b.data))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Len returns number of records in the batch.
|
||||
func (b *Batch) Len() int {
|
||||
return len(b.index)
|
||||
}
|
||||
|
||||
// Reset resets the batch.
|
||||
func (b *Batch) Reset() {
|
||||
b.data = b.data[:0]
|
||||
b.index = b.index[:0]
|
||||
b.internalLen = 0
|
||||
}
|
||||
|
||||
func (b *Batch) replayInternal(fn func(i int, kt keyType, k, v []byte) error) error {
|
||||
for i, index := range b.index {
|
||||
if err := fn(i, index.keyType, index.k(b.data), index.v(b.data)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Batch) append(p *Batch) {
|
||||
ob := len(b.data)
|
||||
oi := len(b.index)
|
||||
b.data = append(b.data, p.data...)
|
||||
b.index = append(b.index, p.index...)
|
||||
b.internalLen += p.internalLen
|
||||
|
||||
// Updating index offset.
|
||||
if ob != 0 {
|
||||
for ; oi < len(b.index); oi++ {
|
||||
index := &b.index[oi]
|
||||
index.keyPos += ob
|
||||
if index.valueLen != 0 {
|
||||
index.valuePos += ob
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Batch) decode(data []byte, expectedLen int) error {
|
||||
b.data = data
|
||||
b.index = b.index[:0]
|
||||
b.internalLen = 0
|
||||
err := decodeBatch(data, func(i int, index batchIndex) error {
|
||||
b.index = append(b.index, index)
|
||||
b.internalLen += index.keyLen + index.valueLen + 8
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if expectedLen >= 0 && len(b.index) != expectedLen {
|
||||
return newErrBatchCorrupted(fmt.Sprintf("invalid records length: %d vs %d", expectedLen, len(b.index)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Batch) putMem(seq uint64, mdb *memdb.DB) error {
|
||||
var ik []byte
|
||||
for i, index := range b.index {
|
||||
ik = makeInternalKey(ik, index.k(b.data), seq+uint64(i), index.keyType)
|
||||
if err := mdb.Put(ik, index.v(b.data)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Batch) revertMem(seq uint64, mdb *memdb.DB) error {
|
||||
var ik []byte
|
||||
for i, index := range b.index {
|
||||
ik = makeInternalKey(ik, index.k(b.data), seq+uint64(i), index.keyType)
|
||||
if err := mdb.Delete(ik); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newBatch() interface{} {
|
||||
return &Batch{}
|
||||
}
|
||||
|
||||
func decodeBatch(data []byte, fn func(i int, index batchIndex) error) error {
|
||||
var index batchIndex
|
||||
for i, o := 0, 0; o < len(data); i++ {
|
||||
// Key type.
|
||||
index.keyType = keyType(data[o])
|
||||
if index.keyType > keyTypeVal {
|
||||
return newErrBatchCorrupted(fmt.Sprintf("bad record: invalid type %#x", uint(index.keyType)))
|
||||
}
|
||||
o++
|
||||
|
||||
// Key.
|
||||
x, n := binary.Uvarint(data[o:])
|
||||
o += n
|
||||
if n <= 0 || o+int(x) > len(data) {
|
||||
return newErrBatchCorrupted("bad record: invalid key length")
|
||||
}
|
||||
index.keyPos = o
|
||||
index.keyLen = int(x)
|
||||
o += index.keyLen
|
||||
|
||||
// Value.
|
||||
if index.keyType == keyTypeVal {
|
||||
x, n = binary.Uvarint(data[o:])
|
||||
o += n
|
||||
if n <= 0 || o+int(x) > len(data) {
|
||||
return newErrBatchCorrupted("bad record: invalid value length")
|
||||
}
|
||||
index.valuePos = o
|
||||
index.valueLen = int(x)
|
||||
o += index.valueLen
|
||||
} else {
|
||||
index.valuePos = 0
|
||||
index.valueLen = 0
|
||||
}
|
||||
|
||||
if err := fn(i, index); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func decodeBatchToMem(data []byte, expectSeq uint64, mdb *memdb.DB) (seq uint64, batchLen int, err error) {
|
||||
seq, batchLen, err = decodeBatchHeader(data)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if seq < expectSeq {
|
||||
return 0, 0, newErrBatchCorrupted("invalid sequence number")
|
||||
}
|
||||
data = data[batchHeaderLen:]
|
||||
var ik []byte
|
||||
var decodedLen int
|
||||
err = decodeBatch(data, func(i int, index batchIndex) error {
|
||||
if i >= batchLen {
|
||||
return newErrBatchCorrupted("invalid records length")
|
||||
}
|
||||
ik = makeInternalKey(ik, index.k(data), seq+uint64(i), index.keyType)
|
||||
if err := mdb.Put(ik, index.v(data)); err != nil {
|
||||
return err
|
||||
}
|
||||
decodedLen++
|
||||
return nil
|
||||
})
|
||||
if err == nil && decodedLen != batchLen {
|
||||
err = newErrBatchCorrupted(fmt.Sprintf("invalid records length: %d vs %d", batchLen, decodedLen))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func encodeBatchHeader(dst []byte, seq uint64, batchLen int) []byte {
|
||||
dst = ensureBuffer(dst, batchHeaderLen)
|
||||
binary.LittleEndian.PutUint64(dst, seq)
|
||||
binary.LittleEndian.PutUint32(dst[8:], uint32(batchLen))
|
||||
return dst
|
||||
}
|
||||
|
||||
func decodeBatchHeader(data []byte) (seq uint64, batchLen int, err error) {
|
||||
if len(data) < batchHeaderLen {
|
||||
return 0, 0, newErrBatchCorrupted("too short")
|
||||
}
|
||||
|
||||
seq = binary.LittleEndian.Uint64(data)
|
||||
batchLen = int(binary.LittleEndian.Uint32(data[8:]))
|
||||
if batchLen < 0 {
|
||||
return 0, 0, newErrBatchCorrupted("invalid records length")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func batchesLen(batches []*Batch) int {
|
||||
batchLen := 0
|
||||
for _, batch := range batches {
|
||||
batchLen += batch.Len()
|
||||
}
|
||||
return batchLen
|
||||
}
|
||||
|
||||
func writeBatchesWithHeader(wr io.Writer, batches []*Batch, seq uint64) error {
|
||||
if _, err := wr.Write(encodeBatchHeader(nil, seq, batchesLen(batches))); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, batch := range batches {
|
||||
if _, err := wr.Write(batch.data); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,704 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package cache provides interface and implementation of a cache algorithms.
|
||||
package cache
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"unsafe"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
// Cacher provides interface to implements a caching functionality.
|
||||
// An implementation must be safe for concurrent use.
|
||||
type Cacher interface {
|
||||
// Capacity returns cache capacity.
|
||||
Capacity() int
|
||||
|
||||
// SetCapacity sets cache capacity.
|
||||
SetCapacity(capacity int)
|
||||
|
||||
// Promote promotes the 'cache node'.
|
||||
Promote(n *Node)
|
||||
|
||||
// Ban evicts the 'cache node' and prevent subsequent 'promote'.
|
||||
Ban(n *Node)
|
||||
|
||||
// Evict evicts the 'cache node'.
|
||||
Evict(n *Node)
|
||||
|
||||
// EvictNS evicts 'cache node' with the given namespace.
|
||||
EvictNS(ns uint64)
|
||||
|
||||
// EvictAll evicts all 'cache node'.
|
||||
EvictAll()
|
||||
|
||||
// Close closes the 'cache tree'
|
||||
Close() error
|
||||
}
|
||||
|
||||
// Value is a 'cacheable object'. It may implements util.Releaser, if
|
||||
// so the the Release method will be called once object is released.
|
||||
type Value interface{}
|
||||
|
||||
// NamespaceGetter provides convenient wrapper for namespace.
|
||||
type NamespaceGetter struct {
|
||||
Cache *Cache
|
||||
NS uint64
|
||||
}
|
||||
|
||||
// Get simply calls Cache.Get() method.
|
||||
func (g *NamespaceGetter) Get(key uint64, setFunc func() (size int, value Value)) *Handle {
|
||||
return g.Cache.Get(g.NS, key, setFunc)
|
||||
}
|
||||
|
||||
// The hash tables implementation is based on:
|
||||
// "Dynamic-Sized Nonblocking Hash Tables", by Yujie Liu,
|
||||
// Kunlong Zhang, and Michael Spear.
|
||||
// ACM Symposium on Principles of Distributed Computing, Jul 2014.
|
||||
|
||||
const (
|
||||
mInitialSize = 1 << 4
|
||||
mOverflowThreshold = 1 << 5
|
||||
mOverflowGrowThreshold = 1 << 7
|
||||
)
|
||||
|
||||
type mBucket struct {
|
||||
mu sync.Mutex
|
||||
node []*Node
|
||||
frozen bool
|
||||
}
|
||||
|
||||
func (b *mBucket) freeze() []*Node {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if !b.frozen {
|
||||
b.frozen = true
|
||||
}
|
||||
return b.node
|
||||
}
|
||||
|
||||
func (b *mBucket) get(r *Cache, h *mNode, hash uint32, ns, key uint64, noset bool) (done, added bool, n *Node) {
|
||||
b.mu.Lock()
|
||||
|
||||
if b.frozen {
|
||||
b.mu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// Scan the node.
|
||||
for _, n := range b.node {
|
||||
if n.hash == hash && n.ns == ns && n.key == key {
|
||||
atomic.AddInt32(&n.ref, 1)
|
||||
b.mu.Unlock()
|
||||
return true, false, n
|
||||
}
|
||||
}
|
||||
|
||||
// Get only.
|
||||
if noset {
|
||||
b.mu.Unlock()
|
||||
return true, false, nil
|
||||
}
|
||||
|
||||
// Create node.
|
||||
n = &Node{
|
||||
r: r,
|
||||
hash: hash,
|
||||
ns: ns,
|
||||
key: key,
|
||||
ref: 1,
|
||||
}
|
||||
// Add node to bucket.
|
||||
b.node = append(b.node, n)
|
||||
bLen := len(b.node)
|
||||
b.mu.Unlock()
|
||||
|
||||
// Update counter.
|
||||
grow := atomic.AddInt32(&r.nodes, 1) >= h.growThreshold
|
||||
if bLen > mOverflowThreshold {
|
||||
grow = grow || atomic.AddInt32(&h.overflow, 1) >= mOverflowGrowThreshold
|
||||
}
|
||||
|
||||
// Grow.
|
||||
if grow && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) {
|
||||
nhLen := len(h.buckets) << 1
|
||||
nh := &mNode{
|
||||
buckets: make([]unsafe.Pointer, nhLen),
|
||||
mask: uint32(nhLen) - 1,
|
||||
pred: unsafe.Pointer(h),
|
||||
growThreshold: int32(nhLen * mOverflowThreshold),
|
||||
shrinkThreshold: int32(nhLen >> 1),
|
||||
}
|
||||
ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh))
|
||||
if !ok {
|
||||
panic("BUG: failed swapping head")
|
||||
}
|
||||
go nh.initBuckets()
|
||||
}
|
||||
|
||||
return true, true, n
|
||||
}
|
||||
|
||||
func (b *mBucket) delete(r *Cache, h *mNode, hash uint32, ns, key uint64) (done, deleted bool) {
|
||||
b.mu.Lock()
|
||||
|
||||
if b.frozen {
|
||||
b.mu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// Scan the node.
|
||||
var (
|
||||
n *Node
|
||||
bLen int
|
||||
)
|
||||
for i := range b.node {
|
||||
n = b.node[i]
|
||||
if n.ns == ns && n.key == key {
|
||||
if atomic.LoadInt32(&n.ref) == 0 {
|
||||
deleted = true
|
||||
|
||||
// Call releaser.
|
||||
if n.value != nil {
|
||||
if r, ok := n.value.(util.Releaser); ok {
|
||||
r.Release()
|
||||
}
|
||||
n.value = nil
|
||||
}
|
||||
|
||||
// Remove node from bucket.
|
||||
b.node = append(b.node[:i], b.node[i+1:]...)
|
||||
bLen = len(b.node)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
|
||||
if deleted {
|
||||
// Call OnDel.
|
||||
for _, f := range n.onDel {
|
||||
f()
|
||||
}
|
||||
|
||||
// Update counter.
|
||||
atomic.AddInt32(&r.size, int32(n.size)*-1)
|
||||
shrink := atomic.AddInt32(&r.nodes, -1) < h.shrinkThreshold
|
||||
if bLen >= mOverflowThreshold {
|
||||
atomic.AddInt32(&h.overflow, -1)
|
||||
}
|
||||
|
||||
// Shrink.
|
||||
if shrink && len(h.buckets) > mInitialSize && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) {
|
||||
nhLen := len(h.buckets) >> 1
|
||||
nh := &mNode{
|
||||
buckets: make([]unsafe.Pointer, nhLen),
|
||||
mask: uint32(nhLen) - 1,
|
||||
pred: unsafe.Pointer(h),
|
||||
growThreshold: int32(nhLen * mOverflowThreshold),
|
||||
shrinkThreshold: int32(nhLen >> 1),
|
||||
}
|
||||
ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh))
|
||||
if !ok {
|
||||
panic("BUG: failed swapping head")
|
||||
}
|
||||
go nh.initBuckets()
|
||||
}
|
||||
}
|
||||
|
||||
return true, deleted
|
||||
}
|
||||
|
||||
type mNode struct {
|
||||
buckets []unsafe.Pointer // []*mBucket
|
||||
mask uint32
|
||||
pred unsafe.Pointer // *mNode
|
||||
resizeInProgess int32
|
||||
|
||||
overflow int32
|
||||
growThreshold int32
|
||||
shrinkThreshold int32
|
||||
}
|
||||
|
||||
func (n *mNode) initBucket(i uint32) *mBucket {
|
||||
if b := (*mBucket)(atomic.LoadPointer(&n.buckets[i])); b != nil {
|
||||
return b
|
||||
}
|
||||
|
||||
p := (*mNode)(atomic.LoadPointer(&n.pred))
|
||||
if p != nil {
|
||||
var node []*Node
|
||||
if n.mask > p.mask {
|
||||
// Grow.
|
||||
pb := (*mBucket)(atomic.LoadPointer(&p.buckets[i&p.mask]))
|
||||
if pb == nil {
|
||||
pb = p.initBucket(i & p.mask)
|
||||
}
|
||||
m := pb.freeze()
|
||||
// Split nodes.
|
||||
for _, x := range m {
|
||||
if x.hash&n.mask == i {
|
||||
node = append(node, x)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Shrink.
|
||||
pb0 := (*mBucket)(atomic.LoadPointer(&p.buckets[i]))
|
||||
if pb0 == nil {
|
||||
pb0 = p.initBucket(i)
|
||||
}
|
||||
pb1 := (*mBucket)(atomic.LoadPointer(&p.buckets[i+uint32(len(n.buckets))]))
|
||||
if pb1 == nil {
|
||||
pb1 = p.initBucket(i + uint32(len(n.buckets)))
|
||||
}
|
||||
m0 := pb0.freeze()
|
||||
m1 := pb1.freeze()
|
||||
// Merge nodes.
|
||||
node = make([]*Node, 0, len(m0)+len(m1))
|
||||
node = append(node, m0...)
|
||||
node = append(node, m1...)
|
||||
}
|
||||
b := &mBucket{node: node}
|
||||
if atomic.CompareAndSwapPointer(&n.buckets[i], nil, unsafe.Pointer(b)) {
|
||||
if len(node) > mOverflowThreshold {
|
||||
atomic.AddInt32(&n.overflow, int32(len(node)-mOverflowThreshold))
|
||||
}
|
||||
return b
|
||||
}
|
||||
}
|
||||
|
||||
return (*mBucket)(atomic.LoadPointer(&n.buckets[i]))
|
||||
}
|
||||
|
||||
func (n *mNode) initBuckets() {
|
||||
for i := range n.buckets {
|
||||
n.initBucket(uint32(i))
|
||||
}
|
||||
atomic.StorePointer(&n.pred, nil)
|
||||
}
|
||||
|
||||
// Cache is a 'cache map'.
|
||||
type Cache struct {
|
||||
mu sync.RWMutex
|
||||
mHead unsafe.Pointer // *mNode
|
||||
nodes int32
|
||||
size int32
|
||||
cacher Cacher
|
||||
closed bool
|
||||
}
|
||||
|
||||
// NewCache creates a new 'cache map'. The cacher is optional and
|
||||
// may be nil.
|
||||
func NewCache(cacher Cacher) *Cache {
|
||||
h := &mNode{
|
||||
buckets: make([]unsafe.Pointer, mInitialSize),
|
||||
mask: mInitialSize - 1,
|
||||
growThreshold: int32(mInitialSize * mOverflowThreshold),
|
||||
shrinkThreshold: 0,
|
||||
}
|
||||
for i := range h.buckets {
|
||||
h.buckets[i] = unsafe.Pointer(&mBucket{})
|
||||
}
|
||||
r := &Cache{
|
||||
mHead: unsafe.Pointer(h),
|
||||
cacher: cacher,
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *Cache) getBucket(hash uint32) (*mNode, *mBucket) {
|
||||
h := (*mNode)(atomic.LoadPointer(&r.mHead))
|
||||
i := hash & h.mask
|
||||
b := (*mBucket)(atomic.LoadPointer(&h.buckets[i]))
|
||||
if b == nil {
|
||||
b = h.initBucket(i)
|
||||
}
|
||||
return h, b
|
||||
}
|
||||
|
||||
func (r *Cache) delete(n *Node) bool {
|
||||
for {
|
||||
h, b := r.getBucket(n.hash)
|
||||
done, deleted := b.delete(r, h, n.hash, n.ns, n.key)
|
||||
if done {
|
||||
return deleted
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Nodes returns number of 'cache node' in the map.
|
||||
func (r *Cache) Nodes() int {
|
||||
return int(atomic.LoadInt32(&r.nodes))
|
||||
}
|
||||
|
||||
// Size returns sums of 'cache node' size in the map.
|
||||
func (r *Cache) Size() int {
|
||||
return int(atomic.LoadInt32(&r.size))
|
||||
}
|
||||
|
||||
// Capacity returns cache capacity.
|
||||
func (r *Cache) Capacity() int {
|
||||
if r.cacher == nil {
|
||||
return 0
|
||||
}
|
||||
return r.cacher.Capacity()
|
||||
}
|
||||
|
||||
// SetCapacity sets cache capacity.
|
||||
func (r *Cache) SetCapacity(capacity int) {
|
||||
if r.cacher != nil {
|
||||
r.cacher.SetCapacity(capacity)
|
||||
}
|
||||
}
|
||||
|
||||
// Get gets 'cache node' with the given namespace and key.
|
||||
// If cache node is not found and setFunc is not nil, Get will atomically creates
|
||||
// the 'cache node' by calling setFunc. Otherwise Get will returns nil.
|
||||
//
|
||||
// The returned 'cache handle' should be released after use by calling Release
|
||||
// method.
|
||||
func (r *Cache) Get(ns, key uint64, setFunc func() (size int, value Value)) *Handle {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
if r.closed {
|
||||
return nil
|
||||
}
|
||||
|
||||
hash := murmur32(ns, key, 0xf00)
|
||||
for {
|
||||
h, b := r.getBucket(hash)
|
||||
done, _, n := b.get(r, h, hash, ns, key, setFunc == nil)
|
||||
if done {
|
||||
if n != nil {
|
||||
n.mu.Lock()
|
||||
if n.value == nil {
|
||||
if setFunc == nil {
|
||||
n.mu.Unlock()
|
||||
n.unref()
|
||||
return nil
|
||||
}
|
||||
|
||||
n.size, n.value = setFunc()
|
||||
if n.value == nil {
|
||||
n.size = 0
|
||||
n.mu.Unlock()
|
||||
n.unref()
|
||||
return nil
|
||||
}
|
||||
atomic.AddInt32(&r.size, int32(n.size))
|
||||
}
|
||||
n.mu.Unlock()
|
||||
if r.cacher != nil {
|
||||
r.cacher.Promote(n)
|
||||
}
|
||||
return &Handle{unsafe.Pointer(n)}
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes and ban 'cache node' with the given namespace and key.
|
||||
// A banned 'cache node' will never inserted into the 'cache tree'. Ban
|
||||
// only attributed to the particular 'cache node', so when a 'cache node'
|
||||
// is recreated it will not be banned.
|
||||
//
|
||||
// If onDel is not nil, then it will be executed if such 'cache node'
|
||||
// doesn't exist or once the 'cache node' is released.
|
||||
//
|
||||
// Delete return true is such 'cache node' exist.
|
||||
func (r *Cache) Delete(ns, key uint64, onDel func()) bool {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
if r.closed {
|
||||
return false
|
||||
}
|
||||
|
||||
hash := murmur32(ns, key, 0xf00)
|
||||
for {
|
||||
h, b := r.getBucket(hash)
|
||||
done, _, n := b.get(r, h, hash, ns, key, true)
|
||||
if done {
|
||||
if n != nil {
|
||||
if onDel != nil {
|
||||
n.mu.Lock()
|
||||
n.onDel = append(n.onDel, onDel)
|
||||
n.mu.Unlock()
|
||||
}
|
||||
if r.cacher != nil {
|
||||
r.cacher.Ban(n)
|
||||
}
|
||||
n.unref()
|
||||
return true
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if onDel != nil {
|
||||
onDel()
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Evict evicts 'cache node' with the given namespace and key. This will
|
||||
// simply call Cacher.Evict.
|
||||
//
|
||||
// Evict return true is such 'cache node' exist.
|
||||
func (r *Cache) Evict(ns, key uint64) bool {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
if r.closed {
|
||||
return false
|
||||
}
|
||||
|
||||
hash := murmur32(ns, key, 0xf00)
|
||||
for {
|
||||
h, b := r.getBucket(hash)
|
||||
done, _, n := b.get(r, h, hash, ns, key, true)
|
||||
if done {
|
||||
if n != nil {
|
||||
if r.cacher != nil {
|
||||
r.cacher.Evict(n)
|
||||
}
|
||||
n.unref()
|
||||
return true
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// EvictNS evicts 'cache node' with the given namespace. This will
|
||||
// simply call Cacher.EvictNS.
|
||||
func (r *Cache) EvictNS(ns uint64) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
if r.closed {
|
||||
return
|
||||
}
|
||||
|
||||
if r.cacher != nil {
|
||||
r.cacher.EvictNS(ns)
|
||||
}
|
||||
}
|
||||
|
||||
// EvictAll evicts all 'cache node'. This will simply call Cacher.EvictAll.
|
||||
func (r *Cache) EvictAll() {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
if r.closed {
|
||||
return
|
||||
}
|
||||
|
||||
if r.cacher != nil {
|
||||
r.cacher.EvictAll()
|
||||
}
|
||||
}
|
||||
|
||||
// Close closes the 'cache map' and forcefully releases all 'cache node'.
|
||||
func (r *Cache) Close() error {
|
||||
r.mu.Lock()
|
||||
if !r.closed {
|
||||
r.closed = true
|
||||
|
||||
h := (*mNode)(r.mHead)
|
||||
h.initBuckets()
|
||||
|
||||
for i := range h.buckets {
|
||||
b := (*mBucket)(h.buckets[i])
|
||||
for _, n := range b.node {
|
||||
// Call releaser.
|
||||
if n.value != nil {
|
||||
if r, ok := n.value.(util.Releaser); ok {
|
||||
r.Release()
|
||||
}
|
||||
n.value = nil
|
||||
}
|
||||
|
||||
// Call OnDel.
|
||||
for _, f := range n.onDel {
|
||||
f()
|
||||
}
|
||||
n.onDel = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
|
||||
// Avoid deadlock.
|
||||
if r.cacher != nil {
|
||||
if err := r.cacher.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CloseWeak closes the 'cache map' and evict all 'cache node' from cacher, but
|
||||
// unlike Close it doesn't forcefully releases 'cache node'.
|
||||
func (r *Cache) CloseWeak() error {
|
||||
r.mu.Lock()
|
||||
if !r.closed {
|
||||
r.closed = true
|
||||
}
|
||||
r.mu.Unlock()
|
||||
|
||||
// Avoid deadlock.
|
||||
if r.cacher != nil {
|
||||
r.cacher.EvictAll()
|
||||
if err := r.cacher.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Node is a 'cache node'.
|
||||
type Node struct {
|
||||
r *Cache
|
||||
|
||||
hash uint32
|
||||
ns, key uint64
|
||||
|
||||
mu sync.Mutex
|
||||
size int
|
||||
value Value
|
||||
|
||||
ref int32
|
||||
onDel []func()
|
||||
|
||||
CacheData unsafe.Pointer
|
||||
}
|
||||
|
||||
// NS returns this 'cache node' namespace.
|
||||
func (n *Node) NS() uint64 {
|
||||
return n.ns
|
||||
}
|
||||
|
||||
// Key returns this 'cache node' key.
|
||||
func (n *Node) Key() uint64 {
|
||||
return n.key
|
||||
}
|
||||
|
||||
// Size returns this 'cache node' size.
|
||||
func (n *Node) Size() int {
|
||||
return n.size
|
||||
}
|
||||
|
||||
// Value returns this 'cache node' value.
|
||||
func (n *Node) Value() Value {
|
||||
return n.value
|
||||
}
|
||||
|
||||
// Ref returns this 'cache node' ref counter.
|
||||
func (n *Node) Ref() int32 {
|
||||
return atomic.LoadInt32(&n.ref)
|
||||
}
|
||||
|
||||
// GetHandle returns an handle for this 'cache node'.
|
||||
func (n *Node) GetHandle() *Handle {
|
||||
if atomic.AddInt32(&n.ref, 1) <= 1 {
|
||||
panic("BUG: Node.GetHandle on zero ref")
|
||||
}
|
||||
return &Handle{unsafe.Pointer(n)}
|
||||
}
|
||||
|
||||
func (n *Node) unref() {
|
||||
if atomic.AddInt32(&n.ref, -1) == 0 {
|
||||
n.r.delete(n)
|
||||
}
|
||||
}
|
||||
|
||||
func (n *Node) unrefLocked() {
|
||||
if atomic.AddInt32(&n.ref, -1) == 0 {
|
||||
n.r.mu.RLock()
|
||||
if !n.r.closed {
|
||||
n.r.delete(n)
|
||||
}
|
||||
n.r.mu.RUnlock()
|
||||
}
|
||||
}
|
||||
|
||||
// Handle is a 'cache handle' of a 'cache node'.
|
||||
type Handle struct {
|
||||
n unsafe.Pointer // *Node
|
||||
}
|
||||
|
||||
// Value returns the value of the 'cache node'.
|
||||
func (h *Handle) Value() Value {
|
||||
n := (*Node)(atomic.LoadPointer(&h.n))
|
||||
if n != nil {
|
||||
return n.value
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Release releases this 'cache handle'.
|
||||
// It is safe to call release multiple times.
|
||||
func (h *Handle) Release() {
|
||||
nPtr := atomic.LoadPointer(&h.n)
|
||||
if nPtr != nil && atomic.CompareAndSwapPointer(&h.n, nPtr, nil) {
|
||||
n := (*Node)(nPtr)
|
||||
n.unrefLocked()
|
||||
}
|
||||
}
|
||||
|
||||
func murmur32(ns, key uint64, seed uint32) uint32 {
|
||||
const (
|
||||
m = uint32(0x5bd1e995)
|
||||
r = 24
|
||||
)
|
||||
|
||||
k1 := uint32(ns >> 32)
|
||||
k2 := uint32(ns)
|
||||
k3 := uint32(key >> 32)
|
||||
k4 := uint32(key)
|
||||
|
||||
k1 *= m
|
||||
k1 ^= k1 >> r
|
||||
k1 *= m
|
||||
|
||||
k2 *= m
|
||||
k2 ^= k2 >> r
|
||||
k2 *= m
|
||||
|
||||
k3 *= m
|
||||
k3 ^= k3 >> r
|
||||
k3 *= m
|
||||
|
||||
k4 *= m
|
||||
k4 ^= k4 >> r
|
||||
k4 *= m
|
||||
|
||||
h := seed
|
||||
|
||||
h *= m
|
||||
h ^= k1
|
||||
h *= m
|
||||
h ^= k2
|
||||
h *= m
|
||||
h ^= k3
|
||||
h *= m
|
||||
h ^= k4
|
||||
|
||||
h ^= h >> 13
|
||||
h *= m
|
||||
h ^= h >> 15
|
||||
|
||||
return h
|
||||
}
|
||||
@ -0,0 +1,195 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package cache
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
type lruNode struct {
|
||||
n *Node
|
||||
h *Handle
|
||||
ban bool
|
||||
|
||||
next, prev *lruNode
|
||||
}
|
||||
|
||||
func (n *lruNode) insert(at *lruNode) {
|
||||
x := at.next
|
||||
at.next = n
|
||||
n.prev = at
|
||||
n.next = x
|
||||
x.prev = n
|
||||
}
|
||||
|
||||
func (n *lruNode) remove() {
|
||||
if n.prev != nil {
|
||||
n.prev.next = n.next
|
||||
n.next.prev = n.prev
|
||||
n.prev = nil
|
||||
n.next = nil
|
||||
} else {
|
||||
panic("BUG: removing removed node")
|
||||
}
|
||||
}
|
||||
|
||||
type lru struct {
|
||||
mu sync.Mutex
|
||||
capacity int
|
||||
used int
|
||||
recent lruNode
|
||||
}
|
||||
|
||||
func (r *lru) reset() {
|
||||
r.recent.next = &r.recent
|
||||
r.recent.prev = &r.recent
|
||||
r.used = 0
|
||||
}
|
||||
|
||||
func (r *lru) Capacity() int {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.capacity
|
||||
}
|
||||
|
||||
func (r *lru) SetCapacity(capacity int) {
|
||||
var evicted []*lruNode
|
||||
|
||||
r.mu.Lock()
|
||||
r.capacity = capacity
|
||||
for r.used > r.capacity {
|
||||
rn := r.recent.prev
|
||||
if rn == nil {
|
||||
panic("BUG: invalid LRU used or capacity counter")
|
||||
}
|
||||
rn.remove()
|
||||
rn.n.CacheData = nil
|
||||
r.used -= rn.n.Size()
|
||||
evicted = append(evicted, rn)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
|
||||
for _, rn := range evicted {
|
||||
rn.h.Release()
|
||||
}
|
||||
}
|
||||
|
||||
func (r *lru) Promote(n *Node) {
|
||||
var evicted []*lruNode
|
||||
|
||||
r.mu.Lock()
|
||||
if n.CacheData == nil {
|
||||
if n.Size() <= r.capacity {
|
||||
rn := &lruNode{n: n, h: n.GetHandle()}
|
||||
rn.insert(&r.recent)
|
||||
n.CacheData = unsafe.Pointer(rn)
|
||||
r.used += n.Size()
|
||||
|
||||
for r.used > r.capacity {
|
||||
rn := r.recent.prev
|
||||
if rn == nil {
|
||||
panic("BUG: invalid LRU used or capacity counter")
|
||||
}
|
||||
rn.remove()
|
||||
rn.n.CacheData = nil
|
||||
r.used -= rn.n.Size()
|
||||
evicted = append(evicted, rn)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
rn := (*lruNode)(n.CacheData)
|
||||
if !rn.ban {
|
||||
rn.remove()
|
||||
rn.insert(&r.recent)
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
|
||||
for _, rn := range evicted {
|
||||
rn.h.Release()
|
||||
}
|
||||
}
|
||||
|
||||
func (r *lru) Ban(n *Node) {
|
||||
r.mu.Lock()
|
||||
if n.CacheData == nil {
|
||||
n.CacheData = unsafe.Pointer(&lruNode{n: n, ban: true})
|
||||
} else {
|
||||
rn := (*lruNode)(n.CacheData)
|
||||
if !rn.ban {
|
||||
rn.remove()
|
||||
rn.ban = true
|
||||
r.used -= rn.n.Size()
|
||||
r.mu.Unlock()
|
||||
|
||||
rn.h.Release()
|
||||
rn.h = nil
|
||||
return
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
func (r *lru) Evict(n *Node) {
|
||||
r.mu.Lock()
|
||||
rn := (*lruNode)(n.CacheData)
|
||||
if rn == nil || rn.ban {
|
||||
r.mu.Unlock()
|
||||
return
|
||||
}
|
||||
n.CacheData = nil
|
||||
r.mu.Unlock()
|
||||
|
||||
rn.h.Release()
|
||||
}
|
||||
|
||||
func (r *lru) EvictNS(ns uint64) {
|
||||
var evicted []*lruNode
|
||||
|
||||
r.mu.Lock()
|
||||
for e := r.recent.prev; e != &r.recent; {
|
||||
rn := e
|
||||
e = e.prev
|
||||
if rn.n.NS() == ns {
|
||||
rn.remove()
|
||||
rn.n.CacheData = nil
|
||||
r.used -= rn.n.Size()
|
||||
evicted = append(evicted, rn)
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
|
||||
for _, rn := range evicted {
|
||||
rn.h.Release()
|
||||
}
|
||||
}
|
||||
|
||||
func (r *lru) EvictAll() {
|
||||
r.mu.Lock()
|
||||
back := r.recent.prev
|
||||
for rn := back; rn != &r.recent; rn = rn.prev {
|
||||
rn.n.CacheData = nil
|
||||
}
|
||||
r.reset()
|
||||
r.mu.Unlock()
|
||||
|
||||
for rn := back; rn != &r.recent; rn = rn.prev {
|
||||
rn.h.Release()
|
||||
}
|
||||
}
|
||||
|
||||
func (r *lru) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewLRU create a new LRU-cache.
|
||||
func NewLRU(capacity int) Cacher {
|
||||
r := &lru{capacity: capacity}
|
||||
r.reset()
|
||||
return r
|
||||
}
|
||||
@ -0,0 +1,67 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"github.com/syndtr/goleveldb/leveldb/comparer"
|
||||
)
|
||||
|
||||
type iComparer struct {
|
||||
ucmp comparer.Comparer
|
||||
}
|
||||
|
||||
func (icmp *iComparer) uName() string {
|
||||
return icmp.ucmp.Name()
|
||||
}
|
||||
|
||||
func (icmp *iComparer) uCompare(a, b []byte) int {
|
||||
return icmp.ucmp.Compare(a, b)
|
||||
}
|
||||
|
||||
func (icmp *iComparer) uSeparator(dst, a, b []byte) []byte {
|
||||
return icmp.ucmp.Separator(dst, a, b)
|
||||
}
|
||||
|
||||
func (icmp *iComparer) uSuccessor(dst, b []byte) []byte {
|
||||
return icmp.ucmp.Successor(dst, b)
|
||||
}
|
||||
|
||||
func (icmp *iComparer) Name() string {
|
||||
return icmp.uName()
|
||||
}
|
||||
|
||||
func (icmp *iComparer) Compare(a, b []byte) int {
|
||||
x := icmp.uCompare(internalKey(a).ukey(), internalKey(b).ukey())
|
||||
if x == 0 {
|
||||
if m, n := internalKey(a).num(), internalKey(b).num(); m > n {
|
||||
return -1
|
||||
} else if m < n {
|
||||
return 1
|
||||
}
|
||||
}
|
||||
return x
|
||||
}
|
||||
|
||||
func (icmp *iComparer) Separator(dst, a, b []byte) []byte {
|
||||
ua, ub := internalKey(a).ukey(), internalKey(b).ukey()
|
||||
dst = icmp.uSeparator(dst, ua, ub)
|
||||
if dst != nil && len(dst) < len(ua) && icmp.uCompare(ua, dst) < 0 {
|
||||
// Append earliest possible number.
|
||||
return append(dst, keyMaxNumBytes...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (icmp *iComparer) Successor(dst, b []byte) []byte {
|
||||
ub := internalKey(b).ukey()
|
||||
dst = icmp.uSuccessor(dst, ub)
|
||||
if dst != nil && len(dst) < len(ub) && icmp.uCompare(ub, dst) < 0 {
|
||||
// Append earliest possible number.
|
||||
return append(dst, keyMaxNumBytes...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,51 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package comparer
|
||||
|
||||
import "bytes"
|
||||
|
||||
type bytesComparer struct{}
|
||||
|
||||
func (bytesComparer) Compare(a, b []byte) int {
|
||||
return bytes.Compare(a, b)
|
||||
}
|
||||
|
||||
func (bytesComparer) Name() string {
|
||||
return "leveldb.BytewiseComparator"
|
||||
}
|
||||
|
||||
func (bytesComparer) Separator(dst, a, b []byte) []byte {
|
||||
i, n := 0, len(a)
|
||||
if n > len(b) {
|
||||
n = len(b)
|
||||
}
|
||||
for ; i < n && a[i] == b[i]; i++ {
|
||||
}
|
||||
if i >= n {
|
||||
// Do not shorten if one string is a prefix of the other
|
||||
} else if c := a[i]; c < 0xff && c+1 < b[i] {
|
||||
dst = append(dst, a[:i+1]...)
|
||||
dst[len(dst)-1]++
|
||||
return dst
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (bytesComparer) Successor(dst, b []byte) []byte {
|
||||
for i, c := range b {
|
||||
if c != 0xff {
|
||||
dst = append(dst, b[:i+1]...)
|
||||
dst[len(dst)-1]++
|
||||
return dst
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DefaultComparer are default implementation of the Comparer interface.
|
||||
// It uses the natural ordering, consistent with bytes.Compare.
|
||||
var DefaultComparer = bytesComparer{}
|
||||
@ -0,0 +1,57 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package comparer provides interface and implementation for ordering
|
||||
// sets of data.
|
||||
package comparer
|
||||
|
||||
// BasicComparer is the interface that wraps the basic Compare method.
|
||||
type BasicComparer interface {
|
||||
// Compare returns -1, 0, or +1 depending on whether a is 'less than',
|
||||
// 'equal to' or 'greater than' b. The two arguments can only be 'equal'
|
||||
// if their contents are exactly equal. Furthermore, the empty slice
|
||||
// must be 'less than' any non-empty slice.
|
||||
Compare(a, b []byte) int
|
||||
}
|
||||
|
||||
// Comparer defines a total ordering over the space of []byte keys: a 'less
|
||||
// than' relationship.
|
||||
type Comparer interface {
|
||||
BasicComparer
|
||||
|
||||
// Name returns name of the comparer.
|
||||
//
|
||||
// The Level-DB on-disk format stores the comparer name, and opening a
|
||||
// database with a different comparer from the one it was created with
|
||||
// will result in an error.
|
||||
//
|
||||
// An implementation to a new name whenever the comparer implementation
|
||||
// changes in a way that will cause the relative ordering of any two keys
|
||||
// to change.
|
||||
//
|
||||
// Names starting with "leveldb." are reserved and should not be used
|
||||
// by any users of this package.
|
||||
Name() string
|
||||
|
||||
// Bellow are advanced functions used to reduce the space requirements
|
||||
// for internal data structures such as index blocks.
|
||||
|
||||
// Separator appends a sequence of bytes x to dst such that a <= x && x < b,
|
||||
// where 'less than' is consistent with Compare. An implementation should
|
||||
// return nil if x equal to a.
|
||||
//
|
||||
// Either contents of a or b should not by any means modified. Doing so
|
||||
// may cause corruption on the internal state.
|
||||
Separator(dst, a, b []byte) []byte
|
||||
|
||||
// Successor appends a sequence of bytes x to dst such that x >= b, where
|
||||
// 'less than' is consistent with Compare. An implementation should return
|
||||
// nil if x equal to b.
|
||||
//
|
||||
// Contents of b should not by any means modified. Doing so may cause
|
||||
// corruption on the internal state.
|
||||
Successor(dst, b []byte) []byte
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,854 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/errors"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
)
|
||||
|
||||
var (
|
||||
errCompactionTransactExiting = errors.New("leveldb: compaction transact exiting")
|
||||
)
|
||||
|
||||
type cStat struct {
|
||||
duration time.Duration
|
||||
read int64
|
||||
write int64
|
||||
}
|
||||
|
||||
func (p *cStat) add(n *cStatStaging) {
|
||||
p.duration += n.duration
|
||||
p.read += n.read
|
||||
p.write += n.write
|
||||
}
|
||||
|
||||
func (p *cStat) get() (duration time.Duration, read, write int64) {
|
||||
return p.duration, p.read, p.write
|
||||
}
|
||||
|
||||
type cStatStaging struct {
|
||||
start time.Time
|
||||
duration time.Duration
|
||||
on bool
|
||||
read int64
|
||||
write int64
|
||||
}
|
||||
|
||||
func (p *cStatStaging) startTimer() {
|
||||
if !p.on {
|
||||
p.start = time.Now()
|
||||
p.on = true
|
||||
}
|
||||
}
|
||||
|
||||
func (p *cStatStaging) stopTimer() {
|
||||
if p.on {
|
||||
p.duration += time.Since(p.start)
|
||||
p.on = false
|
||||
}
|
||||
}
|
||||
|
||||
type cStats struct {
|
||||
lk sync.Mutex
|
||||
stats []cStat
|
||||
}
|
||||
|
||||
func (p *cStats) addStat(level int, n *cStatStaging) {
|
||||
p.lk.Lock()
|
||||
if level >= len(p.stats) {
|
||||
newStats := make([]cStat, level+1)
|
||||
copy(newStats, p.stats)
|
||||
p.stats = newStats
|
||||
}
|
||||
p.stats[level].add(n)
|
||||
p.lk.Unlock()
|
||||
}
|
||||
|
||||
func (p *cStats) getStat(level int) (duration time.Duration, read, write int64) {
|
||||
p.lk.Lock()
|
||||
defer p.lk.Unlock()
|
||||
if level < len(p.stats) {
|
||||
return p.stats[level].get()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (db *DB) compactionError() {
|
||||
var err error
|
||||
noerr:
|
||||
// No error.
|
||||
for {
|
||||
select {
|
||||
case err = <-db.compErrSetC:
|
||||
switch {
|
||||
case err == nil:
|
||||
case err == ErrReadOnly, errors.IsCorrupted(err):
|
||||
goto hasperr
|
||||
default:
|
||||
goto haserr
|
||||
}
|
||||
case <-db.closeC:
|
||||
return
|
||||
}
|
||||
}
|
||||
haserr:
|
||||
// Transient error.
|
||||
for {
|
||||
select {
|
||||
case db.compErrC <- err:
|
||||
case err = <-db.compErrSetC:
|
||||
switch {
|
||||
case err == nil:
|
||||
goto noerr
|
||||
case err == ErrReadOnly, errors.IsCorrupted(err):
|
||||
goto hasperr
|
||||
default:
|
||||
}
|
||||
case <-db.closeC:
|
||||
return
|
||||
}
|
||||
}
|
||||
hasperr:
|
||||
// Persistent error.
|
||||
for {
|
||||
select {
|
||||
case db.compErrC <- err:
|
||||
case db.compPerErrC <- err:
|
||||
case db.writeLockC <- struct{}{}:
|
||||
// Hold write lock, so that write won't pass-through.
|
||||
db.compWriteLocking = true
|
||||
case <-db.closeC:
|
||||
if db.compWriteLocking {
|
||||
// We should release the lock or Close will hang.
|
||||
<-db.writeLockC
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type compactionTransactCounter int
|
||||
|
||||
func (cnt *compactionTransactCounter) incr() {
|
||||
*cnt++
|
||||
}
|
||||
|
||||
type compactionTransactInterface interface {
|
||||
run(cnt *compactionTransactCounter) error
|
||||
revert() error
|
||||
}
|
||||
|
||||
func (db *DB) compactionTransact(name string, t compactionTransactInterface) {
|
||||
defer func() {
|
||||
if x := recover(); x != nil {
|
||||
if x == errCompactionTransactExiting {
|
||||
if err := t.revert(); err != nil {
|
||||
db.logf("%s revert error %q", name, err)
|
||||
}
|
||||
}
|
||||
panic(x)
|
||||
}
|
||||
}()
|
||||
|
||||
const (
|
||||
backoffMin = 1 * time.Second
|
||||
backoffMax = 8 * time.Second
|
||||
backoffMul = 2 * time.Second
|
||||
)
|
||||
var (
|
||||
backoff = backoffMin
|
||||
backoffT = time.NewTimer(backoff)
|
||||
lastCnt = compactionTransactCounter(0)
|
||||
|
||||
disableBackoff = db.s.o.GetDisableCompactionBackoff()
|
||||
)
|
||||
for n := 0; ; n++ {
|
||||
// Check whether the DB is closed.
|
||||
if db.isClosed() {
|
||||
db.logf("%s exiting", name)
|
||||
db.compactionExitTransact()
|
||||
} else if n > 0 {
|
||||
db.logf("%s retrying N·%d", name, n)
|
||||
}
|
||||
|
||||
// Execute.
|
||||
cnt := compactionTransactCounter(0)
|
||||
err := t.run(&cnt)
|
||||
if err != nil {
|
||||
db.logf("%s error I·%d %q", name, cnt, err)
|
||||
}
|
||||
|
||||
// Set compaction error status.
|
||||
select {
|
||||
case db.compErrSetC <- err:
|
||||
case perr := <-db.compPerErrC:
|
||||
if err != nil {
|
||||
db.logf("%s exiting (persistent error %q)", name, perr)
|
||||
db.compactionExitTransact()
|
||||
}
|
||||
case <-db.closeC:
|
||||
db.logf("%s exiting", name)
|
||||
db.compactionExitTransact()
|
||||
}
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
if errors.IsCorrupted(err) {
|
||||
db.logf("%s exiting (corruption detected)", name)
|
||||
db.compactionExitTransact()
|
||||
}
|
||||
|
||||
if !disableBackoff {
|
||||
// Reset backoff duration if counter is advancing.
|
||||
if cnt > lastCnt {
|
||||
backoff = backoffMin
|
||||
lastCnt = cnt
|
||||
}
|
||||
|
||||
// Backoff.
|
||||
backoffT.Reset(backoff)
|
||||
if backoff < backoffMax {
|
||||
backoff *= backoffMul
|
||||
if backoff > backoffMax {
|
||||
backoff = backoffMax
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-backoffT.C:
|
||||
case <-db.closeC:
|
||||
db.logf("%s exiting", name)
|
||||
db.compactionExitTransact()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type compactionTransactFunc struct {
|
||||
runFunc func(cnt *compactionTransactCounter) error
|
||||
revertFunc func() error
|
||||
}
|
||||
|
||||
func (t *compactionTransactFunc) run(cnt *compactionTransactCounter) error {
|
||||
return t.runFunc(cnt)
|
||||
}
|
||||
|
||||
func (t *compactionTransactFunc) revert() error {
|
||||
if t.revertFunc != nil {
|
||||
return t.revertFunc()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *DB) compactionTransactFunc(name string, run func(cnt *compactionTransactCounter) error, revert func() error) {
|
||||
db.compactionTransact(name, &compactionTransactFunc{run, revert})
|
||||
}
|
||||
|
||||
func (db *DB) compactionExitTransact() {
|
||||
panic(errCompactionTransactExiting)
|
||||
}
|
||||
|
||||
func (db *DB) compactionCommit(name string, rec *sessionRecord) {
|
||||
db.compCommitLk.Lock()
|
||||
defer db.compCommitLk.Unlock() // Defer is necessary.
|
||||
db.compactionTransactFunc(name+"@commit", func(cnt *compactionTransactCounter) error {
|
||||
return db.s.commit(rec)
|
||||
}, nil)
|
||||
}
|
||||
|
||||
func (db *DB) memCompaction() {
|
||||
mdb := db.getFrozenMem()
|
||||
if mdb == nil {
|
||||
return
|
||||
}
|
||||
defer mdb.decref()
|
||||
|
||||
db.logf("memdb@flush N·%d S·%s", mdb.Len(), shortenb(mdb.Size()))
|
||||
|
||||
// Don't compact empty memdb.
|
||||
if mdb.Len() == 0 {
|
||||
db.logf("memdb@flush skipping")
|
||||
// drop frozen memdb
|
||||
db.dropFrozenMem()
|
||||
return
|
||||
}
|
||||
|
||||
// Pause table compaction.
|
||||
resumeC := make(chan struct{})
|
||||
select {
|
||||
case db.tcompPauseC <- (chan<- struct{})(resumeC):
|
||||
case <-db.compPerErrC:
|
||||
close(resumeC)
|
||||
resumeC = nil
|
||||
case <-db.closeC:
|
||||
db.compactionExitTransact()
|
||||
}
|
||||
|
||||
var (
|
||||
rec = &sessionRecord{}
|
||||
stats = &cStatStaging{}
|
||||
flushLevel int
|
||||
)
|
||||
|
||||
// Generate tables.
|
||||
db.compactionTransactFunc("memdb@flush", func(cnt *compactionTransactCounter) (err error) {
|
||||
stats.startTimer()
|
||||
flushLevel, err = db.s.flushMemdb(rec, mdb.DB, db.memdbMaxLevel)
|
||||
stats.stopTimer()
|
||||
return
|
||||
}, func() error {
|
||||
for _, r := range rec.addedTables {
|
||||
db.logf("memdb@flush revert @%d", r.num)
|
||||
if err := db.s.stor.Remove(storage.FileDesc{Type: storage.TypeTable, Num: r.num}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
rec.setJournalNum(db.journalFd.Num)
|
||||
rec.setSeqNum(db.frozenSeq)
|
||||
|
||||
// Commit.
|
||||
stats.startTimer()
|
||||
db.compactionCommit("memdb", rec)
|
||||
stats.stopTimer()
|
||||
|
||||
db.logf("memdb@flush committed F·%d T·%v", len(rec.addedTables), stats.duration)
|
||||
|
||||
for _, r := range rec.addedTables {
|
||||
stats.write += r.size
|
||||
}
|
||||
db.compStats.addStat(flushLevel, stats)
|
||||
|
||||
// Drop frozen memdb.
|
||||
db.dropFrozenMem()
|
||||
|
||||
// Resume table compaction.
|
||||
if resumeC != nil {
|
||||
select {
|
||||
case <-resumeC:
|
||||
close(resumeC)
|
||||
case <-db.closeC:
|
||||
db.compactionExitTransact()
|
||||
}
|
||||
}
|
||||
|
||||
// Trigger table compaction.
|
||||
db.compTrigger(db.tcompCmdC)
|
||||
}
|
||||
|
||||
type tableCompactionBuilder struct {
|
||||
db *DB
|
||||
s *session
|
||||
c *compaction
|
||||
rec *sessionRecord
|
||||
stat0, stat1 *cStatStaging
|
||||
|
||||
snapHasLastUkey bool
|
||||
snapLastUkey []byte
|
||||
snapLastSeq uint64
|
||||
snapIter int
|
||||
snapKerrCnt int
|
||||
snapDropCnt int
|
||||
|
||||
kerrCnt int
|
||||
dropCnt int
|
||||
|
||||
minSeq uint64
|
||||
strict bool
|
||||
tableSize int
|
||||
|
||||
tw *tWriter
|
||||
}
|
||||
|
||||
func (b *tableCompactionBuilder) appendKV(key, value []byte) error {
|
||||
// Create new table if not already.
|
||||
if b.tw == nil {
|
||||
// Check for pause event.
|
||||
if b.db != nil {
|
||||
select {
|
||||
case ch := <-b.db.tcompPauseC:
|
||||
b.db.pauseCompaction(ch)
|
||||
case <-b.db.closeC:
|
||||
b.db.compactionExitTransact()
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// Create new table.
|
||||
var err error
|
||||
b.tw, err = b.s.tops.create()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Write key/value into table.
|
||||
return b.tw.append(key, value)
|
||||
}
|
||||
|
||||
func (b *tableCompactionBuilder) needFlush() bool {
|
||||
return b.tw.tw.BytesLen() >= b.tableSize
|
||||
}
|
||||
|
||||
func (b *tableCompactionBuilder) flush() error {
|
||||
t, err := b.tw.finish()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.rec.addTableFile(b.c.sourceLevel+1, t)
|
||||
b.stat1.write += t.size
|
||||
b.s.logf("table@build created L%d@%d N·%d S·%s %q:%q", b.c.sourceLevel+1, t.fd.Num, b.tw.tw.EntriesLen(), shortenb(int(t.size)), t.imin, t.imax)
|
||||
b.tw = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *tableCompactionBuilder) cleanup() {
|
||||
if b.tw != nil {
|
||||
b.tw.drop()
|
||||
b.tw = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (b *tableCompactionBuilder) run(cnt *compactionTransactCounter) error {
|
||||
snapResumed := b.snapIter > 0
|
||||
hasLastUkey := b.snapHasLastUkey // The key might has zero length, so this is necessary.
|
||||
lastUkey := append([]byte{}, b.snapLastUkey...)
|
||||
lastSeq := b.snapLastSeq
|
||||
b.kerrCnt = b.snapKerrCnt
|
||||
b.dropCnt = b.snapDropCnt
|
||||
// Restore compaction state.
|
||||
b.c.restore()
|
||||
|
||||
defer b.cleanup()
|
||||
|
||||
b.stat1.startTimer()
|
||||
defer b.stat1.stopTimer()
|
||||
|
||||
iter := b.c.newIterator()
|
||||
defer iter.Release()
|
||||
for i := 0; iter.Next(); i++ {
|
||||
// Incr transact counter.
|
||||
cnt.incr()
|
||||
|
||||
// Skip until last state.
|
||||
if i < b.snapIter {
|
||||
continue
|
||||
}
|
||||
|
||||
resumed := false
|
||||
if snapResumed {
|
||||
resumed = true
|
||||
snapResumed = false
|
||||
}
|
||||
|
||||
ikey := iter.Key()
|
||||
ukey, seq, kt, kerr := parseInternalKey(ikey)
|
||||
|
||||
if kerr == nil {
|
||||
shouldStop := !resumed && b.c.shouldStopBefore(ikey)
|
||||
|
||||
if !hasLastUkey || b.s.icmp.uCompare(lastUkey, ukey) != 0 {
|
||||
// First occurrence of this user key.
|
||||
|
||||
// Only rotate tables if ukey doesn't hop across.
|
||||
if b.tw != nil && (shouldStop || b.needFlush()) {
|
||||
if err := b.flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Creates snapshot of the state.
|
||||
b.c.save()
|
||||
b.snapHasLastUkey = hasLastUkey
|
||||
b.snapLastUkey = append(b.snapLastUkey[:0], lastUkey...)
|
||||
b.snapLastSeq = lastSeq
|
||||
b.snapIter = i
|
||||
b.snapKerrCnt = b.kerrCnt
|
||||
b.snapDropCnt = b.dropCnt
|
||||
}
|
||||
|
||||
hasLastUkey = true
|
||||
lastUkey = append(lastUkey[:0], ukey...)
|
||||
lastSeq = keyMaxSeq
|
||||
}
|
||||
|
||||
switch {
|
||||
case lastSeq <= b.minSeq:
|
||||
// Dropped because newer entry for same user key exist
|
||||
fallthrough // (A)
|
||||
case kt == keyTypeDel && seq <= b.minSeq && b.c.baseLevelForKey(lastUkey):
|
||||
// For this user key:
|
||||
// (1) there is no data in higher levels
|
||||
// (2) data in lower levels will have larger seq numbers
|
||||
// (3) data in layers that are being compacted here and have
|
||||
// smaller seq numbers will be dropped in the next
|
||||
// few iterations of this loop (by rule (A) above).
|
||||
// Therefore this deletion marker is obsolete and can be dropped.
|
||||
lastSeq = seq
|
||||
b.dropCnt++
|
||||
continue
|
||||
default:
|
||||
lastSeq = seq
|
||||
}
|
||||
} else {
|
||||
if b.strict {
|
||||
return kerr
|
||||
}
|
||||
|
||||
// Don't drop corrupted keys.
|
||||
hasLastUkey = false
|
||||
lastUkey = lastUkey[:0]
|
||||
lastSeq = keyMaxSeq
|
||||
b.kerrCnt++
|
||||
}
|
||||
|
||||
if err := b.appendKV(ikey, iter.Value()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := iter.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Finish last table.
|
||||
if b.tw != nil && !b.tw.empty() {
|
||||
return b.flush()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *tableCompactionBuilder) revert() error {
|
||||
for _, at := range b.rec.addedTables {
|
||||
b.s.logf("table@build revert @%d", at.num)
|
||||
if err := b.s.stor.Remove(storage.FileDesc{Type: storage.TypeTable, Num: at.num}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *DB) tableCompaction(c *compaction, noTrivial bool) {
|
||||
defer c.release()
|
||||
|
||||
rec := &sessionRecord{}
|
||||
rec.addCompPtr(c.sourceLevel, c.imax)
|
||||
|
||||
if !noTrivial && c.trivial() {
|
||||
t := c.levels[0][0]
|
||||
db.logf("table@move L%d@%d -> L%d", c.sourceLevel, t.fd.Num, c.sourceLevel+1)
|
||||
rec.delTable(c.sourceLevel, t.fd.Num)
|
||||
rec.addTableFile(c.sourceLevel+1, t)
|
||||
db.compactionCommit("table-move", rec)
|
||||
return
|
||||
}
|
||||
|
||||
var stats [2]cStatStaging
|
||||
for i, tables := range c.levels {
|
||||
for _, t := range tables {
|
||||
stats[i].read += t.size
|
||||
// Insert deleted tables into record
|
||||
rec.delTable(c.sourceLevel+i, t.fd.Num)
|
||||
}
|
||||
}
|
||||
sourceSize := int(stats[0].read + stats[1].read)
|
||||
minSeq := db.minSeq()
|
||||
db.logf("table@compaction L%d·%d -> L%d·%d S·%s Q·%d", c.sourceLevel, len(c.levels[0]), c.sourceLevel+1, len(c.levels[1]), shortenb(sourceSize), minSeq)
|
||||
|
||||
b := &tableCompactionBuilder{
|
||||
db: db,
|
||||
s: db.s,
|
||||
c: c,
|
||||
rec: rec,
|
||||
stat1: &stats[1],
|
||||
minSeq: minSeq,
|
||||
strict: db.s.o.GetStrict(opt.StrictCompaction),
|
||||
tableSize: db.s.o.GetCompactionTableSize(c.sourceLevel + 1),
|
||||
}
|
||||
db.compactionTransact("table@build", b)
|
||||
|
||||
// Commit.
|
||||
stats[1].startTimer()
|
||||
db.compactionCommit("table", rec)
|
||||
stats[1].stopTimer()
|
||||
|
||||
resultSize := int(stats[1].write)
|
||||
db.logf("table@compaction committed F%s S%s Ke·%d D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), b.kerrCnt, b.dropCnt, stats[1].duration)
|
||||
|
||||
// Save compaction stats
|
||||
for i := range stats {
|
||||
db.compStats.addStat(c.sourceLevel+1, &stats[i])
|
||||
}
|
||||
}
|
||||
|
||||
func (db *DB) tableRangeCompaction(level int, umin, umax []byte) error {
|
||||
db.logf("table@compaction range L%d %q:%q", level, umin, umax)
|
||||
if level >= 0 {
|
||||
if c := db.s.getCompactionRange(level, umin, umax, true); c != nil {
|
||||
db.tableCompaction(c, true)
|
||||
}
|
||||
} else {
|
||||
// Retry until nothing to compact.
|
||||
for {
|
||||
compacted := false
|
||||
|
||||
// Scan for maximum level with overlapped tables.
|
||||
v := db.s.version()
|
||||
m := 1
|
||||
for i := m; i < len(v.levels); i++ {
|
||||
tables := v.levels[i]
|
||||
if tables.overlaps(db.s.icmp, umin, umax, false) {
|
||||
m = i
|
||||
}
|
||||
}
|
||||
v.release()
|
||||
|
||||
for level := 0; level < m; level++ {
|
||||
if c := db.s.getCompactionRange(level, umin, umax, false); c != nil {
|
||||
db.tableCompaction(c, true)
|
||||
compacted = true
|
||||
}
|
||||
}
|
||||
|
||||
if !compacted {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *DB) tableAutoCompaction() {
|
||||
if c := db.s.pickCompaction(); c != nil {
|
||||
db.tableCompaction(c, false)
|
||||
}
|
||||
}
|
||||
|
||||
func (db *DB) tableNeedCompaction() bool {
|
||||
v := db.s.version()
|
||||
defer v.release()
|
||||
return v.needCompaction()
|
||||
}
|
||||
|
||||
// resumeWrite returns an indicator whether we should resume write operation if enough level0 files are compacted.
|
||||
func (db *DB) resumeWrite() bool {
|
||||
v := db.s.version()
|
||||
defer v.release()
|
||||
if v.tLen(0) < db.s.o.GetWriteL0PauseTrigger() {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (db *DB) pauseCompaction(ch chan<- struct{}) {
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
case <-db.closeC:
|
||||
db.compactionExitTransact()
|
||||
}
|
||||
}
|
||||
|
||||
type cCmd interface {
|
||||
ack(err error)
|
||||
}
|
||||
|
||||
type cAuto struct {
|
||||
// Note for table compaction, an non-empty ackC represents it's a compaction waiting command.
|
||||
ackC chan<- error
|
||||
}
|
||||
|
||||
func (r cAuto) ack(err error) {
|
||||
if r.ackC != nil {
|
||||
defer func() {
|
||||
recover()
|
||||
}()
|
||||
r.ackC <- err
|
||||
}
|
||||
}
|
||||
|
||||
type cRange struct {
|
||||
level int
|
||||
min, max []byte
|
||||
ackC chan<- error
|
||||
}
|
||||
|
||||
func (r cRange) ack(err error) {
|
||||
if r.ackC != nil {
|
||||
defer func() {
|
||||
recover()
|
||||
}()
|
||||
r.ackC <- err
|
||||
}
|
||||
}
|
||||
|
||||
// This will trigger auto compaction but will not wait for it.
|
||||
func (db *DB) compTrigger(compC chan<- cCmd) {
|
||||
select {
|
||||
case compC <- cAuto{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// This will trigger auto compaction and/or wait for all compaction to be done.
|
||||
func (db *DB) compTriggerWait(compC chan<- cCmd) (err error) {
|
||||
ch := make(chan error)
|
||||
defer close(ch)
|
||||
// Send cmd.
|
||||
select {
|
||||
case compC <- cAuto{ch}:
|
||||
case err = <-db.compErrC:
|
||||
return
|
||||
case <-db.closeC:
|
||||
return ErrClosed
|
||||
}
|
||||
// Wait cmd.
|
||||
select {
|
||||
case err = <-ch:
|
||||
case err = <-db.compErrC:
|
||||
case <-db.closeC:
|
||||
return ErrClosed
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Send range compaction request.
|
||||
func (db *DB) compTriggerRange(compC chan<- cCmd, level int, min, max []byte) (err error) {
|
||||
ch := make(chan error)
|
||||
defer close(ch)
|
||||
// Send cmd.
|
||||
select {
|
||||
case compC <- cRange{level, min, max, ch}:
|
||||
case err := <-db.compErrC:
|
||||
return err
|
||||
case <-db.closeC:
|
||||
return ErrClosed
|
||||
}
|
||||
// Wait cmd.
|
||||
select {
|
||||
case err = <-ch:
|
||||
case err = <-db.compErrC:
|
||||
case <-db.closeC:
|
||||
return ErrClosed
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) mCompaction() {
|
||||
var x cCmd
|
||||
|
||||
defer func() {
|
||||
if x := recover(); x != nil {
|
||||
if x != errCompactionTransactExiting {
|
||||
panic(x)
|
||||
}
|
||||
}
|
||||
if x != nil {
|
||||
x.ack(ErrClosed)
|
||||
}
|
||||
db.closeW.Done()
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case x = <-db.mcompCmdC:
|
||||
switch x.(type) {
|
||||
case cAuto:
|
||||
db.memCompaction()
|
||||
x.ack(nil)
|
||||
x = nil
|
||||
default:
|
||||
panic("leveldb: unknown command")
|
||||
}
|
||||
case <-db.closeC:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (db *DB) tCompaction() {
|
||||
var (
|
||||
x cCmd
|
||||
waitQ []cCmd
|
||||
)
|
||||
|
||||
defer func() {
|
||||
if x := recover(); x != nil {
|
||||
if x != errCompactionTransactExiting {
|
||||
panic(x)
|
||||
}
|
||||
}
|
||||
for i := range waitQ {
|
||||
waitQ[i].ack(ErrClosed)
|
||||
waitQ[i] = nil
|
||||
}
|
||||
if x != nil {
|
||||
x.ack(ErrClosed)
|
||||
}
|
||||
db.closeW.Done()
|
||||
}()
|
||||
|
||||
for {
|
||||
if db.tableNeedCompaction() {
|
||||
select {
|
||||
case x = <-db.tcompCmdC:
|
||||
case ch := <-db.tcompPauseC:
|
||||
db.pauseCompaction(ch)
|
||||
continue
|
||||
case <-db.closeC:
|
||||
return
|
||||
default:
|
||||
}
|
||||
// Resume write operation as soon as possible.
|
||||
if len(waitQ) > 0 && db.resumeWrite() {
|
||||
for i := range waitQ {
|
||||
waitQ[i].ack(nil)
|
||||
waitQ[i] = nil
|
||||
}
|
||||
waitQ = waitQ[:0]
|
||||
}
|
||||
} else {
|
||||
for i := range waitQ {
|
||||
waitQ[i].ack(nil)
|
||||
waitQ[i] = nil
|
||||
}
|
||||
waitQ = waitQ[:0]
|
||||
select {
|
||||
case x = <-db.tcompCmdC:
|
||||
case ch := <-db.tcompPauseC:
|
||||
db.pauseCompaction(ch)
|
||||
continue
|
||||
case <-db.closeC:
|
||||
return
|
||||
}
|
||||
}
|
||||
if x != nil {
|
||||
switch cmd := x.(type) {
|
||||
case cAuto:
|
||||
if cmd.ackC != nil {
|
||||
// Check the write pause state before caching it.
|
||||
if db.resumeWrite() {
|
||||
x.ack(nil)
|
||||
} else {
|
||||
waitQ = append(waitQ, x)
|
||||
}
|
||||
}
|
||||
case cRange:
|
||||
x.ack(db.tableRangeCompaction(cmd.level, cmd.min, cmd.max))
|
||||
default:
|
||||
panic("leveldb: unknown command")
|
||||
}
|
||||
x = nil
|
||||
}
|
||||
db.tableAutoCompaction()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,360 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math/rand"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
var (
|
||||
errInvalidInternalKey = errors.New("leveldb: Iterator: invalid internal key")
|
||||
)
|
||||
|
||||
type memdbReleaser struct {
|
||||
once sync.Once
|
||||
m *memDB
|
||||
}
|
||||
|
||||
func (mr *memdbReleaser) Release() {
|
||||
mr.once.Do(func() {
|
||||
mr.m.decref()
|
||||
})
|
||||
}
|
||||
|
||||
func (db *DB) newRawIterator(auxm *memDB, auxt tFiles, slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
|
||||
strict := opt.GetStrict(db.s.o.Options, ro, opt.StrictReader)
|
||||
em, fm := db.getMems()
|
||||
v := db.s.version()
|
||||
|
||||
tableIts := v.getIterators(slice, ro)
|
||||
n := len(tableIts) + len(auxt) + 3
|
||||
its := make([]iterator.Iterator, 0, n)
|
||||
|
||||
if auxm != nil {
|
||||
ami := auxm.NewIterator(slice)
|
||||
ami.SetReleaser(&memdbReleaser{m: auxm})
|
||||
its = append(its, ami)
|
||||
}
|
||||
for _, t := range auxt {
|
||||
its = append(its, v.s.tops.newIterator(t, slice, ro))
|
||||
}
|
||||
|
||||
emi := em.NewIterator(slice)
|
||||
emi.SetReleaser(&memdbReleaser{m: em})
|
||||
its = append(its, emi)
|
||||
if fm != nil {
|
||||
fmi := fm.NewIterator(slice)
|
||||
fmi.SetReleaser(&memdbReleaser{m: fm})
|
||||
its = append(its, fmi)
|
||||
}
|
||||
its = append(its, tableIts...)
|
||||
mi := iterator.NewMergedIterator(its, db.s.icmp, strict)
|
||||
mi.SetReleaser(&versionReleaser{v: v})
|
||||
return mi
|
||||
}
|
||||
|
||||
func (db *DB) newIterator(auxm *memDB, auxt tFiles, seq uint64, slice *util.Range, ro *opt.ReadOptions) *dbIter {
|
||||
var islice *util.Range
|
||||
if slice != nil {
|
||||
islice = &util.Range{}
|
||||
if slice.Start != nil {
|
||||
islice.Start = makeInternalKey(nil, slice.Start, keyMaxSeq, keyTypeSeek)
|
||||
}
|
||||
if slice.Limit != nil {
|
||||
islice.Limit = makeInternalKey(nil, slice.Limit, keyMaxSeq, keyTypeSeek)
|
||||
}
|
||||
}
|
||||
rawIter := db.newRawIterator(auxm, auxt, islice, ro)
|
||||
iter := &dbIter{
|
||||
db: db,
|
||||
icmp: db.s.icmp,
|
||||
iter: rawIter,
|
||||
seq: seq,
|
||||
strict: opt.GetStrict(db.s.o.Options, ro, opt.StrictReader),
|
||||
key: make([]byte, 0),
|
||||
value: make([]byte, 0),
|
||||
}
|
||||
atomic.AddInt32(&db.aliveIters, 1)
|
||||
runtime.SetFinalizer(iter, (*dbIter).Release)
|
||||
return iter
|
||||
}
|
||||
|
||||
func (db *DB) iterSamplingRate() int {
|
||||
return rand.Intn(2 * db.s.o.GetIteratorSamplingRate())
|
||||
}
|
||||
|
||||
type dir int
|
||||
|
||||
const (
|
||||
dirReleased dir = iota - 1
|
||||
dirSOI
|
||||
dirEOI
|
||||
dirBackward
|
||||
dirForward
|
||||
)
|
||||
|
||||
// dbIter represent an interator states over a database session.
|
||||
type dbIter struct {
|
||||
db *DB
|
||||
icmp *iComparer
|
||||
iter iterator.Iterator
|
||||
seq uint64
|
||||
strict bool
|
||||
|
||||
smaplingGap int
|
||||
dir dir
|
||||
key []byte
|
||||
value []byte
|
||||
err error
|
||||
releaser util.Releaser
|
||||
}
|
||||
|
||||
func (i *dbIter) sampleSeek() {
|
||||
ikey := i.iter.Key()
|
||||
i.smaplingGap -= len(ikey) + len(i.iter.Value())
|
||||
for i.smaplingGap < 0 {
|
||||
i.smaplingGap += i.db.iterSamplingRate()
|
||||
i.db.sampleSeek(ikey)
|
||||
}
|
||||
}
|
||||
|
||||
func (i *dbIter) setErr(err error) {
|
||||
i.err = err
|
||||
i.key = nil
|
||||
i.value = nil
|
||||
}
|
||||
|
||||
func (i *dbIter) iterErr() {
|
||||
if err := i.iter.Error(); err != nil {
|
||||
i.setErr(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (i *dbIter) Valid() bool {
|
||||
return i.err == nil && i.dir > dirEOI
|
||||
}
|
||||
|
||||
func (i *dbIter) First() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
if i.iter.First() {
|
||||
i.dir = dirSOI
|
||||
return i.next()
|
||||
}
|
||||
i.dir = dirEOI
|
||||
i.iterErr()
|
||||
return false
|
||||
}
|
||||
|
||||
func (i *dbIter) Last() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
if i.iter.Last() {
|
||||
return i.prev()
|
||||
}
|
||||
i.dir = dirSOI
|
||||
i.iterErr()
|
||||
return false
|
||||
}
|
||||
|
||||
func (i *dbIter) Seek(key []byte) bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
ikey := makeInternalKey(nil, key, i.seq, keyTypeSeek)
|
||||
if i.iter.Seek(ikey) {
|
||||
i.dir = dirSOI
|
||||
return i.next()
|
||||
}
|
||||
i.dir = dirEOI
|
||||
i.iterErr()
|
||||
return false
|
||||
}
|
||||
|
||||
func (i *dbIter) next() bool {
|
||||
for {
|
||||
if ukey, seq, kt, kerr := parseInternalKey(i.iter.Key()); kerr == nil {
|
||||
i.sampleSeek()
|
||||
if seq <= i.seq {
|
||||
switch kt {
|
||||
case keyTypeDel:
|
||||
// Skip deleted key.
|
||||
i.key = append(i.key[:0], ukey...)
|
||||
i.dir = dirForward
|
||||
case keyTypeVal:
|
||||
if i.dir == dirSOI || i.icmp.uCompare(ukey, i.key) > 0 {
|
||||
i.key = append(i.key[:0], ukey...)
|
||||
i.value = append(i.value[:0], i.iter.Value()...)
|
||||
i.dir = dirForward
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if i.strict {
|
||||
i.setErr(kerr)
|
||||
break
|
||||
}
|
||||
if !i.iter.Next() {
|
||||
i.dir = dirEOI
|
||||
i.iterErr()
|
||||
break
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (i *dbIter) Next() bool {
|
||||
if i.dir == dirEOI || i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
if !i.iter.Next() || (i.dir == dirBackward && !i.iter.Next()) {
|
||||
i.dir = dirEOI
|
||||
i.iterErr()
|
||||
return false
|
||||
}
|
||||
return i.next()
|
||||
}
|
||||
|
||||
func (i *dbIter) prev() bool {
|
||||
i.dir = dirBackward
|
||||
del := true
|
||||
if i.iter.Valid() {
|
||||
for {
|
||||
if ukey, seq, kt, kerr := parseInternalKey(i.iter.Key()); kerr == nil {
|
||||
i.sampleSeek()
|
||||
if seq <= i.seq {
|
||||
if !del && i.icmp.uCompare(ukey, i.key) < 0 {
|
||||
return true
|
||||
}
|
||||
del = (kt == keyTypeDel)
|
||||
if !del {
|
||||
i.key = append(i.key[:0], ukey...)
|
||||
i.value = append(i.value[:0], i.iter.Value()...)
|
||||
}
|
||||
}
|
||||
} else if i.strict {
|
||||
i.setErr(kerr)
|
||||
return false
|
||||
}
|
||||
if !i.iter.Prev() {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if del {
|
||||
i.dir = dirSOI
|
||||
i.iterErr()
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *dbIter) Prev() bool {
|
||||
if i.dir == dirSOI || i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
switch i.dir {
|
||||
case dirEOI:
|
||||
return i.Last()
|
||||
case dirForward:
|
||||
for i.iter.Prev() {
|
||||
if ukey, _, _, kerr := parseInternalKey(i.iter.Key()); kerr == nil {
|
||||
i.sampleSeek()
|
||||
if i.icmp.uCompare(ukey, i.key) < 0 {
|
||||
goto cont
|
||||
}
|
||||
} else if i.strict {
|
||||
i.setErr(kerr)
|
||||
return false
|
||||
}
|
||||
}
|
||||
i.dir = dirSOI
|
||||
i.iterErr()
|
||||
return false
|
||||
}
|
||||
|
||||
cont:
|
||||
return i.prev()
|
||||
}
|
||||
|
||||
func (i *dbIter) Key() []byte {
|
||||
if i.err != nil || i.dir <= dirEOI {
|
||||
return nil
|
||||
}
|
||||
return i.key
|
||||
}
|
||||
|
||||
func (i *dbIter) Value() []byte {
|
||||
if i.err != nil || i.dir <= dirEOI {
|
||||
return nil
|
||||
}
|
||||
return i.value
|
||||
}
|
||||
|
||||
func (i *dbIter) Release() {
|
||||
if i.dir != dirReleased {
|
||||
// Clear the finalizer.
|
||||
runtime.SetFinalizer(i, nil)
|
||||
|
||||
if i.releaser != nil {
|
||||
i.releaser.Release()
|
||||
i.releaser = nil
|
||||
}
|
||||
|
||||
i.dir = dirReleased
|
||||
i.key = nil
|
||||
i.value = nil
|
||||
i.iter.Release()
|
||||
i.iter = nil
|
||||
atomic.AddInt32(&i.db.aliveIters, -1)
|
||||
i.db = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (i *dbIter) SetReleaser(releaser util.Releaser) {
|
||||
if i.dir == dirReleased {
|
||||
panic(util.ErrReleased)
|
||||
}
|
||||
if i.releaser != nil && releaser != nil {
|
||||
panic(util.ErrHasReleaser)
|
||||
}
|
||||
i.releaser = releaser
|
||||
}
|
||||
|
||||
func (i *dbIter) Error() error {
|
||||
return i.err
|
||||
}
|
||||
@ -0,0 +1,187 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
type snapshotElement struct {
|
||||
seq uint64
|
||||
ref int
|
||||
e *list.Element
|
||||
}
|
||||
|
||||
// Acquires a snapshot, based on latest sequence.
|
||||
func (db *DB) acquireSnapshot() *snapshotElement {
|
||||
db.snapsMu.Lock()
|
||||
defer db.snapsMu.Unlock()
|
||||
|
||||
seq := db.getSeq()
|
||||
|
||||
if e := db.snapsList.Back(); e != nil {
|
||||
se := e.Value.(*snapshotElement)
|
||||
if se.seq == seq {
|
||||
se.ref++
|
||||
return se
|
||||
} else if seq < se.seq {
|
||||
panic("leveldb: sequence number is not increasing")
|
||||
}
|
||||
}
|
||||
se := &snapshotElement{seq: seq, ref: 1}
|
||||
se.e = db.snapsList.PushBack(se)
|
||||
return se
|
||||
}
|
||||
|
||||
// Releases given snapshot element.
|
||||
func (db *DB) releaseSnapshot(se *snapshotElement) {
|
||||
db.snapsMu.Lock()
|
||||
defer db.snapsMu.Unlock()
|
||||
|
||||
se.ref--
|
||||
if se.ref == 0 {
|
||||
db.snapsList.Remove(se.e)
|
||||
se.e = nil
|
||||
} else if se.ref < 0 {
|
||||
panic("leveldb: Snapshot: negative element reference")
|
||||
}
|
||||
}
|
||||
|
||||
// Gets minimum sequence that not being snapshotted.
|
||||
func (db *DB) minSeq() uint64 {
|
||||
db.snapsMu.Lock()
|
||||
defer db.snapsMu.Unlock()
|
||||
|
||||
if e := db.snapsList.Front(); e != nil {
|
||||
return e.Value.(*snapshotElement).seq
|
||||
}
|
||||
|
||||
return db.getSeq()
|
||||
}
|
||||
|
||||
// Snapshot is a DB snapshot.
|
||||
type Snapshot struct {
|
||||
db *DB
|
||||
elem *snapshotElement
|
||||
mu sync.RWMutex
|
||||
released bool
|
||||
}
|
||||
|
||||
// Creates new snapshot object.
|
||||
func (db *DB) newSnapshot() *Snapshot {
|
||||
snap := &Snapshot{
|
||||
db: db,
|
||||
elem: db.acquireSnapshot(),
|
||||
}
|
||||
atomic.AddInt32(&db.aliveSnaps, 1)
|
||||
runtime.SetFinalizer(snap, (*Snapshot).Release)
|
||||
return snap
|
||||
}
|
||||
|
||||
func (snap *Snapshot) String() string {
|
||||
return fmt.Sprintf("leveldb.Snapshot{%d}", snap.elem.seq)
|
||||
}
|
||||
|
||||
// Get gets the value for the given key. It returns ErrNotFound if
|
||||
// the DB does not contains the key.
|
||||
//
|
||||
// The caller should not modify the contents of the returned slice, but
|
||||
// it is safe to modify the contents of the argument after Get returns.
|
||||
func (snap *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
|
||||
err = snap.db.ok()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
snap.mu.RLock()
|
||||
defer snap.mu.RUnlock()
|
||||
if snap.released {
|
||||
err = ErrSnapshotReleased
|
||||
return
|
||||
}
|
||||
return snap.db.get(nil, nil, key, snap.elem.seq, ro)
|
||||
}
|
||||
|
||||
// Has returns true if the DB does contains the given key.
|
||||
//
|
||||
// It is safe to modify the contents of the argument after Get returns.
|
||||
func (snap *Snapshot) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) {
|
||||
err = snap.db.ok()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
snap.mu.RLock()
|
||||
defer snap.mu.RUnlock()
|
||||
if snap.released {
|
||||
err = ErrSnapshotReleased
|
||||
return
|
||||
}
|
||||
return snap.db.has(nil, nil, key, snap.elem.seq, ro)
|
||||
}
|
||||
|
||||
// NewIterator returns an iterator for the snapshot of the underlying DB.
|
||||
// The returned iterator is not safe for concurrent use, but it is safe to use
|
||||
// multiple iterators concurrently, with each in a dedicated goroutine.
|
||||
// It is also safe to use an iterator concurrently with modifying its
|
||||
// underlying DB. The resultant key/value pairs are guaranteed to be
|
||||
// consistent.
|
||||
//
|
||||
// Slice allows slicing the iterator to only contains keys in the given
|
||||
// range. A nil Range.Start is treated as a key before all keys in the
|
||||
// DB. And a nil Range.Limit is treated as a key after all keys in
|
||||
// the DB.
|
||||
//
|
||||
// WARNING: Any slice returned by interator (e.g. slice returned by calling
|
||||
// Iterator.Key() or Iterator.Value() methods), its content should not be
|
||||
// modified unless noted otherwise.
|
||||
//
|
||||
// The iterator must be released after use, by calling Release method.
|
||||
// Releasing the snapshot doesn't mean releasing the iterator too, the
|
||||
// iterator would be still valid until released.
|
||||
//
|
||||
// Also read Iterator documentation of the leveldb/iterator package.
|
||||
func (snap *Snapshot) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
|
||||
if err := snap.db.ok(); err != nil {
|
||||
return iterator.NewEmptyIterator(err)
|
||||
}
|
||||
snap.mu.Lock()
|
||||
defer snap.mu.Unlock()
|
||||
if snap.released {
|
||||
return iterator.NewEmptyIterator(ErrSnapshotReleased)
|
||||
}
|
||||
// Since iterator already hold version ref, it doesn't need to
|
||||
// hold snapshot ref.
|
||||
return snap.db.newIterator(nil, nil, snap.elem.seq, slice, ro)
|
||||
}
|
||||
|
||||
// Release releases the snapshot. This will not release any returned
|
||||
// iterators, the iterators would still be valid until released or the
|
||||
// underlying DB is closed.
|
||||
//
|
||||
// Other methods should not be called after the snapshot has been released.
|
||||
func (snap *Snapshot) Release() {
|
||||
snap.mu.Lock()
|
||||
defer snap.mu.Unlock()
|
||||
|
||||
if !snap.released {
|
||||
// Clear the finalizer.
|
||||
runtime.SetFinalizer(snap, nil)
|
||||
|
||||
snap.released = true
|
||||
snap.db.releaseSnapshot(snap.elem)
|
||||
atomic.AddInt32(&snap.db.aliveSnaps, -1)
|
||||
snap.db = nil
|
||||
snap.elem = nil
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,239 @@
|
||||
// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/journal"
|
||||
"github.com/syndtr/goleveldb/leveldb/memdb"
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
)
|
||||
|
||||
var (
|
||||
errHasFrozenMem = errors.New("has frozen mem")
|
||||
)
|
||||
|
||||
type memDB struct {
|
||||
db *DB
|
||||
*memdb.DB
|
||||
ref int32
|
||||
}
|
||||
|
||||
func (m *memDB) getref() int32 {
|
||||
return atomic.LoadInt32(&m.ref)
|
||||
}
|
||||
|
||||
func (m *memDB) incref() {
|
||||
atomic.AddInt32(&m.ref, 1)
|
||||
}
|
||||
|
||||
func (m *memDB) decref() {
|
||||
if ref := atomic.AddInt32(&m.ref, -1); ref == 0 {
|
||||
// Only put back memdb with std capacity.
|
||||
if m.Capacity() == m.db.s.o.GetWriteBuffer() {
|
||||
m.Reset()
|
||||
m.db.mpoolPut(m.DB)
|
||||
}
|
||||
m.db = nil
|
||||
m.DB = nil
|
||||
} else if ref < 0 {
|
||||
panic("negative memdb ref")
|
||||
}
|
||||
}
|
||||
|
||||
// Get latest sequence number.
|
||||
func (db *DB) getSeq() uint64 {
|
||||
return atomic.LoadUint64(&db.seq)
|
||||
}
|
||||
|
||||
// Atomically adds delta to seq.
|
||||
func (db *DB) addSeq(delta uint64) {
|
||||
atomic.AddUint64(&db.seq, delta)
|
||||
}
|
||||
|
||||
func (db *DB) setSeq(seq uint64) {
|
||||
atomic.StoreUint64(&db.seq, seq)
|
||||
}
|
||||
|
||||
func (db *DB) sampleSeek(ikey internalKey) {
|
||||
v := db.s.version()
|
||||
if v.sampleSeek(ikey) {
|
||||
// Trigger table compaction.
|
||||
db.compTrigger(db.tcompCmdC)
|
||||
}
|
||||
v.release()
|
||||
}
|
||||
|
||||
func (db *DB) mpoolPut(mem *memdb.DB) {
|
||||
if !db.isClosed() {
|
||||
select {
|
||||
case db.memPool <- mem:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (db *DB) mpoolGet(n int) *memDB {
|
||||
var mdb *memdb.DB
|
||||
select {
|
||||
case mdb = <-db.memPool:
|
||||
default:
|
||||
}
|
||||
if mdb == nil || mdb.Capacity() < n {
|
||||
mdb = memdb.New(db.s.icmp, maxInt(db.s.o.GetWriteBuffer(), n))
|
||||
}
|
||||
return &memDB{
|
||||
db: db,
|
||||
DB: mdb,
|
||||
}
|
||||
}
|
||||
|
||||
func (db *DB) mpoolDrain() {
|
||||
ticker := time.NewTicker(30 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
select {
|
||||
case <-db.memPool:
|
||||
default:
|
||||
}
|
||||
case <-db.closeC:
|
||||
ticker.Stop()
|
||||
// Make sure the pool is drained.
|
||||
select {
|
||||
case <-db.memPool:
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
close(db.memPool)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create new memdb and froze the old one; need external synchronization.
|
||||
// newMem only called synchronously by the writer.
|
||||
func (db *DB) newMem(n int) (mem *memDB, err error) {
|
||||
fd := storage.FileDesc{Type: storage.TypeJournal, Num: db.s.allocFileNum()}
|
||||
w, err := db.s.stor.Create(fd)
|
||||
if err != nil {
|
||||
db.s.reuseFileNum(fd.Num)
|
||||
return
|
||||
}
|
||||
|
||||
db.memMu.Lock()
|
||||
defer db.memMu.Unlock()
|
||||
|
||||
if db.frozenMem != nil {
|
||||
return nil, errHasFrozenMem
|
||||
}
|
||||
|
||||
if db.journal == nil {
|
||||
db.journal = journal.NewWriter(w)
|
||||
} else {
|
||||
db.journal.Reset(w)
|
||||
db.journalWriter.Close()
|
||||
db.frozenJournalFd = db.journalFd
|
||||
}
|
||||
db.journalWriter = w
|
||||
db.journalFd = fd
|
||||
db.frozenMem = db.mem
|
||||
mem = db.mpoolGet(n)
|
||||
mem.incref() // for self
|
||||
mem.incref() // for caller
|
||||
db.mem = mem
|
||||
// The seq only incremented by the writer. And whoever called newMem
|
||||
// should hold write lock, so no need additional synchronization here.
|
||||
db.frozenSeq = db.seq
|
||||
return
|
||||
}
|
||||
|
||||
// Get all memdbs.
|
||||
func (db *DB) getMems() (e, f *memDB) {
|
||||
db.memMu.RLock()
|
||||
defer db.memMu.RUnlock()
|
||||
if db.mem != nil {
|
||||
db.mem.incref()
|
||||
} else if !db.isClosed() {
|
||||
panic("nil effective mem")
|
||||
}
|
||||
if db.frozenMem != nil {
|
||||
db.frozenMem.incref()
|
||||
}
|
||||
return db.mem, db.frozenMem
|
||||
}
|
||||
|
||||
// Get effective memdb.
|
||||
func (db *DB) getEffectiveMem() *memDB {
|
||||
db.memMu.RLock()
|
||||
defer db.memMu.RUnlock()
|
||||
if db.mem != nil {
|
||||
db.mem.incref()
|
||||
} else if !db.isClosed() {
|
||||
panic("nil effective mem")
|
||||
}
|
||||
return db.mem
|
||||
}
|
||||
|
||||
// Check whether we has frozen memdb.
|
||||
func (db *DB) hasFrozenMem() bool {
|
||||
db.memMu.RLock()
|
||||
defer db.memMu.RUnlock()
|
||||
return db.frozenMem != nil
|
||||
}
|
||||
|
||||
// Get frozen memdb.
|
||||
func (db *DB) getFrozenMem() *memDB {
|
||||
db.memMu.RLock()
|
||||
defer db.memMu.RUnlock()
|
||||
if db.frozenMem != nil {
|
||||
db.frozenMem.incref()
|
||||
}
|
||||
return db.frozenMem
|
||||
}
|
||||
|
||||
// Drop frozen memdb; assume that frozen memdb isn't nil.
|
||||
func (db *DB) dropFrozenMem() {
|
||||
db.memMu.Lock()
|
||||
if err := db.s.stor.Remove(db.frozenJournalFd); err != nil {
|
||||
db.logf("journal@remove removing @%d %q", db.frozenJournalFd.Num, err)
|
||||
} else {
|
||||
db.logf("journal@remove removed @%d", db.frozenJournalFd.Num)
|
||||
}
|
||||
db.frozenJournalFd = storage.FileDesc{}
|
||||
db.frozenMem.decref()
|
||||
db.frozenMem = nil
|
||||
db.memMu.Unlock()
|
||||
}
|
||||
|
||||
// Clear mems ptr; used by DB.Close().
|
||||
func (db *DB) clearMems() {
|
||||
db.memMu.Lock()
|
||||
db.mem = nil
|
||||
db.frozenMem = nil
|
||||
db.memMu.Unlock()
|
||||
}
|
||||
|
||||
// Set closed flag; return true if not already closed.
|
||||
func (db *DB) setClosed() bool {
|
||||
return atomic.CompareAndSwapUint32(&db.closed, 0, 1)
|
||||
}
|
||||
|
||||
// Check whether DB was closed.
|
||||
func (db *DB) isClosed() bool {
|
||||
return atomic.LoadUint32(&db.closed) != 0
|
||||
}
|
||||
|
||||
// Check read ok status.
|
||||
func (db *DB) ok() error {
|
||||
if db.isClosed() {
|
||||
return ErrClosed
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,329 @@
|
||||
// Copyright (c) 2016, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
var errTransactionDone = errors.New("leveldb: transaction already closed")
|
||||
|
||||
// Transaction is the transaction handle.
|
||||
type Transaction struct {
|
||||
db *DB
|
||||
lk sync.RWMutex
|
||||
seq uint64
|
||||
mem *memDB
|
||||
tables tFiles
|
||||
ikScratch []byte
|
||||
rec sessionRecord
|
||||
stats cStatStaging
|
||||
closed bool
|
||||
}
|
||||
|
||||
// Get gets the value for the given key. It returns ErrNotFound if the
|
||||
// DB does not contains the key.
|
||||
//
|
||||
// The returned slice is its own copy, it is safe to modify the contents
|
||||
// of the returned slice.
|
||||
// It is safe to modify the contents of the argument after Get returns.
|
||||
func (tr *Transaction) Get(key []byte, ro *opt.ReadOptions) ([]byte, error) {
|
||||
tr.lk.RLock()
|
||||
defer tr.lk.RUnlock()
|
||||
if tr.closed {
|
||||
return nil, errTransactionDone
|
||||
}
|
||||
return tr.db.get(tr.mem.DB, tr.tables, key, tr.seq, ro)
|
||||
}
|
||||
|
||||
// Has returns true if the DB does contains the given key.
|
||||
//
|
||||
// It is safe to modify the contents of the argument after Has returns.
|
||||
func (tr *Transaction) Has(key []byte, ro *opt.ReadOptions) (bool, error) {
|
||||
tr.lk.RLock()
|
||||
defer tr.lk.RUnlock()
|
||||
if tr.closed {
|
||||
return false, errTransactionDone
|
||||
}
|
||||
return tr.db.has(tr.mem.DB, tr.tables, key, tr.seq, ro)
|
||||
}
|
||||
|
||||
// NewIterator returns an iterator for the latest snapshot of the transaction.
|
||||
// The returned iterator is not safe for concurrent use, but it is safe to use
|
||||
// multiple iterators concurrently, with each in a dedicated goroutine.
|
||||
// It is also safe to use an iterator concurrently while writes to the
|
||||
// transaction. The resultant key/value pairs are guaranteed to be consistent.
|
||||
//
|
||||
// Slice allows slicing the iterator to only contains keys in the given
|
||||
// range. A nil Range.Start is treated as a key before all keys in the
|
||||
// DB. And a nil Range.Limit is treated as a key after all keys in
|
||||
// the DB.
|
||||
//
|
||||
// WARNING: Any slice returned by interator (e.g. slice returned by calling
|
||||
// Iterator.Key() or Iterator.Key() methods), its content should not be modified
|
||||
// unless noted otherwise.
|
||||
//
|
||||
// The iterator must be released after use, by calling Release method.
|
||||
//
|
||||
// Also read Iterator documentation of the leveldb/iterator package.
|
||||
func (tr *Transaction) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
|
||||
tr.lk.RLock()
|
||||
defer tr.lk.RUnlock()
|
||||
if tr.closed {
|
||||
return iterator.NewEmptyIterator(errTransactionDone)
|
||||
}
|
||||
tr.mem.incref()
|
||||
return tr.db.newIterator(tr.mem, tr.tables, tr.seq, slice, ro)
|
||||
}
|
||||
|
||||
func (tr *Transaction) flush() error {
|
||||
// Flush memdb.
|
||||
if tr.mem.Len() != 0 {
|
||||
tr.stats.startTimer()
|
||||
iter := tr.mem.NewIterator(nil)
|
||||
t, n, err := tr.db.s.tops.createFrom(iter)
|
||||
iter.Release()
|
||||
tr.stats.stopTimer()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tr.mem.getref() == 1 {
|
||||
tr.mem.Reset()
|
||||
} else {
|
||||
tr.mem.decref()
|
||||
tr.mem = tr.db.mpoolGet(0)
|
||||
tr.mem.incref()
|
||||
}
|
||||
tr.tables = append(tr.tables, t)
|
||||
tr.rec.addTableFile(0, t)
|
||||
tr.stats.write += t.size
|
||||
tr.db.logf("transaction@flush created L0@%d N·%d S·%s %q:%q", t.fd.Num, n, shortenb(int(t.size)), t.imin, t.imax)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tr *Transaction) put(kt keyType, key, value []byte) error {
|
||||
tr.ikScratch = makeInternalKey(tr.ikScratch, key, tr.seq+1, kt)
|
||||
if tr.mem.Free() < len(tr.ikScratch)+len(value) {
|
||||
if err := tr.flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := tr.mem.Put(tr.ikScratch, value); err != nil {
|
||||
return err
|
||||
}
|
||||
tr.seq++
|
||||
return nil
|
||||
}
|
||||
|
||||
// Put sets the value for the given key. It overwrites any previous value
|
||||
// for that key; a DB is not a multi-map.
|
||||
// Please note that the transaction is not compacted until committed, so if you
|
||||
// writes 10 same keys, then those 10 same keys are in the transaction.
|
||||
//
|
||||
// It is safe to modify the contents of the arguments after Put returns.
|
||||
func (tr *Transaction) Put(key, value []byte, wo *opt.WriteOptions) error {
|
||||
tr.lk.Lock()
|
||||
defer tr.lk.Unlock()
|
||||
if tr.closed {
|
||||
return errTransactionDone
|
||||
}
|
||||
return tr.put(keyTypeVal, key, value)
|
||||
}
|
||||
|
||||
// Delete deletes the value for the given key.
|
||||
// Please note that the transaction is not compacted until committed, so if you
|
||||
// writes 10 same keys, then those 10 same keys are in the transaction.
|
||||
//
|
||||
// It is safe to modify the contents of the arguments after Delete returns.
|
||||
func (tr *Transaction) Delete(key []byte, wo *opt.WriteOptions) error {
|
||||
tr.lk.Lock()
|
||||
defer tr.lk.Unlock()
|
||||
if tr.closed {
|
||||
return errTransactionDone
|
||||
}
|
||||
return tr.put(keyTypeDel, key, nil)
|
||||
}
|
||||
|
||||
// Write apply the given batch to the transaction. The batch will be applied
|
||||
// sequentially.
|
||||
// Please note that the transaction is not compacted until committed, so if you
|
||||
// writes 10 same keys, then those 10 same keys are in the transaction.
|
||||
//
|
||||
// It is safe to modify the contents of the arguments after Write returns.
|
||||
func (tr *Transaction) Write(b *Batch, wo *opt.WriteOptions) error {
|
||||
if b == nil || b.Len() == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
tr.lk.Lock()
|
||||
defer tr.lk.Unlock()
|
||||
if tr.closed {
|
||||
return errTransactionDone
|
||||
}
|
||||
return b.replayInternal(func(i int, kt keyType, k, v []byte) error {
|
||||
return tr.put(kt, k, v)
|
||||
})
|
||||
}
|
||||
|
||||
func (tr *Transaction) setDone() {
|
||||
tr.closed = true
|
||||
tr.db.tr = nil
|
||||
tr.mem.decref()
|
||||
<-tr.db.writeLockC
|
||||
}
|
||||
|
||||
// Commit commits the transaction. If error is not nil, then the transaction is
|
||||
// not committed, it can then either be retried or discarded.
|
||||
//
|
||||
// Other methods should not be called after transaction has been committed.
|
||||
func (tr *Transaction) Commit() error {
|
||||
if err := tr.db.ok(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tr.lk.Lock()
|
||||
defer tr.lk.Unlock()
|
||||
if tr.closed {
|
||||
return errTransactionDone
|
||||
}
|
||||
if err := tr.flush(); err != nil {
|
||||
// Return error, lets user decide either to retry or discard
|
||||
// transaction.
|
||||
return err
|
||||
}
|
||||
if len(tr.tables) != 0 {
|
||||
// Committing transaction.
|
||||
tr.rec.setSeqNum(tr.seq)
|
||||
tr.db.compCommitLk.Lock()
|
||||
tr.stats.startTimer()
|
||||
var cerr error
|
||||
for retry := 0; retry < 3; retry++ {
|
||||
cerr = tr.db.s.commit(&tr.rec)
|
||||
if cerr != nil {
|
||||
tr.db.logf("transaction@commit error R·%d %q", retry, cerr)
|
||||
select {
|
||||
case <-time.After(time.Second):
|
||||
case <-tr.db.closeC:
|
||||
tr.db.logf("transaction@commit exiting")
|
||||
tr.db.compCommitLk.Unlock()
|
||||
return cerr
|
||||
}
|
||||
} else {
|
||||
// Success. Set db.seq.
|
||||
tr.db.setSeq(tr.seq)
|
||||
break
|
||||
}
|
||||
}
|
||||
tr.stats.stopTimer()
|
||||
if cerr != nil {
|
||||
// Return error, lets user decide either to retry or discard
|
||||
// transaction.
|
||||
return cerr
|
||||
}
|
||||
|
||||
// Update compaction stats. This is safe as long as we hold compCommitLk.
|
||||
tr.db.compStats.addStat(0, &tr.stats)
|
||||
|
||||
// Trigger table auto-compaction.
|
||||
tr.db.compTrigger(tr.db.tcompCmdC)
|
||||
tr.db.compCommitLk.Unlock()
|
||||
|
||||
// Additionally, wait compaction when certain threshold reached.
|
||||
// Ignore error, returns error only if transaction can't be committed.
|
||||
tr.db.waitCompaction()
|
||||
}
|
||||
// Only mark as done if transaction committed successfully.
|
||||
tr.setDone()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tr *Transaction) discard() {
|
||||
// Discard transaction.
|
||||
for _, t := range tr.tables {
|
||||
tr.db.logf("transaction@discard @%d", t.fd.Num)
|
||||
if err1 := tr.db.s.stor.Remove(t.fd); err1 == nil {
|
||||
tr.db.s.reuseFileNum(t.fd.Num)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Discard discards the transaction.
|
||||
//
|
||||
// Other methods should not be called after transaction has been discarded.
|
||||
func (tr *Transaction) Discard() {
|
||||
tr.lk.Lock()
|
||||
if !tr.closed {
|
||||
tr.discard()
|
||||
tr.setDone()
|
||||
}
|
||||
tr.lk.Unlock()
|
||||
}
|
||||
|
||||
func (db *DB) waitCompaction() error {
|
||||
if db.s.tLen(0) >= db.s.o.GetWriteL0PauseTrigger() {
|
||||
return db.compTriggerWait(db.tcompCmdC)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// OpenTransaction opens an atomic DB transaction. Only one transaction can be
|
||||
// opened at a time. Subsequent call to Write and OpenTransaction will be blocked
|
||||
// until in-flight transaction is committed or discarded.
|
||||
// The returned transaction handle is safe for concurrent use.
|
||||
//
|
||||
// Transaction is expensive and can overwhelm compaction, especially if
|
||||
// transaction size is small. Use with caution.
|
||||
//
|
||||
// The transaction must be closed once done, either by committing or discarding
|
||||
// the transaction.
|
||||
// Closing the DB will discard open transaction.
|
||||
func (db *DB) OpenTransaction() (*Transaction, error) {
|
||||
if err := db.ok(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// The write happen synchronously.
|
||||
select {
|
||||
case db.writeLockC <- struct{}{}:
|
||||
case err := <-db.compPerErrC:
|
||||
return nil, err
|
||||
case <-db.closeC:
|
||||
return nil, ErrClosed
|
||||
}
|
||||
|
||||
if db.tr != nil {
|
||||
panic("leveldb: has open transaction")
|
||||
}
|
||||
|
||||
// Flush current memdb.
|
||||
if db.mem != nil && db.mem.Len() != 0 {
|
||||
if _, err := db.rotateMem(0, true); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Wait compaction when certain threshold reached.
|
||||
if err := db.waitCompaction(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tr := &Transaction{
|
||||
db: db,
|
||||
seq: db.seq,
|
||||
mem: db.mpoolGet(0),
|
||||
}
|
||||
tr.mem.incref()
|
||||
db.tr = tr
|
||||
return tr, nil
|
||||
}
|
||||
@ -0,0 +1,102 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"github.com/syndtr/goleveldb/leveldb/errors"
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
// Reader is the interface that wraps basic Get and NewIterator methods.
|
||||
// This interface implemented by both DB and Snapshot.
|
||||
type Reader interface {
|
||||
Get(key []byte, ro *opt.ReadOptions) (value []byte, err error)
|
||||
NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator
|
||||
}
|
||||
|
||||
// Sizes is list of size.
|
||||
type Sizes []int64
|
||||
|
||||
// Sum returns sum of the sizes.
|
||||
func (sizes Sizes) Sum() int64 {
|
||||
var sum int64
|
||||
for _, size := range sizes {
|
||||
sum += size
|
||||
}
|
||||
return sum
|
||||
}
|
||||
|
||||
// Logging.
|
||||
func (db *DB) log(v ...interface{}) { db.s.log(v...) }
|
||||
func (db *DB) logf(format string, v ...interface{}) { db.s.logf(format, v...) }
|
||||
|
||||
// Check and clean files.
|
||||
func (db *DB) checkAndCleanFiles() error {
|
||||
v := db.s.version()
|
||||
defer v.release()
|
||||
|
||||
tmap := make(map[int64]bool)
|
||||
for _, tables := range v.levels {
|
||||
for _, t := range tables {
|
||||
tmap[t.fd.Num] = false
|
||||
}
|
||||
}
|
||||
|
||||
fds, err := db.s.stor.List(storage.TypeAll)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var nt int
|
||||
var rem []storage.FileDesc
|
||||
for _, fd := range fds {
|
||||
keep := true
|
||||
switch fd.Type {
|
||||
case storage.TypeManifest:
|
||||
keep = fd.Num >= db.s.manifestFd.Num
|
||||
case storage.TypeJournal:
|
||||
if !db.frozenJournalFd.Zero() {
|
||||
keep = fd.Num >= db.frozenJournalFd.Num
|
||||
} else {
|
||||
keep = fd.Num >= db.journalFd.Num
|
||||
}
|
||||
case storage.TypeTable:
|
||||
_, keep = tmap[fd.Num]
|
||||
if keep {
|
||||
tmap[fd.Num] = true
|
||||
nt++
|
||||
}
|
||||
}
|
||||
|
||||
if !keep {
|
||||
rem = append(rem, fd)
|
||||
}
|
||||
}
|
||||
|
||||
if nt != len(tmap) {
|
||||
var mfds []storage.FileDesc
|
||||
for num, present := range tmap {
|
||||
if !present {
|
||||
mfds = append(mfds, storage.FileDesc{Type: storage.TypeTable, Num: num})
|
||||
db.logf("db@janitor table missing @%d", num)
|
||||
}
|
||||
}
|
||||
return errors.NewErrCorrupted(storage.FileDesc{}, &errors.ErrMissingFiles{Fds: mfds})
|
||||
}
|
||||
|
||||
db.logf("db@janitor F·%d G·%d", len(fds), len(rem))
|
||||
for _, fd := range rem {
|
||||
db.logf("db@janitor removing %s-%d", fd.Type, fd.Num)
|
||||
if err := db.s.stor.Remove(fd); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,464 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/memdb"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
func (db *DB) writeJournal(batches []*Batch, seq uint64, sync bool) error {
|
||||
wr, err := db.journal.Next()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeBatchesWithHeader(wr, batches, seq); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := db.journal.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
if sync {
|
||||
return db.journalWriter.Sync()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *DB) rotateMem(n int, wait bool) (mem *memDB, err error) {
|
||||
retryLimit := 3
|
||||
retry:
|
||||
// Wait for pending memdb compaction.
|
||||
err = db.compTriggerWait(db.mcompCmdC)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
retryLimit--
|
||||
|
||||
// Create new memdb and journal.
|
||||
mem, err = db.newMem(n)
|
||||
if err != nil {
|
||||
if err == errHasFrozenMem {
|
||||
if retryLimit <= 0 {
|
||||
panic("BUG: still has frozen memdb")
|
||||
}
|
||||
goto retry
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Schedule memdb compaction.
|
||||
if wait {
|
||||
err = db.compTriggerWait(db.mcompCmdC)
|
||||
} else {
|
||||
db.compTrigger(db.mcompCmdC)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (db *DB) flush(n int) (mdb *memDB, mdbFree int, err error) {
|
||||
delayed := false
|
||||
slowdownTrigger := db.s.o.GetWriteL0SlowdownTrigger()
|
||||
pauseTrigger := db.s.o.GetWriteL0PauseTrigger()
|
||||
flush := func() (retry bool) {
|
||||
mdb = db.getEffectiveMem()
|
||||
if mdb == nil {
|
||||
err = ErrClosed
|
||||
return false
|
||||
}
|
||||
defer func() {
|
||||
if retry {
|
||||
mdb.decref()
|
||||
mdb = nil
|
||||
}
|
||||
}()
|
||||
tLen := db.s.tLen(0)
|
||||
mdbFree = mdb.Free()
|
||||
switch {
|
||||
case tLen >= slowdownTrigger && !delayed:
|
||||
delayed = true
|
||||
time.Sleep(time.Millisecond)
|
||||
case mdbFree >= n:
|
||||
return false
|
||||
case tLen >= pauseTrigger:
|
||||
delayed = true
|
||||
// Set the write paused flag explicitly.
|
||||
atomic.StoreInt32(&db.inWritePaused, 1)
|
||||
err = db.compTriggerWait(db.tcompCmdC)
|
||||
// Unset the write paused flag.
|
||||
atomic.StoreInt32(&db.inWritePaused, 0)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
default:
|
||||
// Allow memdb to grow if it has no entry.
|
||||
if mdb.Len() == 0 {
|
||||
mdbFree = n
|
||||
} else {
|
||||
mdb.decref()
|
||||
mdb, err = db.rotateMem(n, false)
|
||||
if err == nil {
|
||||
mdbFree = mdb.Free()
|
||||
} else {
|
||||
mdbFree = 0
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
start := time.Now()
|
||||
for flush() {
|
||||
}
|
||||
if delayed {
|
||||
db.writeDelay += time.Since(start)
|
||||
db.writeDelayN++
|
||||
} else if db.writeDelayN > 0 {
|
||||
db.logf("db@write was delayed N·%d T·%v", db.writeDelayN, db.writeDelay)
|
||||
atomic.AddInt32(&db.cWriteDelayN, int32(db.writeDelayN))
|
||||
atomic.AddInt64(&db.cWriteDelay, int64(db.writeDelay))
|
||||
db.writeDelay = 0
|
||||
db.writeDelayN = 0
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type writeMerge struct {
|
||||
sync bool
|
||||
batch *Batch
|
||||
keyType keyType
|
||||
key, value []byte
|
||||
}
|
||||
|
||||
func (db *DB) unlockWrite(overflow bool, merged int, err error) {
|
||||
for i := 0; i < merged; i++ {
|
||||
db.writeAckC <- err
|
||||
}
|
||||
if overflow {
|
||||
// Pass lock to the next write (that failed to merge).
|
||||
db.writeMergedC <- false
|
||||
} else {
|
||||
// Release lock.
|
||||
<-db.writeLockC
|
||||
}
|
||||
}
|
||||
|
||||
// ourBatch is batch that we can modify.
|
||||
func (db *DB) writeLocked(batch, ourBatch *Batch, merge, sync bool) error {
|
||||
// Try to flush memdb. This method would also trying to throttle writes
|
||||
// if it is too fast and compaction cannot catch-up.
|
||||
mdb, mdbFree, err := db.flush(batch.internalLen)
|
||||
if err != nil {
|
||||
db.unlockWrite(false, 0, err)
|
||||
return err
|
||||
}
|
||||
defer mdb.decref()
|
||||
|
||||
var (
|
||||
overflow bool
|
||||
merged int
|
||||
batches = []*Batch{batch}
|
||||
)
|
||||
|
||||
if merge {
|
||||
// Merge limit.
|
||||
var mergeLimit int
|
||||
if batch.internalLen > 128<<10 {
|
||||
mergeLimit = (1 << 20) - batch.internalLen
|
||||
} else {
|
||||
mergeLimit = 128 << 10
|
||||
}
|
||||
mergeCap := mdbFree - batch.internalLen
|
||||
if mergeLimit > mergeCap {
|
||||
mergeLimit = mergeCap
|
||||
}
|
||||
|
||||
merge:
|
||||
for mergeLimit > 0 {
|
||||
select {
|
||||
case incoming := <-db.writeMergeC:
|
||||
if incoming.batch != nil {
|
||||
// Merge batch.
|
||||
if incoming.batch.internalLen > mergeLimit {
|
||||
overflow = true
|
||||
break merge
|
||||
}
|
||||
batches = append(batches, incoming.batch)
|
||||
mergeLimit -= incoming.batch.internalLen
|
||||
} else {
|
||||
// Merge put.
|
||||
internalLen := len(incoming.key) + len(incoming.value) + 8
|
||||
if internalLen > mergeLimit {
|
||||
overflow = true
|
||||
break merge
|
||||
}
|
||||
if ourBatch == nil {
|
||||
ourBatch = db.batchPool.Get().(*Batch)
|
||||
ourBatch.Reset()
|
||||
batches = append(batches, ourBatch)
|
||||
}
|
||||
// We can use same batch since concurrent write doesn't
|
||||
// guarantee write order.
|
||||
ourBatch.appendRec(incoming.keyType, incoming.key, incoming.value)
|
||||
mergeLimit -= internalLen
|
||||
}
|
||||
sync = sync || incoming.sync
|
||||
merged++
|
||||
db.writeMergedC <- true
|
||||
|
||||
default:
|
||||
break merge
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Release ourBatch if any.
|
||||
if ourBatch != nil {
|
||||
defer db.batchPool.Put(ourBatch)
|
||||
}
|
||||
|
||||
// Seq number.
|
||||
seq := db.seq + 1
|
||||
|
||||
// Write journal.
|
||||
if err := db.writeJournal(batches, seq, sync); err != nil {
|
||||
db.unlockWrite(overflow, merged, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Put batches.
|
||||
for _, batch := range batches {
|
||||
if err := batch.putMem(seq, mdb.DB); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
seq += uint64(batch.Len())
|
||||
}
|
||||
|
||||
// Incr seq number.
|
||||
db.addSeq(uint64(batchesLen(batches)))
|
||||
|
||||
// Rotate memdb if it's reach the threshold.
|
||||
if batch.internalLen >= mdbFree {
|
||||
db.rotateMem(0, false)
|
||||
}
|
||||
|
||||
db.unlockWrite(overflow, merged, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write apply the given batch to the DB. The batch records will be applied
|
||||
// sequentially. Write might be used concurrently, when used concurrently and
|
||||
// batch is small enough, write will try to merge the batches. Set NoWriteMerge
|
||||
// option to true to disable write merge.
|
||||
//
|
||||
// It is safe to modify the contents of the arguments after Write returns but
|
||||
// not before. Write will not modify content of the batch.
|
||||
func (db *DB) Write(batch *Batch, wo *opt.WriteOptions) error {
|
||||
if err := db.ok(); err != nil || batch == nil || batch.Len() == 0 {
|
||||
return err
|
||||
}
|
||||
|
||||
// If the batch size is larger than write buffer, it may justified to write
|
||||
// using transaction instead. Using transaction the batch will be written
|
||||
// into tables directly, skipping the journaling.
|
||||
if batch.internalLen > db.s.o.GetWriteBuffer() && !db.s.o.GetDisableLargeBatchTransaction() {
|
||||
tr, err := db.OpenTransaction()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tr.Write(batch, wo); err != nil {
|
||||
tr.Discard()
|
||||
return err
|
||||
}
|
||||
return tr.Commit()
|
||||
}
|
||||
|
||||
merge := !wo.GetNoWriteMerge() && !db.s.o.GetNoWriteMerge()
|
||||
sync := wo.GetSync() && !db.s.o.GetNoSync()
|
||||
|
||||
// Acquire write lock.
|
||||
if merge {
|
||||
select {
|
||||
case db.writeMergeC <- writeMerge{sync: sync, batch: batch}:
|
||||
if <-db.writeMergedC {
|
||||
// Write is merged.
|
||||
return <-db.writeAckC
|
||||
}
|
||||
// Write is not merged, the write lock is handed to us. Continue.
|
||||
case db.writeLockC <- struct{}{}:
|
||||
// Write lock acquired.
|
||||
case err := <-db.compPerErrC:
|
||||
// Compaction error.
|
||||
return err
|
||||
case <-db.closeC:
|
||||
// Closed
|
||||
return ErrClosed
|
||||
}
|
||||
} else {
|
||||
select {
|
||||
case db.writeLockC <- struct{}{}:
|
||||
// Write lock acquired.
|
||||
case err := <-db.compPerErrC:
|
||||
// Compaction error.
|
||||
return err
|
||||
case <-db.closeC:
|
||||
// Closed
|
||||
return ErrClosed
|
||||
}
|
||||
}
|
||||
|
||||
return db.writeLocked(batch, nil, merge, sync)
|
||||
}
|
||||
|
||||
func (db *DB) putRec(kt keyType, key, value []byte, wo *opt.WriteOptions) error {
|
||||
if err := db.ok(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
merge := !wo.GetNoWriteMerge() && !db.s.o.GetNoWriteMerge()
|
||||
sync := wo.GetSync() && !db.s.o.GetNoSync()
|
||||
|
||||
// Acquire write lock.
|
||||
if merge {
|
||||
select {
|
||||
case db.writeMergeC <- writeMerge{sync: sync, keyType: kt, key: key, value: value}:
|
||||
if <-db.writeMergedC {
|
||||
// Write is merged.
|
||||
return <-db.writeAckC
|
||||
}
|
||||
// Write is not merged, the write lock is handed to us. Continue.
|
||||
case db.writeLockC <- struct{}{}:
|
||||
// Write lock acquired.
|
||||
case err := <-db.compPerErrC:
|
||||
// Compaction error.
|
||||
return err
|
||||
case <-db.closeC:
|
||||
// Closed
|
||||
return ErrClosed
|
||||
}
|
||||
} else {
|
||||
select {
|
||||
case db.writeLockC <- struct{}{}:
|
||||
// Write lock acquired.
|
||||
case err := <-db.compPerErrC:
|
||||
// Compaction error.
|
||||
return err
|
||||
case <-db.closeC:
|
||||
// Closed
|
||||
return ErrClosed
|
||||
}
|
||||
}
|
||||
|
||||
batch := db.batchPool.Get().(*Batch)
|
||||
batch.Reset()
|
||||
batch.appendRec(kt, key, value)
|
||||
return db.writeLocked(batch, batch, merge, sync)
|
||||
}
|
||||
|
||||
// Put sets the value for the given key. It overwrites any previous value
|
||||
// for that key; a DB is not a multi-map. Write merge also applies for Put, see
|
||||
// Write.
|
||||
//
|
||||
// It is safe to modify the contents of the arguments after Put returns but not
|
||||
// before.
|
||||
func (db *DB) Put(key, value []byte, wo *opt.WriteOptions) error {
|
||||
return db.putRec(keyTypeVal, key, value, wo)
|
||||
}
|
||||
|
||||
// Delete deletes the value for the given key. Delete will not returns error if
|
||||
// key doesn't exist. Write merge also applies for Delete, see Write.
|
||||
//
|
||||
// It is safe to modify the contents of the arguments after Delete returns but
|
||||
// not before.
|
||||
func (db *DB) Delete(key []byte, wo *opt.WriteOptions) error {
|
||||
return db.putRec(keyTypeDel, key, nil, wo)
|
||||
}
|
||||
|
||||
func isMemOverlaps(icmp *iComparer, mem *memdb.DB, min, max []byte) bool {
|
||||
iter := mem.NewIterator(nil)
|
||||
defer iter.Release()
|
||||
return (max == nil || (iter.First() && icmp.uCompare(max, internalKey(iter.Key()).ukey()) >= 0)) &&
|
||||
(min == nil || (iter.Last() && icmp.uCompare(min, internalKey(iter.Key()).ukey()) <= 0))
|
||||
}
|
||||
|
||||
// CompactRange compacts the underlying DB for the given key range.
|
||||
// In particular, deleted and overwritten versions are discarded,
|
||||
// and the data is rearranged to reduce the cost of operations
|
||||
// needed to access the data. This operation should typically only
|
||||
// be invoked by users who understand the underlying implementation.
|
||||
//
|
||||
// A nil Range.Start is treated as a key before all keys in the DB.
|
||||
// And a nil Range.Limit is treated as a key after all keys in the DB.
|
||||
// Therefore if both is nil then it will compact entire DB.
|
||||
func (db *DB) CompactRange(r util.Range) error {
|
||||
if err := db.ok(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Lock writer.
|
||||
select {
|
||||
case db.writeLockC <- struct{}{}:
|
||||
case err := <-db.compPerErrC:
|
||||
return err
|
||||
case <-db.closeC:
|
||||
return ErrClosed
|
||||
}
|
||||
|
||||
// Check for overlaps in memdb.
|
||||
mdb := db.getEffectiveMem()
|
||||
if mdb == nil {
|
||||
return ErrClosed
|
||||
}
|
||||
defer mdb.decref()
|
||||
if isMemOverlaps(db.s.icmp, mdb.DB, r.Start, r.Limit) {
|
||||
// Memdb compaction.
|
||||
if _, err := db.rotateMem(0, false); err != nil {
|
||||
<-db.writeLockC
|
||||
return err
|
||||
}
|
||||
<-db.writeLockC
|
||||
if err := db.compTriggerWait(db.mcompCmdC); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
<-db.writeLockC
|
||||
}
|
||||
|
||||
// Table compaction.
|
||||
return db.compTriggerRange(db.tcompCmdC, -1, r.Start, r.Limit)
|
||||
}
|
||||
|
||||
// SetReadOnly makes DB read-only. It will stay read-only until reopened.
|
||||
func (db *DB) SetReadOnly() error {
|
||||
if err := db.ok(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Lock writer.
|
||||
select {
|
||||
case db.writeLockC <- struct{}{}:
|
||||
db.compWriteLocking = true
|
||||
case err := <-db.compPerErrC:
|
||||
return err
|
||||
case <-db.closeC:
|
||||
return ErrClosed
|
||||
}
|
||||
|
||||
// Set compaction read-only.
|
||||
select {
|
||||
case db.compErrSetC <- ErrReadOnly:
|
||||
case perr := <-db.compPerErrC:
|
||||
return perr
|
||||
case <-db.closeC:
|
||||
return ErrClosed
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,92 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package leveldb provides implementation of LevelDB key/value database.
|
||||
//
|
||||
// Create or open a database:
|
||||
//
|
||||
// // The returned DB instance is safe for concurrent use. Which mean that all
|
||||
// // DB's methods may be called concurrently from multiple goroutine.
|
||||
// db, err := leveldb.OpenFile("path/to/db", nil)
|
||||
// ...
|
||||
// defer db.Close()
|
||||
// ...
|
||||
//
|
||||
// Read or modify the database content:
|
||||
//
|
||||
// // Remember that the contents of the returned slice should not be modified.
|
||||
// data, err := db.Get([]byte("key"), nil)
|
||||
// ...
|
||||
// err = db.Put([]byte("key"), []byte("value"), nil)
|
||||
// ...
|
||||
// err = db.Delete([]byte("key"), nil)
|
||||
// ...
|
||||
//
|
||||
// Iterate over database content:
|
||||
//
|
||||
// iter := db.NewIterator(nil, nil)
|
||||
// for iter.Next() {
|
||||
// // Remember that the contents of the returned slice should not be modified, and
|
||||
// // only valid until the next call to Next.
|
||||
// key := iter.Key()
|
||||
// value := iter.Value()
|
||||
// ...
|
||||
// }
|
||||
// iter.Release()
|
||||
// err = iter.Error()
|
||||
// ...
|
||||
//
|
||||
// Iterate over subset of database content with a particular prefix:
|
||||
// iter := db.NewIterator(util.BytesPrefix([]byte("foo-")), nil)
|
||||
// for iter.Next() {
|
||||
// // Use key/value.
|
||||
// ...
|
||||
// }
|
||||
// iter.Release()
|
||||
// err = iter.Error()
|
||||
// ...
|
||||
//
|
||||
// Seek-then-Iterate:
|
||||
//
|
||||
// iter := db.NewIterator(nil, nil)
|
||||
// for ok := iter.Seek(key); ok; ok = iter.Next() {
|
||||
// // Use key/value.
|
||||
// ...
|
||||
// }
|
||||
// iter.Release()
|
||||
// err = iter.Error()
|
||||
// ...
|
||||
//
|
||||
// Iterate over subset of database content:
|
||||
//
|
||||
// iter := db.NewIterator(&util.Range{Start: []byte("foo"), Limit: []byte("xoo")}, nil)
|
||||
// for iter.Next() {
|
||||
// // Use key/value.
|
||||
// ...
|
||||
// }
|
||||
// iter.Release()
|
||||
// err = iter.Error()
|
||||
// ...
|
||||
//
|
||||
// Batch writes:
|
||||
//
|
||||
// batch := new(leveldb.Batch)
|
||||
// batch.Put([]byte("foo"), []byte("value"))
|
||||
// batch.Put([]byte("bar"), []byte("another value"))
|
||||
// batch.Delete([]byte("baz"))
|
||||
// err = db.Write(batch, nil)
|
||||
// ...
|
||||
//
|
||||
// Use bloom filter:
|
||||
//
|
||||
// o := &opt.Options{
|
||||
// Filter: filter.NewBloomFilter(10),
|
||||
// }
|
||||
// db, err := leveldb.OpenFile("path/to/db", o)
|
||||
// ...
|
||||
// defer db.Close()
|
||||
// ...
|
||||
package leveldb
|
||||
@ -0,0 +1,20 @@
|
||||
// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"github.com/syndtr/goleveldb/leveldb/errors"
|
||||
)
|
||||
|
||||
// Common errors.
|
||||
var (
|
||||
ErrNotFound = errors.ErrNotFound
|
||||
ErrReadOnly = errors.New("leveldb: read-only mode")
|
||||
ErrSnapshotReleased = errors.New("leveldb: snapshot released")
|
||||
ErrIterReleased = errors.New("leveldb: iterator released")
|
||||
ErrClosed = errors.New("leveldb: closed")
|
||||
)
|
||||
@ -0,0 +1,78 @@
|
||||
// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package errors provides common error types used throughout leveldb.
|
||||
package errors
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
// Common errors.
|
||||
var (
|
||||
ErrNotFound = New("leveldb: not found")
|
||||
ErrReleased = util.ErrReleased
|
||||
ErrHasReleaser = util.ErrHasReleaser
|
||||
)
|
||||
|
||||
// New returns an error that formats as the given text.
|
||||
func New(text string) error {
|
||||
return errors.New(text)
|
||||
}
|
||||
|
||||
// ErrCorrupted is the type that wraps errors that indicate corruption in
|
||||
// the database.
|
||||
type ErrCorrupted struct {
|
||||
Fd storage.FileDesc
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *ErrCorrupted) Error() string {
|
||||
if !e.Fd.Zero() {
|
||||
return fmt.Sprintf("%v [file=%v]", e.Err, e.Fd)
|
||||
}
|
||||
return e.Err.Error()
|
||||
}
|
||||
|
||||
// NewErrCorrupted creates new ErrCorrupted error.
|
||||
func NewErrCorrupted(fd storage.FileDesc, err error) error {
|
||||
return &ErrCorrupted{fd, err}
|
||||
}
|
||||
|
||||
// IsCorrupted returns a boolean indicating whether the error is indicating
|
||||
// a corruption.
|
||||
func IsCorrupted(err error) bool {
|
||||
switch err.(type) {
|
||||
case *ErrCorrupted:
|
||||
return true
|
||||
case *storage.ErrCorrupted:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ErrMissingFiles is the type that indicating a corruption due to missing
|
||||
// files. ErrMissingFiles always wrapped with ErrCorrupted.
|
||||
type ErrMissingFiles struct {
|
||||
Fds []storage.FileDesc
|
||||
}
|
||||
|
||||
func (e *ErrMissingFiles) Error() string { return "file missing" }
|
||||
|
||||
// SetFd sets 'file info' of the given error with the given file.
|
||||
// Currently only ErrCorrupted is supported, otherwise will do nothing.
|
||||
func SetFd(err error, fd storage.FileDesc) error {
|
||||
switch x := err.(type) {
|
||||
case *ErrCorrupted:
|
||||
x.Fd = fd
|
||||
return x
|
||||
}
|
||||
return err
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"github.com/syndtr/goleveldb/leveldb/filter"
|
||||
)
|
||||
|
||||
type iFilter struct {
|
||||
filter.Filter
|
||||
}
|
||||
|
||||
func (f iFilter) Contains(filter, key []byte) bool {
|
||||
return f.Filter.Contains(filter, internalKey(key).ukey())
|
||||
}
|
||||
|
||||
func (f iFilter) NewGenerator() filter.FilterGenerator {
|
||||
return iFilterGenerator{f.Filter.NewGenerator()}
|
||||
}
|
||||
|
||||
type iFilterGenerator struct {
|
||||
filter.FilterGenerator
|
||||
}
|
||||
|
||||
func (g iFilterGenerator) Add(key []byte) {
|
||||
g.FilterGenerator.Add(internalKey(key).ukey())
|
||||
}
|
||||
@ -0,0 +1,116 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package filter
|
||||
|
||||
import (
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
func bloomHash(key []byte) uint32 {
|
||||
return util.Hash(key, 0xbc9f1d34)
|
||||
}
|
||||
|
||||
type bloomFilter int
|
||||
|
||||
// The bloom filter serializes its parameters and is backward compatible
|
||||
// with respect to them. Therefor, its parameters are not added to its
|
||||
// name.
|
||||
func (bloomFilter) Name() string {
|
||||
return "leveldb.BuiltinBloomFilter"
|
||||
}
|
||||
|
||||
func (f bloomFilter) Contains(filter, key []byte) bool {
|
||||
nBytes := len(filter) - 1
|
||||
if nBytes < 1 {
|
||||
return false
|
||||
}
|
||||
nBits := uint32(nBytes * 8)
|
||||
|
||||
// Use the encoded k so that we can read filters generated by
|
||||
// bloom filters created using different parameters.
|
||||
k := filter[nBytes]
|
||||
if k > 30 {
|
||||
// Reserved for potentially new encodings for short bloom filters.
|
||||
// Consider it a match.
|
||||
return true
|
||||
}
|
||||
|
||||
kh := bloomHash(key)
|
||||
delta := (kh >> 17) | (kh << 15) // Rotate right 17 bits
|
||||
for j := uint8(0); j < k; j++ {
|
||||
bitpos := kh % nBits
|
||||
if (uint32(filter[bitpos/8]) & (1 << (bitpos % 8))) == 0 {
|
||||
return false
|
||||
}
|
||||
kh += delta
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (f bloomFilter) NewGenerator() FilterGenerator {
|
||||
// Round down to reduce probing cost a little bit.
|
||||
k := uint8(f * 69 / 100) // 0.69 =~ ln(2)
|
||||
if k < 1 {
|
||||
k = 1
|
||||
} else if k > 30 {
|
||||
k = 30
|
||||
}
|
||||
return &bloomFilterGenerator{
|
||||
n: int(f),
|
||||
k: k,
|
||||
}
|
||||
}
|
||||
|
||||
type bloomFilterGenerator struct {
|
||||
n int
|
||||
k uint8
|
||||
|
||||
keyHashes []uint32
|
||||
}
|
||||
|
||||
func (g *bloomFilterGenerator) Add(key []byte) {
|
||||
// Use double-hashing to generate a sequence of hash values.
|
||||
// See analysis in [Kirsch,Mitzenmacher 2006].
|
||||
g.keyHashes = append(g.keyHashes, bloomHash(key))
|
||||
}
|
||||
|
||||
func (g *bloomFilterGenerator) Generate(b Buffer) {
|
||||
// Compute bloom filter size (in both bits and bytes)
|
||||
nBits := uint32(len(g.keyHashes) * g.n)
|
||||
// For small n, we can see a very high false positive rate. Fix it
|
||||
// by enforcing a minimum bloom filter length.
|
||||
if nBits < 64 {
|
||||
nBits = 64
|
||||
}
|
||||
nBytes := (nBits + 7) / 8
|
||||
nBits = nBytes * 8
|
||||
|
||||
dest := b.Alloc(int(nBytes) + 1)
|
||||
dest[nBytes] = g.k
|
||||
for _, kh := range g.keyHashes {
|
||||
delta := (kh >> 17) | (kh << 15) // Rotate right 17 bits
|
||||
for j := uint8(0); j < g.k; j++ {
|
||||
bitpos := kh % nBits
|
||||
dest[bitpos/8] |= (1 << (bitpos % 8))
|
||||
kh += delta
|
||||
}
|
||||
}
|
||||
|
||||
g.keyHashes = g.keyHashes[:0]
|
||||
}
|
||||
|
||||
// NewBloomFilter creates a new initialized bloom filter for given
|
||||
// bitsPerKey.
|
||||
//
|
||||
// Since bitsPerKey is persisted individually for each bloom filter
|
||||
// serialization, bloom filters are backwards compatible with respect to
|
||||
// changing bitsPerKey. This means that no big performance penalty will
|
||||
// be experienced when changing the parameter. See documentation for
|
||||
// opt.Options.Filter for more information.
|
||||
func NewBloomFilter(bitsPerKey int) Filter {
|
||||
return bloomFilter(bitsPerKey)
|
||||
}
|
||||
@ -0,0 +1,60 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package filter provides interface and implementation of probabilistic
|
||||
// data structure.
|
||||
//
|
||||
// The filter is resposible for creating small filter from a set of keys.
|
||||
// These filter will then used to test whether a key is a member of the set.
|
||||
// In many cases, a filter can cut down the number of disk seeks from a
|
||||
// handful to a single disk seek per DB.Get call.
|
||||
package filter
|
||||
|
||||
// Buffer is the interface that wraps basic Alloc, Write and WriteByte methods.
|
||||
type Buffer interface {
|
||||
// Alloc allocs n bytes of slice from the buffer. This also advancing
|
||||
// write offset.
|
||||
Alloc(n int) []byte
|
||||
|
||||
// Write appends the contents of p to the buffer.
|
||||
Write(p []byte) (n int, err error)
|
||||
|
||||
// WriteByte appends the byte c to the buffer.
|
||||
WriteByte(c byte) error
|
||||
}
|
||||
|
||||
// Filter is the filter.
|
||||
type Filter interface {
|
||||
// Name returns the name of this policy.
|
||||
//
|
||||
// Note that if the filter encoding changes in an incompatible way,
|
||||
// the name returned by this method must be changed. Otherwise, old
|
||||
// incompatible filters may be passed to methods of this type.
|
||||
Name() string
|
||||
|
||||
// NewGenerator creates a new filter generator.
|
||||
NewGenerator() FilterGenerator
|
||||
|
||||
// Contains returns true if the filter contains the given key.
|
||||
//
|
||||
// The filter are filters generated by the filter generator.
|
||||
Contains(filter, key []byte) bool
|
||||
}
|
||||
|
||||
// FilterGenerator is the filter generator.
|
||||
type FilterGenerator interface {
|
||||
// Add adds a key to the filter generator.
|
||||
//
|
||||
// The key may become invalid after call to this method end, therefor
|
||||
// key must be copied if implementation require keeping key for later
|
||||
// use. The key should not modified directly, doing so may cause
|
||||
// undefined results.
|
||||
Add(key []byte)
|
||||
|
||||
// Generate generates filters based on keys passed so far. After call
|
||||
// to Generate the filter generator maybe resetted, depends on implementation.
|
||||
Generate(b Buffer)
|
||||
}
|
||||
@ -0,0 +1,184 @@
|
||||
// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package iterator
|
||||
|
||||
import (
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
// BasicArray is the interface that wraps basic Len and Search method.
|
||||
type BasicArray interface {
|
||||
// Len returns length of the array.
|
||||
Len() int
|
||||
|
||||
// Search finds smallest index that point to a key that is greater
|
||||
// than or equal to the given key.
|
||||
Search(key []byte) int
|
||||
}
|
||||
|
||||
// Array is the interface that wraps BasicArray and basic Index method.
|
||||
type Array interface {
|
||||
BasicArray
|
||||
|
||||
// Index returns key/value pair with index of i.
|
||||
Index(i int) (key, value []byte)
|
||||
}
|
||||
|
||||
// Array is the interface that wraps BasicArray and basic Get method.
|
||||
type ArrayIndexer interface {
|
||||
BasicArray
|
||||
|
||||
// Get returns a new data iterator with index of i.
|
||||
Get(i int) Iterator
|
||||
}
|
||||
|
||||
type basicArrayIterator struct {
|
||||
util.BasicReleaser
|
||||
array BasicArray
|
||||
pos int
|
||||
err error
|
||||
}
|
||||
|
||||
func (i *basicArrayIterator) Valid() bool {
|
||||
return i.pos >= 0 && i.pos < i.array.Len() && !i.Released()
|
||||
}
|
||||
|
||||
func (i *basicArrayIterator) First() bool {
|
||||
if i.Released() {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
if i.array.Len() == 0 {
|
||||
i.pos = -1
|
||||
return false
|
||||
}
|
||||
i.pos = 0
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *basicArrayIterator) Last() bool {
|
||||
if i.Released() {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
n := i.array.Len()
|
||||
if n == 0 {
|
||||
i.pos = 0
|
||||
return false
|
||||
}
|
||||
i.pos = n - 1
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *basicArrayIterator) Seek(key []byte) bool {
|
||||
if i.Released() {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
n := i.array.Len()
|
||||
if n == 0 {
|
||||
i.pos = 0
|
||||
return false
|
||||
}
|
||||
i.pos = i.array.Search(key)
|
||||
if i.pos >= n {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *basicArrayIterator) Next() bool {
|
||||
if i.Released() {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
i.pos++
|
||||
if n := i.array.Len(); i.pos >= n {
|
||||
i.pos = n
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *basicArrayIterator) Prev() bool {
|
||||
if i.Released() {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
i.pos--
|
||||
if i.pos < 0 {
|
||||
i.pos = -1
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *basicArrayIterator) Error() error { return i.err }
|
||||
|
||||
type arrayIterator struct {
|
||||
basicArrayIterator
|
||||
array Array
|
||||
pos int
|
||||
key, value []byte
|
||||
}
|
||||
|
||||
func (i *arrayIterator) updateKV() {
|
||||
if i.pos == i.basicArrayIterator.pos {
|
||||
return
|
||||
}
|
||||
i.pos = i.basicArrayIterator.pos
|
||||
if i.Valid() {
|
||||
i.key, i.value = i.array.Index(i.pos)
|
||||
} else {
|
||||
i.key = nil
|
||||
i.value = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (i *arrayIterator) Key() []byte {
|
||||
i.updateKV()
|
||||
return i.key
|
||||
}
|
||||
|
||||
func (i *arrayIterator) Value() []byte {
|
||||
i.updateKV()
|
||||
return i.value
|
||||
}
|
||||
|
||||
type arrayIteratorIndexer struct {
|
||||
basicArrayIterator
|
||||
array ArrayIndexer
|
||||
}
|
||||
|
||||
func (i *arrayIteratorIndexer) Get() Iterator {
|
||||
if i.Valid() {
|
||||
return i.array.Get(i.basicArrayIterator.pos)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewArrayIterator returns an iterator from the given array.
|
||||
func NewArrayIterator(array Array) Iterator {
|
||||
return &arrayIterator{
|
||||
basicArrayIterator: basicArrayIterator{array: array, pos: -1},
|
||||
array: array,
|
||||
pos: -1,
|
||||
}
|
||||
}
|
||||
|
||||
// NewArrayIndexer returns an index iterator from the given array.
|
||||
func NewArrayIndexer(array ArrayIndexer) IteratorIndexer {
|
||||
return &arrayIteratorIndexer{
|
||||
basicArrayIterator: basicArrayIterator{array: array, pos: -1},
|
||||
array: array,
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,242 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package iterator
|
||||
|
||||
import (
|
||||
"github.com/syndtr/goleveldb/leveldb/errors"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
// IteratorIndexer is the interface that wraps CommonIterator and basic Get
|
||||
// method. IteratorIndexer provides index for indexed iterator.
|
||||
type IteratorIndexer interface {
|
||||
CommonIterator
|
||||
|
||||
// Get returns a new data iterator for the current position, or nil if
|
||||
// done.
|
||||
Get() Iterator
|
||||
}
|
||||
|
||||
type indexedIterator struct {
|
||||
util.BasicReleaser
|
||||
index IteratorIndexer
|
||||
strict bool
|
||||
|
||||
data Iterator
|
||||
err error
|
||||
errf func(err error)
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (i *indexedIterator) setData() {
|
||||
if i.data != nil {
|
||||
i.data.Release()
|
||||
}
|
||||
i.data = i.index.Get()
|
||||
}
|
||||
|
||||
func (i *indexedIterator) clearData() {
|
||||
if i.data != nil {
|
||||
i.data.Release()
|
||||
}
|
||||
i.data = nil
|
||||
}
|
||||
|
||||
func (i *indexedIterator) indexErr() {
|
||||
if err := i.index.Error(); err != nil {
|
||||
if i.errf != nil {
|
||||
i.errf(err)
|
||||
}
|
||||
i.err = err
|
||||
}
|
||||
}
|
||||
|
||||
func (i *indexedIterator) dataErr() bool {
|
||||
if err := i.data.Error(); err != nil {
|
||||
if i.errf != nil {
|
||||
i.errf(err)
|
||||
}
|
||||
if i.strict || !errors.IsCorrupted(err) {
|
||||
i.err = err
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (i *indexedIterator) Valid() bool {
|
||||
return i.data != nil && i.data.Valid()
|
||||
}
|
||||
|
||||
func (i *indexedIterator) First() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
} else if i.Released() {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
if !i.index.First() {
|
||||
i.indexErr()
|
||||
i.clearData()
|
||||
return false
|
||||
}
|
||||
i.setData()
|
||||
return i.Next()
|
||||
}
|
||||
|
||||
func (i *indexedIterator) Last() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
} else if i.Released() {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
if !i.index.Last() {
|
||||
i.indexErr()
|
||||
i.clearData()
|
||||
return false
|
||||
}
|
||||
i.setData()
|
||||
if !i.data.Last() {
|
||||
if i.dataErr() {
|
||||
return false
|
||||
}
|
||||
i.clearData()
|
||||
return i.Prev()
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *indexedIterator) Seek(key []byte) bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
} else if i.Released() {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
if !i.index.Seek(key) {
|
||||
i.indexErr()
|
||||
i.clearData()
|
||||
return false
|
||||
}
|
||||
i.setData()
|
||||
if !i.data.Seek(key) {
|
||||
if i.dataErr() {
|
||||
return false
|
||||
}
|
||||
i.clearData()
|
||||
return i.Next()
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *indexedIterator) Next() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
} else if i.Released() {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
switch {
|
||||
case i.data != nil && !i.data.Next():
|
||||
if i.dataErr() {
|
||||
return false
|
||||
}
|
||||
i.clearData()
|
||||
fallthrough
|
||||
case i.data == nil:
|
||||
if !i.index.Next() {
|
||||
i.indexErr()
|
||||
return false
|
||||
}
|
||||
i.setData()
|
||||
return i.Next()
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *indexedIterator) Prev() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
} else if i.Released() {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
switch {
|
||||
case i.data != nil && !i.data.Prev():
|
||||
if i.dataErr() {
|
||||
return false
|
||||
}
|
||||
i.clearData()
|
||||
fallthrough
|
||||
case i.data == nil:
|
||||
if !i.index.Prev() {
|
||||
i.indexErr()
|
||||
return false
|
||||
}
|
||||
i.setData()
|
||||
if !i.data.Last() {
|
||||
if i.dataErr() {
|
||||
return false
|
||||
}
|
||||
i.clearData()
|
||||
return i.Prev()
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *indexedIterator) Key() []byte {
|
||||
if i.data == nil {
|
||||
return nil
|
||||
}
|
||||
return i.data.Key()
|
||||
}
|
||||
|
||||
func (i *indexedIterator) Value() []byte {
|
||||
if i.data == nil {
|
||||
return nil
|
||||
}
|
||||
return i.data.Value()
|
||||
}
|
||||
|
||||
func (i *indexedIterator) Release() {
|
||||
i.clearData()
|
||||
i.index.Release()
|
||||
i.BasicReleaser.Release()
|
||||
}
|
||||
|
||||
func (i *indexedIterator) Error() error {
|
||||
if i.err != nil {
|
||||
return i.err
|
||||
}
|
||||
if err := i.index.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (i *indexedIterator) SetErrorCallback(f func(err error)) {
|
||||
i.errf = f
|
||||
}
|
||||
|
||||
// NewIndexedIterator returns an 'indexed iterator'. An index is iterator
|
||||
// that returns another iterator, a 'data iterator'. A 'data iterator' is the
|
||||
// iterator that contains actual key/value pairs.
|
||||
//
|
||||
// If strict is true the any 'corruption errors' (i.e errors.IsCorrupted(err) == true)
|
||||
// won't be ignored and will halt 'indexed iterator', otherwise the iterator will
|
||||
// continue to the next 'data iterator'. Corruption on 'index iterator' will not be
|
||||
// ignored and will halt the iterator.
|
||||
func NewIndexedIterator(index IteratorIndexer, strict bool) Iterator {
|
||||
return &indexedIterator{index: index, strict: strict}
|
||||
}
|
||||
@ -0,0 +1,132 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package iterator provides interface and implementation to traverse over
|
||||
// contents of a database.
|
||||
package iterator
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrIterReleased = errors.New("leveldb/iterator: iterator released")
|
||||
)
|
||||
|
||||
// IteratorSeeker is the interface that wraps the 'seeks method'.
|
||||
type IteratorSeeker interface {
|
||||
// First moves the iterator to the first key/value pair. If the iterator
|
||||
// only contains one key/value pair then First and Last would moves
|
||||
// to the same key/value pair.
|
||||
// It returns whether such pair exist.
|
||||
First() bool
|
||||
|
||||
// Last moves the iterator to the last key/value pair. If the iterator
|
||||
// only contains one key/value pair then First and Last would moves
|
||||
// to the same key/value pair.
|
||||
// It returns whether such pair exist.
|
||||
Last() bool
|
||||
|
||||
// Seek moves the iterator to the first key/value pair whose key is greater
|
||||
// than or equal to the given key.
|
||||
// It returns whether such pair exist.
|
||||
//
|
||||
// It is safe to modify the contents of the argument after Seek returns.
|
||||
Seek(key []byte) bool
|
||||
|
||||
// Next moves the iterator to the next key/value pair.
|
||||
// It returns false if the iterator is exhausted.
|
||||
Next() bool
|
||||
|
||||
// Prev moves the iterator to the previous key/value pair.
|
||||
// It returns false if the iterator is exhausted.
|
||||
Prev() bool
|
||||
}
|
||||
|
||||
// CommonIterator is the interface that wraps common iterator methods.
|
||||
type CommonIterator interface {
|
||||
IteratorSeeker
|
||||
|
||||
// util.Releaser is the interface that wraps basic Release method.
|
||||
// When called Release will releases any resources associated with the
|
||||
// iterator.
|
||||
util.Releaser
|
||||
|
||||
// util.ReleaseSetter is the interface that wraps the basic SetReleaser
|
||||
// method.
|
||||
util.ReleaseSetter
|
||||
|
||||
// TODO: Remove this when ready.
|
||||
Valid() bool
|
||||
|
||||
// Error returns any accumulated error. Exhausting all the key/value pairs
|
||||
// is not considered to be an error.
|
||||
Error() error
|
||||
}
|
||||
|
||||
// Iterator iterates over a DB's key/value pairs in key order.
|
||||
//
|
||||
// When encounter an error any 'seeks method' will return false and will
|
||||
// yield no key/value pairs. The error can be queried by calling the Error
|
||||
// method. Calling Release is still necessary.
|
||||
//
|
||||
// An iterator must be released after use, but it is not necessary to read
|
||||
// an iterator until exhaustion.
|
||||
// Also, an iterator is not necessarily safe for concurrent use, but it is
|
||||
// safe to use multiple iterators concurrently, with each in a dedicated
|
||||
// goroutine.
|
||||
type Iterator interface {
|
||||
CommonIterator
|
||||
|
||||
// Key returns the key of the current key/value pair, or nil if done.
|
||||
// The caller should not modify the contents of the returned slice, and
|
||||
// its contents may change on the next call to any 'seeks method'.
|
||||
Key() []byte
|
||||
|
||||
// Value returns the value of the current key/value pair, or nil if done.
|
||||
// The caller should not modify the contents of the returned slice, and
|
||||
// its contents may change on the next call to any 'seeks method'.
|
||||
Value() []byte
|
||||
}
|
||||
|
||||
// ErrorCallbackSetter is the interface that wraps basic SetErrorCallback
|
||||
// method.
|
||||
//
|
||||
// ErrorCallbackSetter implemented by indexed and merged iterator.
|
||||
type ErrorCallbackSetter interface {
|
||||
// SetErrorCallback allows set an error callback of the corresponding
|
||||
// iterator. Use nil to clear the callback.
|
||||
SetErrorCallback(f func(err error))
|
||||
}
|
||||
|
||||
type emptyIterator struct {
|
||||
util.BasicReleaser
|
||||
err error
|
||||
}
|
||||
|
||||
func (i *emptyIterator) rErr() {
|
||||
if i.err == nil && i.Released() {
|
||||
i.err = ErrIterReleased
|
||||
}
|
||||
}
|
||||
|
||||
func (*emptyIterator) Valid() bool { return false }
|
||||
func (i *emptyIterator) First() bool { i.rErr(); return false }
|
||||
func (i *emptyIterator) Last() bool { i.rErr(); return false }
|
||||
func (i *emptyIterator) Seek(key []byte) bool { i.rErr(); return false }
|
||||
func (i *emptyIterator) Next() bool { i.rErr(); return false }
|
||||
func (i *emptyIterator) Prev() bool { i.rErr(); return false }
|
||||
func (*emptyIterator) Key() []byte { return nil }
|
||||
func (*emptyIterator) Value() []byte { return nil }
|
||||
func (i *emptyIterator) Error() error { return i.err }
|
||||
|
||||
// NewEmptyIterator creates an empty iterator. The err parameter can be
|
||||
// nil, but if not nil the given err will be returned by Error method.
|
||||
func NewEmptyIterator(err error) Iterator {
|
||||
return &emptyIterator{err: err}
|
||||
}
|
||||
@ -0,0 +1,304 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package iterator
|
||||
|
||||
import (
|
||||
"github.com/syndtr/goleveldb/leveldb/comparer"
|
||||
"github.com/syndtr/goleveldb/leveldb/errors"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
type dir int
|
||||
|
||||
const (
|
||||
dirReleased dir = iota - 1
|
||||
dirSOI
|
||||
dirEOI
|
||||
dirBackward
|
||||
dirForward
|
||||
)
|
||||
|
||||
type mergedIterator struct {
|
||||
cmp comparer.Comparer
|
||||
iters []Iterator
|
||||
strict bool
|
||||
|
||||
keys [][]byte
|
||||
index int
|
||||
dir dir
|
||||
err error
|
||||
errf func(err error)
|
||||
releaser util.Releaser
|
||||
}
|
||||
|
||||
func assertKey(key []byte) []byte {
|
||||
if key == nil {
|
||||
panic("leveldb/iterator: nil key")
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
func (i *mergedIterator) iterErr(iter Iterator) bool {
|
||||
if err := iter.Error(); err != nil {
|
||||
if i.errf != nil {
|
||||
i.errf(err)
|
||||
}
|
||||
if i.strict || !errors.IsCorrupted(err) {
|
||||
i.err = err
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (i *mergedIterator) Valid() bool {
|
||||
return i.err == nil && i.dir > dirEOI
|
||||
}
|
||||
|
||||
func (i *mergedIterator) First() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
for x, iter := range i.iters {
|
||||
switch {
|
||||
case iter.First():
|
||||
i.keys[x] = assertKey(iter.Key())
|
||||
case i.iterErr(iter):
|
||||
return false
|
||||
default:
|
||||
i.keys[x] = nil
|
||||
}
|
||||
}
|
||||
i.dir = dirSOI
|
||||
return i.next()
|
||||
}
|
||||
|
||||
func (i *mergedIterator) Last() bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
for x, iter := range i.iters {
|
||||
switch {
|
||||
case iter.Last():
|
||||
i.keys[x] = assertKey(iter.Key())
|
||||
case i.iterErr(iter):
|
||||
return false
|
||||
default:
|
||||
i.keys[x] = nil
|
||||
}
|
||||
}
|
||||
i.dir = dirEOI
|
||||
return i.prev()
|
||||
}
|
||||
|
||||
func (i *mergedIterator) Seek(key []byte) bool {
|
||||
if i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
for x, iter := range i.iters {
|
||||
switch {
|
||||
case iter.Seek(key):
|
||||
i.keys[x] = assertKey(iter.Key())
|
||||
case i.iterErr(iter):
|
||||
return false
|
||||
default:
|
||||
i.keys[x] = nil
|
||||
}
|
||||
}
|
||||
i.dir = dirSOI
|
||||
return i.next()
|
||||
}
|
||||
|
||||
func (i *mergedIterator) next() bool {
|
||||
var key []byte
|
||||
if i.dir == dirForward {
|
||||
key = i.keys[i.index]
|
||||
}
|
||||
for x, tkey := range i.keys {
|
||||
if tkey != nil && (key == nil || i.cmp.Compare(tkey, key) < 0) {
|
||||
key = tkey
|
||||
i.index = x
|
||||
}
|
||||
}
|
||||
if key == nil {
|
||||
i.dir = dirEOI
|
||||
return false
|
||||
}
|
||||
i.dir = dirForward
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *mergedIterator) Next() bool {
|
||||
if i.dir == dirEOI || i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
switch i.dir {
|
||||
case dirSOI:
|
||||
return i.First()
|
||||
case dirBackward:
|
||||
key := append([]byte{}, i.keys[i.index]...)
|
||||
if !i.Seek(key) {
|
||||
return false
|
||||
}
|
||||
return i.Next()
|
||||
}
|
||||
|
||||
x := i.index
|
||||
iter := i.iters[x]
|
||||
switch {
|
||||
case iter.Next():
|
||||
i.keys[x] = assertKey(iter.Key())
|
||||
case i.iterErr(iter):
|
||||
return false
|
||||
default:
|
||||
i.keys[x] = nil
|
||||
}
|
||||
return i.next()
|
||||
}
|
||||
|
||||
func (i *mergedIterator) prev() bool {
|
||||
var key []byte
|
||||
if i.dir == dirBackward {
|
||||
key = i.keys[i.index]
|
||||
}
|
||||
for x, tkey := range i.keys {
|
||||
if tkey != nil && (key == nil || i.cmp.Compare(tkey, key) > 0) {
|
||||
key = tkey
|
||||
i.index = x
|
||||
}
|
||||
}
|
||||
if key == nil {
|
||||
i.dir = dirSOI
|
||||
return false
|
||||
}
|
||||
i.dir = dirBackward
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *mergedIterator) Prev() bool {
|
||||
if i.dir == dirSOI || i.err != nil {
|
||||
return false
|
||||
} else if i.dir == dirReleased {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
switch i.dir {
|
||||
case dirEOI:
|
||||
return i.Last()
|
||||
case dirForward:
|
||||
key := append([]byte{}, i.keys[i.index]...)
|
||||
for x, iter := range i.iters {
|
||||
if x == i.index {
|
||||
continue
|
||||
}
|
||||
seek := iter.Seek(key)
|
||||
switch {
|
||||
case seek && iter.Prev(), !seek && iter.Last():
|
||||
i.keys[x] = assertKey(iter.Key())
|
||||
case i.iterErr(iter):
|
||||
return false
|
||||
default:
|
||||
i.keys[x] = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
x := i.index
|
||||
iter := i.iters[x]
|
||||
switch {
|
||||
case iter.Prev():
|
||||
i.keys[x] = assertKey(iter.Key())
|
||||
case i.iterErr(iter):
|
||||
return false
|
||||
default:
|
||||
i.keys[x] = nil
|
||||
}
|
||||
return i.prev()
|
||||
}
|
||||
|
||||
func (i *mergedIterator) Key() []byte {
|
||||
if i.err != nil || i.dir <= dirEOI {
|
||||
return nil
|
||||
}
|
||||
return i.keys[i.index]
|
||||
}
|
||||
|
||||
func (i *mergedIterator) Value() []byte {
|
||||
if i.err != nil || i.dir <= dirEOI {
|
||||
return nil
|
||||
}
|
||||
return i.iters[i.index].Value()
|
||||
}
|
||||
|
||||
func (i *mergedIterator) Release() {
|
||||
if i.dir != dirReleased {
|
||||
i.dir = dirReleased
|
||||
for _, iter := range i.iters {
|
||||
iter.Release()
|
||||
}
|
||||
i.iters = nil
|
||||
i.keys = nil
|
||||
if i.releaser != nil {
|
||||
i.releaser.Release()
|
||||
i.releaser = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (i *mergedIterator) SetReleaser(releaser util.Releaser) {
|
||||
if i.dir == dirReleased {
|
||||
panic(util.ErrReleased)
|
||||
}
|
||||
if i.releaser != nil && releaser != nil {
|
||||
panic(util.ErrHasReleaser)
|
||||
}
|
||||
i.releaser = releaser
|
||||
}
|
||||
|
||||
func (i *mergedIterator) Error() error {
|
||||
return i.err
|
||||
}
|
||||
|
||||
func (i *mergedIterator) SetErrorCallback(f func(err error)) {
|
||||
i.errf = f
|
||||
}
|
||||
|
||||
// NewMergedIterator returns an iterator that merges its input. Walking the
|
||||
// resultant iterator will return all key/value pairs of all input iterators
|
||||
// in strictly increasing key order, as defined by cmp.
|
||||
// The input's key ranges may overlap, but there are assumed to be no duplicate
|
||||
// keys: if iters[i] contains a key k then iters[j] will not contain that key k.
|
||||
// None of the iters may be nil.
|
||||
//
|
||||
// If strict is true the any 'corruption errors' (i.e errors.IsCorrupted(err) == true)
|
||||
// won't be ignored and will halt 'merged iterator', otherwise the iterator will
|
||||
// continue to the next 'input iterator'.
|
||||
func NewMergedIterator(iters []Iterator, cmp comparer.Comparer, strict bool) Iterator {
|
||||
return &mergedIterator{
|
||||
iters: iters,
|
||||
cmp: cmp,
|
||||
strict: strict,
|
||||
keys: make([][]byte, len(iters)),
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,524 @@
|
||||
// Copyright 2011 The LevelDB-Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Taken from: https://code.google.com/p/leveldb-go/source/browse/leveldb/record/record.go?r=1d5ccbe03246da926391ee12d1c6caae054ff4b0
|
||||
// License, authors and contributors informations can be found at bellow URLs respectively:
|
||||
// https://code.google.com/p/leveldb-go/source/browse/LICENSE
|
||||
// https://code.google.com/p/leveldb-go/source/browse/AUTHORS
|
||||
// https://code.google.com/p/leveldb-go/source/browse/CONTRIBUTORS
|
||||
|
||||
// Package journal reads and writes sequences of journals. Each journal is a stream
|
||||
// of bytes that completes before the next journal starts.
|
||||
//
|
||||
// When reading, call Next to obtain an io.Reader for the next journal. Next will
|
||||
// return io.EOF when there are no more journals. It is valid to call Next
|
||||
// without reading the current journal to exhaustion.
|
||||
//
|
||||
// When writing, call Next to obtain an io.Writer for the next journal. Calling
|
||||
// Next finishes the current journal. Call Close to finish the final journal.
|
||||
//
|
||||
// Optionally, call Flush to finish the current journal and flush the underlying
|
||||
// writer without starting a new journal. To start a new journal after flushing,
|
||||
// call Next.
|
||||
//
|
||||
// Neither Readers or Writers are safe to use concurrently.
|
||||
//
|
||||
// Example code:
|
||||
// func read(r io.Reader) ([]string, error) {
|
||||
// var ss []string
|
||||
// journals := journal.NewReader(r, nil, true, true)
|
||||
// for {
|
||||
// j, err := journals.Next()
|
||||
// if err == io.EOF {
|
||||
// break
|
||||
// }
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// s, err := ioutil.ReadAll(j)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// ss = append(ss, string(s))
|
||||
// }
|
||||
// return ss, nil
|
||||
// }
|
||||
//
|
||||
// func write(w io.Writer, ss []string) error {
|
||||
// journals := journal.NewWriter(w)
|
||||
// for _, s := range ss {
|
||||
// j, err := journals.Next()
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
// if _, err := j.Write([]byte(s)), err != nil {
|
||||
// return err
|
||||
// }
|
||||
// }
|
||||
// return journals.Close()
|
||||
// }
|
||||
//
|
||||
// The wire format is that the stream is divided into 32KiB blocks, and each
|
||||
// block contains a number of tightly packed chunks. Chunks cannot cross block
|
||||
// boundaries. The last block may be shorter than 32 KiB. Any unused bytes in a
|
||||
// block must be zero.
|
||||
//
|
||||
// A journal maps to one or more chunks. Each chunk has a 7 byte header (a 4
|
||||
// byte checksum, a 2 byte little-endian uint16 length, and a 1 byte chunk type)
|
||||
// followed by a payload. The checksum is over the chunk type and the payload.
|
||||
//
|
||||
// There are four chunk types: whether the chunk is the full journal, or the
|
||||
// first, middle or last chunk of a multi-chunk journal. A multi-chunk journal
|
||||
// has one first chunk, zero or more middle chunks, and one last chunk.
|
||||
//
|
||||
// The wire format allows for limited recovery in the face of data corruption:
|
||||
// on a format error (such as a checksum mismatch), the reader moves to the
|
||||
// next block and looks for the next full or first chunk.
|
||||
package journal
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/errors"
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
// These constants are part of the wire format and should not be changed.
|
||||
const (
|
||||
fullChunkType = 1
|
||||
firstChunkType = 2
|
||||
middleChunkType = 3
|
||||
lastChunkType = 4
|
||||
)
|
||||
|
||||
const (
|
||||
blockSize = 32 * 1024
|
||||
headerSize = 7
|
||||
)
|
||||
|
||||
type flusher interface {
|
||||
Flush() error
|
||||
}
|
||||
|
||||
// ErrCorrupted is the error type that generated by corrupted block or chunk.
|
||||
type ErrCorrupted struct {
|
||||
Size int
|
||||
Reason string
|
||||
}
|
||||
|
||||
func (e *ErrCorrupted) Error() string {
|
||||
return fmt.Sprintf("leveldb/journal: block/chunk corrupted: %s (%d bytes)", e.Reason, e.Size)
|
||||
}
|
||||
|
||||
// Dropper is the interface that wrap simple Drop method. The Drop
|
||||
// method will be called when the journal reader dropping a block or chunk.
|
||||
type Dropper interface {
|
||||
Drop(err error)
|
||||
}
|
||||
|
||||
// Reader reads journals from an underlying io.Reader.
|
||||
type Reader struct {
|
||||
// r is the underlying reader.
|
||||
r io.Reader
|
||||
// the dropper.
|
||||
dropper Dropper
|
||||
// strict flag.
|
||||
strict bool
|
||||
// checksum flag.
|
||||
checksum bool
|
||||
// seq is the sequence number of the current journal.
|
||||
seq int
|
||||
// buf[i:j] is the unread portion of the current chunk's payload.
|
||||
// The low bound, i, excludes the chunk header.
|
||||
i, j int
|
||||
// n is the number of bytes of buf that are valid. Once reading has started,
|
||||
// only the final block can have n < blockSize.
|
||||
n int
|
||||
// last is whether the current chunk is the last chunk of the journal.
|
||||
last bool
|
||||
// err is any accumulated error.
|
||||
err error
|
||||
// buf is the buffer.
|
||||
buf [blockSize]byte
|
||||
}
|
||||
|
||||
// NewReader returns a new reader. The dropper may be nil, and if
|
||||
// strict is true then corrupted or invalid chunk will halt the journal
|
||||
// reader entirely.
|
||||
func NewReader(r io.Reader, dropper Dropper, strict, checksum bool) *Reader {
|
||||
return &Reader{
|
||||
r: r,
|
||||
dropper: dropper,
|
||||
strict: strict,
|
||||
checksum: checksum,
|
||||
last: true,
|
||||
}
|
||||
}
|
||||
|
||||
var errSkip = errors.New("leveldb/journal: skipped")
|
||||
|
||||
func (r *Reader) corrupt(n int, reason string, skip bool) error {
|
||||
if r.dropper != nil {
|
||||
r.dropper.Drop(&ErrCorrupted{n, reason})
|
||||
}
|
||||
if r.strict && !skip {
|
||||
r.err = errors.NewErrCorrupted(storage.FileDesc{}, &ErrCorrupted{n, reason})
|
||||
return r.err
|
||||
}
|
||||
return errSkip
|
||||
}
|
||||
|
||||
// nextChunk sets r.buf[r.i:r.j] to hold the next chunk's payload, reading the
|
||||
// next block into the buffer if necessary.
|
||||
func (r *Reader) nextChunk(first bool) error {
|
||||
for {
|
||||
if r.j+headerSize <= r.n {
|
||||
checksum := binary.LittleEndian.Uint32(r.buf[r.j+0 : r.j+4])
|
||||
length := binary.LittleEndian.Uint16(r.buf[r.j+4 : r.j+6])
|
||||
chunkType := r.buf[r.j+6]
|
||||
unprocBlock := r.n - r.j
|
||||
if checksum == 0 && length == 0 && chunkType == 0 {
|
||||
// Drop entire block.
|
||||
r.i = r.n
|
||||
r.j = r.n
|
||||
return r.corrupt(unprocBlock, "zero header", false)
|
||||
}
|
||||
if chunkType < fullChunkType || chunkType > lastChunkType {
|
||||
// Drop entire block.
|
||||
r.i = r.n
|
||||
r.j = r.n
|
||||
return r.corrupt(unprocBlock, fmt.Sprintf("invalid chunk type %#x", chunkType), false)
|
||||
}
|
||||
r.i = r.j + headerSize
|
||||
r.j = r.j + headerSize + int(length)
|
||||
if r.j > r.n {
|
||||
// Drop entire block.
|
||||
r.i = r.n
|
||||
r.j = r.n
|
||||
return r.corrupt(unprocBlock, "chunk length overflows block", false)
|
||||
} else if r.checksum && checksum != util.NewCRC(r.buf[r.i-1:r.j]).Value() {
|
||||
// Drop entire block.
|
||||
r.i = r.n
|
||||
r.j = r.n
|
||||
return r.corrupt(unprocBlock, "checksum mismatch", false)
|
||||
}
|
||||
if first && chunkType != fullChunkType && chunkType != firstChunkType {
|
||||
chunkLength := (r.j - r.i) + headerSize
|
||||
r.i = r.j
|
||||
// Report the error, but skip it.
|
||||
return r.corrupt(chunkLength, "orphan chunk", true)
|
||||
}
|
||||
r.last = chunkType == fullChunkType || chunkType == lastChunkType
|
||||
return nil
|
||||
}
|
||||
|
||||
// The last block.
|
||||
if r.n < blockSize && r.n > 0 {
|
||||
if !first {
|
||||
return r.corrupt(0, "missing chunk part", false)
|
||||
}
|
||||
r.err = io.EOF
|
||||
return r.err
|
||||
}
|
||||
|
||||
// Read block.
|
||||
n, err := io.ReadFull(r.r, r.buf[:])
|
||||
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
return err
|
||||
}
|
||||
if n == 0 {
|
||||
if !first {
|
||||
return r.corrupt(0, "missing chunk part", false)
|
||||
}
|
||||
r.err = io.EOF
|
||||
return r.err
|
||||
}
|
||||
r.i, r.j, r.n = 0, 0, n
|
||||
}
|
||||
}
|
||||
|
||||
// Next returns a reader for the next journal. It returns io.EOF if there are no
|
||||
// more journals. The reader returned becomes stale after the next Next call,
|
||||
// and should no longer be used. If strict is false, the reader will returns
|
||||
// io.ErrUnexpectedEOF error when found corrupted journal.
|
||||
func (r *Reader) Next() (io.Reader, error) {
|
||||
r.seq++
|
||||
if r.err != nil {
|
||||
return nil, r.err
|
||||
}
|
||||
r.i = r.j
|
||||
for {
|
||||
if err := r.nextChunk(true); err == nil {
|
||||
break
|
||||
} else if err != errSkip {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return &singleReader{r, r.seq, nil}, nil
|
||||
}
|
||||
|
||||
// Reset resets the journal reader, allows reuse of the journal reader. Reset returns
|
||||
// last accumulated error.
|
||||
func (r *Reader) Reset(reader io.Reader, dropper Dropper, strict, checksum bool) error {
|
||||
r.seq++
|
||||
err := r.err
|
||||
r.r = reader
|
||||
r.dropper = dropper
|
||||
r.strict = strict
|
||||
r.checksum = checksum
|
||||
r.i = 0
|
||||
r.j = 0
|
||||
r.n = 0
|
||||
r.last = true
|
||||
r.err = nil
|
||||
return err
|
||||
}
|
||||
|
||||
type singleReader struct {
|
||||
r *Reader
|
||||
seq int
|
||||
err error
|
||||
}
|
||||
|
||||
func (x *singleReader) Read(p []byte) (int, error) {
|
||||
r := x.r
|
||||
if r.seq != x.seq {
|
||||
return 0, errors.New("leveldb/journal: stale reader")
|
||||
}
|
||||
if x.err != nil {
|
||||
return 0, x.err
|
||||
}
|
||||
if r.err != nil {
|
||||
return 0, r.err
|
||||
}
|
||||
for r.i == r.j {
|
||||
if r.last {
|
||||
return 0, io.EOF
|
||||
}
|
||||
x.err = r.nextChunk(false)
|
||||
if x.err != nil {
|
||||
if x.err == errSkip {
|
||||
x.err = io.ErrUnexpectedEOF
|
||||
}
|
||||
return 0, x.err
|
||||
}
|
||||
}
|
||||
n := copy(p, r.buf[r.i:r.j])
|
||||
r.i += n
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (x *singleReader) ReadByte() (byte, error) {
|
||||
r := x.r
|
||||
if r.seq != x.seq {
|
||||
return 0, errors.New("leveldb/journal: stale reader")
|
||||
}
|
||||
if x.err != nil {
|
||||
return 0, x.err
|
||||
}
|
||||
if r.err != nil {
|
||||
return 0, r.err
|
||||
}
|
||||
for r.i == r.j {
|
||||
if r.last {
|
||||
return 0, io.EOF
|
||||
}
|
||||
x.err = r.nextChunk(false)
|
||||
if x.err != nil {
|
||||
if x.err == errSkip {
|
||||
x.err = io.ErrUnexpectedEOF
|
||||
}
|
||||
return 0, x.err
|
||||
}
|
||||
}
|
||||
c := r.buf[r.i]
|
||||
r.i++
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// Writer writes journals to an underlying io.Writer.
|
||||
type Writer struct {
|
||||
// w is the underlying writer.
|
||||
w io.Writer
|
||||
// seq is the sequence number of the current journal.
|
||||
seq int
|
||||
// f is w as a flusher.
|
||||
f flusher
|
||||
// buf[i:j] is the bytes that will become the current chunk.
|
||||
// The low bound, i, includes the chunk header.
|
||||
i, j int
|
||||
// buf[:written] has already been written to w.
|
||||
// written is zero unless Flush has been called.
|
||||
written int
|
||||
// first is whether the current chunk is the first chunk of the journal.
|
||||
first bool
|
||||
// pending is whether a chunk is buffered but not yet written.
|
||||
pending bool
|
||||
// err is any accumulated error.
|
||||
err error
|
||||
// buf is the buffer.
|
||||
buf [blockSize]byte
|
||||
}
|
||||
|
||||
// NewWriter returns a new Writer.
|
||||
func NewWriter(w io.Writer) *Writer {
|
||||
f, _ := w.(flusher)
|
||||
return &Writer{
|
||||
w: w,
|
||||
f: f,
|
||||
}
|
||||
}
|
||||
|
||||
// fillHeader fills in the header for the pending chunk.
|
||||
func (w *Writer) fillHeader(last bool) {
|
||||
if w.i+headerSize > w.j || w.j > blockSize {
|
||||
panic("leveldb/journal: bad writer state")
|
||||
}
|
||||
if last {
|
||||
if w.first {
|
||||
w.buf[w.i+6] = fullChunkType
|
||||
} else {
|
||||
w.buf[w.i+6] = lastChunkType
|
||||
}
|
||||
} else {
|
||||
if w.first {
|
||||
w.buf[w.i+6] = firstChunkType
|
||||
} else {
|
||||
w.buf[w.i+6] = middleChunkType
|
||||
}
|
||||
}
|
||||
binary.LittleEndian.PutUint32(w.buf[w.i+0:w.i+4], util.NewCRC(w.buf[w.i+6:w.j]).Value())
|
||||
binary.LittleEndian.PutUint16(w.buf[w.i+4:w.i+6], uint16(w.j-w.i-headerSize))
|
||||
}
|
||||
|
||||
// writeBlock writes the buffered block to the underlying writer, and reserves
|
||||
// space for the next chunk's header.
|
||||
func (w *Writer) writeBlock() {
|
||||
_, w.err = w.w.Write(w.buf[w.written:])
|
||||
w.i = 0
|
||||
w.j = headerSize
|
||||
w.written = 0
|
||||
}
|
||||
|
||||
// writePending finishes the current journal and writes the buffer to the
|
||||
// underlying writer.
|
||||
func (w *Writer) writePending() {
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
if w.pending {
|
||||
w.fillHeader(true)
|
||||
w.pending = false
|
||||
}
|
||||
_, w.err = w.w.Write(w.buf[w.written:w.j])
|
||||
w.written = w.j
|
||||
}
|
||||
|
||||
// Close finishes the current journal and closes the writer.
|
||||
func (w *Writer) Close() error {
|
||||
w.seq++
|
||||
w.writePending()
|
||||
if w.err != nil {
|
||||
return w.err
|
||||
}
|
||||
w.err = errors.New("leveldb/journal: closed Writer")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Flush finishes the current journal, writes to the underlying writer, and
|
||||
// flushes it if that writer implements interface{ Flush() error }.
|
||||
func (w *Writer) Flush() error {
|
||||
w.seq++
|
||||
w.writePending()
|
||||
if w.err != nil {
|
||||
return w.err
|
||||
}
|
||||
if w.f != nil {
|
||||
w.err = w.f.Flush()
|
||||
return w.err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reset resets the journal writer, allows reuse of the journal writer. Reset
|
||||
// will also closes the journal writer if not already.
|
||||
func (w *Writer) Reset(writer io.Writer) (err error) {
|
||||
w.seq++
|
||||
if w.err == nil {
|
||||
w.writePending()
|
||||
err = w.err
|
||||
}
|
||||
w.w = writer
|
||||
w.f, _ = writer.(flusher)
|
||||
w.i = 0
|
||||
w.j = 0
|
||||
w.written = 0
|
||||
w.first = false
|
||||
w.pending = false
|
||||
w.err = nil
|
||||
return
|
||||
}
|
||||
|
||||
// Next returns a writer for the next journal. The writer returned becomes stale
|
||||
// after the next Close, Flush or Next call, and should no longer be used.
|
||||
func (w *Writer) Next() (io.Writer, error) {
|
||||
w.seq++
|
||||
if w.err != nil {
|
||||
return nil, w.err
|
||||
}
|
||||
if w.pending {
|
||||
w.fillHeader(true)
|
||||
}
|
||||
w.i = w.j
|
||||
w.j = w.j + headerSize
|
||||
// Check if there is room in the block for the header.
|
||||
if w.j > blockSize {
|
||||
// Fill in the rest of the block with zeroes.
|
||||
for k := w.i; k < blockSize; k++ {
|
||||
w.buf[k] = 0
|
||||
}
|
||||
w.writeBlock()
|
||||
if w.err != nil {
|
||||
return nil, w.err
|
||||
}
|
||||
}
|
||||
w.first = true
|
||||
w.pending = true
|
||||
return singleWriter{w, w.seq}, nil
|
||||
}
|
||||
|
||||
type singleWriter struct {
|
||||
w *Writer
|
||||
seq int
|
||||
}
|
||||
|
||||
func (x singleWriter) Write(p []byte) (int, error) {
|
||||
w := x.w
|
||||
if w.seq != x.seq {
|
||||
return 0, errors.New("leveldb/journal: stale writer")
|
||||
}
|
||||
if w.err != nil {
|
||||
return 0, w.err
|
||||
}
|
||||
n0 := len(p)
|
||||
for len(p) > 0 {
|
||||
// Write a block, if it is full.
|
||||
if w.j == blockSize {
|
||||
w.fillHeader(false)
|
||||
w.writeBlock()
|
||||
if w.err != nil {
|
||||
return 0, w.err
|
||||
}
|
||||
w.first = false
|
||||
}
|
||||
// Copy bytes into the buffer.
|
||||
n := copy(w.buf[w.j:], p)
|
||||
w.j += n
|
||||
p = p[n:]
|
||||
}
|
||||
return n0, nil
|
||||
}
|
||||
@ -0,0 +1,143 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/errors"
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
)
|
||||
|
||||
// ErrInternalKeyCorrupted records internal key corruption.
|
||||
type ErrInternalKeyCorrupted struct {
|
||||
Ikey []byte
|
||||
Reason string
|
||||
}
|
||||
|
||||
func (e *ErrInternalKeyCorrupted) Error() string {
|
||||
return fmt.Sprintf("leveldb: internal key %q corrupted: %s", e.Ikey, e.Reason)
|
||||
}
|
||||
|
||||
func newErrInternalKeyCorrupted(ikey []byte, reason string) error {
|
||||
return errors.NewErrCorrupted(storage.FileDesc{}, &ErrInternalKeyCorrupted{append([]byte{}, ikey...), reason})
|
||||
}
|
||||
|
||||
type keyType uint
|
||||
|
||||
func (kt keyType) String() string {
|
||||
switch kt {
|
||||
case keyTypeDel:
|
||||
return "d"
|
||||
case keyTypeVal:
|
||||
return "v"
|
||||
}
|
||||
return fmt.Sprintf("<invalid:%#x>", uint(kt))
|
||||
}
|
||||
|
||||
// Value types encoded as the last component of internal keys.
|
||||
// Don't modify; this value are saved to disk.
|
||||
const (
|
||||
keyTypeDel = keyType(0)
|
||||
keyTypeVal = keyType(1)
|
||||
)
|
||||
|
||||
// keyTypeSeek defines the keyType that should be passed when constructing an
|
||||
// internal key for seeking to a particular sequence number (since we
|
||||
// sort sequence numbers in decreasing order and the value type is
|
||||
// embedded as the low 8 bits in the sequence number in internal keys,
|
||||
// we need to use the highest-numbered ValueType, not the lowest).
|
||||
const keyTypeSeek = keyTypeVal
|
||||
|
||||
const (
|
||||
// Maximum value possible for sequence number; the 8-bits are
|
||||
// used by value type, so its can packed together in single
|
||||
// 64-bit integer.
|
||||
keyMaxSeq = (uint64(1) << 56) - 1
|
||||
// Maximum value possible for packed sequence number and type.
|
||||
keyMaxNum = (keyMaxSeq << 8) | uint64(keyTypeSeek)
|
||||
)
|
||||
|
||||
// Maximum number encoded in bytes.
|
||||
var keyMaxNumBytes = make([]byte, 8)
|
||||
|
||||
func init() {
|
||||
binary.LittleEndian.PutUint64(keyMaxNumBytes, keyMaxNum)
|
||||
}
|
||||
|
||||
type internalKey []byte
|
||||
|
||||
func makeInternalKey(dst, ukey []byte, seq uint64, kt keyType) internalKey {
|
||||
if seq > keyMaxSeq {
|
||||
panic("leveldb: invalid sequence number")
|
||||
} else if kt > keyTypeVal {
|
||||
panic("leveldb: invalid type")
|
||||
}
|
||||
|
||||
dst = ensureBuffer(dst, len(ukey)+8)
|
||||
copy(dst, ukey)
|
||||
binary.LittleEndian.PutUint64(dst[len(ukey):], (seq<<8)|uint64(kt))
|
||||
return internalKey(dst)
|
||||
}
|
||||
|
||||
func parseInternalKey(ik []byte) (ukey []byte, seq uint64, kt keyType, err error) {
|
||||
if len(ik) < 8 {
|
||||
return nil, 0, 0, newErrInternalKeyCorrupted(ik, "invalid length")
|
||||
}
|
||||
num := binary.LittleEndian.Uint64(ik[len(ik)-8:])
|
||||
seq, kt = uint64(num>>8), keyType(num&0xff)
|
||||
if kt > keyTypeVal {
|
||||
return nil, 0, 0, newErrInternalKeyCorrupted(ik, "invalid type")
|
||||
}
|
||||
ukey = ik[:len(ik)-8]
|
||||
return
|
||||
}
|
||||
|
||||
func validInternalKey(ik []byte) bool {
|
||||
_, _, _, err := parseInternalKey(ik)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (ik internalKey) assert() {
|
||||
if ik == nil {
|
||||
panic("leveldb: nil internalKey")
|
||||
}
|
||||
if len(ik) < 8 {
|
||||
panic(fmt.Sprintf("leveldb: internal key %q, len=%d: invalid length", []byte(ik), len(ik)))
|
||||
}
|
||||
}
|
||||
|
||||
func (ik internalKey) ukey() []byte {
|
||||
ik.assert()
|
||||
return ik[:len(ik)-8]
|
||||
}
|
||||
|
||||
func (ik internalKey) num() uint64 {
|
||||
ik.assert()
|
||||
return binary.LittleEndian.Uint64(ik[len(ik)-8:])
|
||||
}
|
||||
|
||||
func (ik internalKey) parseNum() (seq uint64, kt keyType) {
|
||||
num := ik.num()
|
||||
seq, kt = uint64(num>>8), keyType(num&0xff)
|
||||
if kt > keyTypeVal {
|
||||
panic(fmt.Sprintf("leveldb: internal key %q, len=%d: invalid type %#x", []byte(ik), len(ik), kt))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (ik internalKey) String() string {
|
||||
if ik == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
|
||||
if ukey, seq, kt, err := parseInternalKey(ik); err == nil {
|
||||
return fmt.Sprintf("%s,%s%d", shorten(string(ukey)), kt, seq)
|
||||
}
|
||||
return fmt.Sprintf("<invalid:%#x>", []byte(ik))
|
||||
}
|
||||
@ -0,0 +1,479 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package memdb provides in-memory key/value database implementation.
|
||||
package memdb
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"sync"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/comparer"
|
||||
"github.com/syndtr/goleveldb/leveldb/errors"
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
// Common errors.
|
||||
var (
|
||||
ErrNotFound = errors.ErrNotFound
|
||||
ErrIterReleased = errors.New("leveldb/memdb: iterator released")
|
||||
)
|
||||
|
||||
const tMaxHeight = 12
|
||||
|
||||
type dbIter struct {
|
||||
util.BasicReleaser
|
||||
p *DB
|
||||
slice *util.Range
|
||||
node int
|
||||
forward bool
|
||||
key, value []byte
|
||||
err error
|
||||
}
|
||||
|
||||
func (i *dbIter) fill(checkStart, checkLimit bool) bool {
|
||||
if i.node != 0 {
|
||||
n := i.p.nodeData[i.node]
|
||||
m := n + i.p.nodeData[i.node+nKey]
|
||||
i.key = i.p.kvData[n:m]
|
||||
if i.slice != nil {
|
||||
switch {
|
||||
case checkLimit && i.slice.Limit != nil && i.p.cmp.Compare(i.key, i.slice.Limit) >= 0:
|
||||
fallthrough
|
||||
case checkStart && i.slice.Start != nil && i.p.cmp.Compare(i.key, i.slice.Start) < 0:
|
||||
i.node = 0
|
||||
goto bail
|
||||
}
|
||||
}
|
||||
i.value = i.p.kvData[m : m+i.p.nodeData[i.node+nVal]]
|
||||
return true
|
||||
}
|
||||
bail:
|
||||
i.key = nil
|
||||
i.value = nil
|
||||
return false
|
||||
}
|
||||
|
||||
func (i *dbIter) Valid() bool {
|
||||
return i.node != 0
|
||||
}
|
||||
|
||||
func (i *dbIter) First() bool {
|
||||
if i.Released() {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
i.forward = true
|
||||
i.p.mu.RLock()
|
||||
defer i.p.mu.RUnlock()
|
||||
if i.slice != nil && i.slice.Start != nil {
|
||||
i.node, _ = i.p.findGE(i.slice.Start, false)
|
||||
} else {
|
||||
i.node = i.p.nodeData[nNext]
|
||||
}
|
||||
return i.fill(false, true)
|
||||
}
|
||||
|
||||
func (i *dbIter) Last() bool {
|
||||
if i.Released() {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
i.forward = false
|
||||
i.p.mu.RLock()
|
||||
defer i.p.mu.RUnlock()
|
||||
if i.slice != nil && i.slice.Limit != nil {
|
||||
i.node = i.p.findLT(i.slice.Limit)
|
||||
} else {
|
||||
i.node = i.p.findLast()
|
||||
}
|
||||
return i.fill(true, false)
|
||||
}
|
||||
|
||||
func (i *dbIter) Seek(key []byte) bool {
|
||||
if i.Released() {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
i.forward = true
|
||||
i.p.mu.RLock()
|
||||
defer i.p.mu.RUnlock()
|
||||
if i.slice != nil && i.slice.Start != nil && i.p.cmp.Compare(key, i.slice.Start) < 0 {
|
||||
key = i.slice.Start
|
||||
}
|
||||
i.node, _ = i.p.findGE(key, false)
|
||||
return i.fill(false, true)
|
||||
}
|
||||
|
||||
func (i *dbIter) Next() bool {
|
||||
if i.Released() {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
if i.node == 0 {
|
||||
if !i.forward {
|
||||
return i.First()
|
||||
}
|
||||
return false
|
||||
}
|
||||
i.forward = true
|
||||
i.p.mu.RLock()
|
||||
defer i.p.mu.RUnlock()
|
||||
i.node = i.p.nodeData[i.node+nNext]
|
||||
return i.fill(false, true)
|
||||
}
|
||||
|
||||
func (i *dbIter) Prev() bool {
|
||||
if i.Released() {
|
||||
i.err = ErrIterReleased
|
||||
return false
|
||||
}
|
||||
|
||||
if i.node == 0 {
|
||||
if i.forward {
|
||||
return i.Last()
|
||||
}
|
||||
return false
|
||||
}
|
||||
i.forward = false
|
||||
i.p.mu.RLock()
|
||||
defer i.p.mu.RUnlock()
|
||||
i.node = i.p.findLT(i.key)
|
||||
return i.fill(true, false)
|
||||
}
|
||||
|
||||
func (i *dbIter) Key() []byte {
|
||||
return i.key
|
||||
}
|
||||
|
||||
func (i *dbIter) Value() []byte {
|
||||
return i.value
|
||||
}
|
||||
|
||||
func (i *dbIter) Error() error { return i.err }
|
||||
|
||||
func (i *dbIter) Release() {
|
||||
if !i.Released() {
|
||||
i.p = nil
|
||||
i.node = 0
|
||||
i.key = nil
|
||||
i.value = nil
|
||||
i.BasicReleaser.Release()
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
nKV = iota
|
||||
nKey
|
||||
nVal
|
||||
nHeight
|
||||
nNext
|
||||
)
|
||||
|
||||
// DB is an in-memory key/value database.
|
||||
type DB struct {
|
||||
cmp comparer.BasicComparer
|
||||
rnd *rand.Rand
|
||||
|
||||
mu sync.RWMutex
|
||||
kvData []byte
|
||||
// Node data:
|
||||
// [0] : KV offset
|
||||
// [1] : Key length
|
||||
// [2] : Value length
|
||||
// [3] : Height
|
||||
// [3..height] : Next nodes
|
||||
nodeData []int
|
||||
prevNode [tMaxHeight]int
|
||||
maxHeight int
|
||||
n int
|
||||
kvSize int
|
||||
}
|
||||
|
||||
func (p *DB) randHeight() (h int) {
|
||||
const branching = 4
|
||||
h = 1
|
||||
for h < tMaxHeight && p.rnd.Int()%branching == 0 {
|
||||
h++
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Must hold RW-lock if prev == true, as it use shared prevNode slice.
|
||||
func (p *DB) findGE(key []byte, prev bool) (int, bool) {
|
||||
node := 0
|
||||
h := p.maxHeight - 1
|
||||
for {
|
||||
next := p.nodeData[node+nNext+h]
|
||||
cmp := 1
|
||||
if next != 0 {
|
||||
o := p.nodeData[next]
|
||||
cmp = p.cmp.Compare(p.kvData[o:o+p.nodeData[next+nKey]], key)
|
||||
}
|
||||
if cmp < 0 {
|
||||
// Keep searching in this list
|
||||
node = next
|
||||
} else {
|
||||
if prev {
|
||||
p.prevNode[h] = node
|
||||
} else if cmp == 0 {
|
||||
return next, true
|
||||
}
|
||||
if h == 0 {
|
||||
return next, cmp == 0
|
||||
}
|
||||
h--
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *DB) findLT(key []byte) int {
|
||||
node := 0
|
||||
h := p.maxHeight - 1
|
||||
for {
|
||||
next := p.nodeData[node+nNext+h]
|
||||
o := p.nodeData[next]
|
||||
if next == 0 || p.cmp.Compare(p.kvData[o:o+p.nodeData[next+nKey]], key) >= 0 {
|
||||
if h == 0 {
|
||||
break
|
||||
}
|
||||
h--
|
||||
} else {
|
||||
node = next
|
||||
}
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
func (p *DB) findLast() int {
|
||||
node := 0
|
||||
h := p.maxHeight - 1
|
||||
for {
|
||||
next := p.nodeData[node+nNext+h]
|
||||
if next == 0 {
|
||||
if h == 0 {
|
||||
break
|
||||
}
|
||||
h--
|
||||
} else {
|
||||
node = next
|
||||
}
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
// Put sets the value for the given key. It overwrites any previous value
|
||||
// for that key; a DB is not a multi-map.
|
||||
//
|
||||
// It is safe to modify the contents of the arguments after Put returns.
|
||||
func (p *DB) Put(key []byte, value []byte) error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
if node, exact := p.findGE(key, true); exact {
|
||||
kvOffset := len(p.kvData)
|
||||
p.kvData = append(p.kvData, key...)
|
||||
p.kvData = append(p.kvData, value...)
|
||||
p.nodeData[node] = kvOffset
|
||||
m := p.nodeData[node+nVal]
|
||||
p.nodeData[node+nVal] = len(value)
|
||||
p.kvSize += len(value) - m
|
||||
return nil
|
||||
}
|
||||
|
||||
h := p.randHeight()
|
||||
if h > p.maxHeight {
|
||||
for i := p.maxHeight; i < h; i++ {
|
||||
p.prevNode[i] = 0
|
||||
}
|
||||
p.maxHeight = h
|
||||
}
|
||||
|
||||
kvOffset := len(p.kvData)
|
||||
p.kvData = append(p.kvData, key...)
|
||||
p.kvData = append(p.kvData, value...)
|
||||
// Node
|
||||
node := len(p.nodeData)
|
||||
p.nodeData = append(p.nodeData, kvOffset, len(key), len(value), h)
|
||||
for i, n := range p.prevNode[:h] {
|
||||
m := n + nNext + i
|
||||
p.nodeData = append(p.nodeData, p.nodeData[m])
|
||||
p.nodeData[m] = node
|
||||
}
|
||||
|
||||
p.kvSize += len(key) + len(value)
|
||||
p.n++
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete deletes the value for the given key. It returns ErrNotFound if
|
||||
// the DB does not contain the key.
|
||||
//
|
||||
// It is safe to modify the contents of the arguments after Delete returns.
|
||||
func (p *DB) Delete(key []byte) error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
node, exact := p.findGE(key, true)
|
||||
if !exact {
|
||||
return ErrNotFound
|
||||
}
|
||||
|
||||
h := p.nodeData[node+nHeight]
|
||||
for i, n := range p.prevNode[:h] {
|
||||
m := n + nNext + i
|
||||
p.nodeData[m] = p.nodeData[p.nodeData[m]+nNext+i]
|
||||
}
|
||||
|
||||
p.kvSize -= p.nodeData[node+nKey] + p.nodeData[node+nVal]
|
||||
p.n--
|
||||
return nil
|
||||
}
|
||||
|
||||
// Contains returns true if the given key are in the DB.
|
||||
//
|
||||
// It is safe to modify the contents of the arguments after Contains returns.
|
||||
func (p *DB) Contains(key []byte) bool {
|
||||
p.mu.RLock()
|
||||
_, exact := p.findGE(key, false)
|
||||
p.mu.RUnlock()
|
||||
return exact
|
||||
}
|
||||
|
||||
// Get gets the value for the given key. It returns error.ErrNotFound if the
|
||||
// DB does not contain the key.
|
||||
//
|
||||
// The caller should not modify the contents of the returned slice, but
|
||||
// it is safe to modify the contents of the argument after Get returns.
|
||||
func (p *DB) Get(key []byte) (value []byte, err error) {
|
||||
p.mu.RLock()
|
||||
if node, exact := p.findGE(key, false); exact {
|
||||
o := p.nodeData[node] + p.nodeData[node+nKey]
|
||||
value = p.kvData[o : o+p.nodeData[node+nVal]]
|
||||
} else {
|
||||
err = ErrNotFound
|
||||
}
|
||||
p.mu.RUnlock()
|
||||
return
|
||||
}
|
||||
|
||||
// Find finds key/value pair whose key is greater than or equal to the
|
||||
// given key. It returns ErrNotFound if the table doesn't contain
|
||||
// such pair.
|
||||
//
|
||||
// The caller should not modify the contents of the returned slice, but
|
||||
// it is safe to modify the contents of the argument after Find returns.
|
||||
func (p *DB) Find(key []byte) (rkey, value []byte, err error) {
|
||||
p.mu.RLock()
|
||||
if node, _ := p.findGE(key, false); node != 0 {
|
||||
n := p.nodeData[node]
|
||||
m := n + p.nodeData[node+nKey]
|
||||
rkey = p.kvData[n:m]
|
||||
value = p.kvData[m : m+p.nodeData[node+nVal]]
|
||||
} else {
|
||||
err = ErrNotFound
|
||||
}
|
||||
p.mu.RUnlock()
|
||||
return
|
||||
}
|
||||
|
||||
// NewIterator returns an iterator of the DB.
|
||||
// The returned iterator is not safe for concurrent use, but it is safe to use
|
||||
// multiple iterators concurrently, with each in a dedicated goroutine.
|
||||
// It is also safe to use an iterator concurrently with modifying its
|
||||
// underlying DB. However, the resultant key/value pairs are not guaranteed
|
||||
// to be a consistent snapshot of the DB at a particular point in time.
|
||||
//
|
||||
// Slice allows slicing the iterator to only contains keys in the given
|
||||
// range. A nil Range.Start is treated as a key before all keys in the
|
||||
// DB. And a nil Range.Limit is treated as a key after all keys in
|
||||
// the DB.
|
||||
//
|
||||
// WARNING: Any slice returned by interator (e.g. slice returned by calling
|
||||
// Iterator.Key() or Iterator.Key() methods), its content should not be modified
|
||||
// unless noted otherwise.
|
||||
//
|
||||
// The iterator must be released after use, by calling Release method.
|
||||
//
|
||||
// Also read Iterator documentation of the leveldb/iterator package.
|
||||
func (p *DB) NewIterator(slice *util.Range) iterator.Iterator {
|
||||
return &dbIter{p: p, slice: slice}
|
||||
}
|
||||
|
||||
// Capacity returns keys/values buffer capacity.
|
||||
func (p *DB) Capacity() int {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
return cap(p.kvData)
|
||||
}
|
||||
|
||||
// Size returns sum of keys and values length. Note that deleted
|
||||
// key/value will not be accounted for, but it will still consume
|
||||
// the buffer, since the buffer is append only.
|
||||
func (p *DB) Size() int {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
return p.kvSize
|
||||
}
|
||||
|
||||
// Free returns keys/values free buffer before need to grow.
|
||||
func (p *DB) Free() int {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
return cap(p.kvData) - len(p.kvData)
|
||||
}
|
||||
|
||||
// Len returns the number of entries in the DB.
|
||||
func (p *DB) Len() int {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
return p.n
|
||||
}
|
||||
|
||||
// Reset resets the DB to initial empty state. Allows reuse the buffer.
|
||||
func (p *DB) Reset() {
|
||||
p.mu.Lock()
|
||||
p.rnd = rand.New(rand.NewSource(0xdeadbeef))
|
||||
p.maxHeight = 1
|
||||
p.n = 0
|
||||
p.kvSize = 0
|
||||
p.kvData = p.kvData[:0]
|
||||
p.nodeData = p.nodeData[:nNext+tMaxHeight]
|
||||
p.nodeData[nKV] = 0
|
||||
p.nodeData[nKey] = 0
|
||||
p.nodeData[nVal] = 0
|
||||
p.nodeData[nHeight] = tMaxHeight
|
||||
for n := 0; n < tMaxHeight; n++ {
|
||||
p.nodeData[nNext+n] = 0
|
||||
p.prevNode[n] = 0
|
||||
}
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
// New creates a new initialized in-memory key/value DB. The capacity
|
||||
// is the initial key/value buffer capacity. The capacity is advisory,
|
||||
// not enforced.
|
||||
//
|
||||
// This DB is append-only, deleting an entry would remove entry node but not
|
||||
// reclaim KV buffer.
|
||||
//
|
||||
// The returned DB instance is safe for concurrent use.
|
||||
func New(cmp comparer.BasicComparer, capacity int) *DB {
|
||||
p := &DB{
|
||||
cmp: cmp,
|
||||
rnd: rand.New(rand.NewSource(0xdeadbeef)),
|
||||
maxHeight: 1,
|
||||
kvData: make([]byte, 0, capacity),
|
||||
nodeData: make([]int, 4+tMaxHeight),
|
||||
}
|
||||
p.nodeData[nHeight] = tMaxHeight
|
||||
return p
|
||||
}
|
||||
@ -0,0 +1,697 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package opt provides sets of options used by LevelDB.
|
||||
package opt
|
||||
|
||||
import (
|
||||
"math"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/cache"
|
||||
"github.com/syndtr/goleveldb/leveldb/comparer"
|
||||
"github.com/syndtr/goleveldb/leveldb/filter"
|
||||
)
|
||||
|
||||
const (
|
||||
KiB = 1024
|
||||
MiB = KiB * 1024
|
||||
GiB = MiB * 1024
|
||||
)
|
||||
|
||||
var (
|
||||
DefaultBlockCacher = LRUCacher
|
||||
DefaultBlockCacheCapacity = 8 * MiB
|
||||
DefaultBlockRestartInterval = 16
|
||||
DefaultBlockSize = 4 * KiB
|
||||
DefaultCompactionExpandLimitFactor = 25
|
||||
DefaultCompactionGPOverlapsFactor = 10
|
||||
DefaultCompactionL0Trigger = 4
|
||||
DefaultCompactionSourceLimitFactor = 1
|
||||
DefaultCompactionTableSize = 2 * MiB
|
||||
DefaultCompactionTableSizeMultiplier = 1.0
|
||||
DefaultCompactionTotalSize = 10 * MiB
|
||||
DefaultCompactionTotalSizeMultiplier = 10.0
|
||||
DefaultCompressionType = SnappyCompression
|
||||
DefaultIteratorSamplingRate = 1 * MiB
|
||||
DefaultOpenFilesCacher = LRUCacher
|
||||
DefaultOpenFilesCacheCapacity = 500
|
||||
DefaultWriteBuffer = 4 * MiB
|
||||
DefaultWriteL0PauseTrigger = 12
|
||||
DefaultWriteL0SlowdownTrigger = 8
|
||||
)
|
||||
|
||||
// Cacher is a caching algorithm.
|
||||
type Cacher interface {
|
||||
New(capacity int) cache.Cacher
|
||||
}
|
||||
|
||||
type CacherFunc struct {
|
||||
NewFunc func(capacity int) cache.Cacher
|
||||
}
|
||||
|
||||
func (f *CacherFunc) New(capacity int) cache.Cacher {
|
||||
if f.NewFunc != nil {
|
||||
return f.NewFunc(capacity)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func noCacher(int) cache.Cacher { return nil }
|
||||
|
||||
var (
|
||||
// LRUCacher is the LRU-cache algorithm.
|
||||
LRUCacher = &CacherFunc{cache.NewLRU}
|
||||
|
||||
// NoCacher is the value to disable caching algorithm.
|
||||
NoCacher = &CacherFunc{}
|
||||
)
|
||||
|
||||
// Compression is the 'sorted table' block compression algorithm to use.
|
||||
type Compression uint
|
||||
|
||||
func (c Compression) String() string {
|
||||
switch c {
|
||||
case DefaultCompression:
|
||||
return "default"
|
||||
case NoCompression:
|
||||
return "none"
|
||||
case SnappyCompression:
|
||||
return "snappy"
|
||||
}
|
||||
return "invalid"
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultCompression Compression = iota
|
||||
NoCompression
|
||||
SnappyCompression
|
||||
nCompression
|
||||
)
|
||||
|
||||
// Strict is the DB 'strict level'.
|
||||
type Strict uint
|
||||
|
||||
const (
|
||||
// If present then a corrupted or invalid chunk or block in manifest
|
||||
// journal will cause an error instead of being dropped.
|
||||
// This will prevent database with corrupted manifest to be opened.
|
||||
StrictManifest Strict = 1 << iota
|
||||
|
||||
// If present then journal chunk checksum will be verified.
|
||||
StrictJournalChecksum
|
||||
|
||||
// If present then a corrupted or invalid chunk or block in journal
|
||||
// will cause an error instead of being dropped.
|
||||
// This will prevent database with corrupted journal to be opened.
|
||||
StrictJournal
|
||||
|
||||
// If present then 'sorted table' block checksum will be verified.
|
||||
// This has effect on both 'read operation' and compaction.
|
||||
StrictBlockChecksum
|
||||
|
||||
// If present then a corrupted 'sorted table' will fails compaction.
|
||||
// The database will enter read-only mode.
|
||||
StrictCompaction
|
||||
|
||||
// If present then a corrupted 'sorted table' will halts 'read operation'.
|
||||
StrictReader
|
||||
|
||||
// If present then leveldb.Recover will drop corrupted 'sorted table'.
|
||||
StrictRecovery
|
||||
|
||||
// This only applicable for ReadOptions, if present then this ReadOptions
|
||||
// 'strict level' will override global ones.
|
||||
StrictOverride
|
||||
|
||||
// StrictAll enables all strict flags.
|
||||
StrictAll = StrictManifest | StrictJournalChecksum | StrictJournal | StrictBlockChecksum | StrictCompaction | StrictReader | StrictRecovery
|
||||
|
||||
// DefaultStrict is the default strict flags. Specify any strict flags
|
||||
// will override default strict flags as whole (i.e. not OR'ed).
|
||||
DefaultStrict = StrictJournalChecksum | StrictBlockChecksum | StrictCompaction | StrictReader
|
||||
|
||||
// NoStrict disables all strict flags. Override default strict flags.
|
||||
NoStrict = ^StrictAll
|
||||
)
|
||||
|
||||
// Options holds the optional parameters for the DB at large.
|
||||
type Options struct {
|
||||
// AltFilters defines one or more 'alternative filters'.
|
||||
// 'alternative filters' will be used during reads if a filter block
|
||||
// does not match with the 'effective filter'.
|
||||
//
|
||||
// The default value is nil
|
||||
AltFilters []filter.Filter
|
||||
|
||||
// BlockCacher provides cache algorithm for LevelDB 'sorted table' block caching.
|
||||
// Specify NoCacher to disable caching algorithm.
|
||||
//
|
||||
// The default value is LRUCacher.
|
||||
BlockCacher Cacher
|
||||
|
||||
// BlockCacheCapacity defines the capacity of the 'sorted table' block caching.
|
||||
// Use -1 for zero, this has same effect as specifying NoCacher to BlockCacher.
|
||||
//
|
||||
// The default value is 8MiB.
|
||||
BlockCacheCapacity int
|
||||
|
||||
// BlockCacheEvictRemoved allows enable forced-eviction on cached block belonging
|
||||
// to removed 'sorted table'.
|
||||
//
|
||||
// The default if false.
|
||||
BlockCacheEvictRemoved bool
|
||||
|
||||
// BlockRestartInterval is the number of keys between restart points for
|
||||
// delta encoding of keys.
|
||||
//
|
||||
// The default value is 16.
|
||||
BlockRestartInterval int
|
||||
|
||||
// BlockSize is the minimum uncompressed size in bytes of each 'sorted table'
|
||||
// block.
|
||||
//
|
||||
// The default value is 4KiB.
|
||||
BlockSize int
|
||||
|
||||
// CompactionExpandLimitFactor limits compaction size after expanded.
|
||||
// This will be multiplied by table size limit at compaction target level.
|
||||
//
|
||||
// The default value is 25.
|
||||
CompactionExpandLimitFactor int
|
||||
|
||||
// CompactionGPOverlapsFactor limits overlaps in grandparent (Level + 2) that a
|
||||
// single 'sorted table' generates.
|
||||
// This will be multiplied by table size limit at grandparent level.
|
||||
//
|
||||
// The default value is 10.
|
||||
CompactionGPOverlapsFactor int
|
||||
|
||||
// CompactionL0Trigger defines number of 'sorted table' at level-0 that will
|
||||
// trigger compaction.
|
||||
//
|
||||
// The default value is 4.
|
||||
CompactionL0Trigger int
|
||||
|
||||
// CompactionSourceLimitFactor limits compaction source size. This doesn't apply to
|
||||
// level-0.
|
||||
// This will be multiplied by table size limit at compaction target level.
|
||||
//
|
||||
// The default value is 1.
|
||||
CompactionSourceLimitFactor int
|
||||
|
||||
// CompactionTableSize limits size of 'sorted table' that compaction generates.
|
||||
// The limits for each level will be calculated as:
|
||||
// CompactionTableSize * (CompactionTableSizeMultiplier ^ Level)
|
||||
// The multiplier for each level can also fine-tuned using CompactionTableSizeMultiplierPerLevel.
|
||||
//
|
||||
// The default value is 2MiB.
|
||||
CompactionTableSize int
|
||||
|
||||
// CompactionTableSizeMultiplier defines multiplier for CompactionTableSize.
|
||||
//
|
||||
// The default value is 1.
|
||||
CompactionTableSizeMultiplier float64
|
||||
|
||||
// CompactionTableSizeMultiplierPerLevel defines per-level multiplier for
|
||||
// CompactionTableSize.
|
||||
// Use zero to skip a level.
|
||||
//
|
||||
// The default value is nil.
|
||||
CompactionTableSizeMultiplierPerLevel []float64
|
||||
|
||||
// CompactionTotalSize limits total size of 'sorted table' for each level.
|
||||
// The limits for each level will be calculated as:
|
||||
// CompactionTotalSize * (CompactionTotalSizeMultiplier ^ Level)
|
||||
// The multiplier for each level can also fine-tuned using
|
||||
// CompactionTotalSizeMultiplierPerLevel.
|
||||
//
|
||||
// The default value is 10MiB.
|
||||
CompactionTotalSize int
|
||||
|
||||
// CompactionTotalSizeMultiplier defines multiplier for CompactionTotalSize.
|
||||
//
|
||||
// The default value is 10.
|
||||
CompactionTotalSizeMultiplier float64
|
||||
|
||||
// CompactionTotalSizeMultiplierPerLevel defines per-level multiplier for
|
||||
// CompactionTotalSize.
|
||||
// Use zero to skip a level.
|
||||
//
|
||||
// The default value is nil.
|
||||
CompactionTotalSizeMultiplierPerLevel []float64
|
||||
|
||||
// Comparer defines a total ordering over the space of []byte keys: a 'less
|
||||
// than' relationship. The same comparison algorithm must be used for reads
|
||||
// and writes over the lifetime of the DB.
|
||||
//
|
||||
// The default value uses the same ordering as bytes.Compare.
|
||||
Comparer comparer.Comparer
|
||||
|
||||
// Compression defines the 'sorted table' block compression to use.
|
||||
//
|
||||
// The default value (DefaultCompression) uses snappy compression.
|
||||
Compression Compression
|
||||
|
||||
// DisableBufferPool allows disable use of util.BufferPool functionality.
|
||||
//
|
||||
// The default value is false.
|
||||
DisableBufferPool bool
|
||||
|
||||
// DisableBlockCache allows disable use of cache.Cache functionality on
|
||||
// 'sorted table' block.
|
||||
//
|
||||
// The default value is false.
|
||||
DisableBlockCache bool
|
||||
|
||||
// DisableCompactionBackoff allows disable compaction retry backoff.
|
||||
//
|
||||
// The default value is false.
|
||||
DisableCompactionBackoff bool
|
||||
|
||||
// DisableLargeBatchTransaction allows disabling switch-to-transaction mode
|
||||
// on large batch write. If enable batch writes large than WriteBuffer will
|
||||
// use transaction.
|
||||
//
|
||||
// The default is false.
|
||||
DisableLargeBatchTransaction bool
|
||||
|
||||
// ErrorIfExist defines whether an error should returned if the DB already
|
||||
// exist.
|
||||
//
|
||||
// The default value is false.
|
||||
ErrorIfExist bool
|
||||
|
||||
// ErrorIfMissing defines whether an error should returned if the DB is
|
||||
// missing. If false then the database will be created if missing, otherwise
|
||||
// an error will be returned.
|
||||
//
|
||||
// The default value is false.
|
||||
ErrorIfMissing bool
|
||||
|
||||
// Filter defines an 'effective filter' to use. An 'effective filter'
|
||||
// if defined will be used to generate per-table filter block.
|
||||
// The filter name will be stored on disk.
|
||||
// During reads LevelDB will try to find matching filter from
|
||||
// 'effective filter' and 'alternative filters'.
|
||||
//
|
||||
// Filter can be changed after a DB has been created. It is recommended
|
||||
// to put old filter to the 'alternative filters' to mitigate lack of
|
||||
// filter during transition period.
|
||||
//
|
||||
// A filter is used to reduce disk reads when looking for a specific key.
|
||||
//
|
||||
// The default value is nil.
|
||||
Filter filter.Filter
|
||||
|
||||
// IteratorSamplingRate defines approximate gap (in bytes) between read
|
||||
// sampling of an iterator. The samples will be used to determine when
|
||||
// compaction should be triggered.
|
||||
//
|
||||
// The default is 1MiB.
|
||||
IteratorSamplingRate int
|
||||
|
||||
// NoSync allows completely disable fsync.
|
||||
//
|
||||
// The default is false.
|
||||
NoSync bool
|
||||
|
||||
// NoWriteMerge allows disabling write merge.
|
||||
//
|
||||
// The default is false.
|
||||
NoWriteMerge bool
|
||||
|
||||
// OpenFilesCacher provides cache algorithm for open files caching.
|
||||
// Specify NoCacher to disable caching algorithm.
|
||||
//
|
||||
// The default value is LRUCacher.
|
||||
OpenFilesCacher Cacher
|
||||
|
||||
// OpenFilesCacheCapacity defines the capacity of the open files caching.
|
||||
// Use -1 for zero, this has same effect as specifying NoCacher to OpenFilesCacher.
|
||||
//
|
||||
// The default value is 500.
|
||||
OpenFilesCacheCapacity int
|
||||
|
||||
// If true then opens DB in read-only mode.
|
||||
//
|
||||
// The default value is false.
|
||||
ReadOnly bool
|
||||
|
||||
// Strict defines the DB strict level.
|
||||
Strict Strict
|
||||
|
||||
// WriteBuffer defines maximum size of a 'memdb' before flushed to
|
||||
// 'sorted table'. 'memdb' is an in-memory DB backed by an on-disk
|
||||
// unsorted journal.
|
||||
//
|
||||
// LevelDB may held up to two 'memdb' at the same time.
|
||||
//
|
||||
// The default value is 4MiB.
|
||||
WriteBuffer int
|
||||
|
||||
// WriteL0StopTrigger defines number of 'sorted table' at level-0 that will
|
||||
// pause write.
|
||||
//
|
||||
// The default value is 12.
|
||||
WriteL0PauseTrigger int
|
||||
|
||||
// WriteL0SlowdownTrigger defines number of 'sorted table' at level-0 that
|
||||
// will trigger write slowdown.
|
||||
//
|
||||
// The default value is 8.
|
||||
WriteL0SlowdownTrigger int
|
||||
}
|
||||
|
||||
func (o *Options) GetAltFilters() []filter.Filter {
|
||||
if o == nil {
|
||||
return nil
|
||||
}
|
||||
return o.AltFilters
|
||||
}
|
||||
|
||||
func (o *Options) GetBlockCacher() Cacher {
|
||||
if o == nil || o.BlockCacher == nil {
|
||||
return DefaultBlockCacher
|
||||
} else if o.BlockCacher == NoCacher {
|
||||
return nil
|
||||
}
|
||||
return o.BlockCacher
|
||||
}
|
||||
|
||||
func (o *Options) GetBlockCacheCapacity() int {
|
||||
if o == nil || o.BlockCacheCapacity == 0 {
|
||||
return DefaultBlockCacheCapacity
|
||||
} else if o.BlockCacheCapacity < 0 {
|
||||
return 0
|
||||
}
|
||||
return o.BlockCacheCapacity
|
||||
}
|
||||
|
||||
func (o *Options) GetBlockCacheEvictRemoved() bool {
|
||||
if o == nil {
|
||||
return false
|
||||
}
|
||||
return o.BlockCacheEvictRemoved
|
||||
}
|
||||
|
||||
func (o *Options) GetBlockRestartInterval() int {
|
||||
if o == nil || o.BlockRestartInterval <= 0 {
|
||||
return DefaultBlockRestartInterval
|
||||
}
|
||||
return o.BlockRestartInterval
|
||||
}
|
||||
|
||||
func (o *Options) GetBlockSize() int {
|
||||
if o == nil || o.BlockSize <= 0 {
|
||||
return DefaultBlockSize
|
||||
}
|
||||
return o.BlockSize
|
||||
}
|
||||
|
||||
func (o *Options) GetCompactionExpandLimit(level int) int {
|
||||
factor := DefaultCompactionExpandLimitFactor
|
||||
if o != nil && o.CompactionExpandLimitFactor > 0 {
|
||||
factor = o.CompactionExpandLimitFactor
|
||||
}
|
||||
return o.GetCompactionTableSize(level+1) * factor
|
||||
}
|
||||
|
||||
func (o *Options) GetCompactionGPOverlaps(level int) int {
|
||||
factor := DefaultCompactionGPOverlapsFactor
|
||||
if o != nil && o.CompactionGPOverlapsFactor > 0 {
|
||||
factor = o.CompactionGPOverlapsFactor
|
||||
}
|
||||
return o.GetCompactionTableSize(level+2) * factor
|
||||
}
|
||||
|
||||
func (o *Options) GetCompactionL0Trigger() int {
|
||||
if o == nil || o.CompactionL0Trigger == 0 {
|
||||
return DefaultCompactionL0Trigger
|
||||
}
|
||||
return o.CompactionL0Trigger
|
||||
}
|
||||
|
||||
func (o *Options) GetCompactionSourceLimit(level int) int {
|
||||
factor := DefaultCompactionSourceLimitFactor
|
||||
if o != nil && o.CompactionSourceLimitFactor > 0 {
|
||||
factor = o.CompactionSourceLimitFactor
|
||||
}
|
||||
return o.GetCompactionTableSize(level+1) * factor
|
||||
}
|
||||
|
||||
func (o *Options) GetCompactionTableSize(level int) int {
|
||||
var (
|
||||
base = DefaultCompactionTableSize
|
||||
mult float64
|
||||
)
|
||||
if o != nil {
|
||||
if o.CompactionTableSize > 0 {
|
||||
base = o.CompactionTableSize
|
||||
}
|
||||
if level < len(o.CompactionTableSizeMultiplierPerLevel) && o.CompactionTableSizeMultiplierPerLevel[level] > 0 {
|
||||
mult = o.CompactionTableSizeMultiplierPerLevel[level]
|
||||
} else if o.CompactionTableSizeMultiplier > 0 {
|
||||
mult = math.Pow(o.CompactionTableSizeMultiplier, float64(level))
|
||||
}
|
||||
}
|
||||
if mult == 0 {
|
||||
mult = math.Pow(DefaultCompactionTableSizeMultiplier, float64(level))
|
||||
}
|
||||
return int(float64(base) * mult)
|
||||
}
|
||||
|
||||
func (o *Options) GetCompactionTotalSize(level int) int64 {
|
||||
var (
|
||||
base = DefaultCompactionTotalSize
|
||||
mult float64
|
||||
)
|
||||
if o != nil {
|
||||
if o.CompactionTotalSize > 0 {
|
||||
base = o.CompactionTotalSize
|
||||
}
|
||||
if level < len(o.CompactionTotalSizeMultiplierPerLevel) && o.CompactionTotalSizeMultiplierPerLevel[level] > 0 {
|
||||
mult = o.CompactionTotalSizeMultiplierPerLevel[level]
|
||||
} else if o.CompactionTotalSizeMultiplier > 0 {
|
||||
mult = math.Pow(o.CompactionTotalSizeMultiplier, float64(level))
|
||||
}
|
||||
}
|
||||
if mult == 0 {
|
||||
mult = math.Pow(DefaultCompactionTotalSizeMultiplier, float64(level))
|
||||
}
|
||||
return int64(float64(base) * mult)
|
||||
}
|
||||
|
||||
func (o *Options) GetComparer() comparer.Comparer {
|
||||
if o == nil || o.Comparer == nil {
|
||||
return comparer.DefaultComparer
|
||||
}
|
||||
return o.Comparer
|
||||
}
|
||||
|
||||
func (o *Options) GetCompression() Compression {
|
||||
if o == nil || o.Compression <= DefaultCompression || o.Compression >= nCompression {
|
||||
return DefaultCompressionType
|
||||
}
|
||||
return o.Compression
|
||||
}
|
||||
|
||||
func (o *Options) GetDisableBufferPool() bool {
|
||||
if o == nil {
|
||||
return false
|
||||
}
|
||||
return o.DisableBufferPool
|
||||
}
|
||||
|
||||
func (o *Options) GetDisableBlockCache() bool {
|
||||
if o == nil {
|
||||
return false
|
||||
}
|
||||
return o.DisableBlockCache
|
||||
}
|
||||
|
||||
func (o *Options) GetDisableCompactionBackoff() bool {
|
||||
if o == nil {
|
||||
return false
|
||||
}
|
||||
return o.DisableCompactionBackoff
|
||||
}
|
||||
|
||||
func (o *Options) GetDisableLargeBatchTransaction() bool {
|
||||
if o == nil {
|
||||
return false
|
||||
}
|
||||
return o.DisableLargeBatchTransaction
|
||||
}
|
||||
|
||||
func (o *Options) GetErrorIfExist() bool {
|
||||
if o == nil {
|
||||
return false
|
||||
}
|
||||
return o.ErrorIfExist
|
||||
}
|
||||
|
||||
func (o *Options) GetErrorIfMissing() bool {
|
||||
if o == nil {
|
||||
return false
|
||||
}
|
||||
return o.ErrorIfMissing
|
||||
}
|
||||
|
||||
func (o *Options) GetFilter() filter.Filter {
|
||||
if o == nil {
|
||||
return nil
|
||||
}
|
||||
return o.Filter
|
||||
}
|
||||
|
||||
func (o *Options) GetIteratorSamplingRate() int {
|
||||
if o == nil || o.IteratorSamplingRate <= 0 {
|
||||
return DefaultIteratorSamplingRate
|
||||
}
|
||||
return o.IteratorSamplingRate
|
||||
}
|
||||
|
||||
func (o *Options) GetNoSync() bool {
|
||||
if o == nil {
|
||||
return false
|
||||
}
|
||||
return o.NoSync
|
||||
}
|
||||
|
||||
func (o *Options) GetNoWriteMerge() bool {
|
||||
if o == nil {
|
||||
return false
|
||||
}
|
||||
return o.NoWriteMerge
|
||||
}
|
||||
|
||||
func (o *Options) GetOpenFilesCacher() Cacher {
|
||||
if o == nil || o.OpenFilesCacher == nil {
|
||||
return DefaultOpenFilesCacher
|
||||
}
|
||||
if o.OpenFilesCacher == NoCacher {
|
||||
return nil
|
||||
}
|
||||
return o.OpenFilesCacher
|
||||
}
|
||||
|
||||
func (o *Options) GetOpenFilesCacheCapacity() int {
|
||||
if o == nil || o.OpenFilesCacheCapacity == 0 {
|
||||
return DefaultOpenFilesCacheCapacity
|
||||
} else if o.OpenFilesCacheCapacity < 0 {
|
||||
return 0
|
||||
}
|
||||
return o.OpenFilesCacheCapacity
|
||||
}
|
||||
|
||||
func (o *Options) GetReadOnly() bool {
|
||||
if o == nil {
|
||||
return false
|
||||
}
|
||||
return o.ReadOnly
|
||||
}
|
||||
|
||||
func (o *Options) GetStrict(strict Strict) bool {
|
||||
if o == nil || o.Strict == 0 {
|
||||
return DefaultStrict&strict != 0
|
||||
}
|
||||
return o.Strict&strict != 0
|
||||
}
|
||||
|
||||
func (o *Options) GetWriteBuffer() int {
|
||||
if o == nil || o.WriteBuffer <= 0 {
|
||||
return DefaultWriteBuffer
|
||||
}
|
||||
return o.WriteBuffer
|
||||
}
|
||||
|
||||
func (o *Options) GetWriteL0PauseTrigger() int {
|
||||
if o == nil || o.WriteL0PauseTrigger == 0 {
|
||||
return DefaultWriteL0PauseTrigger
|
||||
}
|
||||
return o.WriteL0PauseTrigger
|
||||
}
|
||||
|
||||
func (o *Options) GetWriteL0SlowdownTrigger() int {
|
||||
if o == nil || o.WriteL0SlowdownTrigger == 0 {
|
||||
return DefaultWriteL0SlowdownTrigger
|
||||
}
|
||||
return o.WriteL0SlowdownTrigger
|
||||
}
|
||||
|
||||
// ReadOptions holds the optional parameters for 'read operation'. The
|
||||
// 'read operation' includes Get, Find and NewIterator.
|
||||
type ReadOptions struct {
|
||||
// DontFillCache defines whether block reads for this 'read operation'
|
||||
// should be cached. If false then the block will be cached. This does
|
||||
// not affects already cached block.
|
||||
//
|
||||
// The default value is false.
|
||||
DontFillCache bool
|
||||
|
||||
// Strict will be OR'ed with global DB 'strict level' unless StrictOverride
|
||||
// is present. Currently only StrictReader that has effect here.
|
||||
Strict Strict
|
||||
}
|
||||
|
||||
func (ro *ReadOptions) GetDontFillCache() bool {
|
||||
if ro == nil {
|
||||
return false
|
||||
}
|
||||
return ro.DontFillCache
|
||||
}
|
||||
|
||||
func (ro *ReadOptions) GetStrict(strict Strict) bool {
|
||||
if ro == nil {
|
||||
return false
|
||||
}
|
||||
return ro.Strict&strict != 0
|
||||
}
|
||||
|
||||
// WriteOptions holds the optional parameters for 'write operation'. The
|
||||
// 'write operation' includes Write, Put and Delete.
|
||||
type WriteOptions struct {
|
||||
// NoWriteMerge allows disabling write merge.
|
||||
//
|
||||
// The default is false.
|
||||
NoWriteMerge bool
|
||||
|
||||
// Sync is whether to sync underlying writes from the OS buffer cache
|
||||
// through to actual disk, if applicable. Setting Sync can result in
|
||||
// slower writes.
|
||||
//
|
||||
// If false, and the machine crashes, then some recent writes may be lost.
|
||||
// Note that if it is just the process that crashes (and the machine does
|
||||
// not) then no writes will be lost.
|
||||
//
|
||||
// In other words, Sync being false has the same semantics as a write
|
||||
// system call. Sync being true means write followed by fsync.
|
||||
//
|
||||
// The default value is false.
|
||||
Sync bool
|
||||
}
|
||||
|
||||
func (wo *WriteOptions) GetNoWriteMerge() bool {
|
||||
if wo == nil {
|
||||
return false
|
||||
}
|
||||
return wo.NoWriteMerge
|
||||
}
|
||||
|
||||
func (wo *WriteOptions) GetSync() bool {
|
||||
if wo == nil {
|
||||
return false
|
||||
}
|
||||
return wo.Sync
|
||||
}
|
||||
|
||||
func GetStrict(o *Options, ro *ReadOptions, strict Strict) bool {
|
||||
if ro.GetStrict(StrictOverride) {
|
||||
return ro.GetStrict(strict)
|
||||
} else {
|
||||
return o.GetStrict(strict) || ro.GetStrict(strict)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,107 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"github.com/syndtr/goleveldb/leveldb/filter"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
)
|
||||
|
||||
func dupOptions(o *opt.Options) *opt.Options {
|
||||
newo := &opt.Options{}
|
||||
if o != nil {
|
||||
*newo = *o
|
||||
}
|
||||
if newo.Strict == 0 {
|
||||
newo.Strict = opt.DefaultStrict
|
||||
}
|
||||
return newo
|
||||
}
|
||||
|
||||
func (s *session) setOptions(o *opt.Options) {
|
||||
no := dupOptions(o)
|
||||
// Alternative filters.
|
||||
if filters := o.GetAltFilters(); len(filters) > 0 {
|
||||
no.AltFilters = make([]filter.Filter, len(filters))
|
||||
for i, filter := range filters {
|
||||
no.AltFilters[i] = &iFilter{filter}
|
||||
}
|
||||
}
|
||||
// Comparer.
|
||||
s.icmp = &iComparer{o.GetComparer()}
|
||||
no.Comparer = s.icmp
|
||||
// Filter.
|
||||
if filter := o.GetFilter(); filter != nil {
|
||||
no.Filter = &iFilter{filter}
|
||||
}
|
||||
|
||||
s.o = &cachedOptions{Options: no}
|
||||
s.o.cache()
|
||||
}
|
||||
|
||||
const optCachedLevel = 7
|
||||
|
||||
type cachedOptions struct {
|
||||
*opt.Options
|
||||
|
||||
compactionExpandLimit []int
|
||||
compactionGPOverlaps []int
|
||||
compactionSourceLimit []int
|
||||
compactionTableSize []int
|
||||
compactionTotalSize []int64
|
||||
}
|
||||
|
||||
func (co *cachedOptions) cache() {
|
||||
co.compactionExpandLimit = make([]int, optCachedLevel)
|
||||
co.compactionGPOverlaps = make([]int, optCachedLevel)
|
||||
co.compactionSourceLimit = make([]int, optCachedLevel)
|
||||
co.compactionTableSize = make([]int, optCachedLevel)
|
||||
co.compactionTotalSize = make([]int64, optCachedLevel)
|
||||
|
||||
for level := 0; level < optCachedLevel; level++ {
|
||||
co.compactionExpandLimit[level] = co.Options.GetCompactionExpandLimit(level)
|
||||
co.compactionGPOverlaps[level] = co.Options.GetCompactionGPOverlaps(level)
|
||||
co.compactionSourceLimit[level] = co.Options.GetCompactionSourceLimit(level)
|
||||
co.compactionTableSize[level] = co.Options.GetCompactionTableSize(level)
|
||||
co.compactionTotalSize[level] = co.Options.GetCompactionTotalSize(level)
|
||||
}
|
||||
}
|
||||
|
||||
func (co *cachedOptions) GetCompactionExpandLimit(level int) int {
|
||||
if level < optCachedLevel {
|
||||
return co.compactionExpandLimit[level]
|
||||
}
|
||||
return co.Options.GetCompactionExpandLimit(level)
|
||||
}
|
||||
|
||||
func (co *cachedOptions) GetCompactionGPOverlaps(level int) int {
|
||||
if level < optCachedLevel {
|
||||
return co.compactionGPOverlaps[level]
|
||||
}
|
||||
return co.Options.GetCompactionGPOverlaps(level)
|
||||
}
|
||||
|
||||
func (co *cachedOptions) GetCompactionSourceLimit(level int) int {
|
||||
if level < optCachedLevel {
|
||||
return co.compactionSourceLimit[level]
|
||||
}
|
||||
return co.Options.GetCompactionSourceLimit(level)
|
||||
}
|
||||
|
||||
func (co *cachedOptions) GetCompactionTableSize(level int) int {
|
||||
if level < optCachedLevel {
|
||||
return co.compactionTableSize[level]
|
||||
}
|
||||
return co.Options.GetCompactionTableSize(level)
|
||||
}
|
||||
|
||||
func (co *cachedOptions) GetCompactionTotalSize(level int) int64 {
|
||||
if level < optCachedLevel {
|
||||
return co.compactionTotalSize[level]
|
||||
}
|
||||
return co.Options.GetCompactionTotalSize(level)
|
||||
}
|
||||
@ -0,0 +1,210 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/errors"
|
||||
"github.com/syndtr/goleveldb/leveldb/journal"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
)
|
||||
|
||||
// ErrManifestCorrupted records manifest corruption. This error will be
|
||||
// wrapped with errors.ErrCorrupted.
|
||||
type ErrManifestCorrupted struct {
|
||||
Field string
|
||||
Reason string
|
||||
}
|
||||
|
||||
func (e *ErrManifestCorrupted) Error() string {
|
||||
return fmt.Sprintf("leveldb: manifest corrupted (field '%s'): %s", e.Field, e.Reason)
|
||||
}
|
||||
|
||||
func newErrManifestCorrupted(fd storage.FileDesc, field, reason string) error {
|
||||
return errors.NewErrCorrupted(fd, &ErrManifestCorrupted{field, reason})
|
||||
}
|
||||
|
||||
// session represent a persistent database session.
|
||||
type session struct {
|
||||
// Need 64-bit alignment.
|
||||
stNextFileNum int64 // current unused file number
|
||||
stJournalNum int64 // current journal file number; need external synchronization
|
||||
stPrevJournalNum int64 // prev journal file number; no longer used; for compatibility with older version of leveldb
|
||||
stTempFileNum int64
|
||||
stSeqNum uint64 // last mem compacted seq; need external synchronization
|
||||
|
||||
stor *iStorage
|
||||
storLock storage.Locker
|
||||
o *cachedOptions
|
||||
icmp *iComparer
|
||||
tops *tOps
|
||||
fileRef map[int64]int
|
||||
|
||||
manifest *journal.Writer
|
||||
manifestWriter storage.Writer
|
||||
manifestFd storage.FileDesc
|
||||
|
||||
stCompPtrs []internalKey // compaction pointers; need external synchronization
|
||||
stVersion *version // current version
|
||||
vmu sync.Mutex
|
||||
}
|
||||
|
||||
// Creates new initialized session instance.
|
||||
func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) {
|
||||
if stor == nil {
|
||||
return nil, os.ErrInvalid
|
||||
}
|
||||
storLock, err := stor.Lock()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s = &session{
|
||||
stor: newIStorage(stor),
|
||||
storLock: storLock,
|
||||
fileRef: make(map[int64]int),
|
||||
}
|
||||
s.setOptions(o)
|
||||
s.tops = newTableOps(s)
|
||||
s.setVersion(newVersion(s))
|
||||
s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed")
|
||||
return
|
||||
}
|
||||
|
||||
// Close session.
|
||||
func (s *session) close() {
|
||||
s.tops.close()
|
||||
if s.manifest != nil {
|
||||
s.manifest.Close()
|
||||
}
|
||||
if s.manifestWriter != nil {
|
||||
s.manifestWriter.Close()
|
||||
}
|
||||
s.manifest = nil
|
||||
s.manifestWriter = nil
|
||||
s.setVersion(&version{s: s, closing: true})
|
||||
}
|
||||
|
||||
// Release session lock.
|
||||
func (s *session) release() {
|
||||
s.storLock.Unlock()
|
||||
}
|
||||
|
||||
// Create a new database session; need external synchronization.
|
||||
func (s *session) create() error {
|
||||
// create manifest
|
||||
return s.newManifest(nil, nil)
|
||||
}
|
||||
|
||||
// Recover a database session; need external synchronization.
|
||||
func (s *session) recover() (err error) {
|
||||
defer func() {
|
||||
if os.IsNotExist(err) {
|
||||
// Don't return os.ErrNotExist if the underlying storage contains
|
||||
// other files that belong to LevelDB. So the DB won't get trashed.
|
||||
if fds, _ := s.stor.List(storage.TypeAll); len(fds) > 0 {
|
||||
err = &errors.ErrCorrupted{Fd: storage.FileDesc{Type: storage.TypeManifest}, Err: &errors.ErrMissingFiles{}}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
fd, err := s.stor.GetMeta()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
reader, err := s.stor.Open(fd)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
var (
|
||||
// Options.
|
||||
strict = s.o.GetStrict(opt.StrictManifest)
|
||||
|
||||
jr = journal.NewReader(reader, dropper{s, fd}, strict, true)
|
||||
rec = &sessionRecord{}
|
||||
staging = s.stVersion.newStaging()
|
||||
)
|
||||
for {
|
||||
var r io.Reader
|
||||
r, err = jr.Next()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
err = nil
|
||||
break
|
||||
}
|
||||
return errors.SetFd(err, fd)
|
||||
}
|
||||
|
||||
err = rec.decode(r)
|
||||
if err == nil {
|
||||
// save compact pointers
|
||||
for _, r := range rec.compPtrs {
|
||||
s.setCompPtr(r.level, internalKey(r.ikey))
|
||||
}
|
||||
// commit record to version staging
|
||||
staging.commit(rec)
|
||||
} else {
|
||||
err = errors.SetFd(err, fd)
|
||||
if strict || !errors.IsCorrupted(err) {
|
||||
return
|
||||
}
|
||||
s.logf("manifest error: %v (skipped)", errors.SetFd(err, fd))
|
||||
}
|
||||
rec.resetCompPtrs()
|
||||
rec.resetAddedTables()
|
||||
rec.resetDeletedTables()
|
||||
}
|
||||
|
||||
switch {
|
||||
case !rec.has(recComparer):
|
||||
return newErrManifestCorrupted(fd, "comparer", "missing")
|
||||
case rec.comparer != s.icmp.uName():
|
||||
return newErrManifestCorrupted(fd, "comparer", fmt.Sprintf("mismatch: want '%s', got '%s'", s.icmp.uName(), rec.comparer))
|
||||
case !rec.has(recNextFileNum):
|
||||
return newErrManifestCorrupted(fd, "next-file-num", "missing")
|
||||
case !rec.has(recJournalNum):
|
||||
return newErrManifestCorrupted(fd, "journal-file-num", "missing")
|
||||
case !rec.has(recSeqNum):
|
||||
return newErrManifestCorrupted(fd, "seq-num", "missing")
|
||||
}
|
||||
|
||||
s.manifestFd = fd
|
||||
s.setVersion(staging.finish())
|
||||
s.setNextFileNum(rec.nextFileNum)
|
||||
s.recordCommited(rec)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Commit session; need external synchronization.
|
||||
func (s *session) commit(r *sessionRecord) (err error) {
|
||||
v := s.version()
|
||||
defer v.release()
|
||||
|
||||
// spawn new version based on current version
|
||||
nv := v.spawn(r)
|
||||
|
||||
if s.manifest == nil {
|
||||
// manifest journal writer not yet created, create one
|
||||
err = s.newManifest(r, nv)
|
||||
} else {
|
||||
err = s.flushManifest(r)
|
||||
}
|
||||
|
||||
// finally, apply new version if no error rise
|
||||
if err == nil {
|
||||
s.setVersion(nv)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
@ -0,0 +1,302 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/memdb"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
)
|
||||
|
||||
func (s *session) pickMemdbLevel(umin, umax []byte, maxLevel int) int {
|
||||
v := s.version()
|
||||
defer v.release()
|
||||
return v.pickMemdbLevel(umin, umax, maxLevel)
|
||||
}
|
||||
|
||||
func (s *session) flushMemdb(rec *sessionRecord, mdb *memdb.DB, maxLevel int) (int, error) {
|
||||
// Create sorted table.
|
||||
iter := mdb.NewIterator(nil)
|
||||
defer iter.Release()
|
||||
t, n, err := s.tops.createFrom(iter)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Pick level other than zero can cause compaction issue with large
|
||||
// bulk insert and delete on strictly incrementing key-space. The
|
||||
// problem is that the small deletion markers trapped at lower level,
|
||||
// while key/value entries keep growing at higher level. Since the
|
||||
// key-space is strictly incrementing it will not overlaps with
|
||||
// higher level, thus maximum possible level is always picked, while
|
||||
// overlapping deletion marker pushed into lower level.
|
||||
// See: https://github.com/syndtr/goleveldb/issues/127.
|
||||
flushLevel := s.pickMemdbLevel(t.imin.ukey(), t.imax.ukey(), maxLevel)
|
||||
rec.addTableFile(flushLevel, t)
|
||||
|
||||
s.logf("memdb@flush created L%d@%d N·%d S·%s %q:%q", flushLevel, t.fd.Num, n, shortenb(int(t.size)), t.imin, t.imax)
|
||||
return flushLevel, nil
|
||||
}
|
||||
|
||||
// Pick a compaction based on current state; need external synchronization.
|
||||
func (s *session) pickCompaction() *compaction {
|
||||
v := s.version()
|
||||
|
||||
var sourceLevel int
|
||||
var t0 tFiles
|
||||
if v.cScore >= 1 {
|
||||
sourceLevel = v.cLevel
|
||||
cptr := s.getCompPtr(sourceLevel)
|
||||
tables := v.levels[sourceLevel]
|
||||
for _, t := range tables {
|
||||
if cptr == nil || s.icmp.Compare(t.imax, cptr) > 0 {
|
||||
t0 = append(t0, t)
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(t0) == 0 {
|
||||
t0 = append(t0, tables[0])
|
||||
}
|
||||
} else {
|
||||
if p := atomic.LoadPointer(&v.cSeek); p != nil {
|
||||
ts := (*tSet)(p)
|
||||
sourceLevel = ts.level
|
||||
t0 = append(t0, ts.table)
|
||||
} else {
|
||||
v.release()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return newCompaction(s, v, sourceLevel, t0)
|
||||
}
|
||||
|
||||
// Create compaction from given level and range; need external synchronization.
|
||||
func (s *session) getCompactionRange(sourceLevel int, umin, umax []byte, noLimit bool) *compaction {
|
||||
v := s.version()
|
||||
|
||||
if sourceLevel >= len(v.levels) {
|
||||
v.release()
|
||||
return nil
|
||||
}
|
||||
|
||||
t0 := v.levels[sourceLevel].getOverlaps(nil, s.icmp, umin, umax, sourceLevel == 0)
|
||||
if len(t0) == 0 {
|
||||
v.release()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Avoid compacting too much in one shot in case the range is large.
|
||||
// But we cannot do this for level-0 since level-0 files can overlap
|
||||
// and we must not pick one file and drop another older file if the
|
||||
// two files overlap.
|
||||
if !noLimit && sourceLevel > 0 {
|
||||
limit := int64(v.s.o.GetCompactionSourceLimit(sourceLevel))
|
||||
total := int64(0)
|
||||
for i, t := range t0 {
|
||||
total += t.size
|
||||
if total >= limit {
|
||||
s.logf("table@compaction limiting F·%d -> F·%d", len(t0), i+1)
|
||||
t0 = t0[:i+1]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return newCompaction(s, v, sourceLevel, t0)
|
||||
}
|
||||
|
||||
func newCompaction(s *session, v *version, sourceLevel int, t0 tFiles) *compaction {
|
||||
c := &compaction{
|
||||
s: s,
|
||||
v: v,
|
||||
sourceLevel: sourceLevel,
|
||||
levels: [2]tFiles{t0, nil},
|
||||
maxGPOverlaps: int64(s.o.GetCompactionGPOverlaps(sourceLevel)),
|
||||
tPtrs: make([]int, len(v.levels)),
|
||||
}
|
||||
c.expand()
|
||||
c.save()
|
||||
return c
|
||||
}
|
||||
|
||||
// compaction represent a compaction state.
|
||||
type compaction struct {
|
||||
s *session
|
||||
v *version
|
||||
|
||||
sourceLevel int
|
||||
levels [2]tFiles
|
||||
maxGPOverlaps int64
|
||||
|
||||
gp tFiles
|
||||
gpi int
|
||||
seenKey bool
|
||||
gpOverlappedBytes int64
|
||||
imin, imax internalKey
|
||||
tPtrs []int
|
||||
released bool
|
||||
|
||||
snapGPI int
|
||||
snapSeenKey bool
|
||||
snapGPOverlappedBytes int64
|
||||
snapTPtrs []int
|
||||
}
|
||||
|
||||
func (c *compaction) save() {
|
||||
c.snapGPI = c.gpi
|
||||
c.snapSeenKey = c.seenKey
|
||||
c.snapGPOverlappedBytes = c.gpOverlappedBytes
|
||||
c.snapTPtrs = append(c.snapTPtrs[:0], c.tPtrs...)
|
||||
}
|
||||
|
||||
func (c *compaction) restore() {
|
||||
c.gpi = c.snapGPI
|
||||
c.seenKey = c.snapSeenKey
|
||||
c.gpOverlappedBytes = c.snapGPOverlappedBytes
|
||||
c.tPtrs = append(c.tPtrs[:0], c.snapTPtrs...)
|
||||
}
|
||||
|
||||
func (c *compaction) release() {
|
||||
if !c.released {
|
||||
c.released = true
|
||||
c.v.release()
|
||||
}
|
||||
}
|
||||
|
||||
// Expand compacted tables; need external synchronization.
|
||||
func (c *compaction) expand() {
|
||||
limit := int64(c.s.o.GetCompactionExpandLimit(c.sourceLevel))
|
||||
vt0 := c.v.levels[c.sourceLevel]
|
||||
vt1 := tFiles{}
|
||||
if level := c.sourceLevel + 1; level < len(c.v.levels) {
|
||||
vt1 = c.v.levels[level]
|
||||
}
|
||||
|
||||
t0, t1 := c.levels[0], c.levels[1]
|
||||
imin, imax := t0.getRange(c.s.icmp)
|
||||
// We expand t0 here just incase ukey hop across tables.
|
||||
t0 = vt0.getOverlaps(t0, c.s.icmp, imin.ukey(), imax.ukey(), c.sourceLevel == 0)
|
||||
if len(t0) != len(c.levels[0]) {
|
||||
imin, imax = t0.getRange(c.s.icmp)
|
||||
}
|
||||
t1 = vt1.getOverlaps(t1, c.s.icmp, imin.ukey(), imax.ukey(), false)
|
||||
// Get entire range covered by compaction.
|
||||
amin, amax := append(t0, t1...).getRange(c.s.icmp)
|
||||
|
||||
// See if we can grow the number of inputs in "sourceLevel" without
|
||||
// changing the number of "sourceLevel+1" files we pick up.
|
||||
if len(t1) > 0 {
|
||||
exp0 := vt0.getOverlaps(nil, c.s.icmp, amin.ukey(), amax.ukey(), c.sourceLevel == 0)
|
||||
if len(exp0) > len(t0) && t1.size()+exp0.size() < limit {
|
||||
xmin, xmax := exp0.getRange(c.s.icmp)
|
||||
exp1 := vt1.getOverlaps(nil, c.s.icmp, xmin.ukey(), xmax.ukey(), false)
|
||||
if len(exp1) == len(t1) {
|
||||
c.s.logf("table@compaction expanding L%d+L%d (F·%d S·%s)+(F·%d S·%s) -> (F·%d S·%s)+(F·%d S·%s)",
|
||||
c.sourceLevel, c.sourceLevel+1, len(t0), shortenb(int(t0.size())), len(t1), shortenb(int(t1.size())),
|
||||
len(exp0), shortenb(int(exp0.size())), len(exp1), shortenb(int(exp1.size())))
|
||||
imin, imax = xmin, xmax
|
||||
t0, t1 = exp0, exp1
|
||||
amin, amax = append(t0, t1...).getRange(c.s.icmp)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compute the set of grandparent files that overlap this compaction
|
||||
// (parent == sourceLevel+1; grandparent == sourceLevel+2)
|
||||
if level := c.sourceLevel + 2; level < len(c.v.levels) {
|
||||
c.gp = c.v.levels[level].getOverlaps(c.gp, c.s.icmp, amin.ukey(), amax.ukey(), false)
|
||||
}
|
||||
|
||||
c.levels[0], c.levels[1] = t0, t1
|
||||
c.imin, c.imax = imin, imax
|
||||
}
|
||||
|
||||
// Check whether compaction is trivial.
|
||||
func (c *compaction) trivial() bool {
|
||||
return len(c.levels[0]) == 1 && len(c.levels[1]) == 0 && c.gp.size() <= c.maxGPOverlaps
|
||||
}
|
||||
|
||||
func (c *compaction) baseLevelForKey(ukey []byte) bool {
|
||||
for level := c.sourceLevel + 2; level < len(c.v.levels); level++ {
|
||||
tables := c.v.levels[level]
|
||||
for c.tPtrs[level] < len(tables) {
|
||||
t := tables[c.tPtrs[level]]
|
||||
if c.s.icmp.uCompare(ukey, t.imax.ukey()) <= 0 {
|
||||
// We've advanced far enough.
|
||||
if c.s.icmp.uCompare(ukey, t.imin.ukey()) >= 0 {
|
||||
// Key falls in this file's range, so definitely not base level.
|
||||
return false
|
||||
}
|
||||
break
|
||||
}
|
||||
c.tPtrs[level]++
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *compaction) shouldStopBefore(ikey internalKey) bool {
|
||||
for ; c.gpi < len(c.gp); c.gpi++ {
|
||||
gp := c.gp[c.gpi]
|
||||
if c.s.icmp.Compare(ikey, gp.imax) <= 0 {
|
||||
break
|
||||
}
|
||||
if c.seenKey {
|
||||
c.gpOverlappedBytes += gp.size
|
||||
}
|
||||
}
|
||||
c.seenKey = true
|
||||
|
||||
if c.gpOverlappedBytes > c.maxGPOverlaps {
|
||||
// Too much overlap for current output; start new output.
|
||||
c.gpOverlappedBytes = 0
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Creates an iterator.
|
||||
func (c *compaction) newIterator() iterator.Iterator {
|
||||
// Creates iterator slice.
|
||||
icap := len(c.levels)
|
||||
if c.sourceLevel == 0 {
|
||||
// Special case for level-0.
|
||||
icap = len(c.levels[0]) + 1
|
||||
}
|
||||
its := make([]iterator.Iterator, 0, icap)
|
||||
|
||||
// Options.
|
||||
ro := &opt.ReadOptions{
|
||||
DontFillCache: true,
|
||||
Strict: opt.StrictOverride,
|
||||
}
|
||||
strict := c.s.o.GetStrict(opt.StrictCompaction)
|
||||
if strict {
|
||||
ro.Strict |= opt.StrictReader
|
||||
}
|
||||
|
||||
for i, tables := range c.levels {
|
||||
if len(tables) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Level-0 is not sorted and may overlaps each other.
|
||||
if c.sourceLevel+i == 0 {
|
||||
for _, t := range tables {
|
||||
its = append(its, c.s.tops.newIterator(t, nil, ro))
|
||||
}
|
||||
} else {
|
||||
it := iterator.NewIndexedIterator(tables.newIndexIterator(c.s.tops, c.s.icmp, nil, ro), strict)
|
||||
its = append(its, it)
|
||||
}
|
||||
}
|
||||
|
||||
return iterator.NewMergedIterator(its, c.s.icmp, strict)
|
||||
}
|
||||
@ -0,0 +1,323 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/errors"
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
)
|
||||
|
||||
type byteReader interface {
|
||||
io.Reader
|
||||
io.ByteReader
|
||||
}
|
||||
|
||||
// These numbers are written to disk and should not be changed.
|
||||
const (
|
||||
recComparer = 1
|
||||
recJournalNum = 2
|
||||
recNextFileNum = 3
|
||||
recSeqNum = 4
|
||||
recCompPtr = 5
|
||||
recDelTable = 6
|
||||
recAddTable = 7
|
||||
// 8 was used for large value refs
|
||||
recPrevJournalNum = 9
|
||||
)
|
||||
|
||||
type cpRecord struct {
|
||||
level int
|
||||
ikey internalKey
|
||||
}
|
||||
|
||||
type atRecord struct {
|
||||
level int
|
||||
num int64
|
||||
size int64
|
||||
imin internalKey
|
||||
imax internalKey
|
||||
}
|
||||
|
||||
type dtRecord struct {
|
||||
level int
|
||||
num int64
|
||||
}
|
||||
|
||||
type sessionRecord struct {
|
||||
hasRec int
|
||||
comparer string
|
||||
journalNum int64
|
||||
prevJournalNum int64
|
||||
nextFileNum int64
|
||||
seqNum uint64
|
||||
compPtrs []cpRecord
|
||||
addedTables []atRecord
|
||||
deletedTables []dtRecord
|
||||
|
||||
scratch [binary.MaxVarintLen64]byte
|
||||
err error
|
||||
}
|
||||
|
||||
func (p *sessionRecord) has(rec int) bool {
|
||||
return p.hasRec&(1<<uint(rec)) != 0
|
||||
}
|
||||
|
||||
func (p *sessionRecord) setComparer(name string) {
|
||||
p.hasRec |= 1 << recComparer
|
||||
p.comparer = name
|
||||
}
|
||||
|
||||
func (p *sessionRecord) setJournalNum(num int64) {
|
||||
p.hasRec |= 1 << recJournalNum
|
||||
p.journalNum = num
|
||||
}
|
||||
|
||||
func (p *sessionRecord) setPrevJournalNum(num int64) {
|
||||
p.hasRec |= 1 << recPrevJournalNum
|
||||
p.prevJournalNum = num
|
||||
}
|
||||
|
||||
func (p *sessionRecord) setNextFileNum(num int64) {
|
||||
p.hasRec |= 1 << recNextFileNum
|
||||
p.nextFileNum = num
|
||||
}
|
||||
|
||||
func (p *sessionRecord) setSeqNum(num uint64) {
|
||||
p.hasRec |= 1 << recSeqNum
|
||||
p.seqNum = num
|
||||
}
|
||||
|
||||
func (p *sessionRecord) addCompPtr(level int, ikey internalKey) {
|
||||
p.hasRec |= 1 << recCompPtr
|
||||
p.compPtrs = append(p.compPtrs, cpRecord{level, ikey})
|
||||
}
|
||||
|
||||
func (p *sessionRecord) resetCompPtrs() {
|
||||
p.hasRec &= ^(1 << recCompPtr)
|
||||
p.compPtrs = p.compPtrs[:0]
|
||||
}
|
||||
|
||||
func (p *sessionRecord) addTable(level int, num, size int64, imin, imax internalKey) {
|
||||
p.hasRec |= 1 << recAddTable
|
||||
p.addedTables = append(p.addedTables, atRecord{level, num, size, imin, imax})
|
||||
}
|
||||
|
||||
func (p *sessionRecord) addTableFile(level int, t *tFile) {
|
||||
p.addTable(level, t.fd.Num, t.size, t.imin, t.imax)
|
||||
}
|
||||
|
||||
func (p *sessionRecord) resetAddedTables() {
|
||||
p.hasRec &= ^(1 << recAddTable)
|
||||
p.addedTables = p.addedTables[:0]
|
||||
}
|
||||
|
||||
func (p *sessionRecord) delTable(level int, num int64) {
|
||||
p.hasRec |= 1 << recDelTable
|
||||
p.deletedTables = append(p.deletedTables, dtRecord{level, num})
|
||||
}
|
||||
|
||||
func (p *sessionRecord) resetDeletedTables() {
|
||||
p.hasRec &= ^(1 << recDelTable)
|
||||
p.deletedTables = p.deletedTables[:0]
|
||||
}
|
||||
|
||||
func (p *sessionRecord) putUvarint(w io.Writer, x uint64) {
|
||||
if p.err != nil {
|
||||
return
|
||||
}
|
||||
n := binary.PutUvarint(p.scratch[:], x)
|
||||
_, p.err = w.Write(p.scratch[:n])
|
||||
}
|
||||
|
||||
func (p *sessionRecord) putVarint(w io.Writer, x int64) {
|
||||
if x < 0 {
|
||||
panic("invalid negative value")
|
||||
}
|
||||
p.putUvarint(w, uint64(x))
|
||||
}
|
||||
|
||||
func (p *sessionRecord) putBytes(w io.Writer, x []byte) {
|
||||
if p.err != nil {
|
||||
return
|
||||
}
|
||||
p.putUvarint(w, uint64(len(x)))
|
||||
if p.err != nil {
|
||||
return
|
||||
}
|
||||
_, p.err = w.Write(x)
|
||||
}
|
||||
|
||||
func (p *sessionRecord) encode(w io.Writer) error {
|
||||
p.err = nil
|
||||
if p.has(recComparer) {
|
||||
p.putUvarint(w, recComparer)
|
||||
p.putBytes(w, []byte(p.comparer))
|
||||
}
|
||||
if p.has(recJournalNum) {
|
||||
p.putUvarint(w, recJournalNum)
|
||||
p.putVarint(w, p.journalNum)
|
||||
}
|
||||
if p.has(recNextFileNum) {
|
||||
p.putUvarint(w, recNextFileNum)
|
||||
p.putVarint(w, p.nextFileNum)
|
||||
}
|
||||
if p.has(recSeqNum) {
|
||||
p.putUvarint(w, recSeqNum)
|
||||
p.putUvarint(w, p.seqNum)
|
||||
}
|
||||
for _, r := range p.compPtrs {
|
||||
p.putUvarint(w, recCompPtr)
|
||||
p.putUvarint(w, uint64(r.level))
|
||||
p.putBytes(w, r.ikey)
|
||||
}
|
||||
for _, r := range p.deletedTables {
|
||||
p.putUvarint(w, recDelTable)
|
||||
p.putUvarint(w, uint64(r.level))
|
||||
p.putVarint(w, r.num)
|
||||
}
|
||||
for _, r := range p.addedTables {
|
||||
p.putUvarint(w, recAddTable)
|
||||
p.putUvarint(w, uint64(r.level))
|
||||
p.putVarint(w, r.num)
|
||||
p.putVarint(w, r.size)
|
||||
p.putBytes(w, r.imin)
|
||||
p.putBytes(w, r.imax)
|
||||
}
|
||||
return p.err
|
||||
}
|
||||
|
||||
func (p *sessionRecord) readUvarintMayEOF(field string, r io.ByteReader, mayEOF bool) uint64 {
|
||||
if p.err != nil {
|
||||
return 0
|
||||
}
|
||||
x, err := binary.ReadUvarint(r)
|
||||
if err != nil {
|
||||
if err == io.ErrUnexpectedEOF || (mayEOF == false && err == io.EOF) {
|
||||
p.err = errors.NewErrCorrupted(storage.FileDesc{}, &ErrManifestCorrupted{field, "short read"})
|
||||
} else if strings.HasPrefix(err.Error(), "binary:") {
|
||||
p.err = errors.NewErrCorrupted(storage.FileDesc{}, &ErrManifestCorrupted{field, err.Error()})
|
||||
} else {
|
||||
p.err = err
|
||||
}
|
||||
return 0
|
||||
}
|
||||
return x
|
||||
}
|
||||
|
||||
func (p *sessionRecord) readUvarint(field string, r io.ByteReader) uint64 {
|
||||
return p.readUvarintMayEOF(field, r, false)
|
||||
}
|
||||
|
||||
func (p *sessionRecord) readVarint(field string, r io.ByteReader) int64 {
|
||||
x := int64(p.readUvarintMayEOF(field, r, false))
|
||||
if x < 0 {
|
||||
p.err = errors.NewErrCorrupted(storage.FileDesc{}, &ErrManifestCorrupted{field, "invalid negative value"})
|
||||
}
|
||||
return x
|
||||
}
|
||||
|
||||
func (p *sessionRecord) readBytes(field string, r byteReader) []byte {
|
||||
if p.err != nil {
|
||||
return nil
|
||||
}
|
||||
n := p.readUvarint(field, r)
|
||||
if p.err != nil {
|
||||
return nil
|
||||
}
|
||||
x := make([]byte, n)
|
||||
_, p.err = io.ReadFull(r, x)
|
||||
if p.err != nil {
|
||||
if p.err == io.ErrUnexpectedEOF {
|
||||
p.err = errors.NewErrCorrupted(storage.FileDesc{}, &ErrManifestCorrupted{field, "short read"})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return x
|
||||
}
|
||||
|
||||
func (p *sessionRecord) readLevel(field string, r io.ByteReader) int {
|
||||
if p.err != nil {
|
||||
return 0
|
||||
}
|
||||
x := p.readUvarint(field, r)
|
||||
if p.err != nil {
|
||||
return 0
|
||||
}
|
||||
return int(x)
|
||||
}
|
||||
|
||||
func (p *sessionRecord) decode(r io.Reader) error {
|
||||
br, ok := r.(byteReader)
|
||||
if !ok {
|
||||
br = bufio.NewReader(r)
|
||||
}
|
||||
p.err = nil
|
||||
for p.err == nil {
|
||||
rec := p.readUvarintMayEOF("field-header", br, true)
|
||||
if p.err != nil {
|
||||
if p.err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return p.err
|
||||
}
|
||||
switch rec {
|
||||
case recComparer:
|
||||
x := p.readBytes("comparer", br)
|
||||
if p.err == nil {
|
||||
p.setComparer(string(x))
|
||||
}
|
||||
case recJournalNum:
|
||||
x := p.readVarint("journal-num", br)
|
||||
if p.err == nil {
|
||||
p.setJournalNum(x)
|
||||
}
|
||||
case recPrevJournalNum:
|
||||
x := p.readVarint("prev-journal-num", br)
|
||||
if p.err == nil {
|
||||
p.setPrevJournalNum(x)
|
||||
}
|
||||
case recNextFileNum:
|
||||
x := p.readVarint("next-file-num", br)
|
||||
if p.err == nil {
|
||||
p.setNextFileNum(x)
|
||||
}
|
||||
case recSeqNum:
|
||||
x := p.readUvarint("seq-num", br)
|
||||
if p.err == nil {
|
||||
p.setSeqNum(x)
|
||||
}
|
||||
case recCompPtr:
|
||||
level := p.readLevel("comp-ptr.level", br)
|
||||
ikey := p.readBytes("comp-ptr.ikey", br)
|
||||
if p.err == nil {
|
||||
p.addCompPtr(level, internalKey(ikey))
|
||||
}
|
||||
case recAddTable:
|
||||
level := p.readLevel("add-table.level", br)
|
||||
num := p.readVarint("add-table.num", br)
|
||||
size := p.readVarint("add-table.size", br)
|
||||
imin := p.readBytes("add-table.imin", br)
|
||||
imax := p.readBytes("add-table.imax", br)
|
||||
if p.err == nil {
|
||||
p.addTable(level, num, size, imin, imax)
|
||||
}
|
||||
case recDelTable:
|
||||
level := p.readLevel("del-table.level", br)
|
||||
num := p.readVarint("del-table.num", br)
|
||||
if p.err == nil {
|
||||
p.delTable(level, num)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return p.err
|
||||
}
|
||||
@ -0,0 +1,271 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/journal"
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
)
|
||||
|
||||
// Logging.
|
||||
|
||||
type dropper struct {
|
||||
s *session
|
||||
fd storage.FileDesc
|
||||
}
|
||||
|
||||
func (d dropper) Drop(err error) {
|
||||
if e, ok := err.(*journal.ErrCorrupted); ok {
|
||||
d.s.logf("journal@drop %s-%d S·%s %q", d.fd.Type, d.fd.Num, shortenb(e.Size), e.Reason)
|
||||
} else {
|
||||
d.s.logf("journal@drop %s-%d %q", d.fd.Type, d.fd.Num, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *session) log(v ...interface{}) { s.stor.Log(fmt.Sprint(v...)) }
|
||||
func (s *session) logf(format string, v ...interface{}) { s.stor.Log(fmt.Sprintf(format, v...)) }
|
||||
|
||||
// File utils.
|
||||
|
||||
func (s *session) newTemp() storage.FileDesc {
|
||||
num := atomic.AddInt64(&s.stTempFileNum, 1) - 1
|
||||
return storage.FileDesc{Type: storage.TypeTemp, Num: num}
|
||||
}
|
||||
|
||||
func (s *session) addFileRef(fd storage.FileDesc, ref int) int {
|
||||
ref += s.fileRef[fd.Num]
|
||||
if ref > 0 {
|
||||
s.fileRef[fd.Num] = ref
|
||||
} else if ref == 0 {
|
||||
delete(s.fileRef, fd.Num)
|
||||
} else {
|
||||
panic(fmt.Sprintf("negative ref: %v", fd))
|
||||
}
|
||||
return ref
|
||||
}
|
||||
|
||||
// Session state.
|
||||
|
||||
// Get current version. This will incr version ref, must call
|
||||
// version.release (exactly once) after use.
|
||||
func (s *session) version() *version {
|
||||
s.vmu.Lock()
|
||||
defer s.vmu.Unlock()
|
||||
s.stVersion.incref()
|
||||
return s.stVersion
|
||||
}
|
||||
|
||||
func (s *session) tLen(level int) int {
|
||||
s.vmu.Lock()
|
||||
defer s.vmu.Unlock()
|
||||
return s.stVersion.tLen(level)
|
||||
}
|
||||
|
||||
// Set current version to v.
|
||||
func (s *session) setVersion(v *version) {
|
||||
s.vmu.Lock()
|
||||
defer s.vmu.Unlock()
|
||||
// Hold by session. It is important to call this first before releasing
|
||||
// current version, otherwise the still used files might get released.
|
||||
v.incref()
|
||||
if s.stVersion != nil {
|
||||
// Release current version.
|
||||
s.stVersion.releaseNB()
|
||||
}
|
||||
s.stVersion = v
|
||||
}
|
||||
|
||||
// Get current unused file number.
|
||||
func (s *session) nextFileNum() int64 {
|
||||
return atomic.LoadInt64(&s.stNextFileNum)
|
||||
}
|
||||
|
||||
// Set current unused file number to num.
|
||||
func (s *session) setNextFileNum(num int64) {
|
||||
atomic.StoreInt64(&s.stNextFileNum, num)
|
||||
}
|
||||
|
||||
// Mark file number as used.
|
||||
func (s *session) markFileNum(num int64) {
|
||||
nextFileNum := num + 1
|
||||
for {
|
||||
old, x := s.stNextFileNum, nextFileNum
|
||||
if old > x {
|
||||
x = old
|
||||
}
|
||||
if atomic.CompareAndSwapInt64(&s.stNextFileNum, old, x) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Allocate a file number.
|
||||
func (s *session) allocFileNum() int64 {
|
||||
return atomic.AddInt64(&s.stNextFileNum, 1) - 1
|
||||
}
|
||||
|
||||
// Reuse given file number.
|
||||
func (s *session) reuseFileNum(num int64) {
|
||||
for {
|
||||
old, x := s.stNextFileNum, num
|
||||
if old != x+1 {
|
||||
x = old
|
||||
}
|
||||
if atomic.CompareAndSwapInt64(&s.stNextFileNum, old, x) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Set compaction ptr at given level; need external synchronization.
|
||||
func (s *session) setCompPtr(level int, ik internalKey) {
|
||||
if level >= len(s.stCompPtrs) {
|
||||
newCompPtrs := make([]internalKey, level+1)
|
||||
copy(newCompPtrs, s.stCompPtrs)
|
||||
s.stCompPtrs = newCompPtrs
|
||||
}
|
||||
s.stCompPtrs[level] = append(internalKey{}, ik...)
|
||||
}
|
||||
|
||||
// Get compaction ptr at given level; need external synchronization.
|
||||
func (s *session) getCompPtr(level int) internalKey {
|
||||
if level >= len(s.stCompPtrs) {
|
||||
return nil
|
||||
}
|
||||
return s.stCompPtrs[level]
|
||||
}
|
||||
|
||||
// Manifest related utils.
|
||||
|
||||
// Fill given session record obj with current states; need external
|
||||
// synchronization.
|
||||
func (s *session) fillRecord(r *sessionRecord, snapshot bool) {
|
||||
r.setNextFileNum(s.nextFileNum())
|
||||
|
||||
if snapshot {
|
||||
if !r.has(recJournalNum) {
|
||||
r.setJournalNum(s.stJournalNum)
|
||||
}
|
||||
|
||||
if !r.has(recSeqNum) {
|
||||
r.setSeqNum(s.stSeqNum)
|
||||
}
|
||||
|
||||
for level, ik := range s.stCompPtrs {
|
||||
if ik != nil {
|
||||
r.addCompPtr(level, ik)
|
||||
}
|
||||
}
|
||||
|
||||
r.setComparer(s.icmp.uName())
|
||||
}
|
||||
}
|
||||
|
||||
// Mark if record has been committed, this will update session state;
|
||||
// need external synchronization.
|
||||
func (s *session) recordCommited(rec *sessionRecord) {
|
||||
if rec.has(recJournalNum) {
|
||||
s.stJournalNum = rec.journalNum
|
||||
}
|
||||
|
||||
if rec.has(recPrevJournalNum) {
|
||||
s.stPrevJournalNum = rec.prevJournalNum
|
||||
}
|
||||
|
||||
if rec.has(recSeqNum) {
|
||||
s.stSeqNum = rec.seqNum
|
||||
}
|
||||
|
||||
for _, r := range rec.compPtrs {
|
||||
s.setCompPtr(r.level, internalKey(r.ikey))
|
||||
}
|
||||
}
|
||||
|
||||
// Create a new manifest file; need external synchronization.
|
||||
func (s *session) newManifest(rec *sessionRecord, v *version) (err error) {
|
||||
fd := storage.FileDesc{Type: storage.TypeManifest, Num: s.allocFileNum()}
|
||||
writer, err := s.stor.Create(fd)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
jw := journal.NewWriter(writer)
|
||||
|
||||
if v == nil {
|
||||
v = s.version()
|
||||
defer v.release()
|
||||
}
|
||||
if rec == nil {
|
||||
rec = &sessionRecord{}
|
||||
}
|
||||
s.fillRecord(rec, true)
|
||||
v.fillRecord(rec)
|
||||
|
||||
defer func() {
|
||||
if err == nil {
|
||||
s.recordCommited(rec)
|
||||
if s.manifest != nil {
|
||||
s.manifest.Close()
|
||||
}
|
||||
if s.manifestWriter != nil {
|
||||
s.manifestWriter.Close()
|
||||
}
|
||||
if !s.manifestFd.Zero() {
|
||||
s.stor.Remove(s.manifestFd)
|
||||
}
|
||||
s.manifestFd = fd
|
||||
s.manifestWriter = writer
|
||||
s.manifest = jw
|
||||
} else {
|
||||
writer.Close()
|
||||
s.stor.Remove(fd)
|
||||
s.reuseFileNum(fd.Num)
|
||||
}
|
||||
}()
|
||||
|
||||
w, err := jw.Next()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = rec.encode(w)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = jw.Flush()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = s.stor.SetMeta(fd)
|
||||
return
|
||||
}
|
||||
|
||||
// Flush record to disk.
|
||||
func (s *session) flushManifest(rec *sessionRecord) (err error) {
|
||||
s.fillRecord(rec, false)
|
||||
w, err := s.manifest.Next()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = rec.encode(w)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = s.manifest.Flush()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if !s.o.GetNoSync() {
|
||||
err = s.manifestWriter.Sync()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
s.recordCommited(rec)
|
||||
return
|
||||
}
|
||||
@ -0,0 +1,63 @@
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
type iStorage struct {
|
||||
storage.Storage
|
||||
read uint64
|
||||
write uint64
|
||||
}
|
||||
|
||||
func (c *iStorage) Open(fd storage.FileDesc) (storage.Reader, error) {
|
||||
r, err := c.Storage.Open(fd)
|
||||
return &iStorageReader{r, c}, err
|
||||
}
|
||||
|
||||
func (c *iStorage) Create(fd storage.FileDesc) (storage.Writer, error) {
|
||||
w, err := c.Storage.Create(fd)
|
||||
return &iStorageWriter{w, c}, err
|
||||
}
|
||||
|
||||
func (c *iStorage) reads() uint64 {
|
||||
return atomic.LoadUint64(&c.read)
|
||||
}
|
||||
|
||||
func (c *iStorage) writes() uint64 {
|
||||
return atomic.LoadUint64(&c.write)
|
||||
}
|
||||
|
||||
// newIStorage returns the given storage wrapped by iStorage.
|
||||
func newIStorage(s storage.Storage) *iStorage {
|
||||
return &iStorage{s, 0, 0}
|
||||
}
|
||||
|
||||
type iStorageReader struct {
|
||||
storage.Reader
|
||||
c *iStorage
|
||||
}
|
||||
|
||||
func (r *iStorageReader) Read(p []byte) (n int, err error) {
|
||||
n, err = r.Reader.Read(p)
|
||||
atomic.AddUint64(&r.c.read, uint64(n))
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (r *iStorageReader) ReadAt(p []byte, off int64) (n int, err error) {
|
||||
n, err = r.Reader.ReadAt(p, off)
|
||||
atomic.AddUint64(&r.c.read, uint64(n))
|
||||
return n, err
|
||||
}
|
||||
|
||||
type iStorageWriter struct {
|
||||
storage.Writer
|
||||
c *iStorage
|
||||
}
|
||||
|
||||
func (w *iStorageWriter) Write(p []byte) (n int, err error) {
|
||||
n, err = w.Writer.Write(p)
|
||||
atomic.AddUint64(&w.c.write, uint64(n))
|
||||
return n, err
|
||||
}
|
||||
@ -0,0 +1,671 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reservefs.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
errFileOpen = errors.New("leveldb/storage: file still open")
|
||||
errReadOnly = errors.New("leveldb/storage: storage is read-only")
|
||||
)
|
||||
|
||||
type fileLock interface {
|
||||
release() error
|
||||
}
|
||||
|
||||
type fileStorageLock struct {
|
||||
fs *fileStorage
|
||||
}
|
||||
|
||||
func (lock *fileStorageLock) Unlock() {
|
||||
if lock.fs != nil {
|
||||
lock.fs.mu.Lock()
|
||||
defer lock.fs.mu.Unlock()
|
||||
if lock.fs.slock == lock {
|
||||
lock.fs.slock = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type int64Slice []int64
|
||||
|
||||
func (p int64Slice) Len() int { return len(p) }
|
||||
func (p int64Slice) Less(i, j int) bool { return p[i] < p[j] }
|
||||
func (p int64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
|
||||
|
||||
func writeFileSynced(filename string, data []byte, perm os.FileMode) error {
|
||||
f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := f.Write(data)
|
||||
if err == nil && n < len(data) {
|
||||
err = io.ErrShortWrite
|
||||
}
|
||||
if err1 := f.Sync(); err == nil {
|
||||
err = err1
|
||||
}
|
||||
if err1 := f.Close(); err == nil {
|
||||
err = err1
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
const logSizeThreshold = 1024 * 1024 // 1 MiB
|
||||
|
||||
// fileStorage is a file-system backed storage.
|
||||
type fileStorage struct {
|
||||
path string
|
||||
readOnly bool
|
||||
|
||||
mu sync.Mutex
|
||||
flock fileLock
|
||||
slock *fileStorageLock
|
||||
logw *os.File
|
||||
logSize int64
|
||||
buf []byte
|
||||
// Opened file counter; if open < 0 means closed.
|
||||
open int
|
||||
day int
|
||||
}
|
||||
|
||||
// OpenFile returns a new filesystem-backed storage implementation with the given
|
||||
// path. This also acquire a file lock, so any subsequent attempt to open the
|
||||
// same path will fail.
|
||||
//
|
||||
// The storage must be closed after use, by calling Close method.
|
||||
func OpenFile(path string, readOnly bool) (Storage, error) {
|
||||
if fi, err := os.Stat(path); err == nil {
|
||||
if !fi.IsDir() {
|
||||
return nil, fmt.Errorf("leveldb/storage: open %s: not a directory", path)
|
||||
}
|
||||
} else if os.IsNotExist(err) && !readOnly {
|
||||
if err := os.MkdirAll(path, 0755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
flock, err := newFileLock(filepath.Join(path, "LOCK"), readOnly)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
flock.release()
|
||||
}
|
||||
}()
|
||||
|
||||
var (
|
||||
logw *os.File
|
||||
logSize int64
|
||||
)
|
||||
if !readOnly {
|
||||
logw, err = os.OpenFile(filepath.Join(path, "LOG"), os.O_WRONLY|os.O_CREATE, 0644)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logSize, err = logw.Seek(0, os.SEEK_END)
|
||||
if err != nil {
|
||||
logw.Close()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
fs := &fileStorage{
|
||||
path: path,
|
||||
readOnly: readOnly,
|
||||
flock: flock,
|
||||
logw: logw,
|
||||
logSize: logSize,
|
||||
}
|
||||
runtime.SetFinalizer(fs, (*fileStorage).Close)
|
||||
return fs, nil
|
||||
}
|
||||
|
||||
func (fs *fileStorage) Lock() (Locker, error) {
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
if fs.open < 0 {
|
||||
return nil, ErrClosed
|
||||
}
|
||||
if fs.readOnly {
|
||||
return &fileStorageLock{}, nil
|
||||
}
|
||||
if fs.slock != nil {
|
||||
return nil, ErrLocked
|
||||
}
|
||||
fs.slock = &fileStorageLock{fs: fs}
|
||||
return fs.slock, nil
|
||||
}
|
||||
|
||||
func itoa(buf []byte, i int, wid int) []byte {
|
||||
u := uint(i)
|
||||
if u == 0 && wid <= 1 {
|
||||
return append(buf, '0')
|
||||
}
|
||||
|
||||
// Assemble decimal in reverse order.
|
||||
var b [32]byte
|
||||
bp := len(b)
|
||||
for ; u > 0 || wid > 0; u /= 10 {
|
||||
bp--
|
||||
wid--
|
||||
b[bp] = byte(u%10) + '0'
|
||||
}
|
||||
return append(buf, b[bp:]...)
|
||||
}
|
||||
|
||||
func (fs *fileStorage) printDay(t time.Time) {
|
||||
if fs.day == t.Day() {
|
||||
return
|
||||
}
|
||||
fs.day = t.Day()
|
||||
fs.logw.Write([]byte("=============== " + t.Format("Jan 2, 2006 (MST)") + " ===============\n"))
|
||||
}
|
||||
|
||||
func (fs *fileStorage) doLog(t time.Time, str string) {
|
||||
if fs.logSize > logSizeThreshold {
|
||||
// Rotate log file.
|
||||
fs.logw.Close()
|
||||
fs.logw = nil
|
||||
fs.logSize = 0
|
||||
rename(filepath.Join(fs.path, "LOG"), filepath.Join(fs.path, "LOG.old"))
|
||||
}
|
||||
if fs.logw == nil {
|
||||
var err error
|
||||
fs.logw, err = os.OpenFile(filepath.Join(fs.path, "LOG"), os.O_WRONLY|os.O_CREATE, 0644)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// Force printDay on new log file.
|
||||
fs.day = 0
|
||||
}
|
||||
fs.printDay(t)
|
||||
hour, min, sec := t.Clock()
|
||||
msec := t.Nanosecond() / 1e3
|
||||
// time
|
||||
fs.buf = itoa(fs.buf[:0], hour, 2)
|
||||
fs.buf = append(fs.buf, ':')
|
||||
fs.buf = itoa(fs.buf, min, 2)
|
||||
fs.buf = append(fs.buf, ':')
|
||||
fs.buf = itoa(fs.buf, sec, 2)
|
||||
fs.buf = append(fs.buf, '.')
|
||||
fs.buf = itoa(fs.buf, msec, 6)
|
||||
fs.buf = append(fs.buf, ' ')
|
||||
// write
|
||||
fs.buf = append(fs.buf, []byte(str)...)
|
||||
fs.buf = append(fs.buf, '\n')
|
||||
n, _ := fs.logw.Write(fs.buf)
|
||||
fs.logSize += int64(n)
|
||||
}
|
||||
|
||||
func (fs *fileStorage) Log(str string) {
|
||||
if !fs.readOnly {
|
||||
t := time.Now()
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
if fs.open < 0 {
|
||||
return
|
||||
}
|
||||
fs.doLog(t, str)
|
||||
}
|
||||
}
|
||||
|
||||
func (fs *fileStorage) log(str string) {
|
||||
if !fs.readOnly {
|
||||
fs.doLog(time.Now(), str)
|
||||
}
|
||||
}
|
||||
|
||||
func (fs *fileStorage) setMeta(fd FileDesc) error {
|
||||
content := fsGenName(fd) + "\n"
|
||||
// Check and backup old CURRENT file.
|
||||
currentPath := filepath.Join(fs.path, "CURRENT")
|
||||
if _, err := os.Stat(currentPath); err == nil {
|
||||
b, err := ioutil.ReadFile(currentPath)
|
||||
if err != nil {
|
||||
fs.log(fmt.Sprintf("backup CURRENT: %v", err))
|
||||
return err
|
||||
}
|
||||
if string(b) == content {
|
||||
// Content not changed, do nothing.
|
||||
return nil
|
||||
}
|
||||
if err := writeFileSynced(currentPath+".bak", b, 0644); err != nil {
|
||||
fs.log(fmt.Sprintf("backup CURRENT: %v", err))
|
||||
return err
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), fd.Num)
|
||||
if err := writeFileSynced(path, []byte(content), 0644); err != nil {
|
||||
fs.log(fmt.Sprintf("create CURRENT.%d: %v", fd.Num, err))
|
||||
return err
|
||||
}
|
||||
// Replace CURRENT file.
|
||||
if err := rename(path, currentPath); err != nil {
|
||||
fs.log(fmt.Sprintf("rename CURRENT.%d: %v", fd.Num, err))
|
||||
return err
|
||||
}
|
||||
// Sync root directory.
|
||||
if err := syncDir(fs.path); err != nil {
|
||||
fs.log(fmt.Sprintf("syncDir: %v", err))
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fs *fileStorage) SetMeta(fd FileDesc) error {
|
||||
if !FileDescOk(fd) {
|
||||
return ErrInvalidFile
|
||||
}
|
||||
if fs.readOnly {
|
||||
return errReadOnly
|
||||
}
|
||||
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
if fs.open < 0 {
|
||||
return ErrClosed
|
||||
}
|
||||
return fs.setMeta(fd)
|
||||
}
|
||||
|
||||
func (fs *fileStorage) GetMeta() (FileDesc, error) {
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
if fs.open < 0 {
|
||||
return FileDesc{}, ErrClosed
|
||||
}
|
||||
dir, err := os.Open(fs.path)
|
||||
if err != nil {
|
||||
return FileDesc{}, err
|
||||
}
|
||||
names, err := dir.Readdirnames(0)
|
||||
// Close the dir first before checking for Readdirnames error.
|
||||
if ce := dir.Close(); ce != nil {
|
||||
fs.log(fmt.Sprintf("close dir: %v", ce))
|
||||
}
|
||||
if err != nil {
|
||||
return FileDesc{}, err
|
||||
}
|
||||
// Try this in order:
|
||||
// - CURRENT.[0-9]+ ('pending rename' file, descending order)
|
||||
// - CURRENT
|
||||
// - CURRENT.bak
|
||||
//
|
||||
// Skip corrupted file or file that point to a missing target file.
|
||||
type currentFile struct {
|
||||
name string
|
||||
fd FileDesc
|
||||
}
|
||||
tryCurrent := func(name string) (*currentFile, error) {
|
||||
b, err := ioutil.ReadFile(filepath.Join(fs.path, name))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
err = os.ErrNotExist
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var fd FileDesc
|
||||
if len(b) < 1 || b[len(b)-1] != '\n' || !fsParseNamePtr(string(b[:len(b)-1]), &fd) {
|
||||
fs.log(fmt.Sprintf("%s: corrupted content: %q", name, b))
|
||||
err := &ErrCorrupted{
|
||||
Err: errors.New("leveldb/storage: corrupted or incomplete CURRENT file"),
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(fs.path, fsGenName(fd))); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
fs.log(fmt.Sprintf("%s: missing target file: %s", name, fd))
|
||||
err = os.ErrNotExist
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return ¤tFile{name: name, fd: fd}, nil
|
||||
}
|
||||
tryCurrents := func(names []string) (*currentFile, error) {
|
||||
var (
|
||||
cur *currentFile
|
||||
// Last corruption error.
|
||||
lastCerr error
|
||||
)
|
||||
for _, name := range names {
|
||||
var err error
|
||||
cur, err = tryCurrent(name)
|
||||
if err == nil {
|
||||
break
|
||||
} else if err == os.ErrNotExist {
|
||||
// Fallback to the next file.
|
||||
} else if isCorrupted(err) {
|
||||
lastCerr = err
|
||||
// Fallback to the next file.
|
||||
} else {
|
||||
// In case the error is due to permission, etc.
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if cur == nil {
|
||||
err := os.ErrNotExist
|
||||
if lastCerr != nil {
|
||||
err = lastCerr
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return cur, nil
|
||||
}
|
||||
|
||||
// Try 'pending rename' files.
|
||||
var nums []int64
|
||||
for _, name := range names {
|
||||
if strings.HasPrefix(name, "CURRENT.") && name != "CURRENT.bak" {
|
||||
i, err := strconv.ParseInt(name[8:], 10, 64)
|
||||
if err == nil {
|
||||
nums = append(nums, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
var (
|
||||
pendCur *currentFile
|
||||
pendErr = os.ErrNotExist
|
||||
pendNames []string
|
||||
)
|
||||
if len(nums) > 0 {
|
||||
sort.Sort(sort.Reverse(int64Slice(nums)))
|
||||
pendNames = make([]string, len(nums))
|
||||
for i, num := range nums {
|
||||
pendNames[i] = fmt.Sprintf("CURRENT.%d", num)
|
||||
}
|
||||
pendCur, pendErr = tryCurrents(pendNames)
|
||||
if pendErr != nil && pendErr != os.ErrNotExist && !isCorrupted(pendErr) {
|
||||
return FileDesc{}, pendErr
|
||||
}
|
||||
}
|
||||
|
||||
// Try CURRENT and CURRENT.bak.
|
||||
curCur, curErr := tryCurrents([]string{"CURRENT", "CURRENT.bak"})
|
||||
if curErr != nil && curErr != os.ErrNotExist && !isCorrupted(curErr) {
|
||||
return FileDesc{}, curErr
|
||||
}
|
||||
|
||||
// pendCur takes precedence, but guards against obsolete pendCur.
|
||||
if pendCur != nil && (curCur == nil || pendCur.fd.Num > curCur.fd.Num) {
|
||||
curCur = pendCur
|
||||
}
|
||||
|
||||
if curCur != nil {
|
||||
// Restore CURRENT file to proper state.
|
||||
if !fs.readOnly && (curCur.name != "CURRENT" || len(pendNames) != 0) {
|
||||
// Ignore setMeta errors, however don't delete obsolete files if we
|
||||
// catch error.
|
||||
if err := fs.setMeta(curCur.fd); err == nil {
|
||||
// Remove 'pending rename' files.
|
||||
for _, name := range pendNames {
|
||||
if err := os.Remove(filepath.Join(fs.path, name)); err != nil {
|
||||
fs.log(fmt.Sprintf("remove %s: %v", name, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return curCur.fd, nil
|
||||
}
|
||||
|
||||
// Nothing found.
|
||||
if isCorrupted(pendErr) {
|
||||
return FileDesc{}, pendErr
|
||||
}
|
||||
return FileDesc{}, curErr
|
||||
}
|
||||
|
||||
func (fs *fileStorage) List(ft FileType) (fds []FileDesc, err error) {
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
if fs.open < 0 {
|
||||
return nil, ErrClosed
|
||||
}
|
||||
dir, err := os.Open(fs.path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
names, err := dir.Readdirnames(0)
|
||||
// Close the dir first before checking for Readdirnames error.
|
||||
if cerr := dir.Close(); cerr != nil {
|
||||
fs.log(fmt.Sprintf("close dir: %v", cerr))
|
||||
}
|
||||
if err == nil {
|
||||
for _, name := range names {
|
||||
if fd, ok := fsParseName(name); ok && fd.Type&ft != 0 {
|
||||
fds = append(fds, fd)
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (fs *fileStorage) Open(fd FileDesc) (Reader, error) {
|
||||
if !FileDescOk(fd) {
|
||||
return nil, ErrInvalidFile
|
||||
}
|
||||
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
if fs.open < 0 {
|
||||
return nil, ErrClosed
|
||||
}
|
||||
of, err := os.OpenFile(filepath.Join(fs.path, fsGenName(fd)), os.O_RDONLY, 0)
|
||||
if err != nil {
|
||||
if fsHasOldName(fd) && os.IsNotExist(err) {
|
||||
of, err = os.OpenFile(filepath.Join(fs.path, fsGenOldName(fd)), os.O_RDONLY, 0)
|
||||
if err == nil {
|
||||
goto ok
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
ok:
|
||||
fs.open++
|
||||
return &fileWrap{File: of, fs: fs, fd: fd}, nil
|
||||
}
|
||||
|
||||
func (fs *fileStorage) Create(fd FileDesc) (Writer, error) {
|
||||
if !FileDescOk(fd) {
|
||||
return nil, ErrInvalidFile
|
||||
}
|
||||
if fs.readOnly {
|
||||
return nil, errReadOnly
|
||||
}
|
||||
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
if fs.open < 0 {
|
||||
return nil, ErrClosed
|
||||
}
|
||||
of, err := os.OpenFile(filepath.Join(fs.path, fsGenName(fd)), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fs.open++
|
||||
return &fileWrap{File: of, fs: fs, fd: fd}, nil
|
||||
}
|
||||
|
||||
func (fs *fileStorage) Remove(fd FileDesc) error {
|
||||
if !FileDescOk(fd) {
|
||||
return ErrInvalidFile
|
||||
}
|
||||
if fs.readOnly {
|
||||
return errReadOnly
|
||||
}
|
||||
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
if fs.open < 0 {
|
||||
return ErrClosed
|
||||
}
|
||||
err := os.Remove(filepath.Join(fs.path, fsGenName(fd)))
|
||||
if err != nil {
|
||||
if fsHasOldName(fd) && os.IsNotExist(err) {
|
||||
if e1 := os.Remove(filepath.Join(fs.path, fsGenOldName(fd))); !os.IsNotExist(e1) {
|
||||
fs.log(fmt.Sprintf("remove %s: %v (old name)", fd, err))
|
||||
err = e1
|
||||
}
|
||||
} else {
|
||||
fs.log(fmt.Sprintf("remove %s: %v", fd, err))
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (fs *fileStorage) Rename(oldfd, newfd FileDesc) error {
|
||||
if !FileDescOk(oldfd) || !FileDescOk(newfd) {
|
||||
return ErrInvalidFile
|
||||
}
|
||||
if oldfd == newfd {
|
||||
return nil
|
||||
}
|
||||
if fs.readOnly {
|
||||
return errReadOnly
|
||||
}
|
||||
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
if fs.open < 0 {
|
||||
return ErrClosed
|
||||
}
|
||||
return rename(filepath.Join(fs.path, fsGenName(oldfd)), filepath.Join(fs.path, fsGenName(newfd)))
|
||||
}
|
||||
|
||||
func (fs *fileStorage) Close() error {
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
if fs.open < 0 {
|
||||
return ErrClosed
|
||||
}
|
||||
// Clear the finalizer.
|
||||
runtime.SetFinalizer(fs, nil)
|
||||
|
||||
if fs.open > 0 {
|
||||
fs.log(fmt.Sprintf("close: warning, %d files still open", fs.open))
|
||||
}
|
||||
fs.open = -1
|
||||
if fs.logw != nil {
|
||||
fs.logw.Close()
|
||||
}
|
||||
return fs.flock.release()
|
||||
}
|
||||
|
||||
type fileWrap struct {
|
||||
*os.File
|
||||
fs *fileStorage
|
||||
fd FileDesc
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (fw *fileWrap) Sync() error {
|
||||
if err := fw.File.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
if fw.fd.Type == TypeManifest {
|
||||
// Also sync parent directory if file type is manifest.
|
||||
// See: https://code.google.com/p/leveldb/issues/detail?id=190.
|
||||
if err := syncDir(fw.fs.path); err != nil {
|
||||
fw.fs.log(fmt.Sprintf("syncDir: %v", err))
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fw *fileWrap) Close() error {
|
||||
fw.fs.mu.Lock()
|
||||
defer fw.fs.mu.Unlock()
|
||||
if fw.closed {
|
||||
return ErrClosed
|
||||
}
|
||||
fw.closed = true
|
||||
fw.fs.open--
|
||||
err := fw.File.Close()
|
||||
if err != nil {
|
||||
fw.fs.log(fmt.Sprintf("close %s: %v", fw.fd, err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func fsGenName(fd FileDesc) string {
|
||||
switch fd.Type {
|
||||
case TypeManifest:
|
||||
return fmt.Sprintf("MANIFEST-%06d", fd.Num)
|
||||
case TypeJournal:
|
||||
return fmt.Sprintf("%06d.log", fd.Num)
|
||||
case TypeTable:
|
||||
return fmt.Sprintf("%06d.ldb", fd.Num)
|
||||
case TypeTemp:
|
||||
return fmt.Sprintf("%06d.tmp", fd.Num)
|
||||
default:
|
||||
panic("invalid file type")
|
||||
}
|
||||
}
|
||||
|
||||
func fsHasOldName(fd FileDesc) bool {
|
||||
return fd.Type == TypeTable
|
||||
}
|
||||
|
||||
func fsGenOldName(fd FileDesc) string {
|
||||
switch fd.Type {
|
||||
case TypeTable:
|
||||
return fmt.Sprintf("%06d.sst", fd.Num)
|
||||
}
|
||||
return fsGenName(fd)
|
||||
}
|
||||
|
||||
func fsParseName(name string) (fd FileDesc, ok bool) {
|
||||
var tail string
|
||||
_, err := fmt.Sscanf(name, "%d.%s", &fd.Num, &tail)
|
||||
if err == nil {
|
||||
switch tail {
|
||||
case "log":
|
||||
fd.Type = TypeJournal
|
||||
case "ldb", "sst":
|
||||
fd.Type = TypeTable
|
||||
case "tmp":
|
||||
fd.Type = TypeTemp
|
||||
default:
|
||||
return
|
||||
}
|
||||
return fd, true
|
||||
}
|
||||
n, _ := fmt.Sscanf(name, "MANIFEST-%d%s", &fd.Num, &tail)
|
||||
if n == 1 {
|
||||
fd.Type = TypeManifest
|
||||
return fd, true
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func fsParseNamePtr(name string, fd *FileDesc) bool {
|
||||
_fd, ok := fsParseName(name)
|
||||
if fd != nil {
|
||||
*fd = _fd
|
||||
}
|
||||
return ok
|
||||
}
|
||||
@ -0,0 +1,34 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// +build nacl
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func newFileLock(path string, readOnly bool) (fl fileLock, err error) {
|
||||
return nil, syscall.ENOTSUP
|
||||
}
|
||||
|
||||
func setFileLock(f *os.File, readOnly, lock bool) error {
|
||||
return syscall.ENOTSUP
|
||||
}
|
||||
|
||||
func rename(oldpath, newpath string) error {
|
||||
return syscall.ENOTSUP
|
||||
}
|
||||
|
||||
func isErrInvalid(err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func syncDir(name string) error {
|
||||
return syscall.ENOTSUP
|
||||
}
|
||||
@ -0,0 +1,63 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
type plan9FileLock struct {
|
||||
f *os.File
|
||||
}
|
||||
|
||||
func (fl *plan9FileLock) release() error {
|
||||
return fl.f.Close()
|
||||
}
|
||||
|
||||
func newFileLock(path string, readOnly bool) (fl fileLock, err error) {
|
||||
var (
|
||||
flag int
|
||||
perm os.FileMode
|
||||
)
|
||||
if readOnly {
|
||||
flag = os.O_RDONLY
|
||||
} else {
|
||||
flag = os.O_RDWR
|
||||
perm = os.ModeExclusive
|
||||
}
|
||||
f, err := os.OpenFile(path, flag, perm)
|
||||
if os.IsNotExist(err) {
|
||||
f, err = os.OpenFile(path, flag|os.O_CREATE, perm|0644)
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
fl = &plan9FileLock{f: f}
|
||||
return
|
||||
}
|
||||
|
||||
func rename(oldpath, newpath string) error {
|
||||
if _, err := os.Stat(newpath); err == nil {
|
||||
if err := os.Remove(newpath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return os.Rename(oldpath, newpath)
|
||||
}
|
||||
|
||||
func syncDir(name string) error {
|
||||
f, err := os.Open(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
if err := f.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,81 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// +build solaris
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
type unixFileLock struct {
|
||||
f *os.File
|
||||
}
|
||||
|
||||
func (fl *unixFileLock) release() error {
|
||||
if err := setFileLock(fl.f, false, false); err != nil {
|
||||
return err
|
||||
}
|
||||
return fl.f.Close()
|
||||
}
|
||||
|
||||
func newFileLock(path string, readOnly bool) (fl fileLock, err error) {
|
||||
var flag int
|
||||
if readOnly {
|
||||
flag = os.O_RDONLY
|
||||
} else {
|
||||
flag = os.O_RDWR
|
||||
}
|
||||
f, err := os.OpenFile(path, flag, 0)
|
||||
if os.IsNotExist(err) {
|
||||
f, err = os.OpenFile(path, flag|os.O_CREATE, 0644)
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = setFileLock(f, readOnly, true)
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return
|
||||
}
|
||||
fl = &unixFileLock{f: f}
|
||||
return
|
||||
}
|
||||
|
||||
func setFileLock(f *os.File, readOnly, lock bool) error {
|
||||
flock := syscall.Flock_t{
|
||||
Type: syscall.F_UNLCK,
|
||||
Start: 0,
|
||||
Len: 0,
|
||||
Whence: 1,
|
||||
}
|
||||
if lock {
|
||||
if readOnly {
|
||||
flock.Type = syscall.F_RDLCK
|
||||
} else {
|
||||
flock.Type = syscall.F_WRLCK
|
||||
}
|
||||
}
|
||||
return syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &flock)
|
||||
}
|
||||
|
||||
func rename(oldpath, newpath string) error {
|
||||
return os.Rename(oldpath, newpath)
|
||||
}
|
||||
|
||||
func syncDir(name string) error {
|
||||
f, err := os.Open(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
if err := f.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,98 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// +build darwin dragonfly freebsd linux netbsd openbsd
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
type unixFileLock struct {
|
||||
f *os.File
|
||||
}
|
||||
|
||||
func (fl *unixFileLock) release() error {
|
||||
if err := setFileLock(fl.f, false, false); err != nil {
|
||||
return err
|
||||
}
|
||||
return fl.f.Close()
|
||||
}
|
||||
|
||||
func newFileLock(path string, readOnly bool) (fl fileLock, err error) {
|
||||
var flag int
|
||||
if readOnly {
|
||||
flag = os.O_RDONLY
|
||||
} else {
|
||||
flag = os.O_RDWR
|
||||
}
|
||||
f, err := os.OpenFile(path, flag, 0)
|
||||
if os.IsNotExist(err) {
|
||||
f, err = os.OpenFile(path, flag|os.O_CREATE, 0644)
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = setFileLock(f, readOnly, true)
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return
|
||||
}
|
||||
fl = &unixFileLock{f: f}
|
||||
return
|
||||
}
|
||||
|
||||
func setFileLock(f *os.File, readOnly, lock bool) error {
|
||||
how := syscall.LOCK_UN
|
||||
if lock {
|
||||
if readOnly {
|
||||
how = syscall.LOCK_SH
|
||||
} else {
|
||||
how = syscall.LOCK_EX
|
||||
}
|
||||
}
|
||||
return syscall.Flock(int(f.Fd()), how|syscall.LOCK_NB)
|
||||
}
|
||||
|
||||
func rename(oldpath, newpath string) error {
|
||||
return os.Rename(oldpath, newpath)
|
||||
}
|
||||
|
||||
func isErrInvalid(err error) bool {
|
||||
if err == os.ErrInvalid {
|
||||
return true
|
||||
}
|
||||
// Go < 1.8
|
||||
if syserr, ok := err.(*os.SyscallError); ok && syserr.Err == syscall.EINVAL {
|
||||
return true
|
||||
}
|
||||
// Go >= 1.8 returns *os.PathError instead
|
||||
if patherr, ok := err.(*os.PathError); ok && patherr.Err == syscall.EINVAL {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func syncDir(name string) error {
|
||||
// As per fsync manpage, Linux seems to expect fsync on directory, however
|
||||
// some system don't support this, so we will ignore syscall.EINVAL.
|
||||
//
|
||||
// From fsync(2):
|
||||
// Calling fsync() does not necessarily ensure that the entry in the
|
||||
// directory containing the file has also reached disk. For that an
|
||||
// explicit fsync() on a file descriptor for the directory is also needed.
|
||||
f, err := os.Open(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
if err := f.Sync(); err != nil && !isErrInvalid(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,78 @@
|
||||
// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
var (
|
||||
modkernel32 = syscall.NewLazyDLL("kernel32.dll")
|
||||
|
||||
procMoveFileExW = modkernel32.NewProc("MoveFileExW")
|
||||
)
|
||||
|
||||
const (
|
||||
_MOVEFILE_REPLACE_EXISTING = 1
|
||||
)
|
||||
|
||||
type windowsFileLock struct {
|
||||
fd syscall.Handle
|
||||
}
|
||||
|
||||
func (fl *windowsFileLock) release() error {
|
||||
return syscall.Close(fl.fd)
|
||||
}
|
||||
|
||||
func newFileLock(path string, readOnly bool) (fl fileLock, err error) {
|
||||
pathp, err := syscall.UTF16PtrFromString(path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var access, shareMode uint32
|
||||
if readOnly {
|
||||
access = syscall.GENERIC_READ
|
||||
shareMode = syscall.FILE_SHARE_READ
|
||||
} else {
|
||||
access = syscall.GENERIC_READ | syscall.GENERIC_WRITE
|
||||
}
|
||||
fd, err := syscall.CreateFile(pathp, access, shareMode, nil, syscall.OPEN_EXISTING, syscall.FILE_ATTRIBUTE_NORMAL, 0)
|
||||
if err == syscall.ERROR_FILE_NOT_FOUND {
|
||||
fd, err = syscall.CreateFile(pathp, access, shareMode, nil, syscall.OPEN_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0)
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
fl = &windowsFileLock{fd: fd}
|
||||
return
|
||||
}
|
||||
|
||||
func moveFileEx(from *uint16, to *uint16, flags uint32) error {
|
||||
r1, _, e1 := syscall.Syscall(procMoveFileExW.Addr(), 3, uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(to)), uintptr(flags))
|
||||
if r1 == 0 {
|
||||
if e1 != 0 {
|
||||
return error(e1)
|
||||
}
|
||||
return syscall.EINVAL
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func rename(oldpath, newpath string) error {
|
||||
from, err := syscall.UTF16PtrFromString(oldpath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
to, err := syscall.UTF16PtrFromString(newpath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return moveFileEx(from, to, _MOVEFILE_REPLACE_EXISTING)
|
||||
}
|
||||
|
||||
func syncDir(name string) error { return nil }
|
||||
@ -0,0 +1,222 @@
|
||||
// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const typeShift = 4
|
||||
|
||||
// Verify at compile-time that typeShift is large enough to cover all FileType
|
||||
// values by confirming that 0 == 0.
|
||||
var _ [0]struct{} = [TypeAll >> typeShift]struct{}{}
|
||||
|
||||
type memStorageLock struct {
|
||||
ms *memStorage
|
||||
}
|
||||
|
||||
func (lock *memStorageLock) Unlock() {
|
||||
ms := lock.ms
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
if ms.slock == lock {
|
||||
ms.slock = nil
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// memStorage is a memory-backed storage.
|
||||
type memStorage struct {
|
||||
mu sync.Mutex
|
||||
slock *memStorageLock
|
||||
files map[uint64]*memFile
|
||||
meta FileDesc
|
||||
}
|
||||
|
||||
// NewMemStorage returns a new memory-backed storage implementation.
|
||||
func NewMemStorage() Storage {
|
||||
return &memStorage{
|
||||
files: make(map[uint64]*memFile),
|
||||
}
|
||||
}
|
||||
|
||||
func (ms *memStorage) Lock() (Locker, error) {
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
if ms.slock != nil {
|
||||
return nil, ErrLocked
|
||||
}
|
||||
ms.slock = &memStorageLock{ms: ms}
|
||||
return ms.slock, nil
|
||||
}
|
||||
|
||||
func (*memStorage) Log(str string) {}
|
||||
|
||||
func (ms *memStorage) SetMeta(fd FileDesc) error {
|
||||
if !FileDescOk(fd) {
|
||||
return ErrInvalidFile
|
||||
}
|
||||
|
||||
ms.mu.Lock()
|
||||
ms.meta = fd
|
||||
ms.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ms *memStorage) GetMeta() (FileDesc, error) {
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
if ms.meta.Zero() {
|
||||
return FileDesc{}, os.ErrNotExist
|
||||
}
|
||||
return ms.meta, nil
|
||||
}
|
||||
|
||||
func (ms *memStorage) List(ft FileType) ([]FileDesc, error) {
|
||||
ms.mu.Lock()
|
||||
var fds []FileDesc
|
||||
for x := range ms.files {
|
||||
fd := unpackFile(x)
|
||||
if fd.Type&ft != 0 {
|
||||
fds = append(fds, fd)
|
||||
}
|
||||
}
|
||||
ms.mu.Unlock()
|
||||
return fds, nil
|
||||
}
|
||||
|
||||
func (ms *memStorage) Open(fd FileDesc) (Reader, error) {
|
||||
if !FileDescOk(fd) {
|
||||
return nil, ErrInvalidFile
|
||||
}
|
||||
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
if m, exist := ms.files[packFile(fd)]; exist {
|
||||
if m.open {
|
||||
return nil, errFileOpen
|
||||
}
|
||||
m.open = true
|
||||
return &memReader{Reader: bytes.NewReader(m.Bytes()), ms: ms, m: m}, nil
|
||||
}
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
|
||||
func (ms *memStorage) Create(fd FileDesc) (Writer, error) {
|
||||
if !FileDescOk(fd) {
|
||||
return nil, ErrInvalidFile
|
||||
}
|
||||
|
||||
x := packFile(fd)
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
m, exist := ms.files[x]
|
||||
if exist {
|
||||
if m.open {
|
||||
return nil, errFileOpen
|
||||
}
|
||||
m.Reset()
|
||||
} else {
|
||||
m = &memFile{}
|
||||
ms.files[x] = m
|
||||
}
|
||||
m.open = true
|
||||
return &memWriter{memFile: m, ms: ms}, nil
|
||||
}
|
||||
|
||||
func (ms *memStorage) Remove(fd FileDesc) error {
|
||||
if !FileDescOk(fd) {
|
||||
return ErrInvalidFile
|
||||
}
|
||||
|
||||
x := packFile(fd)
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
if _, exist := ms.files[x]; exist {
|
||||
delete(ms.files, x)
|
||||
return nil
|
||||
}
|
||||
return os.ErrNotExist
|
||||
}
|
||||
|
||||
func (ms *memStorage) Rename(oldfd, newfd FileDesc) error {
|
||||
if !FileDescOk(oldfd) || !FileDescOk(newfd) {
|
||||
return ErrInvalidFile
|
||||
}
|
||||
if oldfd == newfd {
|
||||
return nil
|
||||
}
|
||||
|
||||
oldx := packFile(oldfd)
|
||||
newx := packFile(newfd)
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
oldm, exist := ms.files[oldx]
|
||||
if !exist {
|
||||
return os.ErrNotExist
|
||||
}
|
||||
newm, exist := ms.files[newx]
|
||||
if (exist && newm.open) || oldm.open {
|
||||
return errFileOpen
|
||||
}
|
||||
delete(ms.files, oldx)
|
||||
ms.files[newx] = oldm
|
||||
return nil
|
||||
}
|
||||
|
||||
func (*memStorage) Close() error { return nil }
|
||||
|
||||
type memFile struct {
|
||||
bytes.Buffer
|
||||
open bool
|
||||
}
|
||||
|
||||
type memReader struct {
|
||||
*bytes.Reader
|
||||
ms *memStorage
|
||||
m *memFile
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (mr *memReader) Close() error {
|
||||
mr.ms.mu.Lock()
|
||||
defer mr.ms.mu.Unlock()
|
||||
if mr.closed {
|
||||
return ErrClosed
|
||||
}
|
||||
mr.m.open = false
|
||||
return nil
|
||||
}
|
||||
|
||||
type memWriter struct {
|
||||
*memFile
|
||||
ms *memStorage
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (*memWriter) Sync() error { return nil }
|
||||
|
||||
func (mw *memWriter) Close() error {
|
||||
mw.ms.mu.Lock()
|
||||
defer mw.ms.mu.Unlock()
|
||||
if mw.closed {
|
||||
return ErrClosed
|
||||
}
|
||||
mw.memFile.open = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func packFile(fd FileDesc) uint64 {
|
||||
return uint64(fd.Num)<<typeShift | uint64(fd.Type)
|
||||
}
|
||||
|
||||
func unpackFile(x uint64) FileDesc {
|
||||
return FileDesc{FileType(x) & TypeAll, int64(x >> typeShift)}
|
||||
}
|
||||
@ -0,0 +1,187 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package storage provides storage abstraction for LevelDB.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// FileType represent a file type.
|
||||
type FileType int
|
||||
|
||||
// File types.
|
||||
const (
|
||||
TypeManifest FileType = 1 << iota
|
||||
TypeJournal
|
||||
TypeTable
|
||||
TypeTemp
|
||||
|
||||
TypeAll = TypeManifest | TypeJournal | TypeTable | TypeTemp
|
||||
)
|
||||
|
||||
func (t FileType) String() string {
|
||||
switch t {
|
||||
case TypeManifest:
|
||||
return "manifest"
|
||||
case TypeJournal:
|
||||
return "journal"
|
||||
case TypeTable:
|
||||
return "table"
|
||||
case TypeTemp:
|
||||
return "temp"
|
||||
}
|
||||
return fmt.Sprintf("<unknown:%d>", t)
|
||||
}
|
||||
|
||||
// Common error.
|
||||
var (
|
||||
ErrInvalidFile = errors.New("leveldb/storage: invalid file for argument")
|
||||
ErrLocked = errors.New("leveldb/storage: already locked")
|
||||
ErrClosed = errors.New("leveldb/storage: closed")
|
||||
)
|
||||
|
||||
// ErrCorrupted is the type that wraps errors that indicate corruption of
|
||||
// a file. Package storage has its own type instead of using
|
||||
// errors.ErrCorrupted to prevent circular import.
|
||||
type ErrCorrupted struct {
|
||||
Fd FileDesc
|
||||
Err error
|
||||
}
|
||||
|
||||
func isCorrupted(err error) bool {
|
||||
switch err.(type) {
|
||||
case *ErrCorrupted:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (e *ErrCorrupted) Error() string {
|
||||
if !e.Fd.Zero() {
|
||||
return fmt.Sprintf("%v [file=%v]", e.Err, e.Fd)
|
||||
}
|
||||
return e.Err.Error()
|
||||
}
|
||||
|
||||
// Syncer is the interface that wraps basic Sync method.
|
||||
type Syncer interface {
|
||||
// Sync commits the current contents of the file to stable storage.
|
||||
Sync() error
|
||||
}
|
||||
|
||||
// Reader is the interface that groups the basic Read, Seek, ReadAt and Close
|
||||
// methods.
|
||||
type Reader interface {
|
||||
io.ReadSeeker
|
||||
io.ReaderAt
|
||||
io.Closer
|
||||
}
|
||||
|
||||
// Writer is the interface that groups the basic Write, Sync and Close
|
||||
// methods.
|
||||
type Writer interface {
|
||||
io.WriteCloser
|
||||
Syncer
|
||||
}
|
||||
|
||||
// Locker is the interface that wraps Unlock method.
|
||||
type Locker interface {
|
||||
Unlock()
|
||||
}
|
||||
|
||||
// FileDesc is a 'file descriptor'.
|
||||
type FileDesc struct {
|
||||
Type FileType
|
||||
Num int64
|
||||
}
|
||||
|
||||
func (fd FileDesc) String() string {
|
||||
switch fd.Type {
|
||||
case TypeManifest:
|
||||
return fmt.Sprintf("MANIFEST-%06d", fd.Num)
|
||||
case TypeJournal:
|
||||
return fmt.Sprintf("%06d.log", fd.Num)
|
||||
case TypeTable:
|
||||
return fmt.Sprintf("%06d.ldb", fd.Num)
|
||||
case TypeTemp:
|
||||
return fmt.Sprintf("%06d.tmp", fd.Num)
|
||||
default:
|
||||
return fmt.Sprintf("%#x-%d", fd.Type, fd.Num)
|
||||
}
|
||||
}
|
||||
|
||||
// Zero returns true if fd == (FileDesc{}).
|
||||
func (fd FileDesc) Zero() bool {
|
||||
return fd == (FileDesc{})
|
||||
}
|
||||
|
||||
// FileDescOk returns true if fd is a valid 'file descriptor'.
|
||||
func FileDescOk(fd FileDesc) bool {
|
||||
switch fd.Type {
|
||||
case TypeManifest:
|
||||
case TypeJournal:
|
||||
case TypeTable:
|
||||
case TypeTemp:
|
||||
default:
|
||||
return false
|
||||
}
|
||||
return fd.Num >= 0
|
||||
}
|
||||
|
||||
// Storage is the storage. A storage instance must be safe for concurrent use.
|
||||
type Storage interface {
|
||||
// Lock locks the storage. Any subsequent attempt to call Lock will fail
|
||||
// until the last lock released.
|
||||
// Caller should call Unlock method after use.
|
||||
Lock() (Locker, error)
|
||||
|
||||
// Log logs a string. This is used for logging.
|
||||
// An implementation may write to a file, stdout or simply do nothing.
|
||||
Log(str string)
|
||||
|
||||
// SetMeta store 'file descriptor' that can later be acquired using GetMeta
|
||||
// method. The 'file descriptor' should point to a valid file.
|
||||
// SetMeta should be implemented in such way that changes should happen
|
||||
// atomically.
|
||||
SetMeta(fd FileDesc) error
|
||||
|
||||
// GetMeta returns 'file descriptor' stored in meta. The 'file descriptor'
|
||||
// can be updated using SetMeta method.
|
||||
// Returns os.ErrNotExist if meta doesn't store any 'file descriptor', or
|
||||
// 'file descriptor' point to nonexistent file.
|
||||
GetMeta() (FileDesc, error)
|
||||
|
||||
// List returns file descriptors that match the given file types.
|
||||
// The file types may be OR'ed together.
|
||||
List(ft FileType) ([]FileDesc, error)
|
||||
|
||||
// Open opens file with the given 'file descriptor' read-only.
|
||||
// Returns os.ErrNotExist error if the file does not exist.
|
||||
// Returns ErrClosed if the underlying storage is closed.
|
||||
Open(fd FileDesc) (Reader, error)
|
||||
|
||||
// Create creates file with the given 'file descriptor', truncate if already
|
||||
// exist and opens write-only.
|
||||
// Returns ErrClosed if the underlying storage is closed.
|
||||
Create(fd FileDesc) (Writer, error)
|
||||
|
||||
// Remove removes file with the given 'file descriptor'.
|
||||
// Returns ErrClosed if the underlying storage is closed.
|
||||
Remove(fd FileDesc) error
|
||||
|
||||
// Rename renames file from oldfd to newfd.
|
||||
// Returns ErrClosed if the underlying storage is closed.
|
||||
Rename(oldfd, newfd FileDesc) error
|
||||
|
||||
// Close closes the storage.
|
||||
// It is valid to call Close multiple times. Other methods should not be
|
||||
// called after the storage has been closed.
|
||||
Close() error
|
||||
}
|
||||
@ -0,0 +1,531 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/cache"
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
"github.com/syndtr/goleveldb/leveldb/table"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
// tFile holds basic information about a table.
|
||||
type tFile struct {
|
||||
fd storage.FileDesc
|
||||
seekLeft int32
|
||||
size int64
|
||||
imin, imax internalKey
|
||||
}
|
||||
|
||||
// Returns true if given key is after largest key of this table.
|
||||
func (t *tFile) after(icmp *iComparer, ukey []byte) bool {
|
||||
return ukey != nil && icmp.uCompare(ukey, t.imax.ukey()) > 0
|
||||
}
|
||||
|
||||
// Returns true if given key is before smallest key of this table.
|
||||
func (t *tFile) before(icmp *iComparer, ukey []byte) bool {
|
||||
return ukey != nil && icmp.uCompare(ukey, t.imin.ukey()) < 0
|
||||
}
|
||||
|
||||
// Returns true if given key range overlaps with this table key range.
|
||||
func (t *tFile) overlaps(icmp *iComparer, umin, umax []byte) bool {
|
||||
return !t.after(icmp, umin) && !t.before(icmp, umax)
|
||||
}
|
||||
|
||||
// Cosumes one seek and return current seeks left.
|
||||
func (t *tFile) consumeSeek() int32 {
|
||||
return atomic.AddInt32(&t.seekLeft, -1)
|
||||
}
|
||||
|
||||
// Creates new tFile.
|
||||
func newTableFile(fd storage.FileDesc, size int64, imin, imax internalKey) *tFile {
|
||||
f := &tFile{
|
||||
fd: fd,
|
||||
size: size,
|
||||
imin: imin,
|
||||
imax: imax,
|
||||
}
|
||||
|
||||
// We arrange to automatically compact this file after
|
||||
// a certain number of seeks. Let's assume:
|
||||
// (1) One seek costs 10ms
|
||||
// (2) Writing or reading 1MB costs 10ms (100MB/s)
|
||||
// (3) A compaction of 1MB does 25MB of IO:
|
||||
// 1MB read from this level
|
||||
// 10-12MB read from next level (boundaries may be misaligned)
|
||||
// 10-12MB written to next level
|
||||
// This implies that 25 seeks cost the same as the compaction
|
||||
// of 1MB of data. I.e., one seek costs approximately the
|
||||
// same as the compaction of 40KB of data. We are a little
|
||||
// conservative and allow approximately one seek for every 16KB
|
||||
// of data before triggering a compaction.
|
||||
f.seekLeft = int32(size / 16384)
|
||||
if f.seekLeft < 100 {
|
||||
f.seekLeft = 100
|
||||
}
|
||||
|
||||
return f
|
||||
}
|
||||
|
||||
func tableFileFromRecord(r atRecord) *tFile {
|
||||
return newTableFile(storage.FileDesc{Type: storage.TypeTable, Num: r.num}, r.size, r.imin, r.imax)
|
||||
}
|
||||
|
||||
// tFiles hold multiple tFile.
|
||||
type tFiles []*tFile
|
||||
|
||||
func (tf tFiles) Len() int { return len(tf) }
|
||||
func (tf tFiles) Swap(i, j int) { tf[i], tf[j] = tf[j], tf[i] }
|
||||
|
||||
func (tf tFiles) nums() string {
|
||||
x := "[ "
|
||||
for i, f := range tf {
|
||||
if i != 0 {
|
||||
x += ", "
|
||||
}
|
||||
x += fmt.Sprint(f.fd.Num)
|
||||
}
|
||||
x += " ]"
|
||||
return x
|
||||
}
|
||||
|
||||
// Returns true if i smallest key is less than j.
|
||||
// This used for sort by key in ascending order.
|
||||
func (tf tFiles) lessByKey(icmp *iComparer, i, j int) bool {
|
||||
a, b := tf[i], tf[j]
|
||||
n := icmp.Compare(a.imin, b.imin)
|
||||
if n == 0 {
|
||||
return a.fd.Num < b.fd.Num
|
||||
}
|
||||
return n < 0
|
||||
}
|
||||
|
||||
// Returns true if i file number is greater than j.
|
||||
// This used for sort by file number in descending order.
|
||||
func (tf tFiles) lessByNum(i, j int) bool {
|
||||
return tf[i].fd.Num > tf[j].fd.Num
|
||||
}
|
||||
|
||||
// Sorts tables by key in ascending order.
|
||||
func (tf tFiles) sortByKey(icmp *iComparer) {
|
||||
sort.Sort(&tFilesSortByKey{tFiles: tf, icmp: icmp})
|
||||
}
|
||||
|
||||
// Sorts tables by file number in descending order.
|
||||
func (tf tFiles) sortByNum() {
|
||||
sort.Sort(&tFilesSortByNum{tFiles: tf})
|
||||
}
|
||||
|
||||
// Returns sum of all tables size.
|
||||
func (tf tFiles) size() (sum int64) {
|
||||
for _, t := range tf {
|
||||
sum += t.size
|
||||
}
|
||||
return sum
|
||||
}
|
||||
|
||||
// Searches smallest index of tables whose its smallest
|
||||
// key is after or equal with given key.
|
||||
func (tf tFiles) searchMin(icmp *iComparer, ikey internalKey) int {
|
||||
return sort.Search(len(tf), func(i int) bool {
|
||||
return icmp.Compare(tf[i].imin, ikey) >= 0
|
||||
})
|
||||
}
|
||||
|
||||
// Searches smallest index of tables whose its largest
|
||||
// key is after or equal with given key.
|
||||
func (tf tFiles) searchMax(icmp *iComparer, ikey internalKey) int {
|
||||
return sort.Search(len(tf), func(i int) bool {
|
||||
return icmp.Compare(tf[i].imax, ikey) >= 0
|
||||
})
|
||||
}
|
||||
|
||||
// Returns true if given key range overlaps with one or more
|
||||
// tables key range. If unsorted is true then binary search will not be used.
|
||||
func (tf tFiles) overlaps(icmp *iComparer, umin, umax []byte, unsorted bool) bool {
|
||||
if unsorted {
|
||||
// Check against all files.
|
||||
for _, t := range tf {
|
||||
if t.overlaps(icmp, umin, umax) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
i := 0
|
||||
if len(umin) > 0 {
|
||||
// Find the earliest possible internal key for min.
|
||||
i = tf.searchMax(icmp, makeInternalKey(nil, umin, keyMaxSeq, keyTypeSeek))
|
||||
}
|
||||
if i >= len(tf) {
|
||||
// Beginning of range is after all files, so no overlap.
|
||||
return false
|
||||
}
|
||||
return !tf[i].before(icmp, umax)
|
||||
}
|
||||
|
||||
// Returns tables whose its key range overlaps with given key range.
|
||||
// Range will be expanded if ukey found hop across tables.
|
||||
// If overlapped is true then the search will be restarted if umax
|
||||
// expanded.
|
||||
// The dst content will be overwritten.
|
||||
func (tf tFiles) getOverlaps(dst tFiles, icmp *iComparer, umin, umax []byte, overlapped bool) tFiles {
|
||||
dst = dst[:0]
|
||||
for i := 0; i < len(tf); {
|
||||
t := tf[i]
|
||||
if t.overlaps(icmp, umin, umax) {
|
||||
if umin != nil && icmp.uCompare(t.imin.ukey(), umin) < 0 {
|
||||
umin = t.imin.ukey()
|
||||
dst = dst[:0]
|
||||
i = 0
|
||||
continue
|
||||
} else if umax != nil && icmp.uCompare(t.imax.ukey(), umax) > 0 {
|
||||
umax = t.imax.ukey()
|
||||
// Restart search if it is overlapped.
|
||||
if overlapped {
|
||||
dst = dst[:0]
|
||||
i = 0
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
dst = append(dst, t)
|
||||
}
|
||||
i++
|
||||
}
|
||||
|
||||
return dst
|
||||
}
|
||||
|
||||
// Returns tables key range.
|
||||
func (tf tFiles) getRange(icmp *iComparer) (imin, imax internalKey) {
|
||||
for i, t := range tf {
|
||||
if i == 0 {
|
||||
imin, imax = t.imin, t.imax
|
||||
continue
|
||||
}
|
||||
if icmp.Compare(t.imin, imin) < 0 {
|
||||
imin = t.imin
|
||||
}
|
||||
if icmp.Compare(t.imax, imax) > 0 {
|
||||
imax = t.imax
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Creates iterator index from tables.
|
||||
func (tf tFiles) newIndexIterator(tops *tOps, icmp *iComparer, slice *util.Range, ro *opt.ReadOptions) iterator.IteratorIndexer {
|
||||
if slice != nil {
|
||||
var start, limit int
|
||||
if slice.Start != nil {
|
||||
start = tf.searchMax(icmp, internalKey(slice.Start))
|
||||
}
|
||||
if slice.Limit != nil {
|
||||
limit = tf.searchMin(icmp, internalKey(slice.Limit))
|
||||
} else {
|
||||
limit = tf.Len()
|
||||
}
|
||||
tf = tf[start:limit]
|
||||
}
|
||||
return iterator.NewArrayIndexer(&tFilesArrayIndexer{
|
||||
tFiles: tf,
|
||||
tops: tops,
|
||||
icmp: icmp,
|
||||
slice: slice,
|
||||
ro: ro,
|
||||
})
|
||||
}
|
||||
|
||||
// Tables iterator index.
|
||||
type tFilesArrayIndexer struct {
|
||||
tFiles
|
||||
tops *tOps
|
||||
icmp *iComparer
|
||||
slice *util.Range
|
||||
ro *opt.ReadOptions
|
||||
}
|
||||
|
||||
func (a *tFilesArrayIndexer) Search(key []byte) int {
|
||||
return a.searchMax(a.icmp, internalKey(key))
|
||||
}
|
||||
|
||||
func (a *tFilesArrayIndexer) Get(i int) iterator.Iterator {
|
||||
if i == 0 || i == a.Len()-1 {
|
||||
return a.tops.newIterator(a.tFiles[i], a.slice, a.ro)
|
||||
}
|
||||
return a.tops.newIterator(a.tFiles[i], nil, a.ro)
|
||||
}
|
||||
|
||||
// Helper type for sortByKey.
|
||||
type tFilesSortByKey struct {
|
||||
tFiles
|
||||
icmp *iComparer
|
||||
}
|
||||
|
||||
func (x *tFilesSortByKey) Less(i, j int) bool {
|
||||
return x.lessByKey(x.icmp, i, j)
|
||||
}
|
||||
|
||||
// Helper type for sortByNum.
|
||||
type tFilesSortByNum struct {
|
||||
tFiles
|
||||
}
|
||||
|
||||
func (x *tFilesSortByNum) Less(i, j int) bool {
|
||||
return x.lessByNum(i, j)
|
||||
}
|
||||
|
||||
// Table operations.
|
||||
type tOps struct {
|
||||
s *session
|
||||
noSync bool
|
||||
evictRemoved bool
|
||||
cache *cache.Cache
|
||||
bcache *cache.Cache
|
||||
bpool *util.BufferPool
|
||||
}
|
||||
|
||||
// Creates an empty table and returns table writer.
|
||||
func (t *tOps) create() (*tWriter, error) {
|
||||
fd := storage.FileDesc{Type: storage.TypeTable, Num: t.s.allocFileNum()}
|
||||
fw, err := t.s.stor.Create(fd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &tWriter{
|
||||
t: t,
|
||||
fd: fd,
|
||||
w: fw,
|
||||
tw: table.NewWriter(fw, t.s.o.Options),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Builds table from src iterator.
|
||||
func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) {
|
||||
w, err := t.create()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
w.drop()
|
||||
}
|
||||
}()
|
||||
|
||||
for src.Next() {
|
||||
err = w.append(src.Key(), src.Value())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
err = src.Error()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
n = w.tw.EntriesLen()
|
||||
f, err = w.finish()
|
||||
return
|
||||
}
|
||||
|
||||
// Opens table. It returns a cache handle, which should
|
||||
// be released after use.
|
||||
func (t *tOps) open(f *tFile) (ch *cache.Handle, err error) {
|
||||
ch = t.cache.Get(0, uint64(f.fd.Num), func() (size int, value cache.Value) {
|
||||
var r storage.Reader
|
||||
r, err = t.s.stor.Open(f.fd)
|
||||
if err != nil {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var bcache *cache.NamespaceGetter
|
||||
if t.bcache != nil {
|
||||
bcache = &cache.NamespaceGetter{Cache: t.bcache, NS: uint64(f.fd.Num)}
|
||||
}
|
||||
|
||||
var tr *table.Reader
|
||||
tr, err = table.NewReader(r, f.size, f.fd, bcache, t.bpool, t.s.o.Options)
|
||||
if err != nil {
|
||||
r.Close()
|
||||
return 0, nil
|
||||
}
|
||||
return 1, tr
|
||||
|
||||
})
|
||||
if ch == nil && err == nil {
|
||||
err = ErrClosed
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Finds key/value pair whose key is greater than or equal to the
|
||||
// given key.
|
||||
func (t *tOps) find(f *tFile, key []byte, ro *opt.ReadOptions) (rkey, rvalue []byte, err error) {
|
||||
ch, err := t.open(f)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer ch.Release()
|
||||
return ch.Value().(*table.Reader).Find(key, true, ro)
|
||||
}
|
||||
|
||||
// Finds key that is greater than or equal to the given key.
|
||||
func (t *tOps) findKey(f *tFile, key []byte, ro *opt.ReadOptions) (rkey []byte, err error) {
|
||||
ch, err := t.open(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer ch.Release()
|
||||
return ch.Value().(*table.Reader).FindKey(key, true, ro)
|
||||
}
|
||||
|
||||
// Returns approximate offset of the given key.
|
||||
func (t *tOps) offsetOf(f *tFile, key []byte) (offset int64, err error) {
|
||||
ch, err := t.open(f)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer ch.Release()
|
||||
return ch.Value().(*table.Reader).OffsetOf(key)
|
||||
}
|
||||
|
||||
// Creates an iterator from the given table.
|
||||
func (t *tOps) newIterator(f *tFile, slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
|
||||
ch, err := t.open(f)
|
||||
if err != nil {
|
||||
return iterator.NewEmptyIterator(err)
|
||||
}
|
||||
iter := ch.Value().(*table.Reader).NewIterator(slice, ro)
|
||||
iter.SetReleaser(ch)
|
||||
return iter
|
||||
}
|
||||
|
||||
// Removes table from persistent storage. It waits until
|
||||
// no one use the the table.
|
||||
func (t *tOps) remove(f *tFile) {
|
||||
t.cache.Delete(0, uint64(f.fd.Num), func() {
|
||||
if err := t.s.stor.Remove(f.fd); err != nil {
|
||||
t.s.logf("table@remove removing @%d %q", f.fd.Num, err)
|
||||
} else {
|
||||
t.s.logf("table@remove removed @%d", f.fd.Num)
|
||||
}
|
||||
if t.evictRemoved && t.bcache != nil {
|
||||
t.bcache.EvictNS(uint64(f.fd.Num))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Closes the table ops instance. It will close all tables,
|
||||
// regadless still used or not.
|
||||
func (t *tOps) close() {
|
||||
t.bpool.Close()
|
||||
t.cache.Close()
|
||||
if t.bcache != nil {
|
||||
t.bcache.CloseWeak()
|
||||
}
|
||||
}
|
||||
|
||||
// Creates new initialized table ops instance.
|
||||
func newTableOps(s *session) *tOps {
|
||||
var (
|
||||
cacher cache.Cacher
|
||||
bcache *cache.Cache
|
||||
bpool *util.BufferPool
|
||||
)
|
||||
if s.o.GetOpenFilesCacheCapacity() > 0 {
|
||||
cacher = cache.NewLRU(s.o.GetOpenFilesCacheCapacity())
|
||||
}
|
||||
if !s.o.GetDisableBlockCache() {
|
||||
var bcacher cache.Cacher
|
||||
if s.o.GetBlockCacheCapacity() > 0 {
|
||||
bcacher = s.o.GetBlockCacher().New(s.o.GetBlockCacheCapacity())
|
||||
}
|
||||
bcache = cache.NewCache(bcacher)
|
||||
}
|
||||
if !s.o.GetDisableBufferPool() {
|
||||
bpool = util.NewBufferPool(s.o.GetBlockSize() + 5)
|
||||
}
|
||||
return &tOps{
|
||||
s: s,
|
||||
noSync: s.o.GetNoSync(),
|
||||
evictRemoved: s.o.GetBlockCacheEvictRemoved(),
|
||||
cache: cache.NewCache(cacher),
|
||||
bcache: bcache,
|
||||
bpool: bpool,
|
||||
}
|
||||
}
|
||||
|
||||
// tWriter wraps the table writer. It keep track of file descriptor
|
||||
// and added key range.
|
||||
type tWriter struct {
|
||||
t *tOps
|
||||
|
||||
fd storage.FileDesc
|
||||
w storage.Writer
|
||||
tw *table.Writer
|
||||
|
||||
first, last []byte
|
||||
}
|
||||
|
||||
// Append key/value pair to the table.
|
||||
func (w *tWriter) append(key, value []byte) error {
|
||||
if w.first == nil {
|
||||
w.first = append([]byte{}, key...)
|
||||
}
|
||||
w.last = append(w.last[:0], key...)
|
||||
return w.tw.Append(key, value)
|
||||
}
|
||||
|
||||
// Returns true if the table is empty.
|
||||
func (w *tWriter) empty() bool {
|
||||
return w.first == nil
|
||||
}
|
||||
|
||||
// Closes the storage.Writer.
|
||||
func (w *tWriter) close() {
|
||||
if w.w != nil {
|
||||
w.w.Close()
|
||||
w.w = nil
|
||||
}
|
||||
}
|
||||
|
||||
// Finalizes the table and returns table file.
|
||||
func (w *tWriter) finish() (f *tFile, err error) {
|
||||
defer w.close()
|
||||
err = w.tw.Close()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if !w.t.noSync {
|
||||
err = w.w.Sync()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
f = newTableFile(w.fd, int64(w.tw.BytesLen()), internalKey(w.first), internalKey(w.last))
|
||||
return
|
||||
}
|
||||
|
||||
// Drops the table.
|
||||
func (w *tWriter) drop() {
|
||||
w.close()
|
||||
w.t.s.stor.Remove(w.fd)
|
||||
w.t.s.reuseFileNum(w.fd.Num)
|
||||
w.tw = nil
|
||||
w.first = nil
|
||||
w.last = nil
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,177 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package table allows read and write sorted key/value.
|
||||
package table
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
)
|
||||
|
||||
/*
|
||||
Table:
|
||||
|
||||
Table is consist of one or more data blocks, an optional filter block
|
||||
a metaindex block, an index block and a table footer. Metaindex block
|
||||
is a special block used to keep parameters of the table, such as filter
|
||||
block name and its block handle. Index block is a special block used to
|
||||
keep record of data blocks offset and length, index block use one as
|
||||
restart interval. The key used by index block are the last key of preceding
|
||||
block, shorter separator of adjacent blocks or shorter successor of the
|
||||
last key of the last block. Filter block is an optional block contains
|
||||
sequence of filter data generated by a filter generator.
|
||||
|
||||
Table data structure:
|
||||
+ optional
|
||||
/
|
||||
+--------------+--------------+--------------+------+-------+-----------------+-------------+--------+
|
||||
| data block 1 | ... | data block n | filter block | metaindex block | index block | footer |
|
||||
+--------------+--------------+--------------+--------------+-----------------+-------------+--------+
|
||||
|
||||
Each block followed by a 5-bytes trailer contains compression type and checksum.
|
||||
|
||||
Table block trailer:
|
||||
|
||||
+---------------------------+-------------------+
|
||||
| compression type (1-byte) | checksum (4-byte) |
|
||||
+---------------------------+-------------------+
|
||||
|
||||
The checksum is a CRC-32 computed using Castagnoli's polynomial. Compression
|
||||
type also included in the checksum.
|
||||
|
||||
Table footer:
|
||||
|
||||
+------------------- 40-bytes -------------------+
|
||||
/ \
|
||||
+------------------------+--------------------+------+-----------------+
|
||||
| metaindex block handle / index block handle / ---- | magic (8-bytes) |
|
||||
+------------------------+--------------------+------+-----------------+
|
||||
|
||||
The magic are first 64-bit of SHA-1 sum of "http://code.google.com/p/leveldb/".
|
||||
|
||||
NOTE: All fixed-length integer are little-endian.
|
||||
*/
|
||||
|
||||
/*
|
||||
Block:
|
||||
|
||||
Block is consist of one or more key/value entries and a block trailer.
|
||||
Block entry shares key prefix with its preceding key until a restart
|
||||
point reached. A block should contains at least one restart point.
|
||||
First restart point are always zero.
|
||||
|
||||
Block data structure:
|
||||
|
||||
+ restart point + restart point (depends on restart interval)
|
||||
/ /
|
||||
+---------------+---------------+---------------+---------------+---------+
|
||||
| block entry 1 | block entry 2 | ... | block entry n | trailer |
|
||||
+---------------+---------------+---------------+---------------+---------+
|
||||
|
||||
Key/value entry:
|
||||
|
||||
+---- key len ----+
|
||||
/ \
|
||||
+-------+---------+-----------+---------+--------------------+--------------+----------------+
|
||||
| shared (varint) | not shared (varint) | value len (varint) | key (varlen) | value (varlen) |
|
||||
+-----------------+---------------------+--------------------+--------------+----------------+
|
||||
|
||||
Block entry shares key prefix with its preceding key:
|
||||
Conditions:
|
||||
restart_interval=2
|
||||
entry one : key=deck,value=v1
|
||||
entry two : key=dock,value=v2
|
||||
entry three: key=duck,value=v3
|
||||
The entries will be encoded as follow:
|
||||
|
||||
+ restart point (offset=0) + restart point (offset=16)
|
||||
/ /
|
||||
+-----+-----+-----+----------+--------+-----+-----+-----+---------+--------+-----+-----+-----+----------+--------+
|
||||
| 0 | 4 | 2 | "deck" | "v1" | 1 | 3 | 2 | "ock" | "v2" | 0 | 4 | 2 | "duck" | "v3" |
|
||||
+-----+-----+-----+----------+--------+-----+-----+-----+---------+--------+-----+-----+-----+----------+--------+
|
||||
\ / \ / \ /
|
||||
+----------- entry one -----------+ +----------- entry two ----------+ +---------- entry three ----------+
|
||||
|
||||
The block trailer will contains two restart points:
|
||||
|
||||
+------------+-----------+--------+
|
||||
| 0 | 16 | 2 |
|
||||
+------------+-----------+---+----+
|
||||
\ / \
|
||||
+-- restart points --+ + restart points length
|
||||
|
||||
Block trailer:
|
||||
|
||||
+-- 4-bytes --+
|
||||
/ \
|
||||
+-----------------+-----------------+-----------------+------------------------------+
|
||||
| restart point 1 | .... | restart point n | restart points len (4-bytes) |
|
||||
+-----------------+-----------------+-----------------+------------------------------+
|
||||
|
||||
|
||||
NOTE: All fixed-length integer are little-endian.
|
||||
*/
|
||||
|
||||
/*
|
||||
Filter block:
|
||||
|
||||
Filter block consist of one or more filter data and a filter block trailer.
|
||||
The trailer contains filter data offsets, a trailer offset and a 1-byte base Lg.
|
||||
|
||||
Filter block data structure:
|
||||
|
||||
+ offset 1 + offset 2 + offset n + trailer offset
|
||||
/ / / /
|
||||
+---------------+---------------+---------------+---------+
|
||||
| filter data 1 | ... | filter data n | trailer |
|
||||
+---------------+---------------+---------------+---------+
|
||||
|
||||
Filter block trailer:
|
||||
|
||||
+- 4-bytes -+
|
||||
/ \
|
||||
+---------------+---------------+---------------+-------------------------------+------------------+
|
||||
| data 1 offset | .... | data n offset | data-offsets offset (4-bytes) | base Lg (1-byte) |
|
||||
+-------------- +---------------+---------------+-------------------------------+------------------+
|
||||
|
||||
|
||||
NOTE: All fixed-length integer are little-endian.
|
||||
*/
|
||||
|
||||
const (
|
||||
blockTrailerLen = 5
|
||||
footerLen = 48
|
||||
|
||||
magic = "\x57\xfb\x80\x8b\x24\x75\x47\xdb"
|
||||
|
||||
// The block type gives the per-block compression format.
|
||||
// These constants are part of the file format and should not be changed.
|
||||
blockTypeNoCompression = 0
|
||||
blockTypeSnappyCompression = 1
|
||||
|
||||
// Generate new filter every 2KB of data
|
||||
filterBaseLg = 11
|
||||
filterBase = 1 << filterBaseLg
|
||||
)
|
||||
|
||||
type blockHandle struct {
|
||||
offset, length uint64
|
||||
}
|
||||
|
||||
func decodeBlockHandle(src []byte) (blockHandle, int) {
|
||||
offset, n := binary.Uvarint(src)
|
||||
length, m := binary.Uvarint(src[n:])
|
||||
if n == 0 || m == 0 {
|
||||
return blockHandle{}, 0
|
||||
}
|
||||
return blockHandle{offset, length}, n + m
|
||||
}
|
||||
|
||||
func encodeBlockHandle(dst []byte, b blockHandle) int {
|
||||
n := binary.PutUvarint(dst, b.offset)
|
||||
m := binary.PutUvarint(dst[n:], b.length)
|
||||
return n + m
|
||||
}
|
||||
@ -0,0 +1,375 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package table
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/golang/snappy"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/comparer"
|
||||
"github.com/syndtr/goleveldb/leveldb/filter"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
func sharedPrefixLen(a, b []byte) int {
|
||||
i, n := 0, len(a)
|
||||
if n > len(b) {
|
||||
n = len(b)
|
||||
}
|
||||
for i < n && a[i] == b[i] {
|
||||
i++
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
type blockWriter struct {
|
||||
restartInterval int
|
||||
buf util.Buffer
|
||||
nEntries int
|
||||
prevKey []byte
|
||||
restarts []uint32
|
||||
scratch []byte
|
||||
}
|
||||
|
||||
func (w *blockWriter) append(key, value []byte) {
|
||||
nShared := 0
|
||||
if w.nEntries%w.restartInterval == 0 {
|
||||
w.restarts = append(w.restarts, uint32(w.buf.Len()))
|
||||
} else {
|
||||
nShared = sharedPrefixLen(w.prevKey, key)
|
||||
}
|
||||
n := binary.PutUvarint(w.scratch[0:], uint64(nShared))
|
||||
n += binary.PutUvarint(w.scratch[n:], uint64(len(key)-nShared))
|
||||
n += binary.PutUvarint(w.scratch[n:], uint64(len(value)))
|
||||
w.buf.Write(w.scratch[:n])
|
||||
w.buf.Write(key[nShared:])
|
||||
w.buf.Write(value)
|
||||
w.prevKey = append(w.prevKey[:0], key...)
|
||||
w.nEntries++
|
||||
}
|
||||
|
||||
func (w *blockWriter) finish() {
|
||||
// Write restarts entry.
|
||||
if w.nEntries == 0 {
|
||||
// Must have at least one restart entry.
|
||||
w.restarts = append(w.restarts, 0)
|
||||
}
|
||||
w.restarts = append(w.restarts, uint32(len(w.restarts)))
|
||||
for _, x := range w.restarts {
|
||||
buf4 := w.buf.Alloc(4)
|
||||
binary.LittleEndian.PutUint32(buf4, x)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *blockWriter) reset() {
|
||||
w.buf.Reset()
|
||||
w.nEntries = 0
|
||||
w.restarts = w.restarts[:0]
|
||||
}
|
||||
|
||||
func (w *blockWriter) bytesLen() int {
|
||||
restartsLen := len(w.restarts)
|
||||
if restartsLen == 0 {
|
||||
restartsLen = 1
|
||||
}
|
||||
return w.buf.Len() + 4*restartsLen + 4
|
||||
}
|
||||
|
||||
type filterWriter struct {
|
||||
generator filter.FilterGenerator
|
||||
buf util.Buffer
|
||||
nKeys int
|
||||
offsets []uint32
|
||||
}
|
||||
|
||||
func (w *filterWriter) add(key []byte) {
|
||||
if w.generator == nil {
|
||||
return
|
||||
}
|
||||
w.generator.Add(key)
|
||||
w.nKeys++
|
||||
}
|
||||
|
||||
func (w *filterWriter) flush(offset uint64) {
|
||||
if w.generator == nil {
|
||||
return
|
||||
}
|
||||
for x := int(offset / filterBase); x > len(w.offsets); {
|
||||
w.generate()
|
||||
}
|
||||
}
|
||||
|
||||
func (w *filterWriter) finish() {
|
||||
if w.generator == nil {
|
||||
return
|
||||
}
|
||||
// Generate last keys.
|
||||
|
||||
if w.nKeys > 0 {
|
||||
w.generate()
|
||||
}
|
||||
w.offsets = append(w.offsets, uint32(w.buf.Len()))
|
||||
for _, x := range w.offsets {
|
||||
buf4 := w.buf.Alloc(4)
|
||||
binary.LittleEndian.PutUint32(buf4, x)
|
||||
}
|
||||
w.buf.WriteByte(filterBaseLg)
|
||||
}
|
||||
|
||||
func (w *filterWriter) generate() {
|
||||
// Record offset.
|
||||
w.offsets = append(w.offsets, uint32(w.buf.Len()))
|
||||
// Generate filters.
|
||||
if w.nKeys > 0 {
|
||||
w.generator.Generate(&w.buf)
|
||||
w.nKeys = 0
|
||||
}
|
||||
}
|
||||
|
||||
// Writer is a table writer.
|
||||
type Writer struct {
|
||||
writer io.Writer
|
||||
err error
|
||||
// Options
|
||||
cmp comparer.Comparer
|
||||
filter filter.Filter
|
||||
compression opt.Compression
|
||||
blockSize int
|
||||
|
||||
dataBlock blockWriter
|
||||
indexBlock blockWriter
|
||||
filterBlock filterWriter
|
||||
pendingBH blockHandle
|
||||
offset uint64
|
||||
nEntries int
|
||||
// Scratch allocated enough for 5 uvarint. Block writer should not use
|
||||
// first 20-bytes since it will be used to encode block handle, which
|
||||
// then passed to the block writer itself.
|
||||
scratch [50]byte
|
||||
comparerScratch []byte
|
||||
compressionScratch []byte
|
||||
}
|
||||
|
||||
func (w *Writer) writeBlock(buf *util.Buffer, compression opt.Compression) (bh blockHandle, err error) {
|
||||
// Compress the buffer if necessary.
|
||||
var b []byte
|
||||
if compression == opt.SnappyCompression {
|
||||
// Allocate scratch enough for compression and block trailer.
|
||||
if n := snappy.MaxEncodedLen(buf.Len()) + blockTrailerLen; len(w.compressionScratch) < n {
|
||||
w.compressionScratch = make([]byte, n)
|
||||
}
|
||||
compressed := snappy.Encode(w.compressionScratch, buf.Bytes())
|
||||
n := len(compressed)
|
||||
b = compressed[:n+blockTrailerLen]
|
||||
b[n] = blockTypeSnappyCompression
|
||||
} else {
|
||||
tmp := buf.Alloc(blockTrailerLen)
|
||||
tmp[0] = blockTypeNoCompression
|
||||
b = buf.Bytes()
|
||||
}
|
||||
|
||||
// Calculate the checksum.
|
||||
n := len(b) - 4
|
||||
checksum := util.NewCRC(b[:n]).Value()
|
||||
binary.LittleEndian.PutUint32(b[n:], checksum)
|
||||
|
||||
// Write the buffer to the file.
|
||||
_, err = w.writer.Write(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
bh = blockHandle{w.offset, uint64(len(b) - blockTrailerLen)}
|
||||
w.offset += uint64(len(b))
|
||||
return
|
||||
}
|
||||
|
||||
func (w *Writer) flushPendingBH(key []byte) {
|
||||
if w.pendingBH.length == 0 {
|
||||
return
|
||||
}
|
||||
var separator []byte
|
||||
if len(key) == 0 {
|
||||
separator = w.cmp.Successor(w.comparerScratch[:0], w.dataBlock.prevKey)
|
||||
} else {
|
||||
separator = w.cmp.Separator(w.comparerScratch[:0], w.dataBlock.prevKey, key)
|
||||
}
|
||||
if separator == nil {
|
||||
separator = w.dataBlock.prevKey
|
||||
} else {
|
||||
w.comparerScratch = separator
|
||||
}
|
||||
n := encodeBlockHandle(w.scratch[:20], w.pendingBH)
|
||||
// Append the block handle to the index block.
|
||||
w.indexBlock.append(separator, w.scratch[:n])
|
||||
// Reset prev key of the data block.
|
||||
w.dataBlock.prevKey = w.dataBlock.prevKey[:0]
|
||||
// Clear pending block handle.
|
||||
w.pendingBH = blockHandle{}
|
||||
}
|
||||
|
||||
func (w *Writer) finishBlock() error {
|
||||
w.dataBlock.finish()
|
||||
bh, err := w.writeBlock(&w.dataBlock.buf, w.compression)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.pendingBH = bh
|
||||
// Reset the data block.
|
||||
w.dataBlock.reset()
|
||||
// Flush the filter block.
|
||||
w.filterBlock.flush(w.offset)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Append appends key/value pair to the table. The keys passed must
|
||||
// be in increasing order.
|
||||
//
|
||||
// It is safe to modify the contents of the arguments after Append returns.
|
||||
func (w *Writer) Append(key, value []byte) error {
|
||||
if w.err != nil {
|
||||
return w.err
|
||||
}
|
||||
if w.nEntries > 0 && w.cmp.Compare(w.dataBlock.prevKey, key) >= 0 {
|
||||
w.err = fmt.Errorf("leveldb/table: Writer: keys are not in increasing order: %q, %q", w.dataBlock.prevKey, key)
|
||||
return w.err
|
||||
}
|
||||
|
||||
w.flushPendingBH(key)
|
||||
// Append key/value pair to the data block.
|
||||
w.dataBlock.append(key, value)
|
||||
// Add key to the filter block.
|
||||
w.filterBlock.add(key)
|
||||
|
||||
// Finish the data block if block size target reached.
|
||||
if w.dataBlock.bytesLen() >= w.blockSize {
|
||||
if err := w.finishBlock(); err != nil {
|
||||
w.err = err
|
||||
return w.err
|
||||
}
|
||||
}
|
||||
w.nEntries++
|
||||
return nil
|
||||
}
|
||||
|
||||
// BlocksLen returns number of blocks written so far.
|
||||
func (w *Writer) BlocksLen() int {
|
||||
n := w.indexBlock.nEntries
|
||||
if w.pendingBH.length > 0 {
|
||||
// Includes the pending block.
|
||||
n++
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// EntriesLen returns number of entries added so far.
|
||||
func (w *Writer) EntriesLen() int {
|
||||
return w.nEntries
|
||||
}
|
||||
|
||||
// BytesLen returns number of bytes written so far.
|
||||
func (w *Writer) BytesLen() int {
|
||||
return int(w.offset)
|
||||
}
|
||||
|
||||
// Close will finalize the table. Calling Append is not possible
|
||||
// after Close, but calling BlocksLen, EntriesLen and BytesLen
|
||||
// is still possible.
|
||||
func (w *Writer) Close() error {
|
||||
if w.err != nil {
|
||||
return w.err
|
||||
}
|
||||
|
||||
// Write the last data block. Or empty data block if there
|
||||
// aren't any data blocks at all.
|
||||
if w.dataBlock.nEntries > 0 || w.nEntries == 0 {
|
||||
if err := w.finishBlock(); err != nil {
|
||||
w.err = err
|
||||
return w.err
|
||||
}
|
||||
}
|
||||
w.flushPendingBH(nil)
|
||||
|
||||
// Write the filter block.
|
||||
var filterBH blockHandle
|
||||
w.filterBlock.finish()
|
||||
if buf := &w.filterBlock.buf; buf.Len() > 0 {
|
||||
filterBH, w.err = w.writeBlock(buf, opt.NoCompression)
|
||||
if w.err != nil {
|
||||
return w.err
|
||||
}
|
||||
}
|
||||
|
||||
// Write the metaindex block.
|
||||
if filterBH.length > 0 {
|
||||
key := []byte("filter." + w.filter.Name())
|
||||
n := encodeBlockHandle(w.scratch[:20], filterBH)
|
||||
w.dataBlock.append(key, w.scratch[:n])
|
||||
}
|
||||
w.dataBlock.finish()
|
||||
metaindexBH, err := w.writeBlock(&w.dataBlock.buf, w.compression)
|
||||
if err != nil {
|
||||
w.err = err
|
||||
return w.err
|
||||
}
|
||||
|
||||
// Write the index block.
|
||||
w.indexBlock.finish()
|
||||
indexBH, err := w.writeBlock(&w.indexBlock.buf, w.compression)
|
||||
if err != nil {
|
||||
w.err = err
|
||||
return w.err
|
||||
}
|
||||
|
||||
// Write the table footer.
|
||||
footer := w.scratch[:footerLen]
|
||||
for i := range footer {
|
||||
footer[i] = 0
|
||||
}
|
||||
n := encodeBlockHandle(footer, metaindexBH)
|
||||
encodeBlockHandle(footer[n:], indexBH)
|
||||
copy(footer[footerLen-len(magic):], magic)
|
||||
if _, err := w.writer.Write(footer); err != nil {
|
||||
w.err = err
|
||||
return w.err
|
||||
}
|
||||
w.offset += footerLen
|
||||
|
||||
w.err = errors.New("leveldb/table: writer is closed")
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewWriter creates a new initialized table writer for the file.
|
||||
//
|
||||
// Table writer is not safe for concurrent use.
|
||||
func NewWriter(f io.Writer, o *opt.Options) *Writer {
|
||||
w := &Writer{
|
||||
writer: f,
|
||||
cmp: o.GetComparer(),
|
||||
filter: o.GetFilter(),
|
||||
compression: o.GetCompression(),
|
||||
blockSize: o.GetBlockSize(),
|
||||
comparerScratch: make([]byte, 0),
|
||||
}
|
||||
// data block
|
||||
w.dataBlock.restartInterval = o.GetBlockRestartInterval()
|
||||
// The first 20-bytes are used for encoding block handle.
|
||||
w.dataBlock.scratch = w.scratch[20:]
|
||||
// index block
|
||||
w.indexBlock.restartInterval = 1
|
||||
w.indexBlock.scratch = w.scratch[20:]
|
||||
// filter block
|
||||
if w.filter != nil {
|
||||
w.filterBlock.generator = w.filter.NewGenerator()
|
||||
w.filterBlock.flush(0)
|
||||
}
|
||||
return w
|
||||
}
|
||||
@ -0,0 +1,98 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
)
|
||||
|
||||
func shorten(str string) string {
|
||||
if len(str) <= 8 {
|
||||
return str
|
||||
}
|
||||
return str[:3] + ".." + str[len(str)-3:]
|
||||
}
|
||||
|
||||
var bunits = [...]string{"", "Ki", "Mi", "Gi", "Ti"}
|
||||
|
||||
func shortenb(bytes int) string {
|
||||
i := 0
|
||||
for ; bytes > 1024 && i < 4; i++ {
|
||||
bytes /= 1024
|
||||
}
|
||||
return fmt.Sprintf("%d%sB", bytes, bunits[i])
|
||||
}
|
||||
|
||||
func sshortenb(bytes int) string {
|
||||
if bytes == 0 {
|
||||
return "~"
|
||||
}
|
||||
sign := "+"
|
||||
if bytes < 0 {
|
||||
sign = "-"
|
||||
bytes *= -1
|
||||
}
|
||||
i := 0
|
||||
for ; bytes > 1024 && i < 4; i++ {
|
||||
bytes /= 1024
|
||||
}
|
||||
return fmt.Sprintf("%s%d%sB", sign, bytes, bunits[i])
|
||||
}
|
||||
|
||||
func sint(x int) string {
|
||||
if x == 0 {
|
||||
return "~"
|
||||
}
|
||||
sign := "+"
|
||||
if x < 0 {
|
||||
sign = "-"
|
||||
x *= -1
|
||||
}
|
||||
return fmt.Sprintf("%s%d", sign, x)
|
||||
}
|
||||
|
||||
func minInt(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func maxInt(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
type fdSorter []storage.FileDesc
|
||||
|
||||
func (p fdSorter) Len() int {
|
||||
return len(p)
|
||||
}
|
||||
|
||||
func (p fdSorter) Less(i, j int) bool {
|
||||
return p[i].Num < p[j].Num
|
||||
}
|
||||
|
||||
func (p fdSorter) Swap(i, j int) {
|
||||
p[i], p[j] = p[j], p[i]
|
||||
}
|
||||
|
||||
func sortFds(fds []storage.FileDesc) {
|
||||
sort.Sort(fdSorter(fds))
|
||||
}
|
||||
|
||||
func ensureBuffer(b []byte, n int) []byte {
|
||||
if cap(b) < n {
|
||||
return make([]byte, n)
|
||||
}
|
||||
return b[:n]
|
||||
}
|
||||
@ -0,0 +1,293 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package util
|
||||
|
||||
// This a copy of Go std bytes.Buffer with some modification
|
||||
// and some features stripped.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
)
|
||||
|
||||
// A Buffer is a variable-sized buffer of bytes with Read and Write methods.
|
||||
// The zero value for Buffer is an empty buffer ready to use.
|
||||
type Buffer struct {
|
||||
buf []byte // contents are the bytes buf[off : len(buf)]
|
||||
off int // read at &buf[off], write at &buf[len(buf)]
|
||||
bootstrap [64]byte // memory to hold first slice; helps small buffers (Printf) avoid allocation.
|
||||
}
|
||||
|
||||
// Bytes returns a slice of the contents of the unread portion of the buffer;
|
||||
// len(b.Bytes()) == b.Len(). If the caller changes the contents of the
|
||||
// returned slice, the contents of the buffer will change provided there
|
||||
// are no intervening method calls on the Buffer.
|
||||
func (b *Buffer) Bytes() []byte { return b.buf[b.off:] }
|
||||
|
||||
// String returns the contents of the unread portion of the buffer
|
||||
// as a string. If the Buffer is a nil pointer, it returns "<nil>".
|
||||
func (b *Buffer) String() string {
|
||||
if b == nil {
|
||||
// Special case, useful in debugging.
|
||||
return "<nil>"
|
||||
}
|
||||
return string(b.buf[b.off:])
|
||||
}
|
||||
|
||||
// Len returns the number of bytes of the unread portion of the buffer;
|
||||
// b.Len() == len(b.Bytes()).
|
||||
func (b *Buffer) Len() int { return len(b.buf) - b.off }
|
||||
|
||||
// Truncate discards all but the first n unread bytes from the buffer.
|
||||
// It panics if n is negative or greater than the length of the buffer.
|
||||
func (b *Buffer) Truncate(n int) {
|
||||
switch {
|
||||
case n < 0 || n > b.Len():
|
||||
panic("leveldb/util.Buffer: truncation out of range")
|
||||
case n == 0:
|
||||
// Reuse buffer space.
|
||||
b.off = 0
|
||||
}
|
||||
b.buf = b.buf[0 : b.off+n]
|
||||
}
|
||||
|
||||
// Reset resets the buffer so it has no content.
|
||||
// b.Reset() is the same as b.Truncate(0).
|
||||
func (b *Buffer) Reset() { b.Truncate(0) }
|
||||
|
||||
// grow grows the buffer to guarantee space for n more bytes.
|
||||
// It returns the index where bytes should be written.
|
||||
// If the buffer can't grow it will panic with bytes.ErrTooLarge.
|
||||
func (b *Buffer) grow(n int) int {
|
||||
m := b.Len()
|
||||
// If buffer is empty, reset to recover space.
|
||||
if m == 0 && b.off != 0 {
|
||||
b.Truncate(0)
|
||||
}
|
||||
if len(b.buf)+n > cap(b.buf) {
|
||||
var buf []byte
|
||||
if b.buf == nil && n <= len(b.bootstrap) {
|
||||
buf = b.bootstrap[0:]
|
||||
} else if m+n <= cap(b.buf)/2 {
|
||||
// We can slide things down instead of allocating a new
|
||||
// slice. We only need m+n <= cap(b.buf) to slide, but
|
||||
// we instead let capacity get twice as large so we
|
||||
// don't spend all our time copying.
|
||||
copy(b.buf[:], b.buf[b.off:])
|
||||
buf = b.buf[:m]
|
||||
} else {
|
||||
// not enough space anywhere
|
||||
buf = makeSlice(2*cap(b.buf) + n)
|
||||
copy(buf, b.buf[b.off:])
|
||||
}
|
||||
b.buf = buf
|
||||
b.off = 0
|
||||
}
|
||||
b.buf = b.buf[0 : b.off+m+n]
|
||||
return b.off + m
|
||||
}
|
||||
|
||||
// Alloc allocs n bytes of slice from the buffer, growing the buffer as
|
||||
// needed. If n is negative, Alloc will panic.
|
||||
// If the buffer can't grow it will panic with bytes.ErrTooLarge.
|
||||
func (b *Buffer) Alloc(n int) []byte {
|
||||
if n < 0 {
|
||||
panic("leveldb/util.Buffer.Alloc: negative count")
|
||||
}
|
||||
m := b.grow(n)
|
||||
return b.buf[m:]
|
||||
}
|
||||
|
||||
// Grow grows the buffer's capacity, if necessary, to guarantee space for
|
||||
// another n bytes. After Grow(n), at least n bytes can be written to the
|
||||
// buffer without another allocation.
|
||||
// If n is negative, Grow will panic.
|
||||
// If the buffer can't grow it will panic with bytes.ErrTooLarge.
|
||||
func (b *Buffer) Grow(n int) {
|
||||
if n < 0 {
|
||||
panic("leveldb/util.Buffer.Grow: negative count")
|
||||
}
|
||||
m := b.grow(n)
|
||||
b.buf = b.buf[0:m]
|
||||
}
|
||||
|
||||
// Write appends the contents of p to the buffer, growing the buffer as
|
||||
// needed. The return value n is the length of p; err is always nil. If the
|
||||
// buffer becomes too large, Write will panic with bytes.ErrTooLarge.
|
||||
func (b *Buffer) Write(p []byte) (n int, err error) {
|
||||
m := b.grow(len(p))
|
||||
return copy(b.buf[m:], p), nil
|
||||
}
|
||||
|
||||
// MinRead is the minimum slice size passed to a Read call by
|
||||
// Buffer.ReadFrom. As long as the Buffer has at least MinRead bytes beyond
|
||||
// what is required to hold the contents of r, ReadFrom will not grow the
|
||||
// underlying buffer.
|
||||
const MinRead = 512
|
||||
|
||||
// ReadFrom reads data from r until EOF and appends it to the buffer, growing
|
||||
// the buffer as needed. The return value n is the number of bytes read. Any
|
||||
// error except io.EOF encountered during the read is also returned. If the
|
||||
// buffer becomes too large, ReadFrom will panic with bytes.ErrTooLarge.
|
||||
func (b *Buffer) ReadFrom(r io.Reader) (n int64, err error) {
|
||||
// If buffer is empty, reset to recover space.
|
||||
if b.off >= len(b.buf) {
|
||||
b.Truncate(0)
|
||||
}
|
||||
for {
|
||||
if free := cap(b.buf) - len(b.buf); free < MinRead {
|
||||
// not enough space at end
|
||||
newBuf := b.buf
|
||||
if b.off+free < MinRead {
|
||||
// not enough space using beginning of buffer;
|
||||
// double buffer capacity
|
||||
newBuf = makeSlice(2*cap(b.buf) + MinRead)
|
||||
}
|
||||
copy(newBuf, b.buf[b.off:])
|
||||
b.buf = newBuf[:len(b.buf)-b.off]
|
||||
b.off = 0
|
||||
}
|
||||
m, e := r.Read(b.buf[len(b.buf):cap(b.buf)])
|
||||
b.buf = b.buf[0 : len(b.buf)+m]
|
||||
n += int64(m)
|
||||
if e == io.EOF {
|
||||
break
|
||||
}
|
||||
if e != nil {
|
||||
return n, e
|
||||
}
|
||||
}
|
||||
return n, nil // err is EOF, so return nil explicitly
|
||||
}
|
||||
|
||||
// makeSlice allocates a slice of size n. If the allocation fails, it panics
|
||||
// with bytes.ErrTooLarge.
|
||||
func makeSlice(n int) []byte {
|
||||
// If the make fails, give a known error.
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
panic(bytes.ErrTooLarge)
|
||||
}
|
||||
}()
|
||||
return make([]byte, n)
|
||||
}
|
||||
|
||||
// WriteTo writes data to w until the buffer is drained or an error occurs.
|
||||
// The return value n is the number of bytes written; it always fits into an
|
||||
// int, but it is int64 to match the io.WriterTo interface. Any error
|
||||
// encountered during the write is also returned.
|
||||
func (b *Buffer) WriteTo(w io.Writer) (n int64, err error) {
|
||||
if b.off < len(b.buf) {
|
||||
nBytes := b.Len()
|
||||
m, e := w.Write(b.buf[b.off:])
|
||||
if m > nBytes {
|
||||
panic("leveldb/util.Buffer.WriteTo: invalid Write count")
|
||||
}
|
||||
b.off += m
|
||||
n = int64(m)
|
||||
if e != nil {
|
||||
return n, e
|
||||
}
|
||||
// all bytes should have been written, by definition of
|
||||
// Write method in io.Writer
|
||||
if m != nBytes {
|
||||
return n, io.ErrShortWrite
|
||||
}
|
||||
}
|
||||
// Buffer is now empty; reset.
|
||||
b.Truncate(0)
|
||||
return
|
||||
}
|
||||
|
||||
// WriteByte appends the byte c to the buffer, growing the buffer as needed.
|
||||
// The returned error is always nil, but is included to match bufio.Writer's
|
||||
// WriteByte. If the buffer becomes too large, WriteByte will panic with
|
||||
// bytes.ErrTooLarge.
|
||||
func (b *Buffer) WriteByte(c byte) error {
|
||||
m := b.grow(1)
|
||||
b.buf[m] = c
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read reads the next len(p) bytes from the buffer or until the buffer
|
||||
// is drained. The return value n is the number of bytes read. If the
|
||||
// buffer has no data to return, err is io.EOF (unless len(p) is zero);
|
||||
// otherwise it is nil.
|
||||
func (b *Buffer) Read(p []byte) (n int, err error) {
|
||||
if b.off >= len(b.buf) {
|
||||
// Buffer is empty, reset to recover space.
|
||||
b.Truncate(0)
|
||||
if len(p) == 0 {
|
||||
return
|
||||
}
|
||||
return 0, io.EOF
|
||||
}
|
||||
n = copy(p, b.buf[b.off:])
|
||||
b.off += n
|
||||
return
|
||||
}
|
||||
|
||||
// Next returns a slice containing the next n bytes from the buffer,
|
||||
// advancing the buffer as if the bytes had been returned by Read.
|
||||
// If there are fewer than n bytes in the buffer, Next returns the entire buffer.
|
||||
// The slice is only valid until the next call to a read or write method.
|
||||
func (b *Buffer) Next(n int) []byte {
|
||||
m := b.Len()
|
||||
if n > m {
|
||||
n = m
|
||||
}
|
||||
data := b.buf[b.off : b.off+n]
|
||||
b.off += n
|
||||
return data
|
||||
}
|
||||
|
||||
// ReadByte reads and returns the next byte from the buffer.
|
||||
// If no byte is available, it returns error io.EOF.
|
||||
func (b *Buffer) ReadByte() (c byte, err error) {
|
||||
if b.off >= len(b.buf) {
|
||||
// Buffer is empty, reset to recover space.
|
||||
b.Truncate(0)
|
||||
return 0, io.EOF
|
||||
}
|
||||
c = b.buf[b.off]
|
||||
b.off++
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// ReadBytes reads until the first occurrence of delim in the input,
|
||||
// returning a slice containing the data up to and including the delimiter.
|
||||
// If ReadBytes encounters an error before finding a delimiter,
|
||||
// it returns the data read before the error and the error itself (often io.EOF).
|
||||
// ReadBytes returns err != nil if and only if the returned data does not end in
|
||||
// delim.
|
||||
func (b *Buffer) ReadBytes(delim byte) (line []byte, err error) {
|
||||
slice, err := b.readSlice(delim)
|
||||
// return a copy of slice. The buffer's backing array may
|
||||
// be overwritten by later calls.
|
||||
line = append(line, slice...)
|
||||
return
|
||||
}
|
||||
|
||||
// readSlice is like ReadBytes but returns a reference to internal buffer data.
|
||||
func (b *Buffer) readSlice(delim byte) (line []byte, err error) {
|
||||
i := bytes.IndexByte(b.buf[b.off:], delim)
|
||||
end := b.off + i + 1
|
||||
if i < 0 {
|
||||
end = len(b.buf)
|
||||
err = io.EOF
|
||||
}
|
||||
line = b.buf[b.off:end]
|
||||
b.off = end
|
||||
return line, err
|
||||
}
|
||||
|
||||
// NewBuffer creates and initializes a new Buffer using buf as its initial
|
||||
// contents. It is intended to prepare a Buffer to read existing data. It
|
||||
// can also be used to size the internal buffer for writing. To do that,
|
||||
// buf should have the desired capacity but a length of zero.
|
||||
//
|
||||
// In most cases, new(Buffer) (or just declaring a Buffer variable) is
|
||||
// sufficient to initialize a Buffer.
|
||||
func NewBuffer(buf []byte) *Buffer { return &Buffer{buf: buf} }
|
||||
@ -0,0 +1,239 @@
|
||||
// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package util
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
type buffer struct {
|
||||
b []byte
|
||||
miss int
|
||||
}
|
||||
|
||||
// BufferPool is a 'buffer pool'.
|
||||
type BufferPool struct {
|
||||
pool [6]chan []byte
|
||||
size [5]uint32
|
||||
sizeMiss [5]uint32
|
||||
sizeHalf [5]uint32
|
||||
baseline [4]int
|
||||
baseline0 int
|
||||
|
||||
mu sync.RWMutex
|
||||
closed bool
|
||||
closeC chan struct{}
|
||||
|
||||
get uint32
|
||||
put uint32
|
||||
half uint32
|
||||
less uint32
|
||||
equal uint32
|
||||
greater uint32
|
||||
miss uint32
|
||||
}
|
||||
|
||||
func (p *BufferPool) poolNum(n int) int {
|
||||
if n <= p.baseline0 && n > p.baseline0/2 {
|
||||
return 0
|
||||
}
|
||||
for i, x := range p.baseline {
|
||||
if n <= x {
|
||||
return i + 1
|
||||
}
|
||||
}
|
||||
return len(p.baseline) + 1
|
||||
}
|
||||
|
||||
// Get returns buffer with length of n.
|
||||
func (p *BufferPool) Get(n int) []byte {
|
||||
if p == nil {
|
||||
return make([]byte, n)
|
||||
}
|
||||
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
|
||||
if p.closed {
|
||||
return make([]byte, n)
|
||||
}
|
||||
|
||||
atomic.AddUint32(&p.get, 1)
|
||||
|
||||
poolNum := p.poolNum(n)
|
||||
pool := p.pool[poolNum]
|
||||
if poolNum == 0 {
|
||||
// Fast path.
|
||||
select {
|
||||
case b := <-pool:
|
||||
switch {
|
||||
case cap(b) > n:
|
||||
if cap(b)-n >= n {
|
||||
atomic.AddUint32(&p.half, 1)
|
||||
select {
|
||||
case pool <- b:
|
||||
default:
|
||||
}
|
||||
return make([]byte, n)
|
||||
} else {
|
||||
atomic.AddUint32(&p.less, 1)
|
||||
return b[:n]
|
||||
}
|
||||
case cap(b) == n:
|
||||
atomic.AddUint32(&p.equal, 1)
|
||||
return b[:n]
|
||||
default:
|
||||
atomic.AddUint32(&p.greater, 1)
|
||||
}
|
||||
default:
|
||||
atomic.AddUint32(&p.miss, 1)
|
||||
}
|
||||
|
||||
return make([]byte, n, p.baseline0)
|
||||
} else {
|
||||
sizePtr := &p.size[poolNum-1]
|
||||
|
||||
select {
|
||||
case b := <-pool:
|
||||
switch {
|
||||
case cap(b) > n:
|
||||
if cap(b)-n >= n {
|
||||
atomic.AddUint32(&p.half, 1)
|
||||
sizeHalfPtr := &p.sizeHalf[poolNum-1]
|
||||
if atomic.AddUint32(sizeHalfPtr, 1) == 20 {
|
||||
atomic.StoreUint32(sizePtr, uint32(cap(b)/2))
|
||||
atomic.StoreUint32(sizeHalfPtr, 0)
|
||||
} else {
|
||||
select {
|
||||
case pool <- b:
|
||||
default:
|
||||
}
|
||||
}
|
||||
return make([]byte, n)
|
||||
} else {
|
||||
atomic.AddUint32(&p.less, 1)
|
||||
return b[:n]
|
||||
}
|
||||
case cap(b) == n:
|
||||
atomic.AddUint32(&p.equal, 1)
|
||||
return b[:n]
|
||||
default:
|
||||
atomic.AddUint32(&p.greater, 1)
|
||||
if uint32(cap(b)) >= atomic.LoadUint32(sizePtr) {
|
||||
select {
|
||||
case pool <- b:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
atomic.AddUint32(&p.miss, 1)
|
||||
}
|
||||
|
||||
if size := atomic.LoadUint32(sizePtr); uint32(n) > size {
|
||||
if size == 0 {
|
||||
atomic.CompareAndSwapUint32(sizePtr, 0, uint32(n))
|
||||
} else {
|
||||
sizeMissPtr := &p.sizeMiss[poolNum-1]
|
||||
if atomic.AddUint32(sizeMissPtr, 1) == 20 {
|
||||
atomic.StoreUint32(sizePtr, uint32(n))
|
||||
atomic.StoreUint32(sizeMissPtr, 0)
|
||||
}
|
||||
}
|
||||
return make([]byte, n)
|
||||
} else {
|
||||
return make([]byte, n, size)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Put adds given buffer to the pool.
|
||||
func (p *BufferPool) Put(b []byte) {
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
|
||||
if p.closed {
|
||||
return
|
||||
}
|
||||
|
||||
atomic.AddUint32(&p.put, 1)
|
||||
|
||||
pool := p.pool[p.poolNum(cap(b))]
|
||||
select {
|
||||
case pool <- b:
|
||||
default:
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (p *BufferPool) Close() {
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
if !p.closed {
|
||||
p.closed = true
|
||||
p.closeC <- struct{}{}
|
||||
}
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
func (p *BufferPool) String() string {
|
||||
if p == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
|
||||
return fmt.Sprintf("BufferPool{B·%d Z·%v Zm·%v Zh·%v G·%d P·%d H·%d <·%d =·%d >·%d M·%d}",
|
||||
p.baseline0, p.size, p.sizeMiss, p.sizeHalf, p.get, p.put, p.half, p.less, p.equal, p.greater, p.miss)
|
||||
}
|
||||
|
||||
func (p *BufferPool) drain() {
|
||||
ticker := time.NewTicker(2 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
for _, ch := range p.pool {
|
||||
select {
|
||||
case <-ch:
|
||||
default:
|
||||
}
|
||||
}
|
||||
case <-p.closeC:
|
||||
close(p.closeC)
|
||||
for _, ch := range p.pool {
|
||||
close(ch)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NewBufferPool creates a new initialized 'buffer pool'.
|
||||
func NewBufferPool(baseline int) *BufferPool {
|
||||
if baseline <= 0 {
|
||||
panic("baseline can't be <= 0")
|
||||
}
|
||||
p := &BufferPool{
|
||||
baseline0: baseline,
|
||||
baseline: [...]int{baseline / 4, baseline / 2, baseline * 2, baseline * 4},
|
||||
closeC: make(chan struct{}, 1),
|
||||
}
|
||||
for i, cap := range []int{2, 2, 4, 4, 2, 1} {
|
||||
p.pool[i] = make(chan []byte, cap)
|
||||
}
|
||||
go p.drain()
|
||||
return p
|
||||
}
|
||||
@ -0,0 +1,30 @@
|
||||
// Copyright 2011 The LevelDB-Go Authors. All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package util
|
||||
|
||||
import (
|
||||
"hash/crc32"
|
||||
)
|
||||
|
||||
var table = crc32.MakeTable(crc32.Castagnoli)
|
||||
|
||||
// CRC is a CRC-32 checksum computed using Castagnoli's polynomial.
|
||||
type CRC uint32
|
||||
|
||||
// NewCRC creates a new crc based on the given bytes.
|
||||
func NewCRC(b []byte) CRC {
|
||||
return CRC(0).Update(b)
|
||||
}
|
||||
|
||||
// Update updates the crc with the given bytes.
|
||||
func (c CRC) Update(b []byte) CRC {
|
||||
return CRC(crc32.Update(uint32(c), table, b))
|
||||
}
|
||||
|
||||
// Value returns a masked crc.
|
||||
func (c CRC) Value() uint32 {
|
||||
return uint32(c>>15|c<<17) + 0xa282ead8
|
||||
}
|
||||
@ -0,0 +1,48 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package util
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
)
|
||||
|
||||
// Hash return hash of the given data.
|
||||
func Hash(data []byte, seed uint32) uint32 {
|
||||
// Similar to murmur hash
|
||||
const (
|
||||
m = uint32(0xc6a4a793)
|
||||
r = uint32(24)
|
||||
)
|
||||
var (
|
||||
h = seed ^ (uint32(len(data)) * m)
|
||||
i int
|
||||
)
|
||||
|
||||
for n := len(data) - len(data)%4; i < n; i += 4 {
|
||||
h += binary.LittleEndian.Uint32(data[i:])
|
||||
h *= m
|
||||
h ^= (h >> 16)
|
||||
}
|
||||
|
||||
switch len(data) - i {
|
||||
default:
|
||||
panic("not reached")
|
||||
case 3:
|
||||
h += uint32(data[i+2]) << 16
|
||||
fallthrough
|
||||
case 2:
|
||||
h += uint32(data[i+1]) << 8
|
||||
fallthrough
|
||||
case 1:
|
||||
h += uint32(data[i])
|
||||
h *= m
|
||||
h ^= (h >> r)
|
||||
case 0:
|
||||
}
|
||||
|
||||
return h
|
||||
}
|
||||
@ -0,0 +1,32 @@
|
||||
// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package util
|
||||
|
||||
// Range is a key range.
|
||||
type Range struct {
|
||||
// Start of the key range, include in the range.
|
||||
Start []byte
|
||||
|
||||
// Limit of the key range, not include in the range.
|
||||
Limit []byte
|
||||
}
|
||||
|
||||
// BytesPrefix returns key range that satisfy the given prefix.
|
||||
// This only applicable for the standard 'bytes comparer'.
|
||||
func BytesPrefix(prefix []byte) *Range {
|
||||
var limit []byte
|
||||
for i := len(prefix) - 1; i >= 0; i-- {
|
||||
c := prefix[i]
|
||||
if c < 0xff {
|
||||
limit = make([]byte, i+1)
|
||||
copy(limit, prefix)
|
||||
limit[i] = c + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
return &Range{prefix, limit}
|
||||
}
|
||||
@ -0,0 +1,73 @@
|
||||
// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package util provides utilities used throughout leveldb.
|
||||
package util
|
||||
|
||||
import (
|
||||
"errors"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrReleased = errors.New("leveldb: resource already relesed")
|
||||
ErrHasReleaser = errors.New("leveldb: releaser already defined")
|
||||
)
|
||||
|
||||
// Releaser is the interface that wraps the basic Release method.
|
||||
type Releaser interface {
|
||||
// Release releases associated resources. Release should always success
|
||||
// and can be called multiple times without causing error.
|
||||
Release()
|
||||
}
|
||||
|
||||
// ReleaseSetter is the interface that wraps the basic SetReleaser method.
|
||||
type ReleaseSetter interface {
|
||||
// SetReleaser associates the given releaser to the resources. The
|
||||
// releaser will be called once coresponding resources released.
|
||||
// Calling SetReleaser with nil will clear the releaser.
|
||||
//
|
||||
// This will panic if a releaser already present or coresponding
|
||||
// resource is already released. Releaser should be cleared first
|
||||
// before assigned a new one.
|
||||
SetReleaser(releaser Releaser)
|
||||
}
|
||||
|
||||
// BasicReleaser provides basic implementation of Releaser and ReleaseSetter.
|
||||
type BasicReleaser struct {
|
||||
releaser Releaser
|
||||
released bool
|
||||
}
|
||||
|
||||
// Released returns whether Release method already called.
|
||||
func (r *BasicReleaser) Released() bool {
|
||||
return r.released
|
||||
}
|
||||
|
||||
// Release implements Releaser.Release.
|
||||
func (r *BasicReleaser) Release() {
|
||||
if !r.released {
|
||||
if r.releaser != nil {
|
||||
r.releaser.Release()
|
||||
r.releaser = nil
|
||||
}
|
||||
r.released = true
|
||||
}
|
||||
}
|
||||
|
||||
// SetReleaser implements ReleaseSetter.SetReleaser.
|
||||
func (r *BasicReleaser) SetReleaser(releaser Releaser) {
|
||||
if r.released {
|
||||
panic(ErrReleased)
|
||||
}
|
||||
if r.releaser != nil && releaser != nil {
|
||||
panic(ErrHasReleaser)
|
||||
}
|
||||
r.releaser = releaser
|
||||
}
|
||||
|
||||
type NoopReleaser struct{}
|
||||
|
||||
func (NoopReleaser) Release() {}
|
||||
@ -0,0 +1,528 @@
|
||||
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package leveldb
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"unsafe"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb/iterator"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
type tSet struct {
|
||||
level int
|
||||
table *tFile
|
||||
}
|
||||
|
||||
type version struct {
|
||||
s *session
|
||||
|
||||
levels []tFiles
|
||||
|
||||
// Level that should be compacted next and its compaction score.
|
||||
// Score < 1 means compaction is not strictly needed. These fields
|
||||
// are initialized by computeCompaction()
|
||||
cLevel int
|
||||
cScore float64
|
||||
|
||||
cSeek unsafe.Pointer
|
||||
|
||||
closing bool
|
||||
ref int
|
||||
released bool
|
||||
}
|
||||
|
||||
func newVersion(s *session) *version {
|
||||
return &version{s: s}
|
||||
}
|
||||
|
||||
func (v *version) incref() {
|
||||
if v.released {
|
||||
panic("already released")
|
||||
}
|
||||
|
||||
v.ref++
|
||||
if v.ref == 1 {
|
||||
// Incr file ref.
|
||||
for _, tt := range v.levels {
|
||||
for _, t := range tt {
|
||||
v.s.addFileRef(t.fd, 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (v *version) releaseNB() {
|
||||
v.ref--
|
||||
if v.ref > 0 {
|
||||
return
|
||||
} else if v.ref < 0 {
|
||||
panic("negative version ref")
|
||||
}
|
||||
|
||||
for _, tt := range v.levels {
|
||||
for _, t := range tt {
|
||||
if v.s.addFileRef(t.fd, -1) == 0 {
|
||||
v.s.tops.remove(t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
v.released = true
|
||||
}
|
||||
|
||||
func (v *version) release() {
|
||||
v.s.vmu.Lock()
|
||||
v.releaseNB()
|
||||
v.s.vmu.Unlock()
|
||||
}
|
||||
|
||||
func (v *version) walkOverlapping(aux tFiles, ikey internalKey, f func(level int, t *tFile) bool, lf func(level int) bool) {
|
||||
ukey := ikey.ukey()
|
||||
|
||||
// Aux level.
|
||||
if aux != nil {
|
||||
for _, t := range aux {
|
||||
if t.overlaps(v.s.icmp, ukey, ukey) {
|
||||
if !f(-1, t) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if lf != nil && !lf(-1) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Walk tables level-by-level.
|
||||
for level, tables := range v.levels {
|
||||
if len(tables) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
if level == 0 {
|
||||
// Level-0 files may overlap each other. Find all files that
|
||||
// overlap ukey.
|
||||
for _, t := range tables {
|
||||
if t.overlaps(v.s.icmp, ukey, ukey) {
|
||||
if !f(level, t) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if i := tables.searchMax(v.s.icmp, ikey); i < len(tables) {
|
||||
t := tables[i]
|
||||
if v.s.icmp.uCompare(ukey, t.imin.ukey()) >= 0 {
|
||||
if !f(level, t) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if lf != nil && !lf(level) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (v *version) get(aux tFiles, ikey internalKey, ro *opt.ReadOptions, noValue bool) (value []byte, tcomp bool, err error) {
|
||||
if v.closing {
|
||||
return nil, false, ErrClosed
|
||||
}
|
||||
|
||||
ukey := ikey.ukey()
|
||||
|
||||
var (
|
||||
tset *tSet
|
||||
tseek bool
|
||||
|
||||
// Level-0.
|
||||
zfound bool
|
||||
zseq uint64
|
||||
zkt keyType
|
||||
zval []byte
|
||||
)
|
||||
|
||||
err = ErrNotFound
|
||||
|
||||
// Since entries never hop across level, finding key/value
|
||||
// in smaller level make later levels irrelevant.
|
||||
v.walkOverlapping(aux, ikey, func(level int, t *tFile) bool {
|
||||
if level >= 0 && !tseek {
|
||||
if tset == nil {
|
||||
tset = &tSet{level, t}
|
||||
} else {
|
||||
tseek = true
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
fikey, fval []byte
|
||||
ferr error
|
||||
)
|
||||
if noValue {
|
||||
fikey, ferr = v.s.tops.findKey(t, ikey, ro)
|
||||
} else {
|
||||
fikey, fval, ferr = v.s.tops.find(t, ikey, ro)
|
||||
}
|
||||
|
||||
switch ferr {
|
||||
case nil:
|
||||
case ErrNotFound:
|
||||
return true
|
||||
default:
|
||||
err = ferr
|
||||
return false
|
||||
}
|
||||
|
||||
if fukey, fseq, fkt, fkerr := parseInternalKey(fikey); fkerr == nil {
|
||||
if v.s.icmp.uCompare(ukey, fukey) == 0 {
|
||||
// Level <= 0 may overlaps each-other.
|
||||
if level <= 0 {
|
||||
if fseq >= zseq {
|
||||
zfound = true
|
||||
zseq = fseq
|
||||
zkt = fkt
|
||||
zval = fval
|
||||
}
|
||||
} else {
|
||||
switch fkt {
|
||||
case keyTypeVal:
|
||||
value = fval
|
||||
err = nil
|
||||
case keyTypeDel:
|
||||
default:
|
||||
panic("leveldb: invalid internalKey type")
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
} else {
|
||||
err = fkerr
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}, func(level int) bool {
|
||||
if zfound {
|
||||
switch zkt {
|
||||
case keyTypeVal:
|
||||
value = zval
|
||||
err = nil
|
||||
case keyTypeDel:
|
||||
default:
|
||||
panic("leveldb: invalid internalKey type")
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
})
|
||||
|
||||
if tseek && tset.table.consumeSeek() <= 0 {
|
||||
tcomp = atomic.CompareAndSwapPointer(&v.cSeek, nil, unsafe.Pointer(tset))
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (v *version) sampleSeek(ikey internalKey) (tcomp bool) {
|
||||
var tset *tSet
|
||||
|
||||
v.walkOverlapping(nil, ikey, func(level int, t *tFile) bool {
|
||||
if tset == nil {
|
||||
tset = &tSet{level, t}
|
||||
return true
|
||||
}
|
||||
if tset.table.consumeSeek() <= 0 {
|
||||
tcomp = atomic.CompareAndSwapPointer(&v.cSeek, nil, unsafe.Pointer(tset))
|
||||
}
|
||||
return false
|
||||
}, nil)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (v *version) getIterators(slice *util.Range, ro *opt.ReadOptions) (its []iterator.Iterator) {
|
||||
strict := opt.GetStrict(v.s.o.Options, ro, opt.StrictReader)
|
||||
for level, tables := range v.levels {
|
||||
if level == 0 {
|
||||
// Merge all level zero files together since they may overlap.
|
||||
for _, t := range tables {
|
||||
its = append(its, v.s.tops.newIterator(t, slice, ro))
|
||||
}
|
||||
} else if len(tables) != 0 {
|
||||
its = append(its, iterator.NewIndexedIterator(tables.newIndexIterator(v.s.tops, v.s.icmp, slice, ro), strict))
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (v *version) newStaging() *versionStaging {
|
||||
return &versionStaging{base: v}
|
||||
}
|
||||
|
||||
// Spawn a new version based on this version.
|
||||
func (v *version) spawn(r *sessionRecord) *version {
|
||||
staging := v.newStaging()
|
||||
staging.commit(r)
|
||||
return staging.finish()
|
||||
}
|
||||
|
||||
func (v *version) fillRecord(r *sessionRecord) {
|
||||
for level, tables := range v.levels {
|
||||
for _, t := range tables {
|
||||
r.addTableFile(level, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (v *version) tLen(level int) int {
|
||||
if level < len(v.levels) {
|
||||
return len(v.levels[level])
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (v *version) offsetOf(ikey internalKey) (n int64, err error) {
|
||||
for level, tables := range v.levels {
|
||||
for _, t := range tables {
|
||||
if v.s.icmp.Compare(t.imax, ikey) <= 0 {
|
||||
// Entire file is before "ikey", so just add the file size
|
||||
n += t.size
|
||||
} else if v.s.icmp.Compare(t.imin, ikey) > 0 {
|
||||
// Entire file is after "ikey", so ignore
|
||||
if level > 0 {
|
||||
// Files other than level 0 are sorted by meta->min, so
|
||||
// no further files in this level will contain data for
|
||||
// "ikey".
|
||||
break
|
||||
}
|
||||
} else {
|
||||
// "ikey" falls in the range for this table. Add the
|
||||
// approximate offset of "ikey" within the table.
|
||||
if m, err := v.s.tops.offsetOf(t, ikey); err == nil {
|
||||
n += m
|
||||
} else {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (v *version) pickMemdbLevel(umin, umax []byte, maxLevel int) (level int) {
|
||||
if maxLevel > 0 {
|
||||
if len(v.levels) == 0 {
|
||||
return maxLevel
|
||||
}
|
||||
if !v.levels[0].overlaps(v.s.icmp, umin, umax, true) {
|
||||
var overlaps tFiles
|
||||
for ; level < maxLevel; level++ {
|
||||
if pLevel := level + 1; pLevel >= len(v.levels) {
|
||||
return maxLevel
|
||||
} else if v.levels[pLevel].overlaps(v.s.icmp, umin, umax, false) {
|
||||
break
|
||||
}
|
||||
if gpLevel := level + 2; gpLevel < len(v.levels) {
|
||||
overlaps = v.levels[gpLevel].getOverlaps(overlaps, v.s.icmp, umin, umax, false)
|
||||
if overlaps.size() > int64(v.s.o.GetCompactionGPOverlaps(level)) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (v *version) computeCompaction() {
|
||||
// Precomputed best level for next compaction
|
||||
bestLevel := int(-1)
|
||||
bestScore := float64(-1)
|
||||
|
||||
statFiles := make([]int, len(v.levels))
|
||||
statSizes := make([]string, len(v.levels))
|
||||
statScore := make([]string, len(v.levels))
|
||||
statTotSize := int64(0)
|
||||
|
||||
for level, tables := range v.levels {
|
||||
var score float64
|
||||
size := tables.size()
|
||||
if level == 0 {
|
||||
// We treat level-0 specially by bounding the number of files
|
||||
// instead of number of bytes for two reasons:
|
||||
//
|
||||
// (1) With larger write-buffer sizes, it is nice not to do too
|
||||
// many level-0 compaction.
|
||||
//
|
||||
// (2) The files in level-0 are merged on every read and
|
||||
// therefore we wish to avoid too many files when the individual
|
||||
// file size is small (perhaps because of a small write-buffer
|
||||
// setting, or very high compression ratios, or lots of
|
||||
// overwrites/deletions).
|
||||
score = float64(len(tables)) / float64(v.s.o.GetCompactionL0Trigger())
|
||||
} else {
|
||||
score = float64(size) / float64(v.s.o.GetCompactionTotalSize(level))
|
||||
}
|
||||
|
||||
if score > bestScore {
|
||||
bestLevel = level
|
||||
bestScore = score
|
||||
}
|
||||
|
||||
statFiles[level] = len(tables)
|
||||
statSizes[level] = shortenb(int(size))
|
||||
statScore[level] = fmt.Sprintf("%.2f", score)
|
||||
statTotSize += size
|
||||
}
|
||||
|
||||
v.cLevel = bestLevel
|
||||
v.cScore = bestScore
|
||||
|
||||
v.s.logf("version@stat F·%v S·%s%v Sc·%v", statFiles, shortenb(int(statTotSize)), statSizes, statScore)
|
||||
}
|
||||
|
||||
func (v *version) needCompaction() bool {
|
||||
return v.cScore >= 1 || atomic.LoadPointer(&v.cSeek) != nil
|
||||
}
|
||||
|
||||
type tablesScratch struct {
|
||||
added map[int64]atRecord
|
||||
deleted map[int64]struct{}
|
||||
}
|
||||
|
||||
type versionStaging struct {
|
||||
base *version
|
||||
levels []tablesScratch
|
||||
}
|
||||
|
||||
func (p *versionStaging) getScratch(level int) *tablesScratch {
|
||||
if level >= len(p.levels) {
|
||||
newLevels := make([]tablesScratch, level+1)
|
||||
copy(newLevels, p.levels)
|
||||
p.levels = newLevels
|
||||
}
|
||||
return &(p.levels[level])
|
||||
}
|
||||
|
||||
func (p *versionStaging) commit(r *sessionRecord) {
|
||||
// Deleted tables.
|
||||
for _, r := range r.deletedTables {
|
||||
scratch := p.getScratch(r.level)
|
||||
if r.level < len(p.base.levels) && len(p.base.levels[r.level]) > 0 {
|
||||
if scratch.deleted == nil {
|
||||
scratch.deleted = make(map[int64]struct{})
|
||||
}
|
||||
scratch.deleted[r.num] = struct{}{}
|
||||
}
|
||||
if scratch.added != nil {
|
||||
delete(scratch.added, r.num)
|
||||
}
|
||||
}
|
||||
|
||||
// New tables.
|
||||
for _, r := range r.addedTables {
|
||||
scratch := p.getScratch(r.level)
|
||||
if scratch.added == nil {
|
||||
scratch.added = make(map[int64]atRecord)
|
||||
}
|
||||
scratch.added[r.num] = r
|
||||
if scratch.deleted != nil {
|
||||
delete(scratch.deleted, r.num)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *versionStaging) finish() *version {
|
||||
// Build new version.
|
||||
nv := newVersion(p.base.s)
|
||||
numLevel := len(p.levels)
|
||||
if len(p.base.levels) > numLevel {
|
||||
numLevel = len(p.base.levels)
|
||||
}
|
||||
nv.levels = make([]tFiles, numLevel)
|
||||
for level := 0; level < numLevel; level++ {
|
||||
var baseTabels tFiles
|
||||
if level < len(p.base.levels) {
|
||||
baseTabels = p.base.levels[level]
|
||||
}
|
||||
|
||||
if level < len(p.levels) {
|
||||
scratch := p.levels[level]
|
||||
|
||||
var nt tFiles
|
||||
// Prealloc list if possible.
|
||||
if n := len(baseTabels) + len(scratch.added) - len(scratch.deleted); n > 0 {
|
||||
nt = make(tFiles, 0, n)
|
||||
}
|
||||
|
||||
// Base tables.
|
||||
for _, t := range baseTabels {
|
||||
if _, ok := scratch.deleted[t.fd.Num]; ok {
|
||||
continue
|
||||
}
|
||||
if _, ok := scratch.added[t.fd.Num]; ok {
|
||||
continue
|
||||
}
|
||||
nt = append(nt, t)
|
||||
}
|
||||
|
||||
// New tables.
|
||||
for _, r := range scratch.added {
|
||||
nt = append(nt, tableFileFromRecord(r))
|
||||
}
|
||||
|
||||
if len(nt) != 0 {
|
||||
// Sort tables.
|
||||
if level == 0 {
|
||||
nt.sortByNum()
|
||||
} else {
|
||||
nt.sortByKey(p.base.s.icmp)
|
||||
}
|
||||
|
||||
nv.levels[level] = nt
|
||||
}
|
||||
} else {
|
||||
nv.levels[level] = baseTabels
|
||||
}
|
||||
}
|
||||
|
||||
// Trim levels.
|
||||
n := len(nv.levels)
|
||||
for ; n > 0 && nv.levels[n-1] == nil; n-- {
|
||||
}
|
||||
nv.levels = nv.levels[:n]
|
||||
|
||||
// Compute compaction score for new version.
|
||||
nv.computeCompaction()
|
||||
|
||||
return nv
|
||||
}
|
||||
|
||||
type versionReleaser struct {
|
||||
v *version
|
||||
once bool
|
||||
}
|
||||
|
||||
func (vr *versionReleaser) Release() {
|
||||
v := vr.v
|
||||
v.s.vmu.Lock()
|
||||
if !vr.once {
|
||||
v.releaseNB()
|
||||
vr.once = true
|
||||
}
|
||||
v.s.vmu.Unlock()
|
||||
}
|
||||
@ -0,0 +1,21 @@
|
||||
---
|
||||
kind: pipeline
|
||||
name: testing
|
||||
|
||||
trigger:
|
||||
ref:
|
||||
- refs/heads/master
|
||||
- refs/pull/*/head
|
||||
|
||||
steps:
|
||||
- name: test
|
||||
pull: default
|
||||
image: golang:1.11
|
||||
commands:
|
||||
- go get -u golang.org/x/lint/golint
|
||||
- golint ./...
|
||||
- go vet
|
||||
- go test -v -race -coverprofile=coverage.txt -covermode=atomic
|
||||
environment:
|
||||
GOPROXY: https://goproxy.io
|
||||
GO111MODULE: "on"
|
||||
@ -0,0 +1 @@
|
||||
.idea
|
||||
@ -0,0 +1,27 @@
|
||||
Copyright (c) 2016 The Xorm Authors
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
* Neither the name of the {organization} nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
@ -0,0 +1,217 @@
|
||||
# SQL builder
|
||||
|
||||
[](https://drone.gitea.com/xorm/builder) [](http://gocover.io/xorm.io/builder)
|
||||
[](https://goreportcard.com/report/xorm.io/builder)
|
||||
|
||||
Package builder is a lightweight and fast SQL builder for Go and XORM.
|
||||
|
||||
Make sure you have installed Go 1.8+ and then:
|
||||
|
||||
go get xorm.io/builder
|
||||
|
||||
# Insert
|
||||
|
||||
```Go
|
||||
sql, args, err := builder.Insert(Eq{"c": 1, "d": 2}).Into("table1").ToSQL()
|
||||
|
||||
// INSERT INTO table1 SELECT * FROM table2
|
||||
sql, err := builder.Insert().Into("table1").Select().From("table2").ToBoundSQL()
|
||||
|
||||
// INSERT INTO table1 (a, b) SELECT b, c FROM table2
|
||||
sql, err = builder.Insert("a, b").Into("table1").Select("b, c").From("table2").ToBoundSQL()
|
||||
```
|
||||
|
||||
# Select
|
||||
|
||||
```Go
|
||||
// Simple Query
|
||||
sql, args, err := Select("c, d").From("table1").Where(Eq{"a": 1}).ToSQL()
|
||||
// With join
|
||||
sql, args, err = Select("c, d").From("table1").LeftJoin("table2", Eq{"table1.id": 1}.And(Lt{"table2.id": 3})).
|
||||
RightJoin("table3", "table2.id = table3.tid").Where(Eq{"a": 1}).ToSQL()
|
||||
// From sub query
|
||||
sql, args, err := Select("sub.id").From(Select("c").From("table1").Where(Eq{"a": 1}), "sub").Where(Eq{"b": 1}).ToSQL()
|
||||
// From union query
|
||||
sql, args, err = Select("sub.id").From(
|
||||
Select("id").From("table1").Where(Eq{"a": 1}).Union("all", Select("id").From("table1").Where(Eq{"a": 2})),"sub").
|
||||
Where(Eq{"b": 1}).ToSQL()
|
||||
// With order by
|
||||
sql, args, err = Select("a", "b", "c").From("table1").Where(Eq{"f1": "v1", "f2": "v2"}).
|
||||
OrderBy("a ASC").ToSQL()
|
||||
// With limit.
|
||||
// Be careful! You should set up specific dialect for builder before performing a query with LIMIT
|
||||
sql, args, err = Dialect(MYSQL).Select("a", "b", "c").From("table1").OrderBy("a ASC").
|
||||
Limit(5, 10).ToSQL()
|
||||
```
|
||||
|
||||
# Update
|
||||
|
||||
```Go
|
||||
sql, args, err := Update(Eq{"a": 2}).From("table1").Where(Eq{"a": 1}).ToSQL()
|
||||
```
|
||||
|
||||
# Delete
|
||||
|
||||
```Go
|
||||
sql, args, err := Delete(Eq{"a": 1}).From("table1").ToSQL()
|
||||
```
|
||||
|
||||
# Union
|
||||
|
||||
```Go
|
||||
sql, args, err := Select("*").From("a").Where(Eq{"status": "1"}).
|
||||
Union("all", Select("*").From("a").Where(Eq{"status": "2"})).
|
||||
Union("distinct", Select("*").From("a").Where(Eq{"status": "3"})).
|
||||
Union("", Select("*").From("a").Where(Eq{"status": "4"})).
|
||||
ToSQL()
|
||||
```
|
||||
|
||||
# Conditions
|
||||
|
||||
* `Eq` is a redefine of a map, you can give one or more conditions to `Eq`
|
||||
|
||||
```Go
|
||||
import . "xorm.io/builder"
|
||||
|
||||
sql, args, _ := ToSQL(Eq{"a":1})
|
||||
// a=? [1]
|
||||
sql, args, _ := ToSQL(Eq{"b":"c"}.And(Eq{"c": 0}))
|
||||
// b=? AND c=? ["c", 0]
|
||||
sql, args, _ := ToSQL(Eq{"b":"c", "c":0})
|
||||
// b=? AND c=? ["c", 0]
|
||||
sql, args, _ := ToSQL(Eq{"b":"c"}.Or(Eq{"b":"d"}))
|
||||
// b=? OR b=? ["c", "d"]
|
||||
sql, args, _ := ToSQL(Eq{"b": []string{"c", "d"}})
|
||||
// b IN (?,?) ["c", "d"]
|
||||
sql, args, _ := ToSQL(Eq{"b": 1, "c":[]int{2, 3}})
|
||||
// b=? AND c IN (?,?) [1, 2, 3]
|
||||
```
|
||||
|
||||
* `Neq` is the same to `Eq`
|
||||
|
||||
```Go
|
||||
import . "xorm.io/builder"
|
||||
|
||||
sql, args, _ := ToSQL(Neq{"a":1})
|
||||
// a<>? [1]
|
||||
sql, args, _ := ToSQL(Neq{"b":"c"}.And(Neq{"c": 0}))
|
||||
// b<>? AND c<>? ["c", 0]
|
||||
sql, args, _ := ToSQL(Neq{"b":"c", "c":0})
|
||||
// b<>? AND c<>? ["c", 0]
|
||||
sql, args, _ := ToSQL(Neq{"b":"c"}.Or(Neq{"b":"d"}))
|
||||
// b<>? OR b<>? ["c", "d"]
|
||||
sql, args, _ := ToSQL(Neq{"b": []string{"c", "d"}})
|
||||
// b NOT IN (?,?) ["c", "d"]
|
||||
sql, args, _ := ToSQL(Neq{"b": 1, "c":[]int{2, 3}})
|
||||
// b<>? AND c NOT IN (?,?) [1, 2, 3]
|
||||
```
|
||||
|
||||
* `Gt`, `Gte`, `Lt`, `Lte`
|
||||
|
||||
```Go
|
||||
import . "xorm.io/builder"
|
||||
|
||||
sql, args, _ := ToSQL(Gt{"a", 1}.And(Gte{"b", 2}))
|
||||
// a>? AND b>=? [1, 2]
|
||||
sql, args, _ := ToSQL(Lt{"a", 1}.Or(Lte{"b", 2}))
|
||||
// a<? OR b<=? [1, 2]
|
||||
```
|
||||
|
||||
* `Like`
|
||||
|
||||
```Go
|
||||
import . "xorm.io/builder"
|
||||
|
||||
sql, args, _ := ToSQL(Like{"a", "c"})
|
||||
// a LIKE ? [%c%]
|
||||
```
|
||||
|
||||
* `Expr` you can customerize your sql with `Expr`
|
||||
|
||||
```Go
|
||||
import . "xorm.io/builder"
|
||||
|
||||
sql, args, _ := ToSQL(Expr("a = ? ", 1))
|
||||
// a = ? [1]
|
||||
sql, args, _ := ToSQL(Eq{"a": Expr("select id from table where c = ?", 1)})
|
||||
// a=(select id from table where c = ?) [1]
|
||||
```
|
||||
|
||||
* `In` and `NotIn`
|
||||
|
||||
```Go
|
||||
import . "xorm.io/builder"
|
||||
|
||||
sql, args, _ := ToSQL(In("a", 1, 2, 3))
|
||||
// a IN (?,?,?) [1,2,3]
|
||||
sql, args, _ := ToSQL(In("a", []int{1, 2, 3}))
|
||||
// a IN (?,?,?) [1,2,3]
|
||||
sql, args, _ := ToSQL(NotIn("a", Expr("select id from b where c = ?", 1))))
|
||||
// a NOT IN (select id from b where c = ?) [1]
|
||||
```
|
||||
|
||||
* `Exists` and `NotExists`
|
||||
|
||||
```Go
|
||||
import . "xorm.io/builder"
|
||||
|
||||
sql, args, _ := ToSQL(Exists(Select("a").From("table")))
|
||||
// EXISTS (SELECT a FROM table)
|
||||
sql, args, _ := ToSQL(NotExists(Select("a").From("table")))
|
||||
// NOT EXISTS (SELECT a FROM table)
|
||||
```
|
||||
|
||||
* `IsNull` and `NotNull`
|
||||
|
||||
```Go
|
||||
import . "xorm.io/builder"
|
||||
|
||||
sql, args, _ := ToSQL(IsNull{"a"})
|
||||
// a IS NULL []
|
||||
sql, args, _ := ToSQL(NotNull{"b"})
|
||||
// b IS NOT NULL []
|
||||
```
|
||||
|
||||
* `And(conds ...Cond)`, And can connect one or more conditions via And
|
||||
|
||||
```Go
|
||||
import . "xorm.io/builder"
|
||||
|
||||
sql, args, _ := ToSQL(And(Eq{"a":1}, Like{"b", "c"}, Neq{"d", 2}))
|
||||
// a=? AND b LIKE ? AND d<>? [1, %c%, 2]
|
||||
```
|
||||
|
||||
* `Or(conds ...Cond)`, Or can connect one or more conditions via Or
|
||||
|
||||
```Go
|
||||
import . "xorm.io/builder"
|
||||
|
||||
sql, args, _ := ToSQL(Or(Eq{"a":1}, Like{"b", "c"}, Neq{"d", 2}))
|
||||
// a=? OR b LIKE ? OR d<>? [1, %c%, 2]
|
||||
sql, args, _ := ToSQL(Or(Eq{"a":1}, And(Like{"b", "c"}, Neq{"d", 2})))
|
||||
// a=? OR (b LIKE ? AND d<>?) [1, %c%, 2]
|
||||
```
|
||||
|
||||
* `Between`
|
||||
|
||||
```Go
|
||||
import . "xorm.io/builder"
|
||||
|
||||
sql, args, _ := ToSQL(Between{"a", 1, 2})
|
||||
// a BETWEEN 1 AND 2
|
||||
```
|
||||
|
||||
* Define yourself conditions
|
||||
|
||||
Since `Cond` is an interface.
|
||||
|
||||
```Go
|
||||
type Cond interface {
|
||||
WriteTo(Writer) error
|
||||
And(...Cond) Cond
|
||||
Or(...Cond) Cond
|
||||
IsValid() bool
|
||||
}
|
||||
```
|
||||
|
||||
You can define yourself conditions and compose with other `Cond`.
|
||||
@ -0,0 +1,320 @@
|
||||
// Copyright 2016 The Xorm Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package builder
|
||||
|
||||
import (
|
||||
sql2 "database/sql"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type optype byte
|
||||
|
||||
const (
|
||||
condType optype = iota // only conditions
|
||||
selectType // select
|
||||
insertType // insert
|
||||
updateType // update
|
||||
deleteType // delete
|
||||
setOpType // set operation
|
||||
)
|
||||
|
||||
// all databasees
|
||||
const (
|
||||
POSTGRES = "postgres"
|
||||
SQLITE = "sqlite3"
|
||||
MYSQL = "mysql"
|
||||
MSSQL = "mssql"
|
||||
ORACLE = "oracle"
|
||||
|
||||
UNION = "union"
|
||||
INTERSECT = "intersect"
|
||||
EXCEPT = "except"
|
||||
)
|
||||
|
||||
type join struct {
|
||||
joinType string
|
||||
joinTable interface{}
|
||||
joinCond Cond
|
||||
}
|
||||
|
||||
type setOp struct {
|
||||
opType string
|
||||
distinctType string
|
||||
builder *Builder
|
||||
}
|
||||
|
||||
type limit struct {
|
||||
limitN int
|
||||
offset int
|
||||
}
|
||||
|
||||
// Builder describes a SQL statement
|
||||
type Builder struct {
|
||||
optype
|
||||
dialect string
|
||||
isNested bool
|
||||
into string
|
||||
from string
|
||||
subQuery *Builder
|
||||
cond Cond
|
||||
selects []string
|
||||
joins []join
|
||||
setOps []setOp
|
||||
limitation *limit
|
||||
insertCols []string
|
||||
insertVals []interface{}
|
||||
updates []UpdateCond
|
||||
orderBy interface{}
|
||||
groupBy string
|
||||
having string
|
||||
}
|
||||
|
||||
// Dialect sets the db dialect of Builder.
|
||||
func Dialect(dialect string) *Builder {
|
||||
builder := &Builder{cond: NewCond(), dialect: dialect}
|
||||
return builder
|
||||
}
|
||||
|
||||
// MySQL is shortcut of Dialect(MySQL)
|
||||
func MySQL() *Builder {
|
||||
return Dialect(MYSQL)
|
||||
}
|
||||
|
||||
// MsSQL is shortcut of Dialect(MsSQL)
|
||||
func MsSQL() *Builder {
|
||||
return Dialect(MSSQL)
|
||||
}
|
||||
|
||||
// Oracle is shortcut of Dialect(Oracle)
|
||||
func Oracle() *Builder {
|
||||
return Dialect(ORACLE)
|
||||
}
|
||||
|
||||
// Postgres is shortcut of Dialect(Postgres)
|
||||
func Postgres() *Builder {
|
||||
return Dialect(POSTGRES)
|
||||
}
|
||||
|
||||
// SQLite is shortcut of Dialect(SQLITE)
|
||||
func SQLite() *Builder {
|
||||
return Dialect(SQLITE)
|
||||
}
|
||||
|
||||
// Where sets where SQL
|
||||
func (b *Builder) Where(cond Cond) *Builder {
|
||||
if b.cond.IsValid() {
|
||||
b.cond = b.cond.And(cond)
|
||||
} else {
|
||||
b.cond = cond
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// From sets from subject(can be a table name in string or a builder pointer) and its alias
|
||||
func (b *Builder) From(subject interface{}, alias ...string) *Builder {
|
||||
switch subject.(type) {
|
||||
case *Builder:
|
||||
b.subQuery = subject.(*Builder)
|
||||
|
||||
if len(alias) > 0 {
|
||||
b.from = alias[0]
|
||||
} else {
|
||||
b.isNested = true
|
||||
}
|
||||
case string:
|
||||
b.from = subject.(string)
|
||||
|
||||
if len(alias) > 0 {
|
||||
b.from = b.from + " " + alias[0]
|
||||
}
|
||||
}
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
// TableName returns the table name
|
||||
func (b *Builder) TableName() string {
|
||||
if b.optype == insertType {
|
||||
return b.into
|
||||
}
|
||||
return b.from
|
||||
}
|
||||
|
||||
// Into sets insert table name
|
||||
func (b *Builder) Into(tableName string) *Builder {
|
||||
b.into = tableName
|
||||
return b
|
||||
}
|
||||
|
||||
// Union sets union conditions
|
||||
func (b *Builder) Union(distinctType string, cond *Builder) *Builder {
|
||||
return b.setOperation(UNION, distinctType, cond)
|
||||
}
|
||||
|
||||
// Intersect sets intersect conditions
|
||||
func (b *Builder) Intersect(distinctType string, cond *Builder) *Builder {
|
||||
return b.setOperation(INTERSECT, distinctType, cond)
|
||||
}
|
||||
|
||||
// Except sets except conditions
|
||||
func (b *Builder) Except(distinctType string, cond *Builder) *Builder {
|
||||
return b.setOperation(EXCEPT, distinctType, cond)
|
||||
}
|
||||
|
||||
func (b *Builder) setOperation(opType, distinctType string, cond *Builder) *Builder {
|
||||
var builder *Builder
|
||||
if b.optype != setOpType {
|
||||
builder = &Builder{cond: NewCond()}
|
||||
builder.optype = setOpType
|
||||
builder.dialect = b.dialect
|
||||
builder.selects = b.selects
|
||||
|
||||
currentSetOps := b.setOps
|
||||
// erase sub setOps (actually append to new Builder.unions)
|
||||
b.setOps = nil
|
||||
|
||||
for e := range currentSetOps {
|
||||
currentSetOps[e].builder.dialect = b.dialect
|
||||
}
|
||||
|
||||
builder.setOps = append(append(builder.setOps, setOp{opType, "", b}), currentSetOps...)
|
||||
} else {
|
||||
builder = b
|
||||
}
|
||||
|
||||
if cond != nil {
|
||||
if cond.dialect == "" && builder.dialect != "" {
|
||||
cond.dialect = builder.dialect
|
||||
}
|
||||
|
||||
builder.setOps = append(builder.setOps, setOp{opType, distinctType, cond})
|
||||
}
|
||||
|
||||
return builder
|
||||
}
|
||||
|
||||
// Limit sets limitN condition
|
||||
func (b *Builder) Limit(limitN int, offset ...int) *Builder {
|
||||
b.limitation = &limit{limitN: limitN}
|
||||
|
||||
if len(offset) > 0 {
|
||||
b.limitation.offset = offset[0]
|
||||
}
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
// Select sets select SQL
|
||||
func (b *Builder) Select(cols ...string) *Builder {
|
||||
b.selects = cols
|
||||
if b.optype == condType {
|
||||
b.optype = selectType
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// And sets AND condition
|
||||
func (b *Builder) And(cond Cond) *Builder {
|
||||
b.cond = And(b.cond, cond)
|
||||
return b
|
||||
}
|
||||
|
||||
// Or sets OR condition
|
||||
func (b *Builder) Or(cond Cond) *Builder {
|
||||
b.cond = Or(b.cond, cond)
|
||||
return b
|
||||
}
|
||||
|
||||
// Update sets update SQL
|
||||
func (b *Builder) Update(updates ...Cond) *Builder {
|
||||
b.updates = make([]UpdateCond, 0, len(updates))
|
||||
for _, update := range updates {
|
||||
if u, ok := update.(UpdateCond); ok && u.IsValid() {
|
||||
b.updates = append(b.updates, u)
|
||||
}
|
||||
}
|
||||
b.optype = updateType
|
||||
return b
|
||||
}
|
||||
|
||||
// Delete sets delete SQL
|
||||
func (b *Builder) Delete(conds ...Cond) *Builder {
|
||||
b.cond = b.cond.And(conds...)
|
||||
b.optype = deleteType
|
||||
return b
|
||||
}
|
||||
|
||||
// WriteTo implements Writer interface
|
||||
func (b *Builder) WriteTo(w Writer) error {
|
||||
switch b.optype {
|
||||
/*case condType:
|
||||
return b.cond.WriteTo(w)*/
|
||||
case selectType:
|
||||
return b.selectWriteTo(w)
|
||||
case insertType:
|
||||
return b.insertWriteTo(w)
|
||||
case updateType:
|
||||
return b.updateWriteTo(w)
|
||||
case deleteType:
|
||||
return b.deleteWriteTo(w)
|
||||
case setOpType:
|
||||
return b.setOpWriteTo(w)
|
||||
}
|
||||
|
||||
return ErrNotSupportType
|
||||
}
|
||||
|
||||
// ToSQL convert a builder to SQL and args
|
||||
func (b *Builder) ToSQL() (string, []interface{}, error) {
|
||||
w := NewWriter()
|
||||
if err := b.WriteTo(w); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
// in case of sql.NamedArg in args
|
||||
for e := range w.args {
|
||||
if namedArg, ok := w.args[e].(sql2.NamedArg); ok {
|
||||
w.args[e] = namedArg.Value
|
||||
}
|
||||
}
|
||||
|
||||
sql := w.String()
|
||||
var err error
|
||||
|
||||
switch b.dialect {
|
||||
case ORACLE, MSSQL:
|
||||
// This is for compatibility with different sql drivers
|
||||
for e := range w.args {
|
||||
w.args[e] = sql2.Named(fmt.Sprintf("p%d", e+1), w.args[e])
|
||||
}
|
||||
|
||||
var prefix string
|
||||
if b.dialect == ORACLE {
|
||||
prefix = ":p"
|
||||
} else {
|
||||
prefix = "@p"
|
||||
}
|
||||
|
||||
if sql, err = ConvertPlaceholder(sql, prefix); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
case POSTGRES:
|
||||
if sql, err = ConvertPlaceholder(sql, "$"); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return sql, w.args, nil
|
||||
}
|
||||
|
||||
// ToBoundSQL generated a bound SQL string
|
||||
func (b *Builder) ToBoundSQL() (string, error) {
|
||||
w := NewWriter()
|
||||
if err := b.WriteTo(w); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return ConvertToBoundSQL(w.String(), w.args)
|
||||
}
|
||||
@ -0,0 +1,27 @@
|
||||
// Copyright 2016 The Xorm Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package builder
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Delete creates a delete Builder
|
||||
func Delete(conds ...Cond) *Builder {
|
||||
builder := &Builder{cond: NewCond()}
|
||||
return builder.Delete(conds...)
|
||||
}
|
||||
|
||||
func (b *Builder) deleteWriteTo(w Writer) error {
|
||||
if len(b.from) <= 0 {
|
||||
return ErrNoTableName
|
||||
}
|
||||
|
||||
if _, err := fmt.Fprintf(w, "DELETE FROM %s WHERE ", b.from); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return b.cond.WriteTo(w)
|
||||
}
|
||||
@ -0,0 +1,149 @@
|
||||
// Copyright 2016 The Xorm Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package builder
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Insert creates an insert Builder
|
||||
func Insert(eq ...interface{}) *Builder {
|
||||
builder := &Builder{cond: NewCond()}
|
||||
return builder.Insert(eq...)
|
||||
}
|
||||
|
||||
func (b *Builder) insertSelectWriteTo(w Writer) error {
|
||||
if _, err := fmt.Fprintf(w, "INSERT INTO %s ", b.into); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(b.insertCols) > 0 {
|
||||
fmt.Fprintf(w, "(")
|
||||
for _, col := range b.insertCols {
|
||||
fmt.Fprintf(w, col)
|
||||
}
|
||||
fmt.Fprintf(w, ") ")
|
||||
}
|
||||
|
||||
return b.selectWriteTo(w)
|
||||
}
|
||||
|
||||
func (b *Builder) insertWriteTo(w Writer) error {
|
||||
if len(b.into) <= 0 {
|
||||
return ErrNoTableName
|
||||
}
|
||||
if len(b.insertCols) <= 0 && b.from == "" {
|
||||
return ErrNoColumnToInsert
|
||||
}
|
||||
|
||||
if b.into != "" && b.from != "" {
|
||||
return b.insertSelectWriteTo(w)
|
||||
}
|
||||
|
||||
if _, err := fmt.Fprintf(w, "INSERT INTO %s (", b.into); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
args := make([]interface{}, 0)
|
||||
var bs []byte
|
||||
valBuffer := bytes.NewBuffer(bs)
|
||||
|
||||
for i, col := range b.insertCols {
|
||||
value := b.insertVals[i]
|
||||
fmt.Fprint(w, col)
|
||||
if e, ok := value.(*Expression); ok {
|
||||
fmt.Fprintf(valBuffer, "(%s)", e.sql)
|
||||
args = append(args, e.args...)
|
||||
} else if value == nil {
|
||||
fmt.Fprintf(valBuffer, `null`)
|
||||
} else {
|
||||
fmt.Fprint(valBuffer, "?")
|
||||
args = append(args, value)
|
||||
}
|
||||
|
||||
if i != len(b.insertCols)-1 {
|
||||
if _, err := fmt.Fprint(w, ","); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := fmt.Fprint(valBuffer, ","); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := fmt.Fprint(w, ") Values ("); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := w.Write(valBuffer.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := fmt.Fprint(w, ")"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.Append(args...)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type insertColsSorter struct {
|
||||
cols []string
|
||||
vals []interface{}
|
||||
}
|
||||
|
||||
func (s insertColsSorter) Len() int {
|
||||
return len(s.cols)
|
||||
}
|
||||
|
||||
func (s insertColsSorter) Swap(i, j int) {
|
||||
s.cols[i], s.cols[j] = s.cols[j], s.cols[i]
|
||||
s.vals[i], s.vals[j] = s.vals[j], s.vals[i]
|
||||
}
|
||||
|
||||
func (s insertColsSorter) Less(i, j int) bool {
|
||||
return s.cols[i] < s.cols[j]
|
||||
}
|
||||
|
||||
// Insert sets insert SQL
|
||||
func (b *Builder) Insert(eq ...interface{}) *Builder {
|
||||
if len(eq) > 0 {
|
||||
paramType := -1
|
||||
for _, e := range eq {
|
||||
switch t := e.(type) {
|
||||
case Eq:
|
||||
if paramType == -1 {
|
||||
paramType = 0
|
||||
}
|
||||
if paramType != 0 {
|
||||
break
|
||||
}
|
||||
for k, v := range t {
|
||||
b.insertCols = append(b.insertCols, k)
|
||||
b.insertVals = append(b.insertVals, v)
|
||||
}
|
||||
case string:
|
||||
if paramType == -1 {
|
||||
paramType = 1
|
||||
}
|
||||
if paramType != 1 {
|
||||
break
|
||||
}
|
||||
b.insertCols = append(b.insertCols, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(b.insertCols) == len(b.insertVals) {
|
||||
sort.Sort(insertColsSorter{
|
||||
cols: b.insertCols,
|
||||
vals: b.insertVals,
|
||||
})
|
||||
}
|
||||
b.optype = insertType
|
||||
return b
|
||||
}
|
||||
@ -0,0 +1,42 @@
|
||||
// Copyright 2019 The Xorm Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package builder
|
||||
|
||||
// InnerJoin sets inner join
|
||||
func (b *Builder) InnerJoin(joinTable, joinCond interface{}) *Builder {
|
||||
return b.Join("INNER", joinTable, joinCond)
|
||||
}
|
||||
|
||||
// LeftJoin sets left join SQL
|
||||
func (b *Builder) LeftJoin(joinTable, joinCond interface{}) *Builder {
|
||||
return b.Join("LEFT", joinTable, joinCond)
|
||||
}
|
||||
|
||||
// RightJoin sets right join SQL
|
||||
func (b *Builder) RightJoin(joinTable, joinCond interface{}) *Builder {
|
||||
return b.Join("RIGHT", joinTable, joinCond)
|
||||
}
|
||||
|
||||
// CrossJoin sets cross join SQL
|
||||
func (b *Builder) CrossJoin(joinTable, joinCond interface{}) *Builder {
|
||||
return b.Join("CROSS", joinTable, joinCond)
|
||||
}
|
||||
|
||||
// FullJoin sets full join SQL
|
||||
func (b *Builder) FullJoin(joinTable, joinCond interface{}) *Builder {
|
||||
return b.Join("FULL", joinTable, joinCond)
|
||||
}
|
||||
|
||||
// Join sets join table and conditions
|
||||
func (b *Builder) Join(joinType string, joinTable, joinCond interface{}) *Builder {
|
||||
switch joinCond.(type) {
|
||||
case Cond:
|
||||
b.joins = append(b.joins, join{joinType, joinTable, joinCond.(Cond)})
|
||||
case string:
|
||||
b.joins = append(b.joins, join{joinType, joinTable, Expr(joinCond.(string))})
|
||||
}
|
||||
|
||||
return b
|
||||
}
|
||||
@ -0,0 +1,103 @@
|
||||
// Copyright 2018 The Xorm Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package builder
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (b *Builder) limitWriteTo(w Writer) error {
|
||||
if strings.TrimSpace(b.dialect) == "" {
|
||||
return ErrDialectNotSetUp
|
||||
}
|
||||
|
||||
if b.limitation != nil {
|
||||
limit := b.limitation
|
||||
if limit.offset < 0 || limit.limitN <= 0 {
|
||||
return ErrInvalidLimitation
|
||||
}
|
||||
// erase limit condition
|
||||
b.limitation = nil
|
||||
defer func() {
|
||||
b.limitation = limit
|
||||
}()
|
||||
ow := w.(*BytesWriter)
|
||||
|
||||
switch strings.ToLower(strings.TrimSpace(b.dialect)) {
|
||||
case ORACLE:
|
||||
if len(b.selects) == 0 {
|
||||
b.selects = append(b.selects, "*")
|
||||
}
|
||||
|
||||
var final *Builder
|
||||
selects := b.selects
|
||||
b.selects = append(selects, "ROWNUM RN")
|
||||
|
||||
var wb *Builder
|
||||
if b.optype == setOpType {
|
||||
wb = Dialect(b.dialect).Select("at.*", "ROWNUM RN").
|
||||
From(b, "at")
|
||||
} else {
|
||||
wb = b
|
||||
}
|
||||
|
||||
if limit.offset == 0 {
|
||||
final = Dialect(b.dialect).Select(selects...).From(wb, "at").
|
||||
Where(Lte{"at.RN": limit.limitN})
|
||||
} else {
|
||||
sub := Dialect(b.dialect).Select("*").
|
||||
From(b, "at").Where(Lte{"at.RN": limit.offset + limit.limitN})
|
||||
|
||||
final = Dialect(b.dialect).Select(selects...).From(sub, "att").
|
||||
Where(Gt{"att.RN": limit.offset})
|
||||
}
|
||||
|
||||
return final.WriteTo(ow)
|
||||
case SQLITE, MYSQL, POSTGRES:
|
||||
// if type UNION, we need to write previous content back to current writer
|
||||
if b.optype == setOpType {
|
||||
if err := b.WriteTo(ow); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if limit.offset == 0 {
|
||||
fmt.Fprint(ow, " LIMIT ", limit.limitN)
|
||||
} else {
|
||||
fmt.Fprintf(ow, " LIMIT %v OFFSET %v", limit.limitN, limit.offset)
|
||||
}
|
||||
case MSSQL:
|
||||
if len(b.selects) == 0 {
|
||||
b.selects = append(b.selects, "*")
|
||||
}
|
||||
|
||||
var final *Builder
|
||||
selects := b.selects
|
||||
b.selects = append(append([]string{fmt.Sprintf("TOP %d %v", limit.limitN+limit.offset, b.selects[0])},
|
||||
b.selects[1:]...), "ROW_NUMBER() OVER (ORDER BY (SELECT 1)) AS RN")
|
||||
|
||||
var wb *Builder
|
||||
if b.optype == setOpType {
|
||||
wb = Dialect(b.dialect).Select("*", "ROW_NUMBER() OVER (ORDER BY (SELECT 1)) AS RN").
|
||||
From(b, "at")
|
||||
} else {
|
||||
wb = b
|
||||
}
|
||||
|
||||
if limit.offset == 0 {
|
||||
final = Dialect(b.dialect).Select(selects...).From(wb, "at")
|
||||
} else {
|
||||
final = Dialect(b.dialect).Select(selects...).From(wb, "at").Where(Gt{"at.RN": limit.offset})
|
||||
}
|
||||
|
||||
return final.WriteTo(ow)
|
||||
default:
|
||||
return ErrNotSupportType
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,172 @@
|
||||
// Copyright 2016 The Xorm Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package builder
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Select creates a select Builder
|
||||
func Select(cols ...string) *Builder {
|
||||
builder := &Builder{cond: NewCond()}
|
||||
return builder.Select(cols...)
|
||||
}
|
||||
|
||||
func (b *Builder) selectWriteTo(w Writer) error {
|
||||
if len(b.from) <= 0 && !b.isNested {
|
||||
return ErrNoTableName
|
||||
}
|
||||
|
||||
// perform limit before writing to writer when b.dialect between ORACLE and MSSQL
|
||||
// this avoid a duplicate writing problem in simple limit query
|
||||
if b.limitation != nil && (b.dialect == ORACLE || b.dialect == MSSQL) {
|
||||
return b.limitWriteTo(w)
|
||||
}
|
||||
|
||||
if _, err := fmt.Fprint(w, "SELECT "); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(b.selects) > 0 {
|
||||
for i, s := range b.selects {
|
||||
if _, err := fmt.Fprint(w, s); err != nil {
|
||||
return err
|
||||
}
|
||||
if i != len(b.selects)-1 {
|
||||
if _, err := fmt.Fprint(w, ","); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if _, err := fmt.Fprint(w, "*"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if b.subQuery == nil {
|
||||
if _, err := fmt.Fprint(w, " FROM ", b.from); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if b.cond.IsValid() && len(b.from) <= 0 {
|
||||
return ErrUnnamedDerivedTable
|
||||
}
|
||||
if b.subQuery.dialect != "" && b.dialect != b.subQuery.dialect {
|
||||
return ErrInconsistentDialect
|
||||
}
|
||||
|
||||
// dialect of sub-query will inherit from the main one (if not set up)
|
||||
if b.dialect != "" && b.subQuery.dialect == "" {
|
||||
b.subQuery.dialect = b.dialect
|
||||
}
|
||||
|
||||
switch b.subQuery.optype {
|
||||
case selectType, setOpType:
|
||||
fmt.Fprint(w, " FROM (")
|
||||
if err := b.subQuery.WriteTo(w); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(b.from) == 0 {
|
||||
fmt.Fprintf(w, ")")
|
||||
} else {
|
||||
fmt.Fprintf(w, ") %v", b.from)
|
||||
}
|
||||
default:
|
||||
return ErrUnexpectedSubQuery
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range b.joins {
|
||||
b, ok := v.joinTable.(*Builder)
|
||||
if ok {
|
||||
if _, err := fmt.Fprintf(w, " %s JOIN (", v.joinType); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := b.WriteTo(w); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := fmt.Fprintf(w, ") ON "); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if _, err := fmt.Fprintf(w, " %s JOIN %s ON ", v.joinType, v.joinTable); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := v.joinCond.WriteTo(w); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if b.cond.IsValid() {
|
||||
if _, err := fmt.Fprint(w, " WHERE "); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := b.cond.WriteTo(w); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(b.groupBy) > 0 {
|
||||
if _, err := fmt.Fprint(w, " GROUP BY ", b.groupBy); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(b.having) > 0 {
|
||||
if _, err := fmt.Fprint(w, " HAVING ", b.having); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if b.orderBy != nil {
|
||||
switch c := b.orderBy.(type) {
|
||||
case string:
|
||||
if len(c) > 0 {
|
||||
if _, err := fmt.Fprint(w, " ORDER BY ", c); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case *Expression:
|
||||
if _, err := fmt.Fprint(w, " ORDER BY "); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.WriteTo(w); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unknow orderby parameter: %#v", b.orderBy)
|
||||
}
|
||||
}
|
||||
|
||||
if b.limitation != nil {
|
||||
if err := b.limitWriteTo(w); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// OrderBy orderBy SQL
|
||||
func (b *Builder) OrderBy(orderBy interface{}) *Builder {
|
||||
b.orderBy = orderBy
|
||||
return b
|
||||
}
|
||||
|
||||
// GroupBy groupby SQL
|
||||
func (b *Builder) GroupBy(groupby string) *Builder {
|
||||
b.groupBy = groupby
|
||||
return b
|
||||
}
|
||||
|
||||
// Having having SQL
|
||||
func (b *Builder) Having(having string) *Builder {
|
||||
b.having = having
|
||||
return b
|
||||
}
|
||||
@ -0,0 +1,51 @@
|
||||
// Copyright 2018 The Xorm Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package builder
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (b *Builder) setOpWriteTo(w Writer) error {
|
||||
if b.limitation != nil || b.cond.IsValid() ||
|
||||
b.orderBy != nil || b.having != "" || b.groupBy != "" {
|
||||
return ErrNotUnexpectedUnionConditions
|
||||
}
|
||||
|
||||
for idx, o := range b.setOps {
|
||||
current := o.builder
|
||||
if current.optype != selectType {
|
||||
return ErrUnsupportedUnionMembers
|
||||
}
|
||||
|
||||
if len(b.setOps) == 1 {
|
||||
if err := current.selectWriteTo(w); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if b.dialect != "" && b.dialect != current.dialect {
|
||||
return ErrInconsistentDialect
|
||||
}
|
||||
|
||||
if idx != 0 {
|
||||
if o.distinctType == "" {
|
||||
fmt.Fprint(w, fmt.Sprintf(" %s ", strings.ToUpper(o.opType)))
|
||||
} else {
|
||||
fmt.Fprint(w, fmt.Sprintf(" %s %s ", strings.ToUpper(o.opType), strings.ToUpper(o.distinctType)))
|
||||
}
|
||||
}
|
||||
fmt.Fprint(w, "(")
|
||||
|
||||
if err := current.selectWriteTo(w); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Fprint(w, ")")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,57 @@
|
||||
// Copyright 2016 The Xorm Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package builder
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// UpdateCond defines an interface that cond could be used with update
|
||||
type UpdateCond interface {
|
||||
IsValid() bool
|
||||
OpWriteTo(op string, w Writer) error
|
||||
}
|
||||
|
||||
// Update creates an update Builder
|
||||
func Update(updates ...Cond) *Builder {
|
||||
builder := &Builder{cond: NewCond()}
|
||||
return builder.Update(updates...)
|
||||
}
|
||||
|
||||
func (b *Builder) updateWriteTo(w Writer) error {
|
||||
if len(b.from) <= 0 {
|
||||
return ErrNoTableName
|
||||
}
|
||||
if len(b.updates) <= 0 {
|
||||
return ErrNoColumnToUpdate
|
||||
}
|
||||
|
||||
if _, err := fmt.Fprintf(w, "UPDATE %s SET ", b.from); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i, s := range b.updates {
|
||||
|
||||
if err := s.OpWriteTo(",", w); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if i != len(b.updates)-1 {
|
||||
if _, err := fmt.Fprint(w, ","); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !b.cond.IsValid() {
|
||||
return nil
|
||||
}
|
||||
|
||||
if _, err := fmt.Fprint(w, " WHERE "); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return b.cond.WriteTo(w)
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
// Copyright 2016 The Xorm Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package builder
|
||||
|
||||
// Cond defines an interface
|
||||
type Cond interface {
|
||||
WriteTo(Writer) error
|
||||
And(...Cond) Cond
|
||||
Or(...Cond) Cond
|
||||
IsValid() bool
|
||||
}
|
||||
|
||||
type condEmpty struct{}
|
||||
|
||||
var _ Cond = condEmpty{}
|
||||
|
||||
// NewCond creates an empty condition
|
||||
func NewCond() Cond {
|
||||
return condEmpty{}
|
||||
}
|
||||
|
||||
func (condEmpty) WriteTo(w Writer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (condEmpty) And(conds ...Cond) Cond {
|
||||
return And(conds...)
|
||||
}
|
||||
|
||||
func (condEmpty) Or(conds ...Cond) Cond {
|
||||
return Or(conds...)
|
||||
}
|
||||
|
||||
func (condEmpty) IsValid() bool {
|
||||
return false
|
||||
}
|
||||
@ -0,0 +1,61 @@
|
||||
// Copyright 2016 The Xorm Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package builder
|
||||
|
||||
import "fmt"
|
||||
|
||||
type condAnd []Cond
|
||||
|
||||
var _ Cond = condAnd{}
|
||||
|
||||
// And generates AND conditions
|
||||
func And(conds ...Cond) Cond {
|
||||
result := make(condAnd, 0, len(conds))
|
||||
for _, cond := range conds {
|
||||
if cond == nil || !cond.IsValid() {
|
||||
continue
|
||||
}
|
||||
result = append(result, cond)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (and condAnd) WriteTo(w Writer) error {
|
||||
for i, cond := range and {
|
||||
_, isOr := cond.(condOr)
|
||||
_, isExpr := cond.(*Expression)
|
||||
wrap := isOr || isExpr
|
||||
if wrap {
|
||||
fmt.Fprint(w, "(")
|
||||
}
|
||||
|
||||
err := cond.WriteTo(w)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if wrap {
|
||||
fmt.Fprint(w, ")")
|
||||
}
|
||||
|
||||
if i != len(and)-1 {
|
||||
fmt.Fprint(w, " AND ")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (and condAnd) And(conds ...Cond) Cond {
|
||||
return And(and, And(conds...))
|
||||
}
|
||||
|
||||
func (and condAnd) Or(conds ...Cond) Cond {
|
||||
return Or(and, Or(conds...))
|
||||
}
|
||||
|
||||
func (and condAnd) IsValid() bool {
|
||||
return len(and) > 0
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue