// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
|
// Use of this source code is governed by a BSD-style license found in the LICENSE file.
|
|
package codec
|
|
// Contains code shared by both encode and decode.
|
|
import (
|
"encoding/binary"
|
"fmt"
|
"math"
|
"reflect"
|
"sort"
|
"strings"
|
"sync"
|
"time"
|
"unicode"
|
"unicode/utf8"
|
)
|
|
const (
|
structTagName = "codec"
|
|
// Support
|
// encoding.BinaryMarshaler: MarshalBinary() (data []byte, err error)
|
// encoding.BinaryUnmarshaler: UnmarshalBinary(data []byte) error
|
// This constant flag will enable or disable it.
|
supportBinaryMarshal = true
|
|
// Each Encoder or Decoder uses a cache of functions based on conditionals,
|
// so that the conditionals are not run every time.
|
//
|
// Either a map or a slice is used to keep track of the functions.
|
// The map is more natural, but has a higher cost than a slice/array.
|
// This flag (useMapForCodecCache) controls which is used.
|
useMapForCodecCache = false
|
|
// For some common container types, we can short-circuit an elaborate
|
// reflection dance and call encode/decode directly.
|
// The currently supported types are:
|
// - slices of strings, or id's (int64,uint64) or interfaces.
|
// - maps of str->str, str->intf, id(int64,uint64)->intf, intf->intf
|
shortCircuitReflectToFastPath = true
|
|
// for debugging, set this to false, to catch panic traces.
|
// Note that this will always cause rpc tests to fail, since they need io.EOF sent via panic.
|
recoverPanicToErr = true
|
)
|
|
type charEncoding uint8
|
|
const (
|
c_RAW charEncoding = iota
|
c_UTF8
|
c_UTF16LE
|
c_UTF16BE
|
c_UTF32LE
|
c_UTF32BE
|
)
|
|
// valueType is the stream type
|
type valueType uint8
|
|
const (
|
valueTypeUnset valueType = iota
|
valueTypeNil
|
valueTypeInt
|
valueTypeUint
|
valueTypeFloat
|
valueTypeBool
|
valueTypeString
|
valueTypeSymbol
|
valueTypeBytes
|
valueTypeMap
|
valueTypeArray
|
valueTypeTimestamp
|
valueTypeExt
|
|
valueTypeInvalid = 0xff
|
)
|
|
var (
|
bigen = binary.BigEndian
|
structInfoFieldName = "_struct"
|
|
cachedTypeInfo = make(map[uintptr]*typeInfo, 4)
|
cachedTypeInfoMutex sync.RWMutex
|
|
intfSliceTyp = reflect.TypeOf([]interface{}(nil))
|
intfTyp = intfSliceTyp.Elem()
|
|
strSliceTyp = reflect.TypeOf([]string(nil))
|
boolSliceTyp = reflect.TypeOf([]bool(nil))
|
uintSliceTyp = reflect.TypeOf([]uint(nil))
|
uint8SliceTyp = reflect.TypeOf([]uint8(nil))
|
uint16SliceTyp = reflect.TypeOf([]uint16(nil))
|
uint32SliceTyp = reflect.TypeOf([]uint32(nil))
|
uint64SliceTyp = reflect.TypeOf([]uint64(nil))
|
intSliceTyp = reflect.TypeOf([]int(nil))
|
int8SliceTyp = reflect.TypeOf([]int8(nil))
|
int16SliceTyp = reflect.TypeOf([]int16(nil))
|
int32SliceTyp = reflect.TypeOf([]int32(nil))
|
int64SliceTyp = reflect.TypeOf([]int64(nil))
|
float32SliceTyp = reflect.TypeOf([]float32(nil))
|
float64SliceTyp = reflect.TypeOf([]float64(nil))
|
|
mapIntfIntfTyp = reflect.TypeOf(map[interface{}]interface{}(nil))
|
mapStrIntfTyp = reflect.TypeOf(map[string]interface{}(nil))
|
mapStrStrTyp = reflect.TypeOf(map[string]string(nil))
|
|
mapIntIntfTyp = reflect.TypeOf(map[int]interface{}(nil))
|
mapInt64IntfTyp = reflect.TypeOf(map[int64]interface{}(nil))
|
mapUintIntfTyp = reflect.TypeOf(map[uint]interface{}(nil))
|
mapUint64IntfTyp = reflect.TypeOf(map[uint64]interface{}(nil))
|
|
stringTyp = reflect.TypeOf("")
|
timeTyp = reflect.TypeOf(time.Time{})
|
rawExtTyp = reflect.TypeOf(RawExt{})
|
|
mapBySliceTyp = reflect.TypeOf((*MapBySlice)(nil)).Elem()
|
binaryMarshalerTyp = reflect.TypeOf((*binaryMarshaler)(nil)).Elem()
|
binaryUnmarshalerTyp = reflect.TypeOf((*binaryUnmarshaler)(nil)).Elem()
|
|
rawExtTypId = reflect.ValueOf(rawExtTyp).Pointer()
|
intfTypId = reflect.ValueOf(intfTyp).Pointer()
|
timeTypId = reflect.ValueOf(timeTyp).Pointer()
|
|
intfSliceTypId = reflect.ValueOf(intfSliceTyp).Pointer()
|
strSliceTypId = reflect.ValueOf(strSliceTyp).Pointer()
|
|
boolSliceTypId = reflect.ValueOf(boolSliceTyp).Pointer()
|
uintSliceTypId = reflect.ValueOf(uintSliceTyp).Pointer()
|
uint8SliceTypId = reflect.ValueOf(uint8SliceTyp).Pointer()
|
uint16SliceTypId = reflect.ValueOf(uint16SliceTyp).Pointer()
|
uint32SliceTypId = reflect.ValueOf(uint32SliceTyp).Pointer()
|
uint64SliceTypId = reflect.ValueOf(uint64SliceTyp).Pointer()
|
intSliceTypId = reflect.ValueOf(intSliceTyp).Pointer()
|
int8SliceTypId = reflect.ValueOf(int8SliceTyp).Pointer()
|
int16SliceTypId = reflect.ValueOf(int16SliceTyp).Pointer()
|
int32SliceTypId = reflect.ValueOf(int32SliceTyp).Pointer()
|
int64SliceTypId = reflect.ValueOf(int64SliceTyp).Pointer()
|
float32SliceTypId = reflect.ValueOf(float32SliceTyp).Pointer()
|
float64SliceTypId = reflect.ValueOf(float64SliceTyp).Pointer()
|
|
mapStrStrTypId = reflect.ValueOf(mapStrStrTyp).Pointer()
|
mapIntfIntfTypId = reflect.ValueOf(mapIntfIntfTyp).Pointer()
|
mapStrIntfTypId = reflect.ValueOf(mapStrIntfTyp).Pointer()
|
mapIntIntfTypId = reflect.ValueOf(mapIntIntfTyp).Pointer()
|
mapInt64IntfTypId = reflect.ValueOf(mapInt64IntfTyp).Pointer()
|
mapUintIntfTypId = reflect.ValueOf(mapUintIntfTyp).Pointer()
|
mapUint64IntfTypId = reflect.ValueOf(mapUint64IntfTyp).Pointer()
|
// Id = reflect.ValueOf().Pointer()
|
// mapBySliceTypId = reflect.ValueOf(mapBySliceTyp).Pointer()
|
|
binaryMarshalerTypId = reflect.ValueOf(binaryMarshalerTyp).Pointer()
|
binaryUnmarshalerTypId = reflect.ValueOf(binaryUnmarshalerTyp).Pointer()
|
|
intBitsize uint8 = uint8(reflect.TypeOf(int(0)).Bits())
|
uintBitsize uint8 = uint8(reflect.TypeOf(uint(0)).Bits())
|
|
bsAll0x00 = []byte{0, 0, 0, 0, 0, 0, 0, 0}
|
bsAll0xff = []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
|
)
|
|
type binaryUnmarshaler interface {
|
UnmarshalBinary(data []byte) error
|
}
|
|
type binaryMarshaler interface {
|
MarshalBinary() (data []byte, err error)
|
}
|
|
// MapBySlice represents a slice which should be encoded as a map in the stream.
|
// The slice contains a sequence of key-value pairs.
|
type MapBySlice interface {
|
MapBySlice()
|
}
|
|
// WARNING: DO NOT USE DIRECTLY. EXPORTED FOR GODOC BENEFIT. WILL BE REMOVED.
|
//
|
// BasicHandle encapsulates the common options and extension functions.
|
type BasicHandle struct {
|
extHandle
|
EncodeOptions
|
DecodeOptions
|
}
|
|
// Handle is the interface for a specific encoding format.
|
//
|
// Typically, a Handle is pre-configured before first time use,
|
// and not modified while in use. Such a pre-configured Handle
|
// is safe for concurrent access.
|
type Handle interface {
|
writeExt() bool
|
getBasicHandle() *BasicHandle
|
newEncDriver(w encWriter) encDriver
|
newDecDriver(r decReader) decDriver
|
}
|
|
// RawExt represents raw unprocessed extension data.
|
type RawExt struct {
|
Tag byte
|
Data []byte
|
}
|
|
type extTypeTagFn struct {
|
rtid uintptr
|
rt reflect.Type
|
tag byte
|
encFn func(reflect.Value) ([]byte, error)
|
decFn func(reflect.Value, []byte) error
|
}
|
|
type extHandle []*extTypeTagFn
|
|
// AddExt registers an encode and decode function for a reflect.Type.
|
// Note that the type must be a named type, and specifically not
|
// a pointer or Interface. An error is returned if that is not honored.
|
//
|
// To Deregister an ext, call AddExt with 0 tag, nil encfn and nil decfn.
|
func (o *extHandle) AddExt(
|
rt reflect.Type,
|
tag byte,
|
encfn func(reflect.Value) ([]byte, error),
|
decfn func(reflect.Value, []byte) error,
|
) (err error) {
|
// o is a pointer, because we may need to initialize it
|
if rt.PkgPath() == "" || rt.Kind() == reflect.Interface {
|
err = fmt.Errorf("codec.Handle.AddExt: Takes named type, especially not a pointer or interface: %T",
|
reflect.Zero(rt).Interface())
|
return
|
}
|
|
// o cannot be nil, since it is always embedded in a Handle.
|
// if nil, let it panic.
|
// if o == nil {
|
// err = errors.New("codec.Handle.AddExt: extHandle cannot be a nil pointer.")
|
// return
|
// }
|
|
rtid := reflect.ValueOf(rt).Pointer()
|
for _, v := range *o {
|
if v.rtid == rtid {
|
v.tag, v.encFn, v.decFn = tag, encfn, decfn
|
return
|
}
|
}
|
|
*o = append(*o, &extTypeTagFn{rtid, rt, tag, encfn, decfn})
|
return
|
}
|
|
func (o extHandle) getExt(rtid uintptr) *extTypeTagFn {
|
for _, v := range o {
|
if v.rtid == rtid {
|
return v
|
}
|
}
|
return nil
|
}
|
|
func (o extHandle) getExtForTag(tag byte) *extTypeTagFn {
|
for _, v := range o {
|
if v.tag == tag {
|
return v
|
}
|
}
|
return nil
|
}
|
|
func (o extHandle) getDecodeExtForTag(tag byte) (
|
rv reflect.Value, fn func(reflect.Value, []byte) error) {
|
if x := o.getExtForTag(tag); x != nil {
|
// ext is only registered for base
|
rv = reflect.New(x.rt).Elem()
|
fn = x.decFn
|
}
|
return
|
}
|
|
func (o extHandle) getDecodeExt(rtid uintptr) (tag byte, fn func(reflect.Value, []byte) error) {
|
if x := o.getExt(rtid); x != nil {
|
tag = x.tag
|
fn = x.decFn
|
}
|
return
|
}
|
|
func (o extHandle) getEncodeExt(rtid uintptr) (tag byte, fn func(reflect.Value) ([]byte, error)) {
|
if x := o.getExt(rtid); x != nil {
|
tag = x.tag
|
fn = x.encFn
|
}
|
return
|
}
|
|
type structFieldInfo struct {
|
encName string // encode name
|
|
// only one of 'i' or 'is' can be set. If 'i' is -1, then 'is' has been set.
|
|
is []int // (recursive/embedded) field index in struct
|
i int16 // field index in struct
|
omitEmpty bool
|
toArray bool // if field is _struct, is the toArray set?
|
|
// tag string // tag
|
// name string // field name
|
// encNameBs []byte // encoded name as byte stream
|
// ikind int // kind of the field as an int i.e. int(reflect.Kind)
|
}
|
|
func parseStructFieldInfo(fname string, stag string) *structFieldInfo {
|
if fname == "" {
|
panic("parseStructFieldInfo: No Field Name")
|
}
|
si := structFieldInfo{
|
// name: fname,
|
encName: fname,
|
// tag: stag,
|
}
|
|
if stag != "" {
|
for i, s := range strings.Split(stag, ",") {
|
if i == 0 {
|
if s != "" {
|
si.encName = s
|
}
|
} else {
|
switch s {
|
case "omitempty":
|
si.omitEmpty = true
|
case "toarray":
|
si.toArray = true
|
}
|
}
|
}
|
}
|
// si.encNameBs = []byte(si.encName)
|
return &si
|
}
|
|
type sfiSortedByEncName []*structFieldInfo
|
|
func (p sfiSortedByEncName) Len() int {
|
return len(p)
|
}
|
|
func (p sfiSortedByEncName) Less(i, j int) bool {
|
return p[i].encName < p[j].encName
|
}
|
|
func (p sfiSortedByEncName) Swap(i, j int) {
|
p[i], p[j] = p[j], p[i]
|
}
|
|
// typeInfo keeps information about each type referenced in the encode/decode sequence.
|
//
|
// During an encode/decode sequence, we work as below:
|
// - If base is a built in type, en/decode base value
|
// - If base is registered as an extension, en/decode base value
|
// - If type is binary(M/Unm)arshaler, call Binary(M/Unm)arshal method
|
// - Else decode appropriately based on the reflect.Kind
|
type typeInfo struct {
|
sfi []*structFieldInfo // sorted. Used when enc/dec struct to map.
|
sfip []*structFieldInfo // unsorted. Used when enc/dec struct to array.
|
|
rt reflect.Type
|
rtid uintptr
|
|
// baseId gives pointer to the base reflect.Type, after deferencing
|
// the pointers. E.g. base type of ***time.Time is time.Time.
|
base reflect.Type
|
baseId uintptr
|
baseIndir int8 // number of indirections to get to base
|
|
mbs bool // base type (T or *T) is a MapBySlice
|
|
m bool // base type (T or *T) is a binaryMarshaler
|
unm bool // base type (T or *T) is a binaryUnmarshaler
|
mIndir int8 // number of indirections to get to binaryMarshaler type
|
unmIndir int8 // number of indirections to get to binaryUnmarshaler type
|
toArray bool // whether this (struct) type should be encoded as an array
|
}
|
|
func (ti *typeInfo) indexForEncName(name string) int {
|
//tisfi := ti.sfi
|
const binarySearchThreshold = 16
|
if sfilen := len(ti.sfi); sfilen < binarySearchThreshold {
|
// linear search. faster than binary search in my testing up to 16-field structs.
|
for i, si := range ti.sfi {
|
if si.encName == name {
|
return i
|
}
|
}
|
} else {
|
// binary search. adapted from sort/search.go.
|
h, i, j := 0, 0, sfilen
|
for i < j {
|
h = i + (j-i)/2
|
if ti.sfi[h].encName < name {
|
i = h + 1
|
} else {
|
j = h
|
}
|
}
|
if i < sfilen && ti.sfi[i].encName == name {
|
return i
|
}
|
}
|
return -1
|
}
|
|
func getTypeInfo(rtid uintptr, rt reflect.Type) (pti *typeInfo) {
|
var ok bool
|
cachedTypeInfoMutex.RLock()
|
pti, ok = cachedTypeInfo[rtid]
|
cachedTypeInfoMutex.RUnlock()
|
if ok {
|
return
|
}
|
|
cachedTypeInfoMutex.Lock()
|
defer cachedTypeInfoMutex.Unlock()
|
if pti, ok = cachedTypeInfo[rtid]; ok {
|
return
|
}
|
|
ti := typeInfo{rt: rt, rtid: rtid}
|
pti = &ti
|
|
var indir int8
|
if ok, indir = implementsIntf(rt, binaryMarshalerTyp); ok {
|
ti.m, ti.mIndir = true, indir
|
}
|
if ok, indir = implementsIntf(rt, binaryUnmarshalerTyp); ok {
|
ti.unm, ti.unmIndir = true, indir
|
}
|
if ok, _ = implementsIntf(rt, mapBySliceTyp); ok {
|
ti.mbs = true
|
}
|
|
pt := rt
|
var ptIndir int8
|
// for ; pt.Kind() == reflect.Ptr; pt, ptIndir = pt.Elem(), ptIndir+1 { }
|
for pt.Kind() == reflect.Ptr {
|
pt = pt.Elem()
|
ptIndir++
|
}
|
if ptIndir == 0 {
|
ti.base = rt
|
ti.baseId = rtid
|
} else {
|
ti.base = pt
|
ti.baseId = reflect.ValueOf(pt).Pointer()
|
ti.baseIndir = ptIndir
|
}
|
|
if rt.Kind() == reflect.Struct {
|
var siInfo *structFieldInfo
|
if f, ok := rt.FieldByName(structInfoFieldName); ok {
|
siInfo = parseStructFieldInfo(structInfoFieldName, f.Tag.Get(structTagName))
|
ti.toArray = siInfo.toArray
|
}
|
sfip := make([]*structFieldInfo, 0, rt.NumField())
|
rgetTypeInfo(rt, nil, make(map[string]bool), &sfip, siInfo)
|
|
// // try to put all si close together
|
// const tryToPutAllStructFieldInfoTogether = true
|
// if tryToPutAllStructFieldInfoTogether {
|
// sfip2 := make([]structFieldInfo, len(sfip))
|
// for i, si := range sfip {
|
// sfip2[i] = *si
|
// }
|
// for i := range sfip {
|
// sfip[i] = &sfip2[i]
|
// }
|
// }
|
|
ti.sfip = make([]*structFieldInfo, len(sfip))
|
ti.sfi = make([]*structFieldInfo, len(sfip))
|
copy(ti.sfip, sfip)
|
sort.Sort(sfiSortedByEncName(sfip))
|
copy(ti.sfi, sfip)
|
}
|
// sfi = sfip
|
cachedTypeInfo[rtid] = pti
|
return
|
}
|
|
func rgetTypeInfo(rt reflect.Type, indexstack []int, fnameToHastag map[string]bool,
|
sfi *[]*structFieldInfo, siInfo *structFieldInfo,
|
) {
|
// for rt.Kind() == reflect.Ptr {
|
// // indexstack = append(indexstack, 0)
|
// rt = rt.Elem()
|
// }
|
for j := 0; j < rt.NumField(); j++ {
|
f := rt.Field(j)
|
stag := f.Tag.Get(structTagName)
|
if stag == "-" {
|
continue
|
}
|
if r1, _ := utf8.DecodeRuneInString(f.Name); r1 == utf8.RuneError || !unicode.IsUpper(r1) {
|
continue
|
}
|
// if anonymous and there is no struct tag and its a struct (or pointer to struct), inline it.
|
if f.Anonymous && stag == "" {
|
ft := f.Type
|
for ft.Kind() == reflect.Ptr {
|
ft = ft.Elem()
|
}
|
if ft.Kind() == reflect.Struct {
|
indexstack2 := append(append(make([]int, 0, len(indexstack)+4), indexstack...), j)
|
rgetTypeInfo(ft, indexstack2, fnameToHastag, sfi, siInfo)
|
continue
|
}
|
}
|
// do not let fields with same name in embedded structs override field at higher level.
|
// this must be done after anonymous check, to allow anonymous field
|
// still include their child fields
|
if _, ok := fnameToHastag[f.Name]; ok {
|
continue
|
}
|
si := parseStructFieldInfo(f.Name, stag)
|
// si.ikind = int(f.Type.Kind())
|
if len(indexstack) == 0 {
|
si.i = int16(j)
|
} else {
|
si.i = -1
|
si.is = append(append(make([]int, 0, len(indexstack)+4), indexstack...), j)
|
}
|
|
if siInfo != nil {
|
if siInfo.omitEmpty {
|
si.omitEmpty = true
|
}
|
}
|
*sfi = append(*sfi, si)
|
fnameToHastag[f.Name] = stag != ""
|
}
|
}
|
|
func panicToErr(err *error) {
|
if recoverPanicToErr {
|
if x := recover(); x != nil {
|
//debug.PrintStack()
|
panicValToErr(x, err)
|
}
|
}
|
}
|
|
func doPanic(tag string, format string, params ...interface{}) {
|
params2 := make([]interface{}, len(params)+1)
|
params2[0] = tag
|
copy(params2[1:], params)
|
panic(fmt.Errorf("%s: "+format, params2...))
|
}
|
|
func checkOverflowFloat32(f float64, doCheck bool) {
|
if !doCheck {
|
return
|
}
|
// check overflow (logic adapted from std pkg reflect/value.go OverflowFloat()
|
f2 := f
|
if f2 < 0 {
|
f2 = -f
|
}
|
if math.MaxFloat32 < f2 && f2 <= math.MaxFloat64 {
|
decErr("Overflow float32 value: %v", f2)
|
}
|
}
|
|
func checkOverflow(ui uint64, i int64, bitsize uint8) {
|
// check overflow (logic adapted from std pkg reflect/value.go OverflowUint()
|
if bitsize == 0 {
|
return
|
}
|
if i != 0 {
|
if trunc := (i << (64 - bitsize)) >> (64 - bitsize); i != trunc {
|
decErr("Overflow int value: %v", i)
|
}
|
}
|
if ui != 0 {
|
if trunc := (ui << (64 - bitsize)) >> (64 - bitsize); ui != trunc {
|
decErr("Overflow uint value: %v", ui)
|
}
|
}
|
}
|
