Denis Dobrowinski
4 months ago
15 changed files with 1510 additions and 18 deletions
@ -0,0 +1,15 @@ |
|||
package logger |
|||
|
|||
import ( |
|||
"context" |
|||
|
|||
"google.golang.org/grpc/metadata" |
|||
) |
|||
|
|||
func AddToGRPCHeader(ctx context.Context, key, val string) context.Context { |
|||
if _, exists := metadata.FromOutgoingContext(ctx); exists { |
|||
return metadata.AppendToOutgoingContext(ctx, key, val) |
|||
} |
|||
|
|||
return metadata.NewOutgoingContext(ctx, metadata.Pairs(key, val)) |
|||
} |
|||
@ -0,0 +1,31 @@ |
|||
# Compiled Object files, Static and Dynamic libs (Shared Objects) |
|||
*.o |
|||
*.a |
|||
*.so |
|||
|
|||
# Folders |
|||
_obj |
|||
_test |
|||
|
|||
# Architecture specific extensions/prefixes |
|||
*.[568vq] |
|||
[568vq].out |
|||
|
|||
*.cgo1.go |
|||
*.cgo2.c |
|||
_cgo_defun.c |
|||
_cgo_gotypes.go |
|||
_cgo_export.* |
|||
|
|||
_testmain.go |
|||
|
|||
*.exe |
|||
*.test |
|||
*.prof |
|||
/ksuid |
|||
|
|||
# Emacs |
|||
*~ |
|||
|
|||
# govendor |
|||
/vendor/*/ |
|||
@ -0,0 +1,21 @@ |
|||
MIT License |
|||
|
|||
Copyright (c) 2017 Segment.io |
|||
|
|||
Permission is hereby granted, free of charge, to any person obtaining a copy |
|||
of this software and associated documentation files (the "Software"), to deal |
|||
in the Software without restriction, including without limitation the rights |
|||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
|||
copies of the Software, and to permit persons to whom the Software is |
|||
furnished to do so, subject to the following conditions: |
|||
|
|||
The above copyright notice and this permission notice shall be included in all |
|||
copies or substantial portions of the Software. |
|||
|
|||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
|||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
|||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
|||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
|||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
|||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
|||
SOFTWARE. |
|||
@ -0,0 +1,234 @@ |
|||
# ksuid [](https://goreportcard.com/report/github.com/segmentio/ksuid) [](https://godoc.org/github.com/segmentio/ksuid) [](https://circleci.com/gh/segmentio/ksuid.svg?style=shield) |
|||
|
|||
ksuid is an efficient, comprehensive, battle-tested Go library for |
|||
generating and parsing a specific kind of globally unique identifier |
|||
called a *KSUID*. This library serves as its reference implementation. |
|||
|
|||
## Install |
|||
```sh |
|||
go get -u github.com/segmentio/ksuid |
|||
``` |
|||
|
|||
## What is a KSUID? |
|||
|
|||
KSUID is for K-Sortable Unique IDentifier. It is a kind of globally |
|||
unique identifier similar to a [RFC 4122 UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier), built from the ground-up to be "naturally" |
|||
sorted by generation timestamp without any special type-aware logic. |
|||
|
|||
In short, running a set of KSUIDs through the UNIX `sort` command will result |
|||
in a list ordered by generation time. |
|||
|
|||
## Why use KSUIDs? |
|||
|
|||
There are numerous methods for generating unique identifiers, so why KSUID? |
|||
|
|||
1. Naturally ordered by generation time |
|||
2. Collision-free, coordination-free, dependency-free |
|||
3. Highly portable representations |
|||
|
|||
Even if only one of these properties are important to you, KSUID is a great |
|||
choice! :) Many projects chose to use KSUIDs *just* because the text |
|||
representation is copy-and-paste friendly. |
|||
|
|||
### 1. Naturally Ordered By Generation Time |
|||
|
|||
Unlike the more ubiquitous UUIDv4, a KSUID contains a timestamp component |
|||
that allows them to be loosely sorted by generation time. This is not a strong |
|||
guarantee (an invariant) as it depends on wall clocks, but is still incredibly |
|||
useful in practice. Both the binary and text representations will sort by |
|||
creation time without any special sorting logic. |
|||
|
|||
### 2. Collision-free, Coordination-free, Dependency-free |
|||
|
|||
While RFC 4122 UUIDv1s *do* include a time component, there aren't enough |
|||
bytes of randomness to provide strong protection against collisions |
|||
(duplicates). With such a low amount of entropy, it is feasible for a |
|||
malicious party to guess generated IDs, creating a problem for systems whose |
|||
security is, implicitly or explicitly, sensitive to an adversary guessing |
|||
identifiers. |
|||
|
|||
To fit into a 64-bit number space, [Snowflake IDs](https://blog.twitter.com/2010/announcing-snowflake) |
|||
and its derivatives require coordination to avoid collisions, which |
|||
significantly increases the deployment complexity and operational burden. |
|||
|
|||
A KSUID includes 128 bits of pseudorandom data ("entropy"). This number space |
|||
is 64 times larger than the 122 bits used by the well-accepted RFC 4122 UUIDv4 |
|||
standard. The additional timestamp component can be considered "bonus entropy" |
|||
which further decreases the probability of collisions, to the point of physical |
|||
infeasibility in any practical implementation. |
|||
|
|||
### Highly Portable Representations |
|||
|
|||
The text *and* binary representations are lexicographically sortable, which |
|||
allows them to be dropped into systems which do not natively support KSUIDs |
|||
and retain their time-ordered property. |
|||
|
|||
The text representation is an alphanumeric base62 encoding, so it "fits" |
|||
anywhere alphanumeric strings are accepted. No delimiters are used, so |
|||
stringified KSUIDs won't be inadvertently truncated or tokenized when |
|||
interpreted by software that is designed for human-readable text, a common |
|||
problem for the text representation of RFC 4122 UUIDs. |
|||
|
|||
## How do KSUIDs work? |
|||
|
|||
Binary KSUIDs are 20-bytes: a 32-bit unsigned integer UTC timestamp and |
|||
a 128-bit randomly generated payload. The timestamp uses big-endian |
|||
encoding, to support lexicographic sorting. The timestamp epoch is adjusted |
|||
to March 5th, 2014, providing over 100 years of life. The payload is |
|||
generated by a cryptographically-strong pseudorandom number generator. |
|||
|
|||
The text representation is always 27 characters, encoded in alphanumeric |
|||
base62 that will lexicographically sort by timestamp. |
|||
|
|||
## High Performance |
|||
|
|||
This library is designed to be used in code paths that are performance |
|||
critical. Its code has been tuned to eliminate all non-essential |
|||
overhead. The `KSUID` type is derived from a fixed-size array, which |
|||
eliminates the additional reference chasing and allocation involved in |
|||
a variable-width type. |
|||
|
|||
The API provides an interface for use in code paths which are sensitive |
|||
to allocation. For example, the `Append` method can be used to parse the |
|||
text representation and replace the contents of a `KSUID` value |
|||
without additional heap allocation. |
|||
|
|||
All public package level "pure" functions are concurrency-safe, protected |
|||
by a global mutex. For hot loops that generate a large amount of KSUIDs |
|||
from a single Goroutine, the `Sequence` type is provided to elide the |
|||
potential contention. |
|||
|
|||
By default, out of an abundance of caution, the cryptographically-secure |
|||
PRNG is used to generate the random bits of a KSUID. This can be relaxed |
|||
in extremely performance-critical code using the included `FastRander` |
|||
type. `FastRander` uses the standard PRNG with a seed generated by the |
|||
cryptographically-secure PRNG. |
|||
|
|||
*_NOTE:_ While there is no evidence that `FastRander` will increase the |
|||
probability of a collision, it shouldn't be used in scenarios where |
|||
uniqueness is important to security, as there is an increased chance |
|||
the generated IDs can be predicted by an adversary.* |
|||
|
|||
## Battle Tested |
|||
|
|||
This code has been used in production at Segment for several years, |
|||
across a diverse array of projects. Trillions upon trillions of |
|||
KSUIDs have been generated in some of Segment's most |
|||
performance-critical, large-scale distributed systems. |
|||
|
|||
## Plays Well With Others |
|||
|
|||
Designed to be integrated with other libraries, the `KSUID` type |
|||
implements many standard library interfaces, including: |
|||
|
|||
* `Stringer` |
|||
* `database/sql.Scanner` and `database/sql/driver.Valuer` |
|||
* `encoding.BinaryMarshal` and `encoding.BinaryUnmarshal` |
|||
* `encoding.TextMarshal` and `encoding.TextUnmarshal` |
|||
(`encoding/json` friendly!) |
|||
|
|||
## Command Line Tool |
|||
|
|||
This package comes with a command-line tool `ksuid`, useful for |
|||
generating KSUIDs as well as inspecting the internal components of |
|||
existing KSUIDs. Machine-friendly output is provided for scripting |
|||
use cases. |
|||
|
|||
Given a Go build environment, it can be installed with the command: |
|||
|
|||
```sh |
|||
$ go install github.com/segmentio/ksuid/cmd/ksuid |
|||
``` |
|||
|
|||
## CLI Usage Examples |
|||
|
|||
### Generate a KSUID |
|||
|
|||
```sh |
|||
$ ksuid |
|||
0ujsswThIGTUYm2K8FjOOfXtY1K |
|||
``` |
|||
|
|||
### Generate 4 KSUIDs |
|||
|
|||
```sh |
|||
$ ksuid -n 4 |
|||
0ujsszwN8NRY24YaXiTIE2VWDTS |
|||
0ujsswThIGTUYm2K8FjOOfXtY1K |
|||
0ujssxh0cECutqzMgbtXSGnjorm |
|||
0ujsszgFvbiEr7CDgE3z8MAUPFt |
|||
``` |
|||
|
|||
### Inspect the components of a KSUID |
|||
|
|||
```sh |
|||
$ ksuid -f inspect 0ujtsYcgvSTl8PAuAdqWYSMnLOv |
|||
|
|||
REPRESENTATION: |
|||
|
|||
String: 0ujtsYcgvSTl8PAuAdqWYSMnLOv |
|||
Raw: 0669F7EFB5A1CD34B5F99D1154FB6853345C9735 |
|||
|
|||
COMPONENTS: |
|||
|
|||
Time: 2017-10-09 21:00:47 -0700 PDT |
|||
Timestamp: 107608047 |
|||
Payload: B5A1CD34B5F99D1154FB6853345C9735 |
|||
``` |
|||
|
|||
### Generate a KSUID and inspect its components |
|||
|
|||
```sh |
|||
$ ksuid -f inspect |
|||
|
|||
REPRESENTATION: |
|||
|
|||
String: 0ujzPyRiIAffKhBux4PvQdDqMHY |
|||
Raw: 066A029C73FC1AA3B2446246D6E89FCD909E8FE8 |
|||
|
|||
COMPONENTS: |
|||
|
|||
Time: 2017-10-09 21:46:20 -0700 PDT |
|||
Timestamp: 107610780 |
|||
Payload: 73FC1AA3B2446246D6E89FCD909E8FE8 |
|||
|
|||
``` |
|||
|
|||
### Inspect a KSUID with template formatted inspection output |
|||
|
|||
```sh |
|||
$ ksuid -f template -t '{{ .Time }}: {{ .Payload }}' 0ujtsYcgvSTl8PAuAdqWYSMnLOv |
|||
2017-10-09 21:00:47 -0700 PDT: B5A1CD34B5F99D1154FB6853345C9735 |
|||
``` |
|||
|
|||
### Inspect multiple KSUIDs with template formatted output |
|||
|
|||
```sh |
|||
$ ksuid -f template -t '{{ .Time }}: {{ .Payload }}' $(ksuid -n 4) |
|||
2017-10-09 21:05:37 -0700 PDT: 304102BC687E087CC3A811F21D113CCF |
|||
2017-10-09 21:05:37 -0700 PDT: EAF0B240A9BFA55E079D887120D962F0 |
|||
2017-10-09 21:05:37 -0700 PDT: DF0761769909ABB0C7BB9D66F79FC041 |
|||
2017-10-09 21:05:37 -0700 PDT: 1A8F0E3D0BDEB84A5FAD702876F46543 |
|||
``` |
|||
|
|||
### Generate KSUIDs and output JSON using template formatting |
|||
|
|||
```sh |
|||
$ ksuid -f template -t '{ "timestamp": "{{ .Timestamp }}", "payload": "{{ .Payload }}", "ksuid": "{{.String}}"}' -n 4 |
|||
{ "timestamp": "107611700", "payload": "9850EEEC191BF4FF26F99315CE43B0C8", "ksuid": "0uk1Hbc9dQ9pxyTqJ93IUrfhdGq"} |
|||
{ "timestamp": "107611700", "payload": "CC55072555316F45B8CA2D2979D3ED0A", "ksuid": "0uk1HdCJ6hUZKDgcxhpJwUl5ZEI"} |
|||
{ "timestamp": "107611700", "payload": "BA1C205D6177F0992D15EE606AE32238", "ksuid": "0uk1HcdvF0p8C20KtTfdRSB9XIm"} |
|||
{ "timestamp": "107611700", "payload": "67517BA309EA62AE7991B27BB6F2FCAC", "ksuid": "0uk1Ha7hGJ1Q9Xbnkt0yZgNwg3g"} |
|||
``` |
|||
|
|||
## Implementations for other languages |
|||
|
|||
- Python: [svix-ksuid](https://github.com/svixhq/python-ksuid/) |
|||
- Ruby: [ksuid-ruby](https://github.com/michaelherold/ksuid-ruby) |
|||
- Java: [ksuid](https://github.com/ksuid/ksuid) |
|||
- Rust: [rksuid](https://github.com/nharring/rksuid) |
|||
- dotNet: [Ksuid.Net](https://github.com/JoyMoe/Ksuid.Net) |
|||
|
|||
## License |
|||
|
|||
ksuid source code is available under an MIT [License](/LICENSE.md). |
|||
@ -0,0 +1,202 @@ |
|||
package ksuid |
|||
|
|||
import ( |
|||
"encoding/binary" |
|||
"errors" |
|||
) |
|||
|
|||
const ( |
|||
// lexographic ordering (based on Unicode table) is 0-9A-Za-z
|
|||
base62Characters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" |
|||
zeroString = "000000000000000000000000000" |
|||
offsetUppercase = 10 |
|||
offsetLowercase = 36 |
|||
) |
|||
|
|||
var ( |
|||
errShortBuffer = errors.New("the output buffer is too small to hold to decoded value") |
|||
) |
|||
|
|||
// Converts a base 62 byte into the number value that it represents.
|
|||
func base62Value(digit byte) byte { |
|||
switch { |
|||
case digit >= '0' && digit <= '9': |
|||
return digit - '0' |
|||
case digit >= 'A' && digit <= 'Z': |
|||
return offsetUppercase + (digit - 'A') |
|||
default: |
|||
return offsetLowercase + (digit - 'a') |
|||
} |
|||
} |
|||
|
|||
// This function encodes the base 62 representation of the src KSUID in binary
|
|||
// form into dst.
|
|||
//
|
|||
// In order to support a couple of optimizations the function assumes that src
|
|||
// is 20 bytes long and dst is 27 bytes long.
|
|||
//
|
|||
// Any unused bytes in dst will be set to the padding '0' byte.
|
|||
func fastEncodeBase62(dst []byte, src []byte) { |
|||
const srcBase = 4294967296 |
|||
const dstBase = 62 |
|||
|
|||
// Split src into 5 4-byte words, this is where most of the efficiency comes
|
|||
// from because this is a O(N^2) algorithm, and we make N = N / 4 by working
|
|||
// on 32 bits at a time.
|
|||
parts := [5]uint32{ |
|||
binary.BigEndian.Uint32(src[0:4]), |
|||
binary.BigEndian.Uint32(src[4:8]), |
|||
binary.BigEndian.Uint32(src[8:12]), |
|||
binary.BigEndian.Uint32(src[12:16]), |
|||
binary.BigEndian.Uint32(src[16:20]), |
|||
} |
|||
|
|||
n := len(dst) |
|||
bp := parts[:] |
|||
bq := [5]uint32{} |
|||
|
|||
for len(bp) != 0 { |
|||
quotient := bq[:0] |
|||
remainder := uint64(0) |
|||
|
|||
for _, c := range bp { |
|||
value := uint64(c) + uint64(remainder)*srcBase |
|||
digit := value / dstBase |
|||
remainder = value % dstBase |
|||
|
|||
if len(quotient) != 0 || digit != 0 { |
|||
quotient = append(quotient, uint32(digit)) |
|||
} |
|||
} |
|||
|
|||
// Writes at the end of the destination buffer because we computed the
|
|||
// lowest bits first.
|
|||
n-- |
|||
dst[n] = base62Characters[remainder] |
|||
bp = quotient |
|||
} |
|||
|
|||
// Add padding at the head of the destination buffer for all bytes that were
|
|||
// not set.
|
|||
copy(dst[:n], zeroString) |
|||
} |
|||
|
|||
// This function appends the base 62 representation of the KSUID in src to dst,
|
|||
// and returns the extended byte slice.
|
|||
// The result is left-padded with '0' bytes to always append 27 bytes to the
|
|||
// destination buffer.
|
|||
func fastAppendEncodeBase62(dst []byte, src []byte) []byte { |
|||
dst = reserve(dst, stringEncodedLength) |
|||
n := len(dst) |
|||
fastEncodeBase62(dst[n:n+stringEncodedLength], src) |
|||
return dst[:n+stringEncodedLength] |
|||
} |
|||
|
|||
// This function decodes the base 62 representation of the src KSUID to the
|
|||
// binary form into dst.
|
|||
//
|
|||
// In order to support a couple of optimizations the function assumes that src
|
|||
// is 27 bytes long and dst is 20 bytes long.
|
|||
//
|
|||
// Any unused bytes in dst will be set to zero.
|
|||
func fastDecodeBase62(dst []byte, src []byte) error { |
|||
const srcBase = 62 |
|||
const dstBase = 4294967296 |
|||
|
|||
// This line helps BCE (Bounds Check Elimination).
|
|||
// It may be safely removed.
|
|||
_ = src[26] |
|||
|
|||
parts := [27]byte{ |
|||
base62Value(src[0]), |
|||
base62Value(src[1]), |
|||
base62Value(src[2]), |
|||
base62Value(src[3]), |
|||
base62Value(src[4]), |
|||
base62Value(src[5]), |
|||
base62Value(src[6]), |
|||
base62Value(src[7]), |
|||
base62Value(src[8]), |
|||
base62Value(src[9]), |
|||
|
|||
base62Value(src[10]), |
|||
base62Value(src[11]), |
|||
base62Value(src[12]), |
|||
base62Value(src[13]), |
|||
base62Value(src[14]), |
|||
base62Value(src[15]), |
|||
base62Value(src[16]), |
|||
base62Value(src[17]), |
|||
base62Value(src[18]), |
|||
base62Value(src[19]), |
|||
|
|||
base62Value(src[20]), |
|||
base62Value(src[21]), |
|||
base62Value(src[22]), |
|||
base62Value(src[23]), |
|||
base62Value(src[24]), |
|||
base62Value(src[25]), |
|||
base62Value(src[26]), |
|||
} |
|||
|
|||
n := len(dst) |
|||
bp := parts[:] |
|||
bq := [stringEncodedLength]byte{} |
|||
|
|||
for len(bp) > 0 { |
|||
quotient := bq[:0] |
|||
remainder := uint64(0) |
|||
|
|||
for _, c := range bp { |
|||
value := uint64(c) + uint64(remainder)*srcBase |
|||
digit := value / dstBase |
|||
remainder = value % dstBase |
|||
|
|||
if len(quotient) != 0 || digit != 0 { |
|||
quotient = append(quotient, byte(digit)) |
|||
} |
|||
} |
|||
|
|||
if n < 4 { |
|||
return errShortBuffer |
|||
} |
|||
|
|||
dst[n-4] = byte(remainder >> 24) |
|||
dst[n-3] = byte(remainder >> 16) |
|||
dst[n-2] = byte(remainder >> 8) |
|||
dst[n-1] = byte(remainder) |
|||
n -= 4 |
|||
bp = quotient |
|||
} |
|||
|
|||
var zero [20]byte |
|||
copy(dst[:n], zero[:]) |
|||
return nil |
|||
} |
|||
|
|||
// This function appends the base 62 decoded version of src into dst.
|
|||
func fastAppendDecodeBase62(dst []byte, src []byte) []byte { |
|||
dst = reserve(dst, byteLength) |
|||
n := len(dst) |
|||
fastDecodeBase62(dst[n:n+byteLength], src) |
|||
return dst[:n+byteLength] |
|||
} |
|||
|
|||
// Ensures that at least nbytes are available in the remaining capacity of the
|
|||
// destination slice, if not, a new copy is made and returned by the function.
|
|||
func reserve(dst []byte, nbytes int) []byte { |
|||
c := cap(dst) |
|||
n := len(dst) |
|||
|
|||
if avail := c - n; avail < nbytes { |
|||
c *= 2 |
|||
if (c - n) < nbytes { |
|||
c = n + nbytes |
|||
} |
|||
b := make([]byte, n, c) |
|||
copy(b, dst) |
|||
dst = b |
|||
} |
|||
|
|||
return dst |
|||
} |
|||
@ -0,0 +1,352 @@ |
|||
package ksuid |
|||
|
|||
import ( |
|||
"bytes" |
|||
"crypto/rand" |
|||
"database/sql/driver" |
|||
"encoding/binary" |
|||
"fmt" |
|||
"io" |
|||
"math" |
|||
"sync" |
|||
"time" |
|||
) |
|||
|
|||
const ( |
|||
// KSUID's epoch starts more recently so that the 32-bit number space gives a
|
|||
// significantly higher useful lifetime of around 136 years from March 2017.
|
|||
// This number (14e8) was picked to be easy to remember.
|
|||
epochStamp int64 = 1400000000 |
|||
|
|||
// Timestamp is a uint32
|
|||
timestampLengthInBytes = 4 |
|||
|
|||
// Payload is 16-bytes
|
|||
payloadLengthInBytes = 16 |
|||
|
|||
// KSUIDs are 20 bytes when binary encoded
|
|||
byteLength = timestampLengthInBytes + payloadLengthInBytes |
|||
|
|||
// The length of a KSUID when string (base62) encoded
|
|||
stringEncodedLength = 27 |
|||
|
|||
// A string-encoded minimum value for a KSUID
|
|||
minStringEncoded = "000000000000000000000000000" |
|||
|
|||
// A string-encoded maximum value for a KSUID
|
|||
maxStringEncoded = "aWgEPTl1tmebfsQzFP4bxwgy80V" |
|||
) |
|||
|
|||
// KSUIDs are 20 bytes:
|
|||
// 00-03 byte: uint32 BE UTC timestamp with custom epoch
|
|||
// 04-19 byte: random "payload"
|
|||
type KSUID [byteLength]byte |
|||
|
|||
var ( |
|||
rander = rand.Reader |
|||
randMutex = sync.Mutex{} |
|||
randBuffer = [payloadLengthInBytes]byte{} |
|||
|
|||
errSize = fmt.Errorf("Valid KSUIDs are %v bytes", byteLength) |
|||
errStrSize = fmt.Errorf("Valid encoded KSUIDs are %v characters", stringEncodedLength) |
|||
errStrValue = fmt.Errorf("Valid encoded KSUIDs are bounded by %s and %s", minStringEncoded, maxStringEncoded) |
|||
errPayloadSize = fmt.Errorf("Valid KSUID payloads are %v bytes", payloadLengthInBytes) |
|||
|
|||
// Represents a completely empty (invalid) KSUID
|
|||
Nil KSUID |
|||
// Represents the highest value a KSUID can have
|
|||
Max = KSUID{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255} |
|||
) |
|||
|
|||
// Append appends the string representation of i to b, returning a slice to a
|
|||
// potentially larger memory area.
|
|||
func (i KSUID) Append(b []byte) []byte { |
|||
return fastAppendEncodeBase62(b, i[:]) |
|||
} |
|||
|
|||
// The timestamp portion of the ID as a Time object
|
|||
func (i KSUID) Time() time.Time { |
|||
return correctedUTCTimestampToTime(i.Timestamp()) |
|||
} |
|||
|
|||
// The timestamp portion of the ID as a bare integer which is uncorrected
|
|||
// for KSUID's special epoch.
|
|||
func (i KSUID) Timestamp() uint32 { |
|||
return binary.BigEndian.Uint32(i[:timestampLengthInBytes]) |
|||
} |
|||
|
|||
// The 16-byte random payload without the timestamp
|
|||
func (i KSUID) Payload() []byte { |
|||
return i[timestampLengthInBytes:] |
|||
} |
|||
|
|||
// String-encoded representation that can be passed through Parse()
|
|||
func (i KSUID) String() string { |
|||
return string(i.Append(make([]byte, 0, stringEncodedLength))) |
|||
} |
|||
|
|||
// Raw byte representation of KSUID
|
|||
func (i KSUID) Bytes() []byte { |
|||
// Safe because this is by-value
|
|||
return i[:] |
|||
} |
|||
|
|||
// IsNil returns true if this is a "nil" KSUID
|
|||
func (i KSUID) IsNil() bool { |
|||
return i == Nil |
|||
} |
|||
|
|||
// Get satisfies the flag.Getter interface, making it possible to use KSUIDs as
|
|||
// part of of the command line options of a program.
|
|||
func (i KSUID) Get() interface{} { |
|||
return i |
|||
} |
|||
|
|||
// Set satisfies the flag.Value interface, making it possible to use KSUIDs as
|
|||
// part of of the command line options of a program.
|
|||
func (i *KSUID) Set(s string) error { |
|||
return i.UnmarshalText([]byte(s)) |
|||
} |
|||
|
|||
func (i KSUID) MarshalText() ([]byte, error) { |
|||
return []byte(i.String()), nil |
|||
} |
|||
|
|||
func (i KSUID) MarshalBinary() ([]byte, error) { |
|||
return i.Bytes(), nil |
|||
} |
|||
|
|||
func (i *KSUID) UnmarshalText(b []byte) error { |
|||
id, err := Parse(string(b)) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
*i = id |
|||
return nil |
|||
} |
|||
|
|||
func (i *KSUID) UnmarshalBinary(b []byte) error { |
|||
id, err := FromBytes(b) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
*i = id |
|||
return nil |
|||
} |
|||
|
|||
// Value converts the KSUID into a SQL driver value which can be used to
|
|||
// directly use the KSUID as parameter to a SQL query.
|
|||
func (i KSUID) Value() (driver.Value, error) { |
|||
if i.IsNil() { |
|||
return nil, nil |
|||
} |
|||
return i.String(), nil |
|||
} |
|||
|
|||
// Scan implements the sql.Scanner interface. It supports converting from
|
|||
// string, []byte, or nil into a KSUID value. Attempting to convert from
|
|||
// another type will return an error.
|
|||
func (i *KSUID) Scan(src interface{}) error { |
|||
switch v := src.(type) { |
|||
case nil: |
|||
return i.scan(nil) |
|||
case []byte: |
|||
return i.scan(v) |
|||
case string: |
|||
return i.scan([]byte(v)) |
|||
default: |
|||
return fmt.Errorf("Scan: unable to scan type %T into KSUID", v) |
|||
} |
|||
} |
|||
|
|||
func (i *KSUID) scan(b []byte) error { |
|||
switch len(b) { |
|||
case 0: |
|||
*i = Nil |
|||
return nil |
|||
case byteLength: |
|||
return i.UnmarshalBinary(b) |
|||
case stringEncodedLength: |
|||
return i.UnmarshalText(b) |
|||
default: |
|||
return errSize |
|||
} |
|||
} |
|||
|
|||
// Parse decodes a string-encoded representation of a KSUID object
|
|||
func Parse(s string) (KSUID, error) { |
|||
if len(s) != stringEncodedLength { |
|||
return Nil, errStrSize |
|||
} |
|||
|
|||
src := [stringEncodedLength]byte{} |
|||
dst := [byteLength]byte{} |
|||
|
|||
copy(src[:], s[:]) |
|||
|
|||
if err := fastDecodeBase62(dst[:], src[:]); err != nil { |
|||
return Nil, errStrValue |
|||
} |
|||
|
|||
return FromBytes(dst[:]) |
|||
} |
|||
|
|||
func timeToCorrectedUTCTimestamp(t time.Time) uint32 { |
|||
return uint32(t.Unix() - epochStamp) |
|||
} |
|||
|
|||
func correctedUTCTimestampToTime(ts uint32) time.Time { |
|||
return time.Unix(int64(ts)+epochStamp, 0) |
|||
} |
|||
|
|||
// Generates a new KSUID. In the strange case that random bytes
|
|||
// can't be read, it will panic.
|
|||
func New() KSUID { |
|||
ksuid, err := NewRandom() |
|||
if err != nil { |
|||
panic(fmt.Sprintf("Couldn't generate KSUID, inconceivable! error: %v", err)) |
|||
} |
|||
return ksuid |
|||
} |
|||
|
|||
// Generates a new KSUID
|
|||
func NewRandom() (ksuid KSUID, err error) { |
|||
return NewRandomWithTime(time.Now()) |
|||
} |
|||
|
|||
func NewRandomWithTime(t time.Time) (ksuid KSUID, err error) { |
|||
// Go's default random number generators are not safe for concurrent use by
|
|||
// multiple goroutines, the use of the rander and randBuffer are explicitly
|
|||
// synchronized here.
|
|||
randMutex.Lock() |
|||
|
|||
_, err = io.ReadAtLeast(rander, randBuffer[:], len(randBuffer)) |
|||
copy(ksuid[timestampLengthInBytes:], randBuffer[:]) |
|||
|
|||
randMutex.Unlock() |
|||
|
|||
if err != nil { |
|||
ksuid = Nil // don't leak random bytes on error
|
|||
return |
|||
} |
|||
|
|||
ts := timeToCorrectedUTCTimestamp(t) |
|||
binary.BigEndian.PutUint32(ksuid[:timestampLengthInBytes], ts) |
|||
return |
|||
} |
|||
|
|||
// Constructs a KSUID from constituent parts
|
|||
func FromParts(t time.Time, payload []byte) (KSUID, error) { |
|||
if len(payload) != payloadLengthInBytes { |
|||
return Nil, errPayloadSize |
|||
} |
|||
|
|||
var ksuid KSUID |
|||
|
|||
ts := timeToCorrectedUTCTimestamp(t) |
|||
binary.BigEndian.PutUint32(ksuid[:timestampLengthInBytes], ts) |
|||
|
|||
copy(ksuid[timestampLengthInBytes:], payload) |
|||
|
|||
return ksuid, nil |
|||
} |
|||
|
|||
// Constructs a KSUID from a 20-byte binary representation
|
|||
func FromBytes(b []byte) (KSUID, error) { |
|||
var ksuid KSUID |
|||
|
|||
if len(b) != byteLength { |
|||
return Nil, errSize |
|||
} |
|||
|
|||
copy(ksuid[:], b) |
|||
return ksuid, nil |
|||
} |
|||
|
|||
// Sets the global source of random bytes for KSUID generation. This
|
|||
// should probably only be set once globally. While this is technically
|
|||
// thread-safe as in it won't cause corruption, there's no guarantee
|
|||
// on ordering.
|
|||
func SetRand(r io.Reader) { |
|||
if r == nil { |
|||
rander = rand.Reader |
|||
return |
|||
} |
|||
rander = r |
|||
} |
|||
|
|||
// Implements comparison for KSUID type
|
|||
func Compare(a, b KSUID) int { |
|||
return bytes.Compare(a[:], b[:]) |
|||
} |
|||
|
|||
// Sorts the given slice of KSUIDs
|
|||
func Sort(ids []KSUID) { |
|||
quickSort(ids, 0, len(ids)-1) |
|||
} |
|||
|
|||
// IsSorted checks whether a slice of KSUIDs is sorted
|
|||
func IsSorted(ids []KSUID) bool { |
|||
if len(ids) != 0 { |
|||
min := ids[0] |
|||
for _, id := range ids[1:] { |
|||
if bytes.Compare(min[:], id[:]) > 0 { |
|||
return false |
|||
} |
|||
min = id |
|||
} |
|||
} |
|||
return true |
|||
} |
|||
|
|||
func quickSort(a []KSUID, lo int, hi int) { |
|||
if lo < hi { |
|||
pivot := a[hi] |
|||
i := lo - 1 |
|||
|
|||
for j, n := lo, hi; j != n; j++ { |
|||
if bytes.Compare(a[j][:], pivot[:]) < 0 { |
|||
i++ |
|||
a[i], a[j] = a[j], a[i] |
|||
} |
|||
} |
|||
|
|||
i++ |
|||
if bytes.Compare(a[hi][:], a[i][:]) < 0 { |
|||
a[i], a[hi] = a[hi], a[i] |
|||
} |
|||
|
|||
quickSort(a, lo, i-1) |
|||
quickSort(a, i+1, hi) |
|||
} |
|||
} |
|||
|
|||
// Next returns the next KSUID after id.
|
|||
func (id KSUID) Next() KSUID { |
|||
zero := makeUint128(0, 0) |
|||
|
|||
t := id.Timestamp() |
|||
u := uint128Payload(id) |
|||
v := add128(u, makeUint128(0, 1)) |
|||
|
|||
if v == zero { // overflow
|
|||
t++ |
|||
} |
|||
|
|||
return v.ksuid(t) |
|||
} |
|||
|
|||
// Prev returns the previoud KSUID before id.
|
|||
func (id KSUID) Prev() KSUID { |
|||
max := makeUint128(math.MaxUint64, math.MaxUint64) |
|||
|
|||
t := id.Timestamp() |
|||
u := uint128Payload(id) |
|||
v := sub128(u, makeUint128(0, 1)) |
|||
|
|||
if v == max { // overflow
|
|||
t-- |
|||
} |
|||
|
|||
return v.ksuid(t) |
|||
} |
|||
@ -0,0 +1,55 @@ |
|||
package ksuid |
|||
|
|||
import ( |
|||
cryptoRand "crypto/rand" |
|||
"encoding/binary" |
|||
"io" |
|||
"math/rand" |
|||
) |
|||
|
|||
// FastRander is an io.Reader that uses math/rand and is optimized for
|
|||
// generating 16 bytes KSUID payloads. It is intended to be used as a
|
|||
// performance improvements for programs that have no need for
|
|||
// cryptographically secure KSUIDs and are generating a lot of them.
|
|||
var FastRander = newRBG() |
|||
|
|||
func newRBG() io.Reader { |
|||
r, err := newRandomBitsGenerator() |
|||
if err != nil { |
|||
panic(err) |
|||
} |
|||
return r |
|||
} |
|||
|
|||
func newRandomBitsGenerator() (r io.Reader, err error) { |
|||
var seed int64 |
|||
|
|||
if seed, err = readCryptoRandomSeed(); err != nil { |
|||
return |
|||
} |
|||
|
|||
r = &randSourceReader{source: rand.NewSource(seed).(rand.Source64)} |
|||
return |
|||
} |
|||
|
|||
func readCryptoRandomSeed() (seed int64, err error) { |
|||
var b [8]byte |
|||
|
|||
if _, err = io.ReadFull(cryptoRand.Reader, b[:]); err != nil { |
|||
return |
|||
} |
|||
|
|||
seed = int64(binary.LittleEndian.Uint64(b[:])) |
|||
return |
|||
} |
|||
|
|||
type randSourceReader struct { |
|||
source rand.Source64 |
|||
} |
|||
|
|||
func (r *randSourceReader) Read(b []byte) (int, error) { |
|||
// optimized for generating 16 bytes payloads
|
|||
binary.LittleEndian.PutUint64(b[:8], r.source.Uint64()) |
|||
binary.LittleEndian.PutUint64(b[8:], r.source.Uint64()) |
|||
return 16, nil |
|||
} |
|||
@ -0,0 +1,55 @@ |
|||
package ksuid |
|||
|
|||
import ( |
|||
"encoding/binary" |
|||
"errors" |
|||
"math" |
|||
) |
|||
|
|||
// Sequence is a KSUID generator which produces a sequence of ordered KSUIDs
|
|||
// from a seed.
|
|||
//
|
|||
// Up to 65536 KSUIDs can be generated by for a single seed.
|
|||
//
|
|||
// A typical usage of a Sequence looks like this:
|
|||
//
|
|||
// seq := ksuid.Sequence{
|
|||
// Seed: ksuid.New(),
|
|||
// }
|
|||
// id, err := seq.Next()
|
|||
//
|
|||
// Sequence values are not safe to use concurrently from multiple goroutines.
|
|||
type Sequence struct { |
|||
// The seed is used as base for the KSUID generator, all generated KSUIDs
|
|||
// share the same leading 18 bytes of the seed.
|
|||
Seed KSUID |
|||
count uint32 // uint32 for overflow, only 2 bytes are used
|
|||
} |
|||
|
|||
// Next produces the next KSUID in the sequence, or returns an error if the
|
|||
// sequence has been exhausted.
|
|||
func (seq *Sequence) Next() (KSUID, error) { |
|||
id := seq.Seed // copy
|
|||
count := seq.count |
|||
if count > math.MaxUint16 { |
|||
return Nil, errors.New("too many IDs were generated") |
|||
} |
|||
seq.count++ |
|||
return withSequenceNumber(id, uint16(count)), nil |
|||
} |
|||
|
|||
// Bounds returns the inclusive min and max bounds of the KSUIDs that may be
|
|||
// generated by the sequence. If all ids have been generated already then the
|
|||
// returned min value is equal to the max.
|
|||
func (seq *Sequence) Bounds() (min KSUID, max KSUID) { |
|||
count := seq.count |
|||
if count > math.MaxUint16 { |
|||
count = math.MaxUint16 |
|||
} |
|||
return withSequenceNumber(seq.Seed, uint16(count)), withSequenceNumber(seq.Seed, math.MaxUint16) |
|||
} |
|||
|
|||
func withSequenceNumber(id KSUID, n uint16) KSUID { |
|||
binary.BigEndian.PutUint16(id[len(id)-2:], n) |
|||
return id |
|||
} |
|||
@ -0,0 +1,343 @@ |
|||
package ksuid |
|||
|
|||
import ( |
|||
"bytes" |
|||
"encoding/binary" |
|||
) |
|||
|
|||
// CompressedSet is an immutable data type which stores a set of KSUIDs.
|
|||
type CompressedSet []byte |
|||
|
|||
// Iter returns an iterator that produces all KSUIDs in the set.
|
|||
func (set CompressedSet) Iter() CompressedSetIter { |
|||
return CompressedSetIter{ |
|||
content: []byte(set), |
|||
} |
|||
} |
|||
|
|||
// String satisfies the fmt.Stringer interface, returns a human-readable string
|
|||
// representation of the set.
|
|||
func (set CompressedSet) String() string { |
|||
b := bytes.Buffer{} |
|||
b.WriteByte('[') |
|||
set.writeTo(&b) |
|||
b.WriteByte(']') |
|||
return b.String() |
|||
} |
|||
|
|||
// String satisfies the fmt.GoStringer interface, returns a Go representation of
|
|||
// the set.
|
|||
func (set CompressedSet) GoString() string { |
|||
b := bytes.Buffer{} |
|||
b.WriteString("ksuid.CompressedSet{") |
|||
set.writeTo(&b) |
|||
b.WriteByte('}') |
|||
return b.String() |
|||
} |
|||
|
|||
func (set CompressedSet) writeTo(b *bytes.Buffer) { |
|||
a := [27]byte{} |
|||
|
|||
for i, it := 0, set.Iter(); it.Next(); i++ { |
|||
if i != 0 { |
|||
b.WriteString(", ") |
|||
} |
|||
b.WriteByte('"') |
|||
it.KSUID.Append(a[:0]) |
|||
b.Write(a[:]) |
|||
b.WriteByte('"') |
|||
} |
|||
} |
|||
|
|||
// Compress creates and returns a compressed set of KSUIDs from the list given
|
|||
// as arguments.
|
|||
func Compress(ids ...KSUID) CompressedSet { |
|||
c := 1 + byteLength + (len(ids) / 5) |
|||
b := make([]byte, 0, c) |
|||
return AppendCompressed(b, ids...) |
|||
} |
|||
|
|||
// AppendCompressed uses the given byte slice as pre-allocated storage space to
|
|||
// build a KSUID set.
|
|||
//
|
|||
// Note that the set uses a compression technique to store the KSUIDs, so the
|
|||
// resuling length is not 20 x len(ids). The rule of thumb here is for the given
|
|||
// byte slice to reserve the amount of memory that the application would be OK
|
|||
// to waste.
|
|||
func AppendCompressed(set []byte, ids ...KSUID) CompressedSet { |
|||
if len(ids) != 0 { |
|||
if !IsSorted(ids) { |
|||
Sort(ids) |
|||
} |
|||
one := makeUint128(0, 1) |
|||
|
|||
// The first KSUID is always written to the set, this is the starting
|
|||
// point for all deltas.
|
|||
set = append(set, byte(rawKSUID)) |
|||
set = append(set, ids[0][:]...) |
|||
|
|||
timestamp := ids[0].Timestamp() |
|||
lastKSUID := ids[0] |
|||
lastValue := uint128Payload(ids[0]) |
|||
|
|||
for i := 1; i != len(ids); i++ { |
|||
id := ids[i] |
|||
|
|||
if id == lastKSUID { |
|||
continue |
|||
} |
|||
|
|||
t := id.Timestamp() |
|||
v := uint128Payload(id) |
|||
|
|||
if t != timestamp { |
|||
d := t - timestamp |
|||
n := varintLength32(d) |
|||
|
|||
set = append(set, timeDelta|byte(n)) |
|||
set = appendVarint32(set, d, n) |
|||
set = append(set, id[timestampLengthInBytes:]...) |
|||
|
|||
timestamp = t |
|||
} else { |
|||
d := sub128(v, lastValue) |
|||
|
|||
if d != one { |
|||
n := varintLength128(d) |
|||
|
|||
set = append(set, payloadDelta|byte(n)) |
|||
set = appendVarint128(set, d, n) |
|||
} else { |
|||
l, c := rangeLength(ids[i+1:], t, id, v) |
|||
m := uint64(l + 1) |
|||
n := varintLength64(m) |
|||
|
|||
set = append(set, payloadRange|byte(n)) |
|||
set = appendVarint64(set, m, n) |
|||
|
|||
i += c |
|||
id = ids[i] |
|||
v = uint128Payload(id) |
|||
} |
|||
} |
|||
|
|||
lastKSUID = id |
|||
lastValue = v |
|||
} |
|||
} |
|||
return CompressedSet(set) |
|||
} |
|||
|
|||
func rangeLength(ids []KSUID, timestamp uint32, lastKSUID KSUID, lastValue uint128) (length int, count int) { |
|||
one := makeUint128(0, 1) |
|||
|
|||
for i := range ids { |
|||
id := ids[i] |
|||
|
|||
if id == lastKSUID { |
|||
continue |
|||
} |
|||
|
|||
if id.Timestamp() != timestamp { |
|||
count = i |
|||
return |
|||
} |
|||
|
|||
v := uint128Payload(id) |
|||
|
|||
if sub128(v, lastValue) != one { |
|||
count = i |
|||
return |
|||
} |
|||
|
|||
lastKSUID = id |
|||
lastValue = v |
|||
length++ |
|||
} |
|||
|
|||
count = len(ids) |
|||
return |
|||
} |
|||
|
|||
func appendVarint128(b []byte, v uint128, n int) []byte { |
|||
c := v.bytes() |
|||
return append(b, c[len(c)-n:]...) |
|||
} |
|||
|
|||
func appendVarint64(b []byte, v uint64, n int) []byte { |
|||
c := [8]byte{} |
|||
binary.BigEndian.PutUint64(c[:], v) |
|||
return append(b, c[len(c)-n:]...) |
|||
} |
|||
|
|||
func appendVarint32(b []byte, v uint32, n int) []byte { |
|||
c := [4]byte{} |
|||
binary.BigEndian.PutUint32(c[:], v) |
|||
return append(b, c[len(c)-n:]...) |
|||
} |
|||
|
|||
func varint128(b []byte) uint128 { |
|||
a := [16]byte{} |
|||
copy(a[16-len(b):], b) |
|||
return makeUint128FromPayload(a[:]) |
|||
} |
|||
|
|||
func varint64(b []byte) uint64 { |
|||
a := [8]byte{} |
|||
copy(a[8-len(b):], b) |
|||
return binary.BigEndian.Uint64(a[:]) |
|||
} |
|||
|
|||
func varint32(b []byte) uint32 { |
|||
a := [4]byte{} |
|||
copy(a[4-len(b):], b) |
|||
return binary.BigEndian.Uint32(a[:]) |
|||
} |
|||
|
|||
func varintLength128(v uint128) int { |
|||
if v[1] != 0 { |
|||
return 8 + varintLength64(v[1]) |
|||
} |
|||
return varintLength64(v[0]) |
|||
} |
|||
|
|||
func varintLength64(v uint64) int { |
|||
switch { |
|||
case (v & 0xFFFFFFFFFFFFFF00) == 0: |
|||
return 1 |
|||
case (v & 0xFFFFFFFFFFFF0000) == 0: |
|||
return 2 |
|||
case (v & 0xFFFFFFFFFF000000) == 0: |
|||
return 3 |
|||
case (v & 0xFFFFFFFF00000000) == 0: |
|||
return 4 |
|||
case (v & 0xFFFFFF0000000000) == 0: |
|||
return 5 |
|||
case (v & 0xFFFF000000000000) == 0: |
|||
return 6 |
|||
case (v & 0xFF00000000000000) == 0: |
|||
return 7 |
|||
default: |
|||
return 8 |
|||
} |
|||
} |
|||
|
|||
func varintLength32(v uint32) int { |
|||
switch { |
|||
case (v & 0xFFFFFF00) == 0: |
|||
return 1 |
|||
case (v & 0xFFFF0000) == 0: |
|||
return 2 |
|||
case (v & 0xFF000000) == 0: |
|||
return 3 |
|||
default: |
|||
return 4 |
|||
} |
|||
} |
|||
|
|||
const ( |
|||
rawKSUID = 0 |
|||
timeDelta = (1 << 6) |
|||
payloadDelta = (1 << 7) |
|||
payloadRange = (1 << 6) | (1 << 7) |
|||
) |
|||
|
|||
// CompressedSetIter is an iterator type returned by Set.Iter to produce the
|
|||
// list of KSUIDs stored in a set.
|
|||
//
|
|||
// Here's is how the iterator type is commonly used:
|
|||
//
|
|||
// for it := set.Iter(); it.Next(); {
|
|||
// id := it.KSUID
|
|||
// // ...
|
|||
// }
|
|||
//
|
|||
// CompressedSetIter values are not safe to use concurrently from multiple
|
|||
// goroutines.
|
|||
type CompressedSetIter struct { |
|||
// KSUID is modified by calls to the Next method to hold the KSUID loaded
|
|||
// by the iterator.
|
|||
KSUID KSUID |
|||
|
|||
content []byte |
|||
offset int |
|||
|
|||
seqlength uint64 |
|||
timestamp uint32 |
|||
lastValue uint128 |
|||
} |
|||
|
|||
// Next moves the iterator forward, returning true if there a KSUID was found,
|
|||
// or false if the iterator as reached the end of the set it was created from.
|
|||
func (it *CompressedSetIter) Next() bool { |
|||
if it.seqlength != 0 { |
|||
value := incr128(it.lastValue) |
|||
it.KSUID = value.ksuid(it.timestamp) |
|||
it.seqlength-- |
|||
it.lastValue = value |
|||
return true |
|||
} |
|||
|
|||
if it.offset == len(it.content) { |
|||
return false |
|||
} |
|||
|
|||
b := it.content[it.offset] |
|||
it.offset++ |
|||
|
|||
const mask = rawKSUID | timeDelta | payloadDelta | payloadRange |
|||
tag := int(b) & mask |
|||
cnt := int(b) & ^mask |
|||
|
|||
switch tag { |
|||
case rawKSUID: |
|||
off0 := it.offset |
|||
off1 := off0 + byteLength |
|||
|
|||
copy(it.KSUID[:], it.content[off0:off1]) |
|||
|
|||
it.offset = off1 |
|||
it.timestamp = it.KSUID.Timestamp() |
|||
it.lastValue = uint128Payload(it.KSUID) |
|||
|
|||
case timeDelta: |
|||
off0 := it.offset |
|||
off1 := off0 + cnt |
|||
off2 := off1 + payloadLengthInBytes |
|||
|
|||
it.timestamp += varint32(it.content[off0:off1]) |
|||
|
|||
binary.BigEndian.PutUint32(it.KSUID[:timestampLengthInBytes], it.timestamp) |
|||
copy(it.KSUID[timestampLengthInBytes:], it.content[off1:off2]) |
|||
|
|||
it.offset = off2 |
|||
it.lastValue = uint128Payload(it.KSUID) |
|||
|
|||
case payloadDelta: |
|||
off0 := it.offset |
|||
off1 := off0 + cnt |
|||
|
|||
delta := varint128(it.content[off0:off1]) |
|||
value := add128(it.lastValue, delta) |
|||
|
|||
it.KSUID = value.ksuid(it.timestamp) |
|||
it.offset = off1 |
|||
it.lastValue = value |
|||
|
|||
case payloadRange: |
|||
off0 := it.offset |
|||
off1 := off0 + cnt |
|||
|
|||
value := incr128(it.lastValue) |
|||
it.KSUID = value.ksuid(it.timestamp) |
|||
it.seqlength = varint64(it.content[off0:off1]) |
|||
it.offset = off1 |
|||
it.seqlength-- |
|||
it.lastValue = value |
|||
|
|||
default: |
|||
panic("KSUID set iterator is reading malformed data") |
|||
} |
|||
|
|||
return true |
|||
} |
|||
@ -0,0 +1,141 @@ |
|||
package ksuid |
|||
|
|||
import "fmt" |
|||
|
|||
// uint128 represents an unsigned 128 bits little endian integer.
|
|||
type uint128 [2]uint64 |
|||
|
|||
func uint128Payload(ksuid KSUID) uint128 { |
|||
return makeUint128FromPayload(ksuid[timestampLengthInBytes:]) |
|||
} |
|||
|
|||
func makeUint128(high uint64, low uint64) uint128 { |
|||
return uint128{low, high} |
|||
} |
|||
|
|||
func makeUint128FromPayload(payload []byte) uint128 { |
|||
return uint128{ |
|||
// low
|
|||
uint64(payload[8])<<56 | |
|||
uint64(payload[9])<<48 | |
|||
uint64(payload[10])<<40 | |
|||
uint64(payload[11])<<32 | |
|||
uint64(payload[12])<<24 | |
|||
uint64(payload[13])<<16 | |
|||
uint64(payload[14])<<8 | |
|||
uint64(payload[15]), |
|||
// high
|
|||
uint64(payload[0])<<56 | |
|||
uint64(payload[1])<<48 | |
|||
uint64(payload[2])<<40 | |
|||
uint64(payload[3])<<32 | |
|||
uint64(payload[4])<<24 | |
|||
uint64(payload[5])<<16 | |
|||
uint64(payload[6])<<8 | |
|||
uint64(payload[7]), |
|||
} |
|||
} |
|||
|
|||
func (v uint128) ksuid(timestamp uint32) KSUID { |
|||
return KSUID{ |
|||
// time
|
|||
byte(timestamp >> 24), |
|||
byte(timestamp >> 16), |
|||
byte(timestamp >> 8), |
|||
byte(timestamp), |
|||
|
|||
// high
|
|||
byte(v[1] >> 56), |
|||
byte(v[1] >> 48), |
|||
byte(v[1] >> 40), |
|||
byte(v[1] >> 32), |
|||
byte(v[1] >> 24), |
|||
byte(v[1] >> 16), |
|||
byte(v[1] >> 8), |
|||
byte(v[1]), |
|||
|
|||
// low
|
|||
byte(v[0] >> 56), |
|||
byte(v[0] >> 48), |
|||
byte(v[0] >> 40), |
|||
byte(v[0] >> 32), |
|||
byte(v[0] >> 24), |
|||
byte(v[0] >> 16), |
|||
byte(v[0] >> 8), |
|||
byte(v[0]), |
|||
} |
|||
} |
|||
|
|||
func (v uint128) bytes() [16]byte { |
|||
return [16]byte{ |
|||
// high
|
|||
byte(v[1] >> 56), |
|||
byte(v[1] >> 48), |
|||
byte(v[1] >> 40), |
|||
byte(v[1] >> 32), |
|||
byte(v[1] >> 24), |
|||
byte(v[1] >> 16), |
|||
byte(v[1] >> 8), |
|||
byte(v[1]), |
|||
|
|||
// low
|
|||
byte(v[0] >> 56), |
|||
byte(v[0] >> 48), |
|||
byte(v[0] >> 40), |
|||
byte(v[0] >> 32), |
|||
byte(v[0] >> 24), |
|||
byte(v[0] >> 16), |
|||
byte(v[0] >> 8), |
|||
byte(v[0]), |
|||
} |
|||
} |
|||
|
|||
func (v uint128) String() string { |
|||
return fmt.Sprintf("0x%016X%016X", v[0], v[1]) |
|||
} |
|||
|
|||
const wordBitSize = 64 |
|||
|
|||
func cmp128(x, y uint128) int { |
|||
if x[1] < y[1] { |
|||
return -1 |
|||
} |
|||
if x[1] > y[1] { |
|||
return 1 |
|||
} |
|||
if x[0] < y[0] { |
|||
return -1 |
|||
} |
|||
if x[0] > y[0] { |
|||
return 1 |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func add128(x, y uint128) (z uint128) { |
|||
x0 := x[0] |
|||
y0 := y[0] |
|||
z0 := x0 + y0 |
|||
z[0] = z0 |
|||
|
|||
c := (x0&y0 | (x0|y0)&^z0) >> (wordBitSize - 1) |
|||
|
|||
z[1] = x[1] + y[1] + c |
|||
return |
|||
} |
|||
|
|||
func sub128(x, y uint128) (z uint128) { |
|||
x0 := x[0] |
|||
y0 := y[0] |
|||
z0 := x0 - y0 |
|||
z[0] = z0 |
|||
|
|||
c := (y0&^x0 | (y0|^x0)&z0) >> (wordBitSize - 1) |
|||
|
|||
z[1] = x[1] - y[1] - c |
|||
return |
|||
} |
|||
|
|||
func incr128(x uint128) uint128 { |
|||
return add128(x, uint128{1, 0}) |
|||
} |
|||
Loading…
Reference in new issue