Merge pull request #3775 from thaJeztah/remove_digestset
Milos Gajdos
GitHub
commit
9bb63e6e46
|
|
@ -0,0 +1,51 @@
|
||||||
|
package digestset
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/opencontainers/go-digest"
|
||||||
|
"github.com/opencontainers/go-digest/digestset"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrDigestNotFound is used when a matching digest
|
||||||
|
// could not be found in a set.
|
||||||
|
//
|
||||||
|
// Deprecated: use [digestset.ErrDigestNotFound].
|
||||||
|
var ErrDigestNotFound = digestset.ErrDigestNotFound
|
||||||
|
|
||||||
|
// ErrDigestAmbiguous is used when multiple digests
|
||||||
|
// are found in a set. None of the matching digests
|
||||||
|
// should be considered valid matches.
|
||||||
|
//
|
||||||
|
// Deprecated: use [digestset.ErrDigestAmbiguous].
|
||||||
|
var ErrDigestAmbiguous = digestset.ErrDigestAmbiguous
|
||||||
|
|
||||||
|
// Set is used to hold a unique set of digests which
|
||||||
|
// may be easily referenced by a string
|
||||||
|
// representation of the digest as well as short representation.
|
||||||
|
// The uniqueness of the short representation is based on other
|
||||||
|
// digests in the set. If digests are omitted from this set,
|
||||||
|
// collisions in a larger set may not be detected, therefore it
|
||||||
|
// is important to always do short representation lookups on
|
||||||
|
// the complete set of digests. To mitigate collisions, an
|
||||||
|
// appropriately long short code should be used.
|
||||||
|
//
|
||||||
|
// Deprecated: use [digestset.Set].
|
||||||
|
type Set = digestset.Set
|
||||||
|
|
||||||
|
// NewSet creates an empty set of digests
|
||||||
|
// which may have digests added.
|
||||||
|
//
|
||||||
|
// Deprecated: use [digestset.NewSet].
|
||||||
|
func NewSet() *digestset.Set {
|
||||||
|
return digestset.NewSet()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ShortCodeTable returns a map of Digest to unique short codes. The
|
||||||
|
// length represents the minimum value, the maximum length may be the
|
||||||
|
// entire value of digest if uniqueness cannot be achieved without the
|
||||||
|
// full value. This function will attempt to make short codes as short
|
||||||
|
// as possible to be unique.
|
||||||
|
//
|
||||||
|
// Deprecated: use [digestset.ShortCodeTable].
|
||||||
|
func ShortCodeTable(dst *digestset.Set, length int) map[digest.Digest]string {
|
||||||
|
return digestset.ShortCodeTable(dst, length)
|
||||||
|
}
|
||||||
|
|
@ -1,372 +0,0 @@
|
||||||
package digestset
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/sha256"
|
|
||||||
_ "crypto/sha512"
|
|
||||||
"encoding/binary"
|
|
||||||
"math/rand"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
digest "github.com/opencontainers/go-digest"
|
|
||||||
)
|
|
||||||
|
|
||||||
func assertEqualDigests(t *testing.T, d1, d2 digest.Digest) {
|
|
||||||
if d1 != d2 {
|
|
||||||
t.Fatalf("Digests do not match:\n\tActual: %s\n\tExpected: %s", d1, d2)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLookup(t *testing.T) {
|
|
||||||
digests := []digest.Digest{
|
|
||||||
"sha256:1234511111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha256:1234111111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha256:1234611111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha256:5432111111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha256:6543111111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha256:6432111111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha256:6542111111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha256:6532111111111111111111111111111111111111111111111111111111111111",
|
|
||||||
}
|
|
||||||
|
|
||||||
dset := NewSet()
|
|
||||||
for i := range digests {
|
|
||||||
if err := dset.Add(digests[i]); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
dgst, err := dset.Lookup("54")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
assertEqualDigests(t, dgst, digests[3])
|
|
||||||
|
|
||||||
_, err = dset.Lookup("1234")
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("Expected ambiguous error looking up: 1234")
|
|
||||||
}
|
|
||||||
if err != ErrDigestAmbiguous {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = dset.Lookup("9876")
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("Expected not found error looking up: 9876")
|
|
||||||
}
|
|
||||||
if err != ErrDigestNotFound {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = dset.Lookup("sha256:1234")
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("Expected ambiguous error looking up: sha256:1234")
|
|
||||||
}
|
|
||||||
if err != ErrDigestAmbiguous {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
dgst, err = dset.Lookup("sha256:12345")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
assertEqualDigests(t, dgst, digests[0])
|
|
||||||
|
|
||||||
dgst, err = dset.Lookup("sha256:12346")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
assertEqualDigests(t, dgst, digests[2])
|
|
||||||
|
|
||||||
dgst, err = dset.Lookup("12346")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
assertEqualDigests(t, dgst, digests[2])
|
|
||||||
|
|
||||||
dgst, err = dset.Lookup("12345")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
assertEqualDigests(t, dgst, digests[0])
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAddDuplication(t *testing.T) {
|
|
||||||
digests := []digest.Digest{
|
|
||||||
"sha256:1234111111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha256:1234511111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha256:1234611111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha256:5432111111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha256:6543111111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha512:65431111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha512:65421111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha512:65321111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111",
|
|
||||||
}
|
|
||||||
|
|
||||||
dset := NewSet()
|
|
||||||
for i := range digests {
|
|
||||||
if err := dset.Add(digests[i]); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(dset.entries) != 8 {
|
|
||||||
t.Fatal("Invalid dset size")
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := dset.Add(digest.Digest("sha256:1234511111111111111111111111111111111111111111111111111111111111")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(dset.entries) != 8 {
|
|
||||||
t.Fatal("Duplicate digest insert should not increase entries size")
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := dset.Add(digest.Digest("sha384:123451111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(dset.entries) != 9 {
|
|
||||||
t.Fatal("Insert with different algorithm should be allowed")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRemove(t *testing.T) {
|
|
||||||
digests, err := createDigests(10)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
dset := NewSet()
|
|
||||||
for i := range digests {
|
|
||||||
if err := dset.Add(digests[i]); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
dgst, err := dset.Lookup(digests[0].String())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if dgst != digests[0] {
|
|
||||||
t.Fatalf("Unexpected digest value:\n\tExpected: %s\n\tActual: %s", digests[0], dgst)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := dset.Remove(digests[0]); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := dset.Lookup(digests[0].String()); err != ErrDigestNotFound {
|
|
||||||
t.Fatalf("Expected error %v when looking up removed digest, got %v", ErrDigestNotFound, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAll(t *testing.T) {
|
|
||||||
digests, err := createDigests(100)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
dset := NewSet()
|
|
||||||
for i := range digests {
|
|
||||||
if err := dset.Add(digests[i]); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
all := map[digest.Digest]struct{}{}
|
|
||||||
for _, dgst := range dset.All() {
|
|
||||||
all[dgst] = struct{}{}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(all) != len(digests) {
|
|
||||||
t.Fatalf("Unexpected number of unique digests found:\n\tExpected: %d\n\tActual: %d", len(digests), len(all))
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, dgst := range digests {
|
|
||||||
if _, ok := all[dgst]; !ok {
|
|
||||||
t.Fatalf("Missing element at position %d: %s", i, dgst)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertEqualShort(t *testing.T, actual, expected string) {
|
|
||||||
if actual != expected {
|
|
||||||
t.Fatalf("Unexpected short value:\n\tExpected: %s\n\tActual: %s", expected, actual)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestShortCodeTable(t *testing.T) {
|
|
||||||
digests := []digest.Digest{
|
|
||||||
"sha256:1234111111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha256:1234511111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha256:1234611111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha256:5432111111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha256:6543111111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha256:6432111111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha256:6542111111111111111111111111111111111111111111111111111111111111",
|
|
||||||
"sha256:6532111111111111111111111111111111111111111111111111111111111111",
|
|
||||||
}
|
|
||||||
|
|
||||||
dset := NewSet()
|
|
||||||
for i := range digests {
|
|
||||||
if err := dset.Add(digests[i]); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
dump := ShortCodeTable(dset, 2)
|
|
||||||
|
|
||||||
if len(dump) < len(digests) {
|
|
||||||
t.Fatalf("Error unexpected size: %d, expecting %d", len(dump), len(digests))
|
|
||||||
}
|
|
||||||
assertEqualShort(t, dump[digests[0]], "12341")
|
|
||||||
assertEqualShort(t, dump[digests[1]], "12345")
|
|
||||||
assertEqualShort(t, dump[digests[2]], "12346")
|
|
||||||
assertEqualShort(t, dump[digests[3]], "54")
|
|
||||||
assertEqualShort(t, dump[digests[4]], "6543")
|
|
||||||
assertEqualShort(t, dump[digests[5]], "64")
|
|
||||||
assertEqualShort(t, dump[digests[6]], "6542")
|
|
||||||
assertEqualShort(t, dump[digests[7]], "653")
|
|
||||||
}
|
|
||||||
|
|
||||||
func createDigests(count int) ([]digest.Digest, error) {
|
|
||||||
r := rand.New(rand.NewSource(25823))
|
|
||||||
digests := make([]digest.Digest, count)
|
|
||||||
for i := range digests {
|
|
||||||
h := sha256.New()
|
|
||||||
if err := binary.Write(h, binary.BigEndian, r.Int63()); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
digests[i] = digest.NewDigest("sha256", h)
|
|
||||||
}
|
|
||||||
return digests, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func benchAddNTable(b *testing.B, n int) {
|
|
||||||
digests, err := createDigests(n)
|
|
||||||
if err != nil {
|
|
||||||
b.Fatal(err)
|
|
||||||
}
|
|
||||||
b.ResetTimer()
|
|
||||||
for i := 0; i < b.N; i++ {
|
|
||||||
dset := &Set{entries: digestEntries(make([]*digestEntry, 0, n))}
|
|
||||||
for j := range digests {
|
|
||||||
if err = dset.Add(digests[j]); err != nil {
|
|
||||||
b.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func benchLookupNTable(b *testing.B, n int, shortLen int) {
|
|
||||||
digests, err := createDigests(n)
|
|
||||||
if err != nil {
|
|
||||||
b.Fatal(err)
|
|
||||||
}
|
|
||||||
dset := &Set{entries: digestEntries(make([]*digestEntry, 0, n))}
|
|
||||||
for i := range digests {
|
|
||||||
if err := dset.Add(digests[i]); err != nil {
|
|
||||||
b.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
shorts := make([]string, 0, n)
|
|
||||||
for _, short := range ShortCodeTable(dset, shortLen) {
|
|
||||||
shorts = append(shorts, short)
|
|
||||||
}
|
|
||||||
|
|
||||||
b.ResetTimer()
|
|
||||||
for i := 0; i < b.N; i++ {
|
|
||||||
if _, err = dset.Lookup(shorts[i%n]); err != nil {
|
|
||||||
b.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func benchRemoveNTable(b *testing.B, n int) {
|
|
||||||
digests, err := createDigests(n)
|
|
||||||
if err != nil {
|
|
||||||
b.Fatal(err)
|
|
||||||
}
|
|
||||||
b.ResetTimer()
|
|
||||||
for i := 0; i < b.N; i++ {
|
|
||||||
dset := &Set{entries: digestEntries(make([]*digestEntry, 0, n))}
|
|
||||||
b.StopTimer()
|
|
||||||
for j := range digests {
|
|
||||||
if err = dset.Add(digests[j]); err != nil {
|
|
||||||
b.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
b.StartTimer()
|
|
||||||
for j := range digests {
|
|
||||||
if err = dset.Remove(digests[j]); err != nil {
|
|
||||||
b.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func benchShortCodeNTable(b *testing.B, n int, shortLen int) {
|
|
||||||
digests, err := createDigests(n)
|
|
||||||
if err != nil {
|
|
||||||
b.Fatal(err)
|
|
||||||
}
|
|
||||||
dset := &Set{entries: digestEntries(make([]*digestEntry, 0, n))}
|
|
||||||
for i := range digests {
|
|
||||||
if err := dset.Add(digests[i]); err != nil {
|
|
||||||
b.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
b.ResetTimer()
|
|
||||||
for i := 0; i < b.N; i++ {
|
|
||||||
ShortCodeTable(dset, shortLen)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkAdd10(b *testing.B) {
|
|
||||||
benchAddNTable(b, 10)
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkAdd100(b *testing.B) {
|
|
||||||
benchAddNTable(b, 100)
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkAdd1000(b *testing.B) {
|
|
||||||
benchAddNTable(b, 1000)
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkRemove10(b *testing.B) {
|
|
||||||
benchRemoveNTable(b, 10)
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkRemove100(b *testing.B) {
|
|
||||||
benchRemoveNTable(b, 100)
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkRemove1000(b *testing.B) {
|
|
||||||
benchRemoveNTable(b, 1000)
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkLookup10(b *testing.B) {
|
|
||||||
benchLookupNTable(b, 10, 12)
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkLookup100(b *testing.B) {
|
|
||||||
benchLookupNTable(b, 100, 12)
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkLookup1000(b *testing.B) {
|
|
||||||
benchLookupNTable(b, 1000, 12)
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkShortCode10(b *testing.B) {
|
|
||||||
benchShortCodeNTable(b, 10, 12)
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkShortCode100(b *testing.B) {
|
|
||||||
benchShortCodeNTable(b, 100, 12)
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkShortCode1000(b *testing.B) {
|
|
||||||
benchShortCodeNTable(b, 1000, 12)
|
|
||||||
}
|
|
||||||
15
digestset/set.go → vendor/github.com/opencontainers/go-digest/digestset/set.go
generated
vendored
15
digestset/set.go → vendor/github.com/opencontainers/go-digest/digestset/set.go
generated
vendored
|
|
@ -1,3 +1,18 @@
|
||||||
|
// Copyright 2020, 2020 OCI Contributors
|
||||||
|
// Copyright 2017 Docker, Inc.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
package digestset
|
package digestset
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|
@ -181,6 +181,7 @@ github.com/ncw/swift/swifttest
|
||||||
# github.com/opencontainers/go-digest v1.0.0
|
# github.com/opencontainers/go-digest v1.0.0
|
||||||
## explicit; go 1.13
|
## explicit; go 1.13
|
||||||
github.com/opencontainers/go-digest
|
github.com/opencontainers/go-digest
|
||||||
|
github.com/opencontainers/go-digest/digestset
|
||||||
# github.com/opencontainers/image-spec v1.0.2
|
# github.com/opencontainers/image-spec v1.0.2
|
||||||
## explicit
|
## explicit
|
||||||
github.com/opencontainers/image-spec/specs-go
|
github.com/opencontainers/image-spec/specs-go
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue