daniel
/
distribution
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for manifest list ("fat manifest") 

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>
master
Aaron Lehmann 2015-12-16 17:26:13 -08:00
parent 66a33baa36
commit 7ef71988a8
6 changed files with 313 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

193
docs/handlers/api_test.go
View File

@ -23,6 +23,7 @@ import (
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest"
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/docker/distribution/registry/api/errcode"
@ -702,12 +703,14 @@ func TestManifestAPI(t *testing.T) {
deleteEnabled := false
env := newTestEnv(t, deleteEnabled)
testManifestAPISchema1(t, env, "foo/schema1")
testManifestAPISchema2(t, env, "foo/schema2")
schema2Args := testManifestAPISchema2(t, env, "foo/schema2")
testManifestAPIManifestList(t, env, schema2Args)
deleteEnabled = true
env = newTestEnv(t, deleteEnabled)
testManifestAPISchema1(t, env, "foo/schema1")
testManifestAPISchema2(t, env, "foo/schema2")
schema2Args = testManifestAPISchema2(t, env, "foo/schema2")
testManifestAPIManifestList(t, env, schema2Args)
}
func TestManifestDelete(t *testing.T) {
@ -1393,6 +1396,179 @@ func testManifestAPISchema2(t *testing.T, env *testEnv, imageName string) manife
return args
}
func testManifestAPIManifestList(t *testing.T, env *testEnv, args manifestArgs) {
imageName := args.imageName
tag := "manifestlisttag"
manifestURL, err := env.builder.BuildManifestURL(imageName, tag)
if err != nil {
t.Fatalf("unexpected error getting manifest url: %v", err)
}
// --------------------------------
// Attempt to push manifest list that refers to an unknown manifest
manifestList := &manifestlist.ManifestList{
Versioned: manifest.Versioned{
SchemaVersion: 2,
},
MediaType: manifestlist.MediaTypeManifestList,
Manifests: []manifestlist.ManifestDescriptor{
{
Descriptor: distribution.Descriptor{
Digest: "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b",
Size: 3253,
MediaType: schema2.MediaTypeManifest,
},
Platform: manifestlist.PlatformSpec{
Architecture: "amd64",
OS: "linux",
},
},
},
}
resp := putManifest(t, "putting missing manifest manifestlist", manifestURL, manifestlist.MediaTypeManifestList, manifestList)
defer resp.Body.Close()
checkResponse(t, "putting missing manifest manifestlist", resp, http.StatusBadRequest)
_, p, counts := checkBodyHasErrorCodes(t, "putting missing manifest manifestlist", resp, v2.ErrorCodeManifestBlobUnknown)
expectedCounts := map[errcode.ErrorCode]int{
v2.ErrorCodeManifestBlobUnknown: 1,
}
if !reflect.DeepEqual(counts, expectedCounts) {
t.Fatalf("unexpected number of error codes encountered: %v\n!=\n%v\n---\n%s", counts, expectedCounts, string(p))
}
// -------------------
// Push a manifest list that references an actual manifest
manifestList.Manifests[0].Digest = args.dgst
deserializedManifestList, err := manifestlist.FromDescriptors(manifestList.Manifests)
if err != nil {
t.Fatalf("could not create DeserializedManifestList: %v", err)
}
_, canonical, err := deserializedManifestList.Payload()
if err != nil {
t.Fatalf("could not get manifest list payload: %v", err)
}
dgst := digest.FromBytes(canonical)
manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String())
checkErr(t, err, "building manifest url")
resp = putManifest(t, "putting manifest list no error", manifestURL, manifestlist.MediaTypeManifestList, deserializedManifestList)
checkResponse(t, "putting manifest list no error", resp, http.StatusCreated)
checkHeaders(t, resp, http.Header{
"Location": []string{manifestDigestURL},
"Docker-Content-Digest": []string{dgst.String()},
})
// --------------------
// Push by digest -- should get same result
resp = putManifest(t, "putting manifest list by digest", manifestDigestURL, manifestlist.MediaTypeManifestList, deserializedManifestList)
checkResponse(t, "putting manifest list by digest", resp, http.StatusCreated)
checkHeaders(t, resp, http.Header{
"Location": []string{manifestDigestURL},
"Docker-Content-Digest": []string{dgst.String()},
})
// ------------------
// Fetch by tag name
req, err := http.NewRequest("GET", manifestURL, nil)
if err != nil {
t.Fatalf("Error constructing request: %s", err)
}
req.Header.Set("Accept", manifestlist.MediaTypeManifestList)
req.Header.Add("Accept", schema1.MediaTypeManifest)
req.Header.Add("Accept", schema2.MediaTypeManifest)
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("unexpected error fetching manifest list: %v", err)
}
defer resp.Body.Close()
checkResponse(t, "fetching uploaded manifest list", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Docker-Content-Digest": []string{dgst.String()},
"ETag": []string{fmt.Sprintf(`"%s"`, dgst)},
})
var fetchedManifestList manifestlist.DeserializedManifestList
dec := json.NewDecoder(resp.Body)
if err := dec.Decode(&fetchedManifestList); err != nil {
t.Fatalf("error decoding fetched manifest list: %v", err)
}
_, fetchedCanonical, err := fetchedManifestList.Payload()
if err != nil {
t.Fatalf("error getting manifest list payload: %v", err)
}
if !bytes.Equal(fetchedCanonical, canonical) {
t.Fatalf("manifest lists do not match")
}
// ---------------
// Fetch by digest
req, err = http.NewRequest("GET", manifestDigestURL, nil)
if err != nil {
t.Fatalf("Error constructing request: %s", err)
}
req.Header.Set("Accept", manifestlist.MediaTypeManifestList)
resp, err = http.DefaultClient.Do(req)
checkErr(t, err, "fetching manifest list by digest")
defer resp.Body.Close()
checkResponse(t, "fetching uploaded manifest list", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Docker-Content-Digest": []string{dgst.String()},
"ETag": []string{fmt.Sprintf(`"%s"`, dgst)},
})
var fetchedManifestListByDigest manifestlist.DeserializedManifestList
dec = json.NewDecoder(resp.Body)
if err := dec.Decode(&fetchedManifestListByDigest); err != nil {
t.Fatalf("error decoding fetched manifest: %v", err)
}
_, fetchedCanonical, err = fetchedManifestListByDigest.Payload()
if err != nil {
t.Fatalf("error getting manifest list payload: %v", err)
}
if !bytes.Equal(fetchedCanonical, canonical) {
t.Fatalf("manifests do not match")
}
// Get by name with etag, gives 304
etag := resp.Header.Get("Etag")
req, err = http.NewRequest("GET", manifestURL, nil)
if err != nil {
t.Fatalf("Error constructing request: %s", err)
}
req.Header.Set("If-None-Match", etag)
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("Error constructing request: %s", err)
}
checkResponse(t, "fetching manifest by name with etag", resp, http.StatusNotModified)
// Get by digest with etag, gives 304
req, err = http.NewRequest("GET", manifestDigestURL, nil)
if err != nil {
t.Fatalf("Error constructing request: %s", err)
}
req.Header.Set("If-None-Match", etag)
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("Error constructing request: %s", err)
}
checkResponse(t, "fetching manifest by dgst with etag", resp, http.StatusNotModified)
}
func testManifestDelete(t *testing.T, env *testEnv, args manifestArgs) {
imageName := args.imageName
dgst := args.dgst
@ -1521,13 +1697,20 @@ func newTestEnvWithConfig(t *testing.T, config *configuration.Configuration) *te
func putManifest(t *testing.T, msg, url, contentType string, v interface{}) *http.Response {
var body []byte
if sm, ok := v.(*schema1.SignedManifest); ok {
_, pl, err := sm.Payload()
switch m := v.(type) {
case *schema1.SignedManifest:
_, pl, err := m.Payload()
if err != nil {
t.Fatalf("error getting payload: %v", err)
}
body = pl
} else {
case *manifestlist.DeserializedManifestList:
_, pl, err := m.Payload()
if err != nil {
t.Fatalf("error getting payload: %v", err)
}
body = pl
default:
var err error
body, err = json.MarshalIndent(v, "", " ")
if err != nil {

96
docs/storage/manifestlisthandler.go Normal file
View File

@ -0,0 +1,96 @@
package storage
import (
"fmt"
"encoding/json"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/manifestlist"
)
// manifestListHandler is a ManifestHandler that covers schema2 manifest lists.
type manifestListHandler struct {
repository *repository
blobStore *linkedBlobStore
ctx context.Context
}
var _ ManifestHandler = &manifestListHandler{}
func (ms *manifestListHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) {
context.GetLogger(ms.ctx).Debug("(*manifestListHandler).Unmarshal")
var m manifestlist.DeserializedManifestList
if err := json.Unmarshal(content, &m); err != nil {
return nil, err
}
return &m, nil
}
func (ms *manifestListHandler) Put(ctx context.Context, manifestList distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) {
context.GetLogger(ms.ctx).Debug("(*manifestListHandler).Put")
m, ok := manifestList.(*manifestlist.DeserializedManifestList)
if !ok {
return "", fmt.Errorf("wrong type put to manifestListHandler: %T", manifestList)
}
if err := ms.verifyManifest(ms.ctx, *m, skipDependencyVerification); err != nil {
return "", err
}
mt, payload, err := m.Payload()
if err != nil {
return "", err
}
revision, err := ms.blobStore.Put(ctx, mt, payload)
if err != nil {
context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err)
return "", err
}
// Link the revision into the repository.
if err := ms.blobStore.linkBlob(ctx, revision); err != nil {
return "", err
}
return revision.Digest, nil
}
// verifyManifest ensures that the manifest content is valid from the
// perspective of the registry. As a policy, the registry only tries to
// store valid content, leaving trust policies of that content up to
// consumers.
func (ms *manifestListHandler) verifyManifest(ctx context.Context, mnfst manifestlist.DeserializedManifestList, skipDependencyVerification bool) error {
var errs distribution.ErrManifestVerification
if !skipDependencyVerification {
// This manifest service is different from the blob service
// returned by Blob. It uses a linked blob store to ensure that
// only manifests are accessible.
manifestService, err := ms.repository.Manifests(ctx)
if err != nil {
return err
}
for _, manifestDescriptor := range mnfst.References() {
exists, err := manifestService.Exists(ctx, manifestDescriptor.Digest)
if err != nil && err != distribution.ErrBlobUnknown {
errs = append(errs, err)
}
if err != nil || !exists {
// On error here, we always append unknown blob errors.
errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: manifestDescriptor.Digest})
}
}
}
if len(errs) != 0 {
return errs
}
return nil
}

24
docs/storage/manifeststore.go
View File

@ -8,6 +8,7 @@ import (
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest"
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
)
@ -44,8 +45,9 @@ type manifestStore struct {
skipDependencyVerification bool
schema1Handler ManifestHandler
schema2Handler ManifestHandler
schema1Handler ManifestHandler
schema2Handler ManifestHandler
manifestListHandler ManifestHandler
}
var _ distribution.ManifestService = &manifestStore{}
@ -92,7 +94,21 @@ func (ms *manifestStore) Get(ctx context.Context, dgst digest.Digest, options ..
case 1:
return ms.schema1Handler.Unmarshal(ctx, dgst, content)
case 2:
return ms.schema2Handler.Unmarshal(ctx, dgst, content)
// This can be an image manifest or a manifest list
var mediaType struct {
MediaType string `json:"mediaType"`
}
if err = json.Unmarshal(content, &mediaType); err != nil {
return nil, err
}
switch mediaType.MediaType {
case schema2.MediaTypeManifest:
return ms.schema2Handler.Unmarshal(ctx, dgst, content)
case manifestlist.MediaTypeManifestList:
return ms.manifestListHandler.Unmarshal(ctx, dgst, content)
default:
return nil, distribution.ErrManifestVerification{fmt.Errorf("unrecognized manifest content type %s", mediaType.MediaType)}
}
}
return nil, fmt.Errorf("unrecognized manifest schema version %d", versioned.SchemaVersion)
@ -106,6 +122,8 @@ func (ms *manifestStore) Put(ctx context.Context, manifest distribution.Manifest
return ms.schema1Handler.Put(ctx, manifest, ms.skipDependencyVerification)
case *schema2.DeserializedManifest:
return ms.schema2Handler.Put(ctx, manifest, ms.skipDependencyVerification)
case *manifestlist.DeserializedManifestList:
return ms.manifestListHandler.Put(ctx, manifest, ms.skipDependencyVerification)
}
return "", fmt.Errorf("unrecognized manifest type %T", manifest)

5
docs/storage/registry.go
View File

@ -200,6 +200,11 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M
repository: repo,
blobStore: blobStore,
},
manifestListHandler: &manifestListHandler{
ctx: ctx,
repository: repo,
blobStore: blobStore,
},
}
// Apply options

5
docs/storage/schema2manifesthandler.go
View File

@ -62,9 +62,8 @@ func (ms *schema2ManifestHandler) Put(ctx context.Context, manifest distribution
}
// verifyManifest ensures that the manifest content is valid from the
// perspective of the registry. It ensures that the signature is valid for the
// enclosed payload. As a policy, the registry only tries to store valid
// content, leaving trust policies of that content up to consumems.
// perspective of the registry. As a policy, the registry only tries to store
// valid content, leaving trust policies of that content up to consumers.
func (ms *schema2ManifestHandler) verifyManifest(ctx context.Context, mnfst schema2.DeserializedManifest, skipDependencyVerification bool) error {
var errs distribution.ErrManifestVerification

2
docs/storage/signedmanifesthandler.go
View File

@ -91,7 +91,7 @@ func (ms *signedManifestHandler) Put(ctx context.Context, manifest distribution.
// verifyManifest ensures that the manifest content is valid from the
// perspective of the registry. It ensures that the signature is valid for the
// enclosed payload. As a policy, the registry only tries to store valid
// content, leaving trust policies of that content up to consumems.
// content, leaving trust policies of that content up to consumers.
func (ms *signedManifestHandler) verifyManifest(ctx context.Context, mnfst schema1.SignedManifest, skipDependencyVerification bool) error {
var errs distribution.ErrManifestVerification