Merge pull request #73 from BrianBland/godeps

Updates Godeps to a compiling/stable state

Godeps/Godeps.json

@@ -30,15 +30,15 @@
 		},
 		{
 			"ImportPath": "github.com/crowdmob/goamz/aws",
-			"Rev": "cd22d9897beff6f3de22cec4bdb7d46b9e2dee67"
+			"Rev": "962cedbbde5e1af59fb0b4ab681c848e61676941"
 		},
 		{
 			"ImportPath": "github.com/crowdmob/goamz/cloudfront",
-			"Rev": "cd22d9897beff6f3de22cec4bdb7d46b9e2dee67"
+			"Rev": "962cedbbde5e1af59fb0b4ab681c848e61676941"
 		},
 		{
 			"ImportPath": "github.com/crowdmob/goamz/s3",
-			"Rev": "cd22d9897beff6f3de22cec4bdb7d46b9e2dee67"
+			"Rev": "962cedbbde5e1af59fb0b4ab681c848e61676941"
 		},
 		{
 			"ImportPath": "github.com/docker/docker/pkg/tarsum",
@@ -52,7 +52,7 @@
 		},
 		{
 			"ImportPath": "github.com/docker/libtrust",
-			"Rev": "a9625ce37e2dc5fed2e51eec2d39c39e4ac4c1df"
+			"Rev": "c54fbb67c1f1e68d7d6f8d2ad7c9360404616a41"
 		},
 		{
 			"ImportPath": "github.com/gorilla/context",

Godeps/_workspace/src/github.com/crowdmob/goamz/aws/aws.go

@@ -518,7 +518,7 @@ func dialTimeout(network, addr string) (net.Conn, error) {
 	return net.DialTimeout(network, addr, time.Duration(2*time.Second))
 }
 
-func InstanceRegion() string {
+func AvailabilityZone() string {
 	transport := http.Transport{Dial: dialTimeout}
 	client := http.Client{
 		Transport: &transport,
@@ -532,13 +532,21 @@ func InstanceRegion() string {
 		if err != nil {
 			return "unknown"
 		} else {
-			b := string(body)
-			region := b[:len(b)-1]
-			return region
+			return string(body)
 		}
 	}
 }
 
+func InstanceRegion() string {
+	az := AvailabilityZone()
+	if az == "unknown" {
+		return az
+	} else {
+		region := az[:len(az)-1]
+		return region
+	}
+}
+
 func InstanceId() string {
 	transport := http.Transport{Dial: dialTimeout}
 	client := http.Client{

Godeps/_workspace/src/github.com/crowdmob/goamz/aws/sign.go

@@ -151,16 +151,37 @@ Any changes to the request after signing the request will invalidate the signatu
 */
 func (s *V4Signer) Sign(req *http.Request) {
 	req.Header.Set("host", req.Host) // host header must be included as a signed header
-	payloadHash := s.payloadHash(req)
+	t := s.requestTime(req)          // Get request time
+
+	payloadHash := ""
+
+	if _, ok := req.Form["X-Amz-Expires"]; ok {
+		// We are authenticating the the request by using query params
+		// (also known as pre-signing a url, http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html)
+		payloadHash = "UNSIGNED-PAYLOAD"
+		req.Header.Del("x-amz-date")
+
+		req.Form["X-Amz-SignedHeaders"] = []string{s.signedHeaders(req.Header)}
+		req.Form["X-Amz-Algorithm"] = []string{"AWS4-HMAC-SHA256"}
+		req.Form["X-Amz-Credential"] = []string{s.auth.AccessKey + "/" + s.credentialScope(t)}
+		req.Form["X-Amz-Date"] = []string{t.Format(ISO8601BasicFormat)}
+		req.URL.RawQuery = req.Form.Encode()
+	} else {
+		payloadHash = s.payloadHash(req)
 		if s.IncludeXAmzContentSha256 {
 			req.Header.Set("x-amz-content-sha256", payloadHash) // x-amz-content-sha256 contains the payload hash
 		}
-	t := s.requestTime(req)                           // Get request time
+	}
 	creq := s.canonicalRequest(req, payloadHash)      // Build canonical request
 	sts := s.stringToSign(t, creq)                    // Build string to sign
 	signature := s.signature(t, sts)                  // Calculate the AWS Signature Version 4
 	auth := s.authorization(req.Header, t, signature) // Create Authorization header value
+
+	if _, ok := req.Form["X-Amz-Expires"]; ok {
+		req.Form["X-Amz-Signature"] = []string{signature}
+	} else {
 		req.Header.Set("Authorization", auth) // Add Authorization header to request
+	}
 	return
 }
 
@@ -266,8 +287,20 @@ func (s *V4Signer) canonicalQueryString(u *url.URL) string {
 }
 
 func (s *V4Signer) canonicalHeaders(h http.Header) string {
-	i, a := 0, make([]string, len(h))
+	i, a, lowerCase := 0, make([]string, len(h)), make(map[string][]string)
+
 	for k, v := range h {
+		lowerCase[strings.ToLower(k)] = v
+	}
+
+	var keys []string
+	for k := range lowerCase {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	for _, k := range keys {
+		v := lowerCase[k]
 		for j, w := range v {
 			v[j] = strings.Trim(w, " ")
 		}
@@ -275,7 +308,6 @@ func (s *V4Signer) canonicalHeaders(h http.Header) string {
 		a[i] = strings.ToLower(k) + ":" + strings.Join(v, ",")
 		i++
 	}
-	sort.Strings(a)
 	return strings.Join(a, "\n")
 }
 

Godeps/_workspace/src/github.com/crowdmob/goamz/s3/multi.go

@@ -165,7 +165,7 @@ func (m *Multi) PutPartCopy(n int, options CopyOptions, source string) (*CopyObj
 			params:  params,
 		}
 		resp := &CopyObjectResult{}
-		err := m.Bucket.S3.query(req, resp)
+		err = m.Bucket.S3.query(req, resp)
 		if shouldRetry(err) && attempt.HasNext() {
 			continue
 		}

Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3.go

@@ -40,6 +40,7 @@ type S3 struct {
 	aws.Region
 	ConnectTimeout time.Duration
 	ReadTimeout    time.Duration
+	Signature      int
 	private        byte // Reserve the right of using private data.
 }
 
@@ -95,7 +96,7 @@ var attempts = aws.AttemptStrategy{
 
 // New creates a new S3.
 func New(auth aws.Auth, region aws.Region) *S3 {
-	return &S3{auth, region, 0, 0, 0}
+	return &S3{auth, region, 0, 0, 0, aws.V2Signature}
 }
 
 // Bucket returns a Bucket with the given name.
@@ -772,15 +773,26 @@ func (b *Bucket) SignedURL(path string, expires time.Time) string {
 // SignedURLWithArgs returns a signed URL that allows anyone holding the URL
 // to retrieve the object at path. The signature is valid until expires.
 func (b *Bucket) SignedURLWithArgs(path string, expires time.Time, params url.Values, headers http.Header) string {
+	return b.SignedURLWithMethod("GET", path, expires, params, headers)
+}
+
+// SignedURLWithMethod returns a signed URL that allows anyone holding the URL
+// to either retrieve the object at path or make a HEAD request against it. The signature is valid until expires.
+func (b *Bucket) SignedURLWithMethod(method, path string, expires time.Time, params url.Values, headers http.Header) string {
 	var uv = url.Values{}
 
 	if params != nil {
 		uv = params
 	}
 
+	if b.S3.Signature == aws.V2Signature {
 		uv.Set("Expires", strconv.FormatInt(expires.Unix(), 10))
+	} else {
+		uv.Set("X-Amz-Expires", strconv.FormatInt(expires.Unix()-time.Now().Unix(), 10))
+	}
 
 	req := &request{
+		method:  method,
 		bucket:  b.Name,
 		path:    path,
 		params:  uv,
@@ -810,9 +822,15 @@ func (b *Bucket) UploadSignedURL(path, method, content_type string, expires time
 	if method != "POST" {
 		method = "PUT"
 	}
-	stringToSign := method + "\n\n" + content_type + "\n" + strconv.FormatInt(expire_date, 10) + "\n/" + b.Name + "/" + path
-	fmt.Println("String to sign:\n", stringToSign)
+
 	a := b.S3.Auth
+	tokenData := ""
+
+	if a.Token() != "" {
+		tokenData = "x-amz-security-token:" + a.Token() + "\n"
+	}
+
+	stringToSign := method + "\n\n" + content_type + "\n" + strconv.FormatInt(expire_date, 10) + "\n" + tokenData + "/" + b.Name + "/" + path
 	secretKey := a.SecretKey
 	accessId := a.AccessKey
 	mac := hmac.New(sha1.New, []byte(secretKey))
@@ -832,7 +850,7 @@ func (b *Bucket) UploadSignedURL(path, method, content_type string, expires time
 	params.Add("Expires", strconv.FormatInt(expire_date, 10))
 	params.Add("Signature", signature)
 	if a.Token() != "" {
-		params.Add("token", a.Token())
+		params.Add("x-amz-security-token", a.Token())
 	}
 
 	signedurl.RawQuery = params.Encode()
@@ -924,7 +942,10 @@ func (s3 *S3) queryV4Sign(req *request, resp interface{}) error {
 		req.headers = map[string][]string{}
 	}
 
-	s3.setBaseURL(req)
+	err := s3.setBaseURL(req)
+	if err != nil {
+		return err
+	}
 
 	hreq, err := s3.setupHttpRequest(req)
 	if err != nil {
@@ -992,13 +1013,6 @@ func partiallyEscapedPath(path string) string {
 
 // prepare sets up req to be delivered to S3.
 func (s3 *S3) prepare(req *request) error {
-	var signpath = req.path
-
-	if !req.prepared {
-		req.prepared = true
-		if req.method == "" {
-			req.method = "GET"
-		}
 	// Copy so they can be mutated without affecting on retries.
 	params := make(url.Values)
 	headers := make(http.Header)
@@ -1010,39 +1024,68 @@ func (s3 *S3) prepare(req *request) error {
 	}
 	req.params = params
 	req.headers = headers
+
+	if !req.prepared {
+		req.prepared = true
+		if req.method == "" {
+			req.method = "GET"
+		}
+
 		if !strings.HasPrefix(req.path, "/") {
 			req.path = "/" + req.path
 		}
-		signpath = req.path
 
 		err := s3.setBaseURL(req)
 		if err != nil {
 			return err
 		}
-		if req.bucket != "" {
-			signpath = "/" + req.bucket + signpath
-		}
 	}
 
+	if s3.Auth.Token() != "" {
+		req.headers["X-Amz-Security-Token"] = []string{s3.Auth.Token()}
+	}
+
+	if s3.Signature == aws.V2Signature {
 		// Always sign again as it's not clear how far the
 		// server has handled a previous attempt.
 		u, err := url.Parse(req.baseurl)
 		if err != nil {
-		return fmt.Errorf("bad S3 endpoint URL %q: %v", req.baseurl, err)
+			return err
 		}
 
-	signpathPatiallyEscaped := partiallyEscapedPath(signpath)
+		signpathPatiallyEscaped := partiallyEscapedPath(req.path)
 		req.headers["Host"] = []string{u.Host}
 		req.headers["Date"] = []string{time.Now().In(time.UTC).Format(time.RFC1123)}
-	if s3.Auth.Token() != "" {
-		req.headers["X-Amz-Security-Token"] = []string{s3.Auth.Token()}
-	}
+
 		sign(s3.Auth, req.method, signpathPatiallyEscaped, req.params, req.headers)
+	} else {
+		hreq, err := s3.setupHttpRequest(req)
+		if err != nil {
+			return err
+		}
+
+		hreq.Host = hreq.URL.Host
+		signer := aws.NewV4Signer(s3.Auth, "s3", s3.Region)
+		signer.IncludeXAmzContentSha256 = true
+		signer.Sign(hreq)
+
+		req.payload = hreq.Body
+		if _, ok := headers["Content-Length"]; ok {
+			req.headers["Content-Length"] = headers["Content-Length"]
+		}
+	}
 	return nil
 }
 
 // Prepares an *http.Request for doHttpRequest
 func (s3 *S3) setupHttpRequest(req *request) (*http.Request, error) {
+	// Copy so that signing the http request will not mutate it
+	headers := make(http.Header)
+	for k, v := range req.headers {
+		headers[k] = v
+	}
+	req.headers = headers
+
 	u, err := req.url()
 	if err != nil {
 		return nil, err
@@ -1056,6 +1099,7 @@ func (s3 *S3) setupHttpRequest(req *request) (*http.Request, error) {
 		ProtoMinor: 1,
 		Close:      true,
 		Header:     req.headers,
+		Form:       req.params,
 	}
 
 	if v, ok := req.headers["Content-Length"]; ok {

Godeps/_workspace/src/github.com/docker/libtrust/jsonsign.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"sort"
 	"time"
 	"unicode"
 )
@@ -31,11 +32,27 @@ type jsHeader struct {
 }
 
 type jsSignature struct {
-	Header    *jsHeader `json:"header"`
+	Header    jsHeader `json:"header"`
 	Signature string   `json:"signature"`
 	Protected string   `json:"protected,omitempty"`
 }
 
+type jsSignaturesSorted []jsSignature
+
+func (jsbkid jsSignaturesSorted) Swap(i, j int) { jsbkid[i], jsbkid[j] = jsbkid[j], jsbkid[i] }
+func (jsbkid jsSignaturesSorted) Len() int      { return len(jsbkid) }
+
+func (jsbkid jsSignaturesSorted) Less(i, j int) bool {
+	ki, kj := jsbkid[i].Header.JWK.KeyID(), jsbkid[j].Header.JWK.KeyID()
+	si, sj := jsbkid[i].Signature, jsbkid[j].Signature
+
+	if ki == kj {
+		return si < sj
+	}
+
+	return ki < kj
+}
+
 type signKey struct {
 	PrivateKey
 	Chain []*x509.Certificate
@@ -44,7 +61,7 @@ type signKey struct {
 // JSONSignature represents a signature of a json object.
 type JSONSignature struct {
 	payload      string
-	signatures   []*jsSignature
+	signatures   []jsSignature
 	indent       string
 	formatLength int
 	formatTail   []byte
@@ -52,7 +69,7 @@ type JSONSignature struct {
 
 func newJSONSignature() *JSONSignature {
 	return &JSONSignature{
-		signatures: make([]*jsSignature, 0, 1),
+		signatures: make([]jsSignature, 0, 1),
 	}
 }
 
@@ -99,17 +116,14 @@ func (js *JSONSignature) Sign(key PrivateKey) error {
 		return err
 	}
 
-	header := &jsHeader{
+	js.signatures = append(js.signatures, jsSignature{
+		Header: jsHeader{
 			JWK:       key.PublicKey(),
 			Algorithm: algorithm,
-	}
-	sig := &jsSignature{
-		Header:    header,
+		},
 		Signature: joseBase64UrlEncode(sigBytes),
 		Protected: protected,
-	}
-
-	js.signatures = append(js.signatures, sig)
+	})
 
 	return nil
 }
@@ -136,7 +150,7 @@ func (js *JSONSignature) SignWithChain(key PrivateKey, chain []*x509.Certificate
 		return err
 	}
 
-	header := &jsHeader{
+	header := jsHeader{
 		Chain:     make([]string, len(chain)),
 		Algorithm: algorithm,
 	}
@@ -145,13 +159,11 @@ func (js *JSONSignature) SignWithChain(key PrivateKey, chain []*x509.Certificate
 		header.Chain[i] = base64.StdEncoding.EncodeToString(cert.Raw)
 	}
 
-	sig := &jsSignature{
+	js.signatures = append(js.signatures, jsSignature{
 		Header:    header,
 		Signature: joseBase64UrlEncode(sigBytes),
 		Protected: protected,
-	}
-
-	js.signatures = append(js.signatures, sig)
+	})
 
 	return nil
 }
@@ -272,6 +284,9 @@ func (js *JSONSignature) JWS() ([]byte, error) {
 	if len(js.signatures) == 0 {
 		return nil, errors.New("missing signature")
 	}
+
+	sort.Sort(jsSignaturesSorted(js.signatures))
+
 	jsonMap := map[string]interface{}{
 		"payload":    js.payload,
 		"signatures": js.signatures,
@@ -301,7 +316,7 @@ type jsParsedHeader struct {
 }
 
 type jsParsedSignature struct {
-	Header    *jsParsedHeader `json:"header"`
+	Header    jsParsedHeader `json:"header"`
 	Signature string         `json:"signature"`
 	Protected string         `json:"protected"`
 }
@@ -310,7 +325,7 @@ type jsParsedSignature struct {
 func ParseJWS(content []byte) (*JSONSignature, error) {
 	type jsParsed struct {
 		Payload    string              `json:"payload"`
-		Signatures []*jsParsedSignature `json:"signatures"`
+		Signatures []jsParsedSignature `json:"signatures"`
 	}
 	parsed := &jsParsed{}
 	err := json.Unmarshal(content, parsed)
@@ -329,9 +344,9 @@ func ParseJWS(content []byte) (*JSONSignature, error) {
 	if err != nil {
 		return nil, err
 	}
-	js.signatures = make([]*jsSignature, len(parsed.Signatures))
+	js.signatures = make([]jsSignature, len(parsed.Signatures))
 	for i, signature := range parsed.Signatures {
-		header := &jsHeader{
+		header := jsHeader{
 			Algorithm: signature.Header.Algorithm,
 		}
 		if signature.Header.Chain != nil {
@@ -344,7 +359,7 @@ func ParseJWS(content []byte) (*JSONSignature, error) {
 			}
 			header.JWK = publicKey
 		}
-		js.signatures[i] = &jsSignature{
+		js.signatures[i] = jsSignature{
 			Header:    header,
 			Signature: signature.Signature,
 			Protected: signature.Protected,
@@ -356,7 +371,11 @@ func ParseJWS(content []byte) (*JSONSignature, error) {
 
 // NewJSONSignature returns a new unsigned JWS from a json byte array.
 // JSONSignature will need to be signed before serializing or storing.
-func NewJSONSignature(content []byte) (*JSONSignature, error) {
+// Optionally, one or more signatures can be provided as byte buffers,
+// containing serialized JWS signatures, to assemble a fully signed JWS
+// package. It is the callers responsibility to ensure uniqueness of the
+// provided signatures.
+func NewJSONSignature(content []byte, signatures ...[]byte) (*JSONSignature, error) {
 	var dataMap map[string]interface{}
 	err := json.Unmarshal(content, &dataMap)
 	if err != nil {
@@ -380,6 +399,40 @@ func NewJSONSignature(content []byte) (*JSONSignature, error) {
 	js.formatLength = lastRuneIndex + 1
 	js.formatTail = content[js.formatLength:]
 
+	if len(signatures) > 0 {
+		for _, signature := range signatures {
+			var parsedJSig jsParsedSignature
+
+			if err := json.Unmarshal(signature, &parsedJSig); err != nil {
+				return nil, err
+			}
+
+			// TODO(stevvooe): A lot of the code below is repeated in
+			// ParseJWS. It will require more refactoring to fix that.
+			jsig := jsSignature{
+				Header: jsHeader{
+					Algorithm: parsedJSig.Header.Algorithm,
+				},
+				Signature: parsedJSig.Signature,
+				Protected: parsedJSig.Protected,
+			}
+
+			if parsedJSig.Header.Chain != nil {
+				jsig.Header.Chain = parsedJSig.Header.Chain
+			}
+
+			if parsedJSig.Header.JWK != nil {
+				publicKey, err := UnmarshalPublicKeyJWK([]byte(parsedJSig.Header.JWK))
+				if err != nil {
+					return nil, err
+				}
+				jsig.Header.JWK = publicKey
+			}
+
+			js.signatures = append(js.signatures, jsig)
+		}
+	}
+
 	return js, nil
 }
 
@@ -455,7 +508,7 @@ func ParsePrettySignature(content []byte, signatureKey string) (*JSONSignature,
 	}
 
 	js := newJSONSignature()
-	js.signatures = make([]*jsSignature, len(signatureBlocks))
+	js.signatures = make([]jsSignature, len(signatureBlocks))
 
 	for i, signatureBlock := range signatureBlocks {
 		protectedBytes, err := joseBase64UrlDecode(signatureBlock.Protected)
@@ -491,7 +544,7 @@ func ParsePrettySignature(content []byte, signatureKey string) (*JSONSignature,
 			return nil, errors.New("conflicting format tail")
 		}
 
-		header := &jsHeader{
+		header := jsHeader{
 			Algorithm: signatureBlock.Header.Algorithm,
 			Chain:     signatureBlock.Header.Chain,
 		}
@@ -502,7 +555,7 @@ func ParsePrettySignature(content []byte, signatureKey string) (*JSONSignature,
 			}
 			header.JWK = publicKey
 		}
-		js.signatures[i] = &jsSignature{
+		js.signatures[i] = jsSignature{
 			Header:    header,
 			Signature: signatureBlock.Signature,
 			Protected: signatureBlock.Protected,
@@ -532,6 +585,8 @@ func (js *JSONSignature) PrettySignature(signatureKey string) ([]byte, error) {
 	}
 	payload = payload[:js.formatLength]
 
+	sort.Sort(jsSignaturesSorted(js.signatures))
+
 	var marshalled []byte
 	var marshallErr error
 	if js.indent != "" {
@@ -565,6 +620,26 @@ func (js *JSONSignature) PrettySignature(signatureKey string) ([]byte, error) {
 	return buf.Bytes(), nil
 }
 
+// Signatures provides the signatures on this JWS as opaque blobs, sorted by
+// keyID. These blobs can be stored and reassembled with payloads. Internally,
+// they are simply marshaled json web signatures but implementations should
+// not rely on this.
+func (js *JSONSignature) Signatures() ([][]byte, error) {
+	sort.Sort(jsSignaturesSorted(js.signatures))
+
+	var sb [][]byte
+	for _, jsig := range js.signatures {
+		p, err := json.Marshal(jsig)
+		if err != nil {
+			return nil, err
+		}
+
+		sb = append(sb, p)
+	}
+
+	return sb, nil
+}
+
 // Merge combines the signatures from one or more other signatures into the
 // method receiver. If the payloads differ for any argument, an error will be
 // returned and the receiver will not be modified.