Go 实现 AWS4 请求认证 | Go主题月

DevOpsxAI
25,197 阅读5分钟

什么是 Amazon S3？它是 AWS 提供的一种对象存储服务，提供行业领先的可扩展性、数据可用性、安全性和性能。Amazon S3 可达到 99.999999999%（11 个 9）的持久性。

Amazon S3 里用到了 AWS Signature Version 4（下面简称 AWS4）做认证请求，这篇文章将会讲解如何使用 Go 实现 AWS4 请求认证。

AWS4 是一种用于对所有 AWS 区域服务的入站 API 请求进行身份验证的协议。

AWS4

AWS4 对请求进行签名有以下优势（但这也取决于你如何使用）：

  • 验证请求者的身份 - 经过身份验证的请求需要使用 AccessKeyID 和 SecretAccessKey 创建签名。
  • 保护传输中的数据 - 为了防止在传输过程中对请求进行篡改，可以使用一些请求元素（比如请求路径、请求头等）来计算请求签名。Amazon S3 在收到请求后，使用相同的请求元素来计算签名。如果 Amazon S3 接收到的任何请求组件与用于计算签名的组件不匹配，Amazon S3 将拒绝该请求。
  • 防止重用请求的签名部分 - 请求的签名部分在请求中的时间戳的一段时间内有效。

授权方式

  • HTTP 身份验证头，例如 Authorization 请求头：
Authorization: AWS4-HMAC-SHA256 
Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, 
SignedHeaders=host;range;x-amz-date,
Signature=fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024
  • URL 查询字符串参数，例如预签名 URL：
https://s3.amazonaws.com/examplebucket/test.txt
?X-Amz-Algorithm=AWS4-HMAC-SHA256
&X-Amz-Credential=<your-access-key-id>/20130721/us-east-1/s3/aws4_request
&X-Amz-Date=20130721T201207Z
&X-Amz-Expires=86400
&X-Amz-SignedHeaders=host
&X-Amz-Signature=<signature-value>

Go 实现 HTTP 身份验证头

HTTP 身份验证头组成部分

  • AWS4-HMAC-SHA256 - 该字符串指定 AWS4 和签名算法（HMAC-SHA256）。
  • Credential - 指定 AccessKeyID、计算签名的日期、区域和服务。它的格式是 <your-access-key-id>/<date>/<aws-region>/<aws-service>/aws4_request，date 的格式是 YYYYMMDD。
  • SignedHeaders - 指定用于计算签名的请求头列表，以分号分隔。仅包含请求头的名称，且必须是小写，例如：host;range;x-amz-date。
  • Signature - 表示为 64 个小写十六进制字符的 256 位签名，例如：fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024。

上传图片至掘金

根据对 AWS4 请求认证的学习，并参考 simples3 的代码实现了 上传图片至掘金（字节跳动的存储服务，其服务名称是 imagex）的功能，下面是部分的代码实现（主要省略了 Client 部分的实现，完整代码有待后续完善后进行开源）：

package juejin

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"hash/crc32"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"time"

	"github.com/tidwall/gjson"
)

const (
	amzDateISO8601TimeFormat = "20060102T150405Z"
	shortTimeFormat          = "20060102"
	algorithm                = "AWS4-HMAC-SHA256"
	serviceName              = "imagex"
	serviceID                = "k3u1fbpfcp"
	version                  = "2018-08-01"
	uploadURLFormat          = "https://%s/%s"

	RegionCNNorth = "cn-north-1"

	actionApplyImageUpload  = "ApplyImageUpload"
	actionCommitImageUpload = "CommitImageUpload"

	polynomialCRC32 = 0xEDB88320
)

var (
	newLine = []byte{'\n'}

	// if object matches reserved string, no need to encode them
	reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
)

type ImageX struct {
	AccessKey string
	SecretKey string
	Region    string
	Client    *http.Client

	Token   string
	Version string
	BaseURL string
}

type UploadToken struct {
	AccessKeyID     string `json:"AccessKeyID"`
	SecretAccessKey string `json:"SecretAccessKey"`
	SessionToken    string `json:"SessionToken"`
}

func (c *Client) UploadImage(region, imgPath string) (string, error) {
	uploadToken, err := c.GetUploadToken()
	if err != nil {
		return "", err
	}

	ix := &ImageX{
		AccessKey: uploadToken.AccessKeyID,
		SecretKey: uploadToken.SecretAccessKey,
		Token:     uploadToken.SessionToken,
		Region:    region,
	}

	applyRes, err := ix.ApplyImageUpload()
	if err != nil {
		return "", err
	}

	storeInfo := gjson.Get(applyRes, "Result.UploadAddress.StoreInfos.0")
	storeURI := storeInfo.Get("StoreUri").String()
	storeAuth := storeInfo.Get("Auth").String()
	uploadHost := gjson.Get(applyRes, "Result.UploadAddress.UploadHosts.0").String()
	uploadURL := fmt.Sprintf(uploadURLFormat, uploadHost, storeURI)
	if err := ix.Upload(uploadURL, imgPath, storeAuth); err != nil {
		return "", err
	}

	sessionKey := gjson.Get(applyRes, "Result.UploadAddress.SessionKey").String()
	if _, err = ix.CommitImageUpload(sessionKey); err != nil {
		return "", err
	}

	return c.GetImageURL(storeURI)
}

func (c *Client) GetImageURL(uri string) (string, error) {
	endpoint := "/imagex/get_img_url"
	params := &url.Values{
		"uri": []string{uri},
	}
	raw, err := c.Get(APIBaseURL, endpoint, params)
	if err != nil {
		return "", err
	}
	rawurl := gjson.Get(raw, "data.main_url").String()
	return rawurl, nil
}

func (c *Client) GetUploadToken() (*UploadToken, error) {
	endpoint := "/imagex/gen_token"
	params := &url.Values{
		"client": []string{"web"},
	}
	raw, err := c.Get(APIBaseURL, endpoint, params)
	if err != nil {
		return nil, err
	}
	var token *UploadToken
	err = json.Unmarshal([]byte(gjson.Get(raw, "data.token").String()), &token)
	return token, err
}

func (ix *ImageX) ApplyImageUpload() (string, error) {
	rawurl := fmt.Sprintf("https://imagex.bytedanceapi.com/?Action=%s&Version=%s&ServiceId=%s",
		actionApplyImageUpload, version, serviceID)
	req, err := http.NewRequest(http.MethodGet, rawurl, nil)
	if err != nil {
		return "", err
	}

	if err := ix.signRequest(req); err != nil {
		return "", err
	}

	res, err := ix.getClient().Do(req)
	if err != nil {
		return "", err
	}
	defer res.Body.Close()
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return "", err
	}
	raw := string(b)
	if res.StatusCode != 200 || gjson.Get(raw, "ResponseMetadata.Error").Exists() {
		return "", fmt.Errorf("raw: %s, response: %+v", raw, res)
	}
	return raw, nil
}

func (ix *ImageX) CommitImageUpload(sessionKey string) (string, error) {
	rawurl := fmt.Sprintf("https://imagex.bytedanceapi.com/?Action=%s&Version=%s&SessionKey=%s&ServiceId=%s",
		actionCommitImageUpload, version, sessionKey, serviceID)
	req, err := http.NewRequest(http.MethodPost, rawurl, nil)
	if err != nil {
		return "", err
	}

	if err := ix.signRequest(req); err != nil {
		return "", err
	}

	res, err := ix.getClient().Do(req)
	if err != nil {
		return "", err
	}
	defer res.Body.Close()
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return "", err
	}
	raw := string(b)
	if res.StatusCode != 200 || gjson.Get(raw, "ResponseMetadata.Error").Exists() {
		return "", fmt.Errorf("raw: %s, response: %+v", raw, res)
	}
	return raw, nil
}

func (ix *ImageX) getClient() *http.Client {
	if ix.Client == nil {
		return http.DefaultClient
	}
	return ix.Client
}

func (ix *ImageX) signKeys(t time.Time) []byte {
	h := makeHMac([]byte("AWS4"+ix.SecretKey), []byte(t.Format(shortTimeFormat)))
	h = makeHMac(h, []byte(ix.Region))
	h = makeHMac(h, []byte(serviceName))
	h = makeHMac(h, []byte("aws4_request"))
	return h
}

func (ix *ImageX) writeRequest(w io.Writer, r *http.Request) error {
	r.Header.Set("host", r.Host)

	w.Write([]byte(r.Method))
	w.Write(newLine)
	writeURI(w, r)
	w.Write(newLine)
	writeQuery(w, r)
	w.Write(newLine)
	writeHeader(w, r)
	w.Write(newLine)
	w.Write(newLine)
	writeHeaderList(w, r)
	w.Write(newLine)
	return writeBody(w, r)
}

func (ix *ImageX) writeStringToSign(w io.Writer, t time.Time, r *http.Request) error {
	w.Write([]byte(algorithm))
	w.Write(newLine)
	w.Write([]byte(t.Format(amzDateISO8601TimeFormat)))
	w.Write(newLine)

	w.Write([]byte(ix.creds(t)))
	w.Write(newLine)

	h := sha256.New()
	if err := ix.writeRequest(h, r); err != nil {
		return err
	}
	fmt.Fprintf(w, "%x", h.Sum(nil))
	return nil
}

func (ix *ImageX) creds(t time.Time) string {
	return t.Format(shortTimeFormat) + "/" + ix.Region + "/" + serviceName + "/aws4_request"
}

func (ix *ImageX) signRequest(req *http.Request) error {
	t := time.Now().UTC()
	req.Header.Set("x-amz-date", t.Format(amzDateISO8601TimeFormat))

	req.Header.Set("x-amz-security-token", ix.Token)

	k := ix.signKeys(t)
	h := hmac.New(sha256.New, k)

	if err := ix.writeStringToSign(h, t, req); err != nil {
		return err
	}

	auth := bytes.NewBufferString(algorithm)
	auth.Write([]byte(" Credential=" + ix.AccessKey + "/" + ix.creds(t)))
	auth.Write([]byte{',', ' '})
	auth.Write([]byte("SignedHeaders="))
	writeHeaderList(auth, req)
	auth.Write([]byte{',', ' '})
	auth.Write([]byte("Signature=" + fmt.Sprintf("%x", h.Sum(nil))))

	req.Header.Set("authorization", auth.String())
	return nil
}

func writeURI(w io.Writer, r *http.Request) {
	path := r.URL.RequestURI()
	if r.URL.RawQuery != "" {
		path = path[:len(path)-len(r.URL.RawQuery)-1]
	}
	slash := strings.HasSuffix(path, "/")
	path = filepath.Clean(path)
	if path != "/" && slash {
		path += "/"
	}
	w.Write([]byte(path))
}
func writeQuery(w io.Writer, r *http.Request) {
	var a []string
	for k, vs := range r.URL.Query() {
		k = url.QueryEscape(k)
		for _, v := range vs {
			if v == "" {
				a = append(a, k)
			} else {
				v = url.QueryEscape(v)
				a = append(a, k+"="+v)
			}
		}
	}
	sort.Strings(a)
	for i, s := range a {
		if i > 0 {
			w.Write([]byte{'&'})
		}
		w.Write([]byte(s))
	}
}

func writeHeader(w io.Writer, r *http.Request) {
	i, a := 0, make([]string, len(r.Header))
	for k, v := range r.Header {
		sort.Strings(v)
		a[i] = strings.ToLower(k) + ":" + strings.Join(v, ",")
		i++
	}
	sort.Strings(a)
	for i, s := range a {
		if i > 0 {
			w.Write(newLine)
		}
		io.WriteString(w, s)
	}
}

func writeHeaderList(w io.Writer, r *http.Request) {
	i, a := 0, make([]string, len(r.Header))
	for k := range r.Header {
		a[i] = strings.ToLower(k)
		i++
	}
	sort.Strings(a)
	for i, s := range a {
		if i > 0 {
			w.Write([]byte{';'})
		}
		w.Write([]byte(s))
	}
}

func writeBody(w io.Writer, r *http.Request) error {
	var (
		b   []byte
		err error
	)
	// If the payload is empty, use the empty string as the input to the SHA256 function
	// http://docs.amazonwebservices.com/general/latest/gr/sigv4-create-canonical-request.html
	if r.Body == nil {
		b = []byte("")
	} else {
		b, err = ioutil.ReadAll(r.Body)
		if err != nil {
			return err
		}
		r.Body = ioutil.NopCloser(bytes.NewBuffer(b))
	}

	h := sha256.New()
	h.Write(b)
	fmt.Fprintf(w, "%x", h.Sum(nil))
	return nil
}

func makeHMac(key []byte, data []byte) []byte {
	hash := hmac.New(sha256.New, key)
	hash.Write(data)
	return hash.Sum(nil)
}

func (ix *ImageX) Upload(rawurl, fp, auth string) error {
	crc32, err := hashFileCRC32(fp)
	if err != nil {
		return err
	}
	file, err := os.Open(fp)
	if err != nil {
		return err
	}
	defer file.Close()

	req, err := http.NewRequest(http.MethodPost, rawurl, file)
	if err != nil {
		return err
	}
	req.Header.Add("authorization", auth)
	req.Header.Add("Content-Type", "application/octet-stream")
	req.Header.Add("content-crc32", crc32)
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}
	raw := string(b)
	if gjson.Get(raw, "success").Int() != 0 {
		return fmt.Errorf("raw: %s, response: %+v", raw, res)
	}
	return nil
}

// hashFileCRC32 generate CRC32 hash of a file
// Refer https://mrwaggel.be/post/generate-crc32-hash-of-a-file-in-golang-turorial/
func hashFileCRC32(filePath string) (string, error) {
	file, err := os.Open(filePath)
	if err != nil {
		return "", err
	}
	defer file.Close()
	tablePolynomial := crc32.MakeTable(polynomialCRC32)
	hash := crc32.New(tablePolynomial)
	if _, err := io.Copy(hash, file); err != nil {
		return "", err
	}
	return hex.EncodeToString(hash.Sum(nil)), nil
}

总结

就一句话，掘金牛啊牛啊！

avatar
文章
阅读
粉丝
相关推荐
docker阿里云镜像加速
6.9k阅读
 · 
1点赞
搭建国外服务器
704阅读
 · 
4点赞
一条命令激活 Windows 系统！再也不用找激活工具激活码了
3.4k阅读
 · 
4点赞
一次 Go json 序列化踩坑经历
619阅读
 · 
0点赞
快速部署Odoo，开启高效企业管理
793阅读
 · 
0点赞
QMT行情接口以及历史行情数据下载
1.7k阅读
 · 
0点赞
【机器学习|Python】sklearn中的逻辑回归模型
4.3k阅读
 · 
1点赞
双11中阿里云服务器价格最便宜的一款推荐给你
429阅读
 · 
2点赞
「译文」Google SRE 二十年的经验教训
491阅读
 · 
0点赞

友情链接：