Feat: slave policy creating upload session API

wujizun · Feb 27, 2022 · 2811ee3 · 2811ee3
1 parent 7dd636d
commit 2811ee3
Show file tree

Hide file tree

Showing 21 changed files with 236 additions and 106 deletions.
diff --git a/models/policy.go b/models/policy.go
@@ -219,16 +219,16 @@ func (policy *Policy) IsTransitUpload(size uint64) bool {
 	return policy.Type == "local"
 }
 
-// IsPathGenerateNeeded 返回此策略是否需要在生成上传凭证时生成存储路径
-func (policy *Policy) IsPathGenerateNeeded() bool {
-	return policy.Type != "remote"
-}
-
 // IsThumbGenerateNeeded 返回此策略是否需要在上传后生成缩略图
 func (policy *Policy) IsThumbGenerateNeeded() bool {
 	return policy.Type == "local"
 }
 
+// IsUploadPlaceholderWithSize 返回此策略创建上传会话时是否需要预留空间
+func (policy *Policy) IsUploadPlaceholderWithSize() bool {
+	return policy.Type == "remote"
+}
+
 // CanStructureBeListed 返回存储策略是否能被前台列物理目录
 func (policy *Policy) CanStructureBeListed() bool {
 	return policy.Type != "local" && policy.Type != "remote"

diff --git a/pkg/filesystem/driver/cos/handler.go b/pkg/filesystem/driver/cos/handler.go
@@ -326,7 +326,7 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64,
 }
 
 // Token 获取上传策略和认证Token
-func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (serializer.UploadCredential, error) {
+func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
 	// 生成回调地址
 	siteURL := model.GetSiteURL()
 	apiBaseURI, _ := url.Parse("/api/v3/callback/cos/" + uploadSession.Key)
@@ -383,38 +383,38 @@ func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error)
 	}, nil
 }
 
-func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, keyTime string, savePath string) (serializer.UploadCredential, error) {
+func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, keyTime string, savePath string) (*serializer.UploadCredential, error) {
 	// 编码上传策略
 	policyJSON, err := json.Marshal(policy)
 	if err != nil {
-		return serializer.UploadCredential{}, err
+		return nil, err
 	}
 	policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
 
 	// 签名上传策略
 	hmacSign := hmac.New(sha1.New, []byte(handler.Policy.SecretKey))
 	_, err = io.WriteString(hmacSign, keyTime)
 	if err != nil {
-		return serializer.UploadCredential{}, err
+		return nil, err
 	}
 	signKey := fmt.Sprintf("%x", hmacSign.Sum(nil))
 
 	sha1Sign := sha1.New()
 	_, err = sha1Sign.Write(policyJSON)
 	if err != nil {
-		return serializer.UploadCredential{}, err
+		return nil, err
 	}
 	stringToSign := fmt.Sprintf("%x", sha1Sign.Sum(nil))
 
 	// 最终签名
 	hmacFinalSign := hmac.New(sha1.New, []byte(signKey))
 	_, err = hmacFinalSign.Write([]byte(stringToSign))
 	if err != nil {
-		return serializer.UploadCredential{}, err
+		return nil, err
 	}
 	signature := hmacFinalSign.Sum(nil)
 
-	return serializer.UploadCredential{
+	return &serializer.UploadCredential{
 		Policy:    policyEncoded,
 		Path:      savePath,
 		AccessKey: handler.Policy.AccessKey,

diff --git a/pkg/filesystem/driver/handler.go b/pkg/filesystem/driver/handler.go
@@ -30,7 +30,7 @@ type Handler interface {
 	Source(ctx context.Context, path string, url url.URL, ttl int64, isDownload bool, speed int) (string, error)
 
 	// Token 获取有效期为ttl的上传凭证和签名
-	Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (serializer.UploadCredential, error)
+	Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error)
 
 	// CancelToken 取消已经创建的有状态上传凭证
 	CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error

diff --git a/pkg/filesystem/driver/local/handler.go b/pkg/filesystem/driver/local/handler.go
@@ -254,8 +254,8 @@ func (handler Driver) Source(
 }
 
 // Token 获取上传策略和认证Token，本地策略直接返回空值
-func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (serializer.UploadCredential, error) {
-	return serializer.UploadCredential{
+func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
+	return &serializer.UploadCredential{
 		SessionID: uploadSession.Key,
 		ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
 	}, nil

diff --git a/pkg/filesystem/driver/onedrive/handler.go b/pkg/filesystem/driver/onedrive/handler.go
@@ -223,12 +223,12 @@ func (handler Driver) replaceSourceHost(origin string) (string, error) {
 }
 
 // Token 获取上传会话URL
-func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (serializer.UploadCredential, error) {
+func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
 	fileInfo := file.Info()
 
 	// 如果小于4MB，则由服务端中转
 	if fileInfo.Size <= SmallFileSize {
-		return serializer.UploadCredential{}, nil
+		return nil, nil
 	}
 
 	// 生成回调地址
@@ -238,13 +238,13 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
 
 	uploadURL, err := handler.Client.CreateUploadSession(ctx, fileInfo.SavePath, WithConflictBehavior("fail"))
 	if err != nil {
-		return serializer.UploadCredential{}, err
+		return nil, err
 	}
 
 	// 监控回调及上传
 	go handler.Client.MonitorUpload(uploadURL, uploadSession.Key, fileInfo.SavePath, fileInfo.Size, ttl)
 
-	return serializer.UploadCredential{
+	return &serializer.UploadCredential{
 		Policy: uploadURL,
 		Token:  apiURL.String(),
 	}, nil

diff --git a/pkg/filesystem/driver/oss/handler.go b/pkg/filesystem/driver/oss/handler.go
@@ -398,7 +398,7 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64,
 }
 
 // Token 获取上传策略和认证Token
-func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (serializer.UploadCredential, error) {
+func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
 	// 生成回调地址
 	siteURL := model.GetSiteURL()
 	apiBaseURI, _ := url.Parse("/api/v3/callback/oss/" + uploadSession.Key)
@@ -429,13 +429,13 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
 	return handler.getUploadCredential(ctx, postPolicy, callbackPolicy, ttl, savePath)
 }
 
-func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, callback CallbackPolicy, TTL int64, savePath string) (serializer.UploadCredential, error) {
+func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, callback CallbackPolicy, TTL int64, savePath string) (*serializer.UploadCredential, error) {
 	// 处理回调策略
 	callbackPolicyEncoded := ""
 	if callback.CallbackURL != "" {
 		callbackPolicyJSON, err := json.Marshal(callback)
 		if err != nil {
-			return serializer.UploadCredential{}, err
+			return nil, err
 		}
 		callbackPolicyEncoded = base64.StdEncoding.EncodeToString(callbackPolicyJSON)
 		policy.Conditions = append(policy.Conditions, map[string]string{"callback": callbackPolicyEncoded})
@@ -444,19 +444,19 @@ func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPoli
 	// 编码上传策略
 	policyJSON, err := json.Marshal(policy)
 	if err != nil {
-		return serializer.UploadCredential{}, err
+		return nil, err
 	}
 	policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
 
 	// 签名上传策略
 	hmacSign := hmac.New(sha1.New, []byte(handler.Policy.SecretKey))
 	_, err = io.WriteString(hmacSign, policyEncoded)
 	if err != nil {
-		return serializer.UploadCredential{}, err
+		return nil, err
 	}
 	signature := base64.StdEncoding.EncodeToString(hmacSign.Sum(nil))
 
-	return serializer.UploadCredential{
+	return &serializer.UploadCredential{
 		Policy:    fmt.Sprintf("%s:%s", callbackPolicyEncoded, policyEncoded),
 		Path:      savePath,
 		AccessKey: handler.Policy.AccessKey,

diff --git a/pkg/filesystem/driver/qiniu/handler.go b/pkg/filesystem/driver/qiniu/handler.go
@@ -274,7 +274,7 @@ func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64)
 }
 
 // Token 获取上传策略和认证Token
-func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (serializer.UploadCredential, error) {
+func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
 	// 生成回调地址
 	siteURL := model.GetSiteURL()
 	apiBaseURI, _ := url.Parse("/api/v3/callback/qiniu/" + uploadSession.Key)
@@ -299,12 +299,12 @@ func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *seria
 }
 
 // getUploadCredential 签名上传策略
-func (handler Driver) getUploadCredential(ctx context.Context, policy storage.PutPolicy, TTL int64) (serializer.UploadCredential, error) {
+func (handler Driver) getUploadCredential(ctx context.Context, policy storage.PutPolicy, TTL int64) (*serializer.UploadCredential, error) {
 	policy.Expires = uint64(TTL)
 	mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
 	upToken := policy.UploadToken(mac)
 
-	return serializer.UploadCredential{
+	return &serializer.UploadCredential{
 		Token: upToken,
 	}, nil
 }

diff --git a/pkg/filesystem/driver/remote/client.go b/pkg/filesystem/driver/remote/client.go
@@ -0,0 +1,70 @@
+package remote
+
+import (
+	"encoding/json"
+	model "github.com/cloudreve/Cloudreve/v3/models"
+	"github.com/cloudreve/Cloudreve/v3/pkg/auth"
+	"github.com/cloudreve/Cloudreve/v3/pkg/request"
+	"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
+	"net/url"
+	"strings"
+)
+
+// Client to operate remote slave server
+type Client interface {
+	CreateUploadSession(session *serializer.UploadSession, ttl int64) error
+}
+
+// NewClient creates new Client from given policy
+func NewClient(policy *model.Policy) (Client, error) {
+	authInstance := auth.HMACAuth{[]byte(policy.SecretKey)}
+	serverURL, err := url.Parse(policy.Server)
+	if err != nil {
+		return nil, err
+	}
+
+	base, _ := url.Parse("/api/v3/slave")
+	signTTL := model.GetIntSetting("slave_api_timeout", 60)
+
+	return &remoteClient{
+		policy:       policy,
+		authInstance: authInstance,
+		httpClient: request.NewClient(
+			request.WithEndpoint(serverURL.ResolveReference(base).String()),
+			request.WithCredential(authInstance, int64(signTTL)),
+			request.WithMasterMeta(),
+		),
+	}, nil
+}
+
+type remoteClient struct {
+	policy       *model.Policy
+	authInstance auth.Auth
+	httpClient   request.Client
+}
+
+func (c *remoteClient) CreateUploadSession(session *serializer.UploadSession, ttl int64) error {
+	reqBodyEncoded, err := json.Marshal(map[string]interface{}{
+		"session": session,
+		"ttl":     ttl,
+	})
+	if err != nil {
+		return err
+	}
+
+	bodyReader := strings.NewReader(string(reqBodyEncoded))
+	resp, err := c.httpClient.Request(
+		"PUT",
+		"upload",
+		bodyReader,
+	).CheckHTTPResponse(200).DecodeResponse()
+	if err != nil {
+		return err
+	}
+
+	if resp.Code != 0 {
+		return serializer.NewErrorFromResponse(resp)
+	}
+
+	return nil
+}
diff --git a/pkg/filesystem/driver/remote/handler.go b/pkg/filesystem/driver/remote/handler.go
@@ -25,6 +25,23 @@ type Driver struct {
 	Client       request.Client
 	Policy       *model.Policy
 	AuthInstance auth.Auth
+
+	client Client
+}
+
+// NewDriver initializes a new Driver from policy
+func NewDriver(policy *model.Policy) (*Driver, error) {
+	client, err := NewClient(policy)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Driver{
+		Policy:       policy,
+		Client:       request.NewClient(),
+		AuthInstance: auth.HMACAuth{[]byte(policy.SecretKey)},
+		client:       client,
+	}, nil
 }
 
 // List 列取文件
@@ -305,22 +322,30 @@ func (handler Driver) Source(
 }
 
 // Token 获取上传策略和认证Token
-func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (serializer.UploadCredential, error) {
-	// 生成回调地址
-	siteURL := model.GetSiteURL()
-	apiBaseURI, _ := url.Parse("/api/v3/callback/remote/" + uploadSession.Key)
-	apiURL := siteURL.ResolveReference(apiBaseURI)
+func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
+	if err := handler.client.CreateUploadSession(uploadSession, ttl); err != nil {
+		return nil, err
+	}
 
-	// 生成上传策略
-	policy := serializer.UploadPolicy{
-		SavePath:         handler.Policy.DirNameRule,
-		FileName:         handler.Policy.FileNameRule,
-		AutoRename:       handler.Policy.AutoRename,
-		MaxSize:          handler.Policy.MaxSize,
-		AllowedExtension: handler.Policy.OptionsSerialized.FileType,
-		CallbackURL:      apiURL.String(),
-	}
-	return handler.getUploadCredential(ctx, policy, ttl)
+	return &serializer.UploadCredential{
+		SessionID: uploadSession.Key,
+		ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
+	}, nil
+	//// 生成回调地址
+	//siteURL := model.GetSiteURL()
+	//apiBaseURI, _ := url.Parse("/api/v3/callback/remote/" + uploadSession.Key)
+	//apiURL := siteURL.ResolveReference(apiBaseURI)
+	//
+	//// 生成上传策略
+	//policy := serializer.UploadPolicy{
+	//	SavePath:         handler.Policy.DirNameRule,
+	//	FileName:         handler.Policy.FileNameRule,
+	//	AutoRename:       handler.Policy.AutoRename,
+	//	MaxSize:          handler.Policy.MaxSize,
+	//	AllowedExtension: handler.Policy.OptionsSerialized.FileType,
+	//	CallbackURL:      apiURL.String(),
+	//}
+	//return handler.getUploadCredential(ctx, policy, ttl)
 }
 
 func (handler Driver) getUploadCredential(ctx context.Context, policy serializer.UploadPolicy, TTL int64) (serializer.UploadCredential, error) {

diff --git a/pkg/filesystem/driver/s3/handler.go b/pkg/filesystem/driver/s3/handler.go
@@ -325,7 +325,7 @@ func (handler Driver) Source(
 }
 
 // Token 获取上传策略和认证Token
-func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (serializer.UploadCredential, error) {
+func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
 	// 生成回调地址
 	siteURL := model.GetSiteURL()
 	apiBaseURI, _ := url.Parse("/api/v3/callback/s3/" + uploadSession.Key)
@@ -378,7 +378,7 @@ func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error)
 
 }
 
-func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, callback *url.URL, savePath string) (serializer.UploadCredential, error) {
+func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, callback *url.URL, savePath string) (*serializer.UploadCredential, error) {
 
 	longDate := time.Now().UTC().Format("20060102T150405Z")
 	shortDate := time.Now().UTC().Format("20060102")
@@ -390,7 +390,7 @@ func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPoli
 	// 编码上传策略
 	policyJSON, err := json.Marshal(policy)
 	if err != nil {
-		return serializer.UploadCredential{}, err
+		return nil, err
 	}
 	policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
 
@@ -401,7 +401,7 @@ func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPoli
 	signature = getHMAC(signature, []byte("aws4_request"))
 	signature = getHMAC(signature, []byte(policyEncoded))
 
-	return serializer.UploadCredential{
+	return &serializer.UploadCredential{
 		Policy:    policyEncoded,
 		Callback:  callback.String(),
 		Token:     hex.EncodeToString(signature),

diff --git a/pkg/filesystem/driver/shadow/masterinslave/handler.go b/pkg/filesystem/driver/shadow/masterinslave/handler.go
@@ -47,8 +47,8 @@ func (d *Driver) Source(ctx context.Context, path string, url url.URL, ttl int64
 	return "", ErrNotImplemented
 }
 
-func (d *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (serializer.UploadCredential, error) {
-	return serializer.UploadCredential{}, ErrNotImplemented
+func (d *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
+	return nil, ErrNotImplemented
 }
 
 func (d *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {

diff --git a/pkg/filesystem/driver/shadow/slaveinmaster/handler.go b/pkg/filesystem/driver/shadow/slaveinmaster/handler.go
@@ -113,8 +113,8 @@ func (d *Driver) Source(ctx context.Context, path string, url url.URL, ttl int64
 	return "", ErrNotImplemented
 }
 
-func (d *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (serializer.UploadCredential, error) {
-	return serializer.UploadCredential{}, ErrNotImplemented
+func (d *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
+	return nil, ErrNotImplemented
 }
 
 func (d *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {