整体平台上传功能大改造

This commit is contained in:
2026-09-09 10:49:57 +08:00
parent d2643a3697
commit de7225b61e
21 changed files with 3835 additions and 2622 deletions
+440 -252
View File
@@ -1,252 +1,440 @@
package services
import (
"context"
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"mime/multipart"
"os"
"path/filepath"
"strings"
"time"
"server/models"
"github.com/qiniu/go-sdk/v7/auth/qbox"
"github.com/qiniu/go-sdk/v7/storage"
)
// StorageService 存储服务接口
type StorageService interface {
Upload(file multipart.File, header *multipart.FileHeader) (*UploadResult, error)
GetPublicURL(key string) string
Delete(key string) error
}
// UploadResult 上传结果
type UploadResult struct {
URL string // 完整访问URL
Key string // 存储key/路径
Size int64 // 文件大小
MD5 string // 文件MD5
MimeType string // 文件类型
}
// LocalStorage 本地存储实现
type LocalStorage struct {
BaseDir string // 基础目录,默认 "uploads"
BaseURL string // 基础URL,默认 "/"
}
// NewLocalStorage 创建本地存储服务
func NewLocalStorage() *LocalStorage {
return &LocalStorage{
BaseDir: "uploads",
BaseURL: "/",
}
}
// Upload 上传文件到本地
func (s *LocalStorage) Upload(file multipart.File, header *multipart.FileHeader) (*UploadResult, error) {
// 生成存储路径
ext := filepath.Ext(header.Filename)
datePath := time.Now().Format("2006/01/02")
fileName := fmt.Sprintf("%d%s", time.Now().UnixNano(), ext)
savePath := filepath.Join(datePath, fileName)
// 创建目录
destDir := filepath.Join(s.BaseDir, filepath.FromSlash(datePath))
if err := os.MkdirAll(destDir, 0755); err != nil {
return nil, fmt.Errorf("创建目录失败: %w", err)
}
// 保存文件
destPath := filepath.Join(s.BaseDir, filepath.FromSlash(savePath))
dst, err := os.Create(destPath)
if err != nil {
return nil, fmt.Errorf("创建文件失败: %w", err)
}
defer dst.Close()
// 计算MD5并复制文件
hash := md5.New()
size, err := io.Copy(io.MultiWriter(dst, hash), file)
if err != nil {
_ = os.Remove(destPath)
return nil, fmt.Errorf("保存文件失败: %w", err)
}
md5Sum := hex.EncodeToString(hash.Sum(nil))
webURL := s.BaseURL + strings.ReplaceAll(filepath.ToSlash(destPath), "\\", "/")
return &UploadResult{
URL: webURL,
Key: savePath,
Size: size,
MD5: md5Sum,
MimeType: header.Header.Get("Content-Type"),
}, nil
}
// GetPublicURL 获取公开访问URL
func (s *LocalStorage) GetPublicURL(key string) string {
return s.BaseURL + filepath.ToSlash(filepath.Join(s.BaseDir, key))
}
// Delete 删除本地文件
func (s *LocalStorage) Delete(key string) error {
filePath := filepath.Join(s.BaseDir, filepath.FromSlash(key))
return os.Remove(filePath)
}
// QiniuStorage 七牛云存储实现
type QiniuStorage struct {
AccessKey string
SecretKey string
Bucket string
Domain string
Region string
}
// NewQiniuStorage 创建七牛云存储服务
func NewQiniuStorage(cfg *models.StorageConfig) *QiniuStorage {
return &QiniuStorage{
AccessKey: cfg.QiniuAccessKey,
SecretKey: cfg.QiniuSecretKey,
Bucket: cfg.QiniuBucket,
Domain: cfg.QiniuDomain,
Region: cfg.QiniuRegion,
}
}
// getZone 根据区域代码获取存储区域
func (s *QiniuStorage) getZone() *storage.Region {
switch s.Region {
case "z0":
return &storage.ZoneHuadong
case "z1":
return &storage.ZoneHuabei
case "z2":
return &storage.ZoneHuanan
case "na0":
return &storage.ZoneBeimei
case "as0":
return &storage.ZoneXinjiapo
case "cn-east-2":
return &storage.ZoneHuadongZheJiang2
default:
return &storage.ZoneHuadong // 默认华东
}
}
// Upload 上传文件到七牛云
func (s *QiniuStorage) Upload(file multipart.File, header *multipart.FileHeader) (*UploadResult, error) {
// 生成存储key
ext := filepath.Ext(header.Filename)
datePath := time.Now().Format("2006/01/02")
fileName := fmt.Sprintf("%d%s", time.Now().UnixNano(), ext)
key := filepath.ToSlash(filepath.Join(datePath, fileName))
// 创建上传凭证
mac := qbox.NewMac(s.AccessKey, s.SecretKey)
putPolicy := storage.PutPolicy{
Scope: s.Bucket,
}
upToken := putPolicy.UploadToken(mac)
// 配置上传参数
cfg := storage.Config{
Region: s.getZone(),
UseHTTPS: true,
UseCdnDomains: false,
}
// 创建表单上传器
formUploader := storage.NewFormUploader(&cfg)
ret := storage.PutRet{}
putExtra := storage.PutExtra{}
// 计算文件大小和MD5
tmpFile, err := os.CreateTemp("", "qiniu_upload_*")
if err != nil {
return nil, fmt.Errorf("创建临时文件失败: %w", err)
}
defer os.Remove(tmpFile.Name())
defer tmpFile.Close()
hash := md5.New()
size, err := io.Copy(io.MultiWriter(tmpFile, hash), file)
if err != nil {
return nil, fmt.Errorf("读取文件失败: %w", err)
}
md5Sum := hex.EncodeToString(hash.Sum(nil))
// 重置文件指针
if _, err := tmpFile.Seek(0, 0); err != nil {
return nil, fmt.Errorf("重置文件指针失败: %w", err)
}
// 执行上传
err = formUploader.Put(context.Background(), &ret, upToken, key, tmpFile, size, &putExtra)
if err != nil {
return nil, fmt.Errorf("上传到七牛云失败: %w", err)
}
// 构建完整URL
domain := strings.TrimRight(s.Domain, "/")
url := fmt.Sprintf("%s/%s", domain, ret.Key)
return &UploadResult{
URL: url,
Key: ret.Key,
Size: size,
MD5: md5Sum,
MimeType: header.Header.Get("Content-Type"),
}, nil
}
// GetPublicURL 获取七牛云公开访问URL
func (s *QiniuStorage) GetPublicURL(key string) string {
domain := strings.TrimRight(s.Domain, "/")
return fmt.Sprintf("%s/%s", domain, key)
}
// Delete 删除七牛云文件
func (s *QiniuStorage) Delete(key string) error {
mac := qbox.NewMac(s.AccessKey, s.SecretKey)
cfg := storage.Config{
Region: s.getZone(),
UseHTTPS: true,
}
bucketManager := storage.NewBucketManager(mac, &cfg)
err := bucketManager.Delete(s.Bucket, key)
if err != nil {
return fmt.Errorf("删除七牛云文件失败: %w", err)
}
return nil
}
// GetStorageService 根据配置获取存储服务
func GetStorageService() (StorageService, error) {
cfg, err := models.GetStorageConfig()
if err != nil {
// 默认使用本地存储
return NewLocalStorage(), nil
}
switch cfg.StorageType {
case "qiniu":
if cfg.QiniuAccessKey == "" || cfg.QiniuSecretKey == "" ||
cfg.QiniuBucket == "" || cfg.QiniuDomain == "" {
return nil, fmt.Errorf("七牛云配置不完整")
}
return NewQiniuStorage(cfg), nil
case "local":
return NewLocalStorage(), nil
default:
return NewLocalStorage(), nil
}
}
package services
import (
"context"
"crypto/md5"
"crypto/rand"
"encoding/hex"
"fmt"
"io"
"mime/multipart"
"os"
"path"
"path/filepath"
"strings"
"time"
"server/models"
"github.com/qiniu/go-sdk/v7/auth/qbox"
"github.com/qiniu/go-sdk/v7/storage"
)
// 来源端
const (
SourceBackend = "backend" // 租户后台端
SourcePlatform = "platform" // 平台端
)
// 归属范围
const (
ScopeTenant = "tenant" // 租户共享文件
ScopeUser = "user" // 用户个人文件
)
// 存储类型
const (
StorageTypeLocal = "local"
StorageTypeQiniu = "qiniu"
)
// UploadContext 上传上下文,决定文件最终的存储路径
type UploadContext struct {
Source string // backend / platform
Tid uint64 // 租户 ID(platform 端可为 0)
Tuid uint64 // 归属用户 ID,0 表示租户共享文件
}
// Scope 返回归属范围:带了归属用户即个人文件,否则为租户共享文件。
// 平台端不做用户分层,统一按租户共享处理。
func (c UploadContext) Scope() string {
if c.Source == SourcePlatform {
return ScopeTenant
}
if c.Tuid > 0 {
return ScopeUser
}
return ScopeTenant
}
// BuildObjectKey 生成存储相对路径(不含域名、不含本地 BaseDir)
//
// backend 共享: backend/234573/2026/09/09/xxx.png
// backend 个人: backend/234573/67091493/2026/09/09/xxx.png
// platform : platform/2026/09/09/xxx.png
func BuildObjectKey(ctx UploadContext, ext string) string {
datePath := time.Now().Format("2006/01/02")
name := fmt.Sprintf("%d_%s%s", time.Now().UnixNano(), randomHex(6), normalizeExt(ext))
if ctx.Source == SourcePlatform {
return path.Join(SourcePlatform, datePath, name)
}
if ctx.Tuid > 0 {
return path.Join(SourceBackend, fmt.Sprint(ctx.Tid), fmt.Sprint(ctx.Tuid), datePath, name)
}
return path.Join(SourceBackend, fmt.Sprint(ctx.Tid), datePath, name)
}
// normalizeExt 规范扩展名:小写、补前导点
func normalizeExt(ext string) string {
ext = strings.TrimSpace(ext)
if ext == "" {
return ""
}
if !strings.HasPrefix(ext, ".") {
ext = "." + ext
}
return strings.ToLower(ext)
}
// randomHex 生成 n 位十六进制随机串(使用 crypto/rand,无需种子)
func randomHex(n int) string {
b := make([]byte, (n+1)/2)
if _, err := rand.Read(b); err != nil {
return fmt.Sprint(time.Now().UnixNano() % 1000000)
}
return hex.EncodeToString(b)[:n]
}
// StagedFile 已落临时文件、算完 MD5 的待提交文件
type StagedFile struct {
TempPath string // 本地临时文件路径
MD5 string // 文件内容 MD5
Size int64 // 文件大小
MimeType string // 文件类型
Ext string // 扩展名(含点)
}
// UploadResult 上传结果
type UploadResult struct {
URL string // 完整访问URL
Key string // 存储key/相对路径(object_key)
Size int64 // 文件大小
MD5 string // 文件MD5
MimeType string // 文件类型
}
// StorageService 存储服务接口
type StorageService interface {
// Stage 把上传流写入临时文件并计算 MD5,此时文件尚未进入正式存储目录
Stage(file multipart.File, header *multipart.FileHeader) (*StagedFile, error)
// Commit 把临时文件提交到按 ctx 计算出的正式路径;返回访问 URL 与 object_key
Commit(staged *StagedFile, ctx UploadContext) (*UploadResult, error)
// Discard 丢弃临时文件(去重命中或出错时调用)
Discard(staged *StagedFile) error
GetPublicURL(key string) string
Delete(key string) error
// Move 把已存文件从 oldKey 改名到 newKey(迁移用:本地 Rename / 七牛 Move)
Move(oldKey, newKey string) error
// Type 返回存储类型 local / qiniu
Type() string
}
// LocalStorage 本地存储实现
type LocalStorage struct {
BaseDir string // 基础目录,默认 "uploads"
BaseURL string // 基础URL,默认 "/"
}
// NewLocalStorage 创建本地存储服务
func NewLocalStorage() *LocalStorage {
return &LocalStorage{
BaseDir: "uploads",
BaseURL: "/",
}
}
// Type 存储类型
func (s *LocalStorage) Type() string { return StorageTypeLocal }
// tempDir 临时目录:与正式目录同盘,保证 Commit 时 os.Rename 不跨设备
func (s *LocalStorage) tempDir() string {
return filepath.Join(s.BaseDir, ".tmp")
}
// Stage 写入临时文件并计算 MD5
func (s *LocalStorage) Stage(file multipart.File, header *multipart.FileHeader) (*StagedFile, error) {
dir := s.tempDir()
if err := os.MkdirAll(dir, 0755); err != nil {
return nil, fmt.Errorf("创建临时目录失败: %w", err)
}
tmp, err := os.CreateTemp(dir, "stage_*")
if err != nil {
return nil, fmt.Errorf("创建临时文件失败: %w", err)
}
defer tmp.Close()
hash := md5.New()
size, err := io.Copy(io.MultiWriter(tmp, hash), file)
if err != nil {
tmpPath := tmp.Name()
_ = os.Remove(tmpPath)
return nil, fmt.Errorf("读取上传文件失败: %w", err)
}
return &StagedFile{
TempPath: tmp.Name(),
MD5: hex.EncodeToString(hash.Sum(nil)),
Size: size,
MimeType: mimeTypeOf(header),
Ext: normalizeExt(filepath.Ext(header.Filename)),
}, nil
}
// Commit 提交到正式目录
func (s *LocalStorage) Commit(staged *StagedFile, ctx UploadContext) (*UploadResult, error) {
if staged == nil {
return nil, fmt.Errorf("待提交文件为空")
}
key := BuildObjectKey(ctx, staged.Ext)
destPath := filepath.Join(s.BaseDir, filepath.FromSlash(key))
if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
return nil, fmt.Errorf("创建目录失败: %w", err)
}
// 同盘优先 Rename;跨设备失败则回退复制
if err := os.Rename(staged.TempPath, destPath); err != nil {
if copyErr := copyFile(staged.TempPath, destPath); copyErr != nil {
return nil, fmt.Errorf("保存文件失败: %w", copyErr)
}
_ = os.Remove(staged.TempPath)
}
return &UploadResult{
URL: s.GetPublicURL(key),
Key: key,
Size: staged.Size,
MD5: staged.MD5,
MimeType: staged.MimeType,
}, nil
}
// Discard 删除临时文件
func (s *LocalStorage) Discard(staged *StagedFile) error {
if staged == nil || staged.TempPath == "" {
return nil
}
return os.Remove(staged.TempPath)
}
// GetPublicURL 获取公开访问URL
func (s *LocalStorage) GetPublicURL(key string) string {
full := filepath.ToSlash(filepath.Join(s.BaseDir, filepath.FromSlash(key)))
return s.BaseURL + strings.ReplaceAll(full, "\\", "/")
}
// Delete 删除本地文件
func (s *LocalStorage) Delete(key string) error {
filePath := filepath.Join(s.BaseDir, filepath.FromSlash(key))
return os.Remove(filePath)
}
// Move 本地改名(同盘,不搬数据)
func (s *LocalStorage) Move(oldKey, newKey string) error {
oldPath := filepath.Join(s.BaseDir, filepath.FromSlash(oldKey))
newPath := filepath.Join(s.BaseDir, filepath.FromSlash(newKey))
if err := os.MkdirAll(filepath.Dir(newPath), 0755); err != nil {
return fmt.Errorf("创建目标目录失败: %w", err)
}
if err := os.Rename(oldPath, newPath); err != nil {
// 跨设备时回退为复制 + 删除
if copyErr := copyFile(oldPath, newPath); copyErr != nil {
return fmt.Errorf("移动文件失败: %w", copyErr)
}
_ = os.Remove(oldPath)
}
return nil
}
// QiniuStorage 七牛云存储实现
type QiniuStorage struct {
AccessKey string
SecretKey string
Bucket string
Domain string
Region string
}
// NewQiniuStorage 创建七牛云存储服务
func NewQiniuStorage(cfg *models.StorageConfig) *QiniuStorage {
return &QiniuStorage{
AccessKey: cfg.QiniuAccessKey,
SecretKey: cfg.QiniuSecretKey,
Bucket: cfg.QiniuBucket,
Domain: cfg.QiniuDomain,
Region: cfg.QiniuRegion,
}
}
// Type 存储类型
func (s *QiniuStorage) Type() string { return StorageTypeQiniu }
// getZone 根据区域代码获取存储区域
func (s *QiniuStorage) getZone() *storage.Region {
switch s.Region {
case "z0":
return &storage.ZoneHuadong
case "z1":
return &storage.ZoneHuabei
case "z2":
return &storage.ZoneHuanan
case "na0":
return &storage.ZoneBeimei
case "as0":
return &storage.ZoneXinjiapo
case "cn-east-2":
return &storage.ZoneHuadongZheJiang2
default:
return &storage.ZoneHuadong // 默认华东
}
}
// Stage 写入临时文件并计算 MD5(七牛直传拿不到内容,这里统一先落临时文件算 MD5,
// 便于去重;未命中才会真正上传到七牛)
func (s *QiniuStorage) Stage(file multipart.File, header *multipart.FileHeader) (*StagedFile, error) {
tmp, err := os.CreateTemp("", "yz_upload_*")
if err != nil {
return nil, fmt.Errorf("创建临时文件失败: %w", err)
}
defer tmp.Close()
hash := md5.New()
size, err := io.Copy(io.MultiWriter(tmp, hash), file)
if err != nil {
_ = os.Remove(tmp.Name())
return nil, fmt.Errorf("读取上传文件失败: %w", err)
}
return &StagedFile{
TempPath: tmp.Name(),
MD5: hex.EncodeToString(hash.Sum(nil)),
Size: size,
MimeType: mimeTypeOf(header),
Ext: normalizeExt(filepath.Ext(header.Filename)),
}, nil
}
// Commit 上传到七牛云
func (s *QiniuStorage) Commit(staged *StagedFile, ctx UploadContext) (*UploadResult, error) {
if staged == nil {
return nil, fmt.Errorf("待提交文件为空")
}
key := BuildObjectKey(ctx, staged.Ext)
f, err := os.Open(staged.TempPath)
if err != nil {
return nil, fmt.Errorf("打开临时文件失败: %w", err)
}
defer f.Close()
mac := qbox.NewMac(s.AccessKey, s.SecretKey)
putPolicy := storage.PutPolicy{Scope: s.Bucket}
upToken := putPolicy.UploadToken(mac)
cfg := storage.Config{
Region: s.getZone(),
UseHTTPS: true,
UseCdnDomains: false,
}
formUploader := storage.NewFormUploader(&cfg)
ret := storage.PutRet{}
if err := formUploader.Put(context.Background(), &ret, upToken, key, f, staged.Size, &storage.PutExtra{}); err != nil {
return nil, fmt.Errorf("上传到七牛云失败: %w", err)
}
return &UploadResult{
URL: s.GetPublicURL(key),
Key: key,
Size: staged.Size,
MD5: staged.MD5,
MimeType: staged.MimeType,
}, nil
}
// Discard 删除临时文件
func (s *QiniuStorage) Discard(staged *StagedFile) error {
if staged == nil || staged.TempPath == "" {
return nil
}
return os.Remove(staged.TempPath)
}
// GetPublicURL 获取七牛云公开访问URL
func (s *QiniuStorage) GetPublicURL(key string) string {
domain := strings.TrimRight(s.Domain, "/")
return fmt.Sprintf("%s/%s", domain, key)
}
// Delete 删除七牛云文件
func (s *QiniuStorage) Delete(key string) error {
return s.runBucketManager(func(bm *storage.BucketManager) error {
return bm.Delete(s.Bucket, key)
}, "删除七牛云文件失败")
}
// Move 七牛服务端改名(同 bucket 内原子操作,不走流量)
func (s *QiniuStorage) Move(oldKey, newKey string) error {
return s.runBucketManager(func(bm *storage.BucketManager) error {
// force=false:目标 key 已存在时直接报错,避免覆盖,保证迁移可重复执行
return bm.Move(s.Bucket, oldKey, s.Bucket, newKey, false)
}, "移动七牛云文件失败")
}
func (s *QiniuStorage) runBucketManager(fn func(*storage.BucketManager) error, errMsg string) error {
mac := qbox.NewMac(s.AccessKey, s.SecretKey)
cfg := storage.Config{Region: s.getZone(), UseHTTPS: true}
bm := storage.NewBucketManager(mac, &cfg)
if err := fn(bm); err != nil {
return fmt.Errorf("%s: %w", errMsg, err)
}
return nil
}
// GetStorageService 根据配置获取存储服务
func GetStorageService() (StorageService, error) {
cfg, err := models.GetStorageConfig()
if err != nil {
// 默认使用本地存储
return NewLocalStorage(), nil
}
switch cfg.StorageType {
case "qiniu":
if cfg.QiniuAccessKey == "" || cfg.QiniuSecretKey == "" ||
cfg.QiniuBucket == "" || cfg.QiniuDomain == "" {
return nil, fmt.Errorf("七牛云配置不完整")
}
return NewQiniuStorage(cfg), nil
case "local":
return NewLocalStorage(), nil
default:
return NewLocalStorage(), nil
}
}
// mimeTypeOf 从 multipart header 取 Content-Type
func mimeTypeOf(header *multipart.FileHeader) string {
if header == nil {
return ""
}
return header.Header.Get("Content-Type")
}
// copyFile 文件复制(Rename 跨设备失败时的回退方案)
func copyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
return err
}
defer out.Close()
if _, err := io.Copy(out, in); err != nil {
return err
}
return out.Sync()
}