增加七牛云存储

2026-04-09 16:26:38 +08:00
parent 01426eda44
commit 36d2a8945e
21 changed files with 2544 additions and 61 deletions
@@ -0,0 +1,191 @@
+package services
+
+import (
+	"fmt"
+	"mime/multipart"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"server/models"
+)
+
+// MigrationProgress 迁移进度
+type MigrationProgress struct {
+	Total     int
+	Success   int
+	Failed    int
+	Current   string
+	Errors    []string
+	mu        sync.Mutex
+}
+
+// AddSuccess 增加成功计数
+func (p *MigrationProgress) AddSuccess() {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.Success++
+}
+
+// AddFailed 增加失败计数
+func (p *MigrationProgress) AddFailed(err string) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.Failed++
+	p.Errors = append(p.Errors, err)
+}
+
+// SetCurrent 设置当前处理的文件
+func (p *MigrationProgress) SetCurrent(filename string) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.Current = filename
+}
+
+// GetProgress 获取进度信息
+func (p *MigrationProgress) GetProgress() (int, int, int, string) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.Total, p.Success, p.Failed, p.Current
+}
+
+// StorageMigration 存储迁移服务
+type StorageMigration struct {
+	fromService StorageService
+	toService   StorageService
+	progress    *MigrationProgress
+}
+
+// NewStorageMigration 创建存储迁移服务
+func NewStorageMigration(from, to StorageService) *StorageMigration {
+	return &StorageMigration{
+		fromService: from,
+		toService:   to,
+		progress: &MigrationProgress{
+			Errors: make([]string, 0),
+		},
+	}
+}
+
+// MigrateFile 迁移单个文件
+func (m *StorageMigration) MigrateFile(file *models.SystemFile) error {
+	m.progress.SetCurrent(file.Name)
+
+	// 如果是本地存储，从本地读取文件
+	if localFrom, ok := m.fromService.(*LocalStorage); ok {
+		// 从本地文件系统读取
+		localPath := strings.TrimPrefix(file.Src, "/")
+		filePath := filepath.Join(localFrom.BaseDir, localPath)
+		
+		f, err := os.Open(filePath)
+		if err != nil {
+			return fmt.Errorf("打开本地文件失败: %w", err)
+		}
+		defer f.Close()
+
+		// 获取文件信息
+		stat, err := f.Stat()
+		if err != nil {
+			return fmt.Errorf("获取文件信息失败: %w", err)
+		}
+
+		// 创建 multipart.FileHeader
+		header := &multipart.FileHeader{
+			Filename: file.Name,
+			Size:     stat.Size(),
+		}
+
+		// 上传到目标存储
+		result, err := m.toService.Upload(f, header)
+		if err != nil {
+			return fmt.Errorf("上传到目标存储失败: %w", err)
+		}
+
+		// 更新数据库记录
+		_, err = models.Orm.QueryTable(new(models.SystemFile)).
+			Filter("id", file.ID).
+			Update(map[string]interface{}{
+				"src": result.URL,
+			})
+		if err != nil {
+			// 上传成功但更新数据库失败，尝试删除已上传的文件
+			_ = m.toService.Delete(result.Key)
+			return fmt.Errorf("更新数据库失败: %w", err)
+		}
+
+		m.progress.AddSuccess()
+		return nil
+	}
+
+	// 如果是七牛云存储，需要先下载再上传（这里简化处理）
+	return fmt.Errorf("暂不支持从七牛云迁移到本地")
+}
+
+// MigrateAll 迁移所有文件
+func (m *StorageMigration) MigrateAll(tid uint64) error {
+	// 获取所有文件
+	var files []models.SystemFile
+	_, err := models.Orm.QueryTable(new(models.SystemFile)).
+		Filter("tid", tid).
+		Filter("delete_time__isnull", true).
+		All(&files)
+	if err != nil {
+		return fmt.Errorf("获取文件列表失败: %w", err)
+	}
+
+	m.progress.Total = len(files)
+
+	// 并发迁移（限制并发数）
+	concurrency := 5
+	sem := make(chan struct{}, concurrency)
+	var wg sync.WaitGroup
+
+	for i := range files {
+		wg.Add(1)
+		go func(file *models.SystemFile) {
+			defer wg.Done()
+			sem <- struct{}{}        // 获取信号量
+			defer func() { <-sem }() // 释放信号量
+
+			if err := m.MigrateFile(file); err != nil {
+				m.progress.AddFailed(fmt.Sprintf("%s: %v", file.Name, err))
+			}
+		}(&files[i])
+	}
+
+	wg.Wait()
+	return nil
+}
+
+// GetProgress 获取迁移进度
+func (m *StorageMigration) GetProgress() *MigrationProgress {
+	return m.progress
+}
+
+// MigrateLocalToQiniu 从本地存储迁移到七牛云
+func MigrateLocalToQiniu(tid uint64) (*MigrationProgress, error) {
+	// 获取存储配置
+	cfg, err := models.GetStorageConfig()
+	if err != nil {
+		return nil, fmt.Errorf("获取存储配置失败: %w", err)
+	}
+
+	if cfg.StorageType != "qiniu" {
+		return nil, fmt.Errorf("当前存储类型不是七牛云")
+	}
+
+	// 创建存储服务
+	localStorage := NewLocalStorage()
+	qiniuStorage := NewQiniuStorage(cfg)
+
+	// 创建迁移服务
+	migration := NewStorageMigration(localStorage, qiniuStorage)
+
+	// 执行迁移
+	if err := migration.MigrateAll(tid); err != nil {
+		return migration.GetProgress(), err
+	}
+
+	return migration.GetProgress(), nil
+}
@@ -0,0 +1,252 @@
+package services
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"server/models"
+
+	"github.com/qiniu/go-sdk/v7/auth/qbox"
+	"github.com/qiniu/go-sdk/v7/storage"
+)
+
+// StorageService 存储服务接口
+type StorageService interface {
+	Upload(file multipart.File, header *multipart.FileHeader) (*UploadResult, error)
+	GetPublicURL(key string) string
+	Delete(key string) error
+}
+
+// UploadResult 上传结果
+type UploadResult struct {
+	URL      string // 完整访问URL
+	Key      string // 存储key/路径
+	Size     int64  // 文件大小
+	MD5      string // 文件MD5
+	MimeType string // 文件类型
+}
+
+// LocalStorage 本地存储实现
+type LocalStorage struct {
+	BaseDir string // 基础目录，默认 "uploads"
+	BaseURL string // 基础URL，默认 "/"
+}
+
+// NewLocalStorage 创建本地存储服务
+func NewLocalStorage() *LocalStorage {
+	return &LocalStorage{
+		BaseDir: "uploads",
+		BaseURL: "/",
+	}
+}
+
+// Upload 上传文件到本地
+func (s *LocalStorage) Upload(file multipart.File, header *multipart.FileHeader) (*UploadResult, error) {
+	// 生成存储路径
+	ext := filepath.Ext(header.Filename)
+	datePath := time.Now().Format("2006/01/02")
+	fileName := fmt.Sprintf("%d%s", time.Now().UnixNano(), ext)
+	savePath := filepath.Join(datePath, fileName)
+	
+	// 创建目录
+	destDir := filepath.Join(s.BaseDir, filepath.FromSlash(datePath))
+	if err := os.MkdirAll(destDir, 0755); err != nil {
+		return nil, fmt.Errorf("创建目录失败: %w", err)
+	}
+	
+	// 保存文件
+	destPath := filepath.Join(s.BaseDir, filepath.FromSlash(savePath))
+	dst, err := os.Create(destPath)
+	if err != nil {
+		return nil, fmt.Errorf("创建文件失败: %w", err)
+	}
+	defer dst.Close()
+	
+	// 计算MD5并复制文件
+	hash := md5.New()
+	size, err := io.Copy(io.MultiWriter(dst, hash), file)
+	if err != nil {
+		_ = os.Remove(destPath)
+		return nil, fmt.Errorf("保存文件失败: %w", err)
+	}
+	
+	md5Sum := hex.EncodeToString(hash.Sum(nil))
+	webURL := s.BaseURL + strings.ReplaceAll(filepath.ToSlash(destPath), "\\", "/")
+	
+	return &UploadResult{
+		URL:      webURL,
+		Key:      savePath,
+		Size:     size,
+		MD5:      md5Sum,
+		MimeType: header.Header.Get("Content-Type"),
+	}, nil
+}
+
+// GetPublicURL 获取公开访问URL
+func (s *LocalStorage) GetPublicURL(key string) string {
+	return s.BaseURL + filepath.ToSlash(filepath.Join(s.BaseDir, key))
+}
+
+// Delete 删除本地文件
+func (s *LocalStorage) Delete(key string) error {
+	filePath := filepath.Join(s.BaseDir, filepath.FromSlash(key))
+	return os.Remove(filePath)
+}
+
+// QiniuStorage 七牛云存储实现
+type QiniuStorage struct {
+	AccessKey string
+	SecretKey string
+	Bucket    string
+	Domain    string
+	Region    string
+}
+
+// NewQiniuStorage 创建七牛云存储服务
+func NewQiniuStorage(cfg *models.StorageConfig) *QiniuStorage {
+	return &QiniuStorage{
+		AccessKey: cfg.QiniuAccessKey,
+		SecretKey: cfg.QiniuSecretKey,
+		Bucket:    cfg.QiniuBucket,
+		Domain:    cfg.QiniuDomain,
+		Region:    cfg.QiniuRegion,
+	}
+}
+
+// getZone 根据区域代码获取存储区域
+func (s *QiniuStorage) getZone() *storage.Region {
+	switch s.Region {
+	case "z0":
+		return &storage.ZoneHuadong
+	case "z1":
+		return &storage.ZoneHuabei
+	case "z2":
+		return &storage.ZoneHuanan
+	case "na0":
+		return &storage.ZoneBeimei
+	case "as0":
+		return &storage.ZoneXinjiapo
+	case "cn-east-2":
+		return &storage.ZoneHuadongZheJiang2
+	default:
+		return &storage.ZoneHuadong // 默认华东
+	}
+}
+
+// Upload 上传文件到七牛云
+func (s *QiniuStorage) Upload(file multipart.File, header *multipart.FileHeader) (*UploadResult, error) {
+	// 生成存储key
+	ext := filepath.Ext(header.Filename)
+	datePath := time.Now().Format("2006/01/02")
+	fileName := fmt.Sprintf("%d%s", time.Now().UnixNano(), ext)
+	key := filepath.ToSlash(filepath.Join(datePath, fileName))
+	
+	// 创建上传凭证
+	mac := qbox.NewMac(s.AccessKey, s.SecretKey)
+	putPolicy := storage.PutPolicy{
+		Scope: s.Bucket,
+	}
+	upToken := putPolicy.UploadToken(mac)
+	
+	// 配置上传参数
+	cfg := storage.Config{
+		Region:        s.getZone(),
+		UseHTTPS:      true,
+		UseCdnDomains: false,
+	}
+	
+	// 创建表单上传器
+	formUploader := storage.NewFormUploader(&cfg)
+	ret := storage.PutRet{}
+	putExtra := storage.PutExtra{}
+	
+	// 计算文件大小和MD5
+	tmpFile, err := os.CreateTemp("", "qiniu_upload_*")
+	if err != nil {
+		return nil, fmt.Errorf("创建临时文件失败: %w", err)
+	}
+	defer os.Remove(tmpFile.Name())
+	defer tmpFile.Close()
+	
+	hash := md5.New()
+	size, err := io.Copy(io.MultiWriter(tmpFile, hash), file)
+	if err != nil {
+		return nil, fmt.Errorf("读取文件失败: %w", err)
+	}
+	md5Sum := hex.EncodeToString(hash.Sum(nil))
+	
+	// 重置文件指针
+	if _, err := tmpFile.Seek(0, 0); err != nil {
+		return nil, fmt.Errorf("重置文件指针失败: %w", err)
+	}
+	
+	// 执行上传
+	err = formUploader.Put(context.Background(), &ret, upToken, key, tmpFile, size, &putExtra)
+	if err != nil {
+		return nil, fmt.Errorf("上传到七牛云失败: %w", err)
+	}
+	
+	// 构建完整URL
+	domain := strings.TrimRight(s.Domain, "/")
+	url := fmt.Sprintf("%s/%s", domain, ret.Key)
+	
+	return &UploadResult{
+		URL:      url,
+		Key:      ret.Key,
+		Size:     size,
+		MD5:      md5Sum,
+		MimeType: header.Header.Get("Content-Type"),
+	}, nil
+}
+
+// GetPublicURL 获取七牛云公开访问URL
+func (s *QiniuStorage) GetPublicURL(key string) string {
+	domain := strings.TrimRight(s.Domain, "/")
+	return fmt.Sprintf("%s/%s", domain, key)
+}
+
+// Delete 删除七牛云文件
+func (s *QiniuStorage) Delete(key string) error {
+	mac := qbox.NewMac(s.AccessKey, s.SecretKey)
+	cfg := storage.Config{
+		Region:   s.getZone(),
+		UseHTTPS: true,
+	}
+	
+	bucketManager := storage.NewBucketManager(mac, &cfg)
+	err := bucketManager.Delete(s.Bucket, key)
+	if err != nil {
+		return fmt.Errorf("删除七牛云文件失败: %w", err)
+	}
+	return nil
+}
+
+// GetStorageService 根据配置获取存储服务
+func GetStorageService() (StorageService, error) {
+	cfg, err := models.GetStorageConfig()
+	if err != nil {
+		// 默认使用本地存储
+		return NewLocalStorage(), nil
+	}
+	
+	switch cfg.StorageType {
+	case "qiniu":
+		if cfg.QiniuAccessKey == "" || cfg.QiniuSecretKey == "" || 
+		   cfg.QiniuBucket == "" || cfg.QiniuDomain == "" {
+			return nil, fmt.Errorf("七牛云配置不完整")
+		}
+		return NewQiniuStorage(cfg), nil
+	case "local":
+		return NewLocalStorage(), nil
+	default:
+		return NewLocalStorage(), nil
+	}
+}