package middleware import ( "log" "path" "regexp" "strconv" "strings" "sync" "time" "github.com/beego/beego/v2/core/logs" beego "github.com/beego/beego/v2/server/web" "github.com/beego/beego/v2/server/web/context" ) // security_probe.go 扫描器探针的日志降噪与泄露检测。 // // 公网服务器每天都会收到大量针对 .env / config.js / credentials.json 等 // 敏感文件的自动化探测。它们全部 404 时属于背景噪音,逐条打印访问日志只会淹没真正的告警。 // 这里接管 Beego 的访问日志过滤器: // - 探针路径不再逐条打印; // - 持续探测但均未命中时,每 10 分钟汇总提示一次「未发现泄露」; // - 一旦探针路径返回 2xx(文件真实可访问),立即输出 [security][leak] 告警。 // // 注意:该过滤器依赖 Beego 的 access log 流程,app.conf 中不要开启 // EnableStaticLogs=true,否则 Beego 会跳过 DefaultAccessLogFilter。 // filterconsole 是带探针过滤的 console 日志适配器。 // 在 main.go 里把 BConfig.Log.Outputs 指向它,既保留 Beego dev 模式原有的 // `[D] router.go` 访问日志样式,又能把探针/根路径 404 从日志里过滤掉。 func init() { logs.Register("filterconsole", func() logs.Logger { return &probeFilterLogger{inner: logs.NewConsole()} }) } type probeFilterLogger struct { inner logs.Logger } func (l *probeFilterLogger) Init(config string) error { return l.inner.Init(config) } func (l *probeFilterLogger) WriteMsg(lm *logs.LogMsg) error { if shouldSuppressAccessLogMsg(lm.Msg) { return nil } return l.inner.WriteMsg(lm) } func (l *probeFilterLogger) Destroy() { l.inner.Destroy() } func (l *probeFilterLogger) Flush() { l.inner.Flush() } func (l *probeFilterLogger) SetFormatter(f logs.LogFormatter) { l.inner.SetFormatter(f) } var ( ansiColorRe = regexp.MustCompile(`\x1b\[[0-9;]*m`) accessStatusRe = regexp.MustCompile(`\|\s*(\d{3})\s*\|`) accessMethodPath = regexp.MustCompile(`\|\s*(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)\s+(\S+)`) ) // shouldSuppressAccessLogMsg 解析 Beego dev 访问日志行,过滤探针与根路径 404。 func shouldSuppressAccessLogMsg(msg string) bool { clean := ansiColorRe.ReplaceAllString(msg, "") m := accessMethodPath.FindStringSubmatch(clean) if len(m) < 3 { return false } p := path.Clean(m[2]) status := 0 if sm := accessStatusRe.FindStringSubmatch(clean); len(sm) == 2 { status, _ = strconv.Atoi(sm[1]) } if p == "/" && status == 404 { recordRootNoise() return true } if IsScannerProbe(p) { recordProbeNoise() return true } return false } // scannerProbeFiles 常见扫描器探测的完整文件名(小写)。 var scannerProbeFiles = map[string]struct{}{ ".env": {}, "env": {}, "env.js": {}, "env.txt": {}, "config.js": {}, "config.json": {}, "config.yaml": {}, "config.yml": {}, "config.php": {}, "credentials": {}, "credentials.json": {}, "credentials.ini": {}, "secrets.json": {}, "lithic.json": {}, "sardine.json": {}, "database.sql": {}, "phpinfo": {}, "phpinfo.php": {}, "info.php": {}, "wp-config.php": {}, "wp-config.php.bak": {}, "wp-login.php": {}, "docker-compose.yml": {}, "docker-compose.yaml": {}, "composer.json": {}, "gitconfig": {}, "runtime-config.js": {}, "settings.json": {}, "index.js": {}, "aws.yml": {}, "aws.json": {}, } // scannerProbeDirs 目录型探针(源代码/凭证/WordPress 目录)。 var scannerProbeDirs = []string{ "/.git", "/.svn", "/.aws", "/.ssh", "/vendor/composer", "/wp-content", "/wp-admin", // Go 源码目录:日志里出现的 /middleware、/controllers、/routers 等 "/middleware", "/routers", "/controllers", "/models", "/services", "/pkg", "/scripts", "/conf", "/database", "/docs", "/cmd", "/version", } // IsScannerProbe 判断路径是否属于常见的敏感文件扫描探针。 func IsScannerProbe(rawPath string) bool { p := strings.ToLower(path.Clean("/" + strings.TrimPrefix(rawPath, "/"))) if p == "/" { return false } base := path.Base(p) if _, ok := scannerProbeFiles[base]; ok { return true } // .env / .env.local / sendgrid.env / config.env 等变体 if base == ".env" || base == "env" || strings.HasPrefix(base, ".env.") || strings.HasSuffix(base, ".env") { return true } for _, dir := range scannerProbeDirs { if p == dir || strings.HasPrefix(p, dir+"/") { return true } } // 源码/配置文件后缀:/middleware/security_probe.go、/controllers/backend_reimburse.go 等 switch path.Ext(base) { case ".go", ".vue", ".ts", ".sql", ".conf": return true } // WordPress 常见备份文件,如 wp-config.php.bak if strings.HasPrefix(base, "wp-") && strings.HasSuffix(base, ".bak") { return true } return false } const probeSummaryInterval = 10 * time.Minute var probeLogState = struct { mu sync.Mutex suppressed int lastSummary time.Time rootSuppressed int lastRootSummary time.Time leakLogged map[string]time.Time }{ leakLogged: make(map[string]time.Time), } // probeAccessLogFilter 包装 Beego 默认访问日志过滤器。 // Filter 返回 true 时 Beego 跳过该请求的访问日志。 type probeAccessLogFilter struct { fallback beego.FilterHandler } // NewSecurityProbeLogFilter 创建扫描探针访问日志过滤器。 func NewSecurityProbeLogFilter(fallback beego.FilterHandler) beego.FilterHandler { return &probeAccessLogFilter{fallback: fallback} } func (f *probeAccessLogFilter) Filter(ctx *context.Context) bool { p := path.Clean(ctx.Request.URL.Path) status := ctx.ResponseWriter.Status // 当前进程只提供后端 API / 认证中心,根路径没有绑定站点; // 公网扫描器/爬虫大量请求 / 得到 404,单独聚合,不逐条刷日志。 if p == "/" && status == 404 { recordRootNoise() return true } if !IsScannerProbe(p) { if f.fallback != nil { return f.fallback.Filter(ctx) } return false } // 平台后台 SPA 对未知路径会兜底返回 index.html(200), // 这不代表敏感文件可访问,不能当成泄露。 spaFallback, _ := ctx.Input.GetData("spa_fallback").(bool) contentType := strings.TrimSpace(ctx.ResponseWriter.Header().Get("Content-Type")) // 只有「明确 2xx + 有内容类型」才判定为可访问资源。 // status=0 或 Content-Type 为空多为 SPA 兜底/空响应,避免误报泄漏。 if status >= 200 && status < 300 && !spaFallback && contentType != "" { recordProbeLeak(ctx, status, contentType) } else { recordProbeNoise() } // 探针请求不进入常规访问日志 return true } func recordProbeNoise() { probeLogState.mu.Lock() defer probeLogState.mu.Unlock() probeLogState.suppressed++ if !probeLogState.lastSummary.IsZero() && time.Since(probeLogState.lastSummary) < probeSummaryInterval { return } count := probeLogState.suppressed probeLogState.suppressed = 0 probeLogState.lastSummary = time.Now() log.Printf("[security] 已忽略 %d 次敏感文件探测请求(.env/config/credentials 等),未发现泄漏", count) } func recordRootNoise() { probeLogState.mu.Lock() defer probeLogState.mu.Unlock() probeLogState.rootSuppressed++ if !probeLogState.lastRootSummary.IsZero() && time.Since(probeLogState.lastRootSummary) < probeSummaryInterval { return } count := probeLogState.rootSuppressed probeLogState.rootSuppressed = 0 probeLogState.lastRootSummary = time.Now() log.Printf("[http] 已忽略 %d 次根路径 404(当前 Host 未绑定站点),如需排查请用正式域名访问", count) } func recordProbeLeak(ctx *context.Context, status int, contentType string) { key := ctx.Request.Method + " " + ctx.Request.URL.Path now := time.Now() probeLogState.mu.Lock() last := probeLogState.leakLogged[key] if !last.IsZero() && now.Sub(last) < probeSummaryInterval { probeLogState.mu.Unlock() return } probeLogState.leakLogged[key] = now probeLogState.mu.Unlock() log.Printf( "[security][leak] 敏感文件探测命中可访问资源: %s %s -> %d (Content-Type: %s),疑似泄漏,请立即检查该路径是否应公开", ctx.Request.Method, ctx.Request.URL.Path, status, contentType, ) }