284 lines
8.3 KiB
Go
284 lines
8.3 KiB
Go
package middleware
|
||
|
||
import (
|
||
"log"
|
||
"path"
|
||
"regexp"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/beego/beego/v2/core/logs"
|
||
beego "github.com/beego/beego/v2/server/web"
|
||
"github.com/beego/beego/v2/server/web/context"
|
||
)
|
||
|
||
// security_probe.go 扫描器探针的日志降噪与泄露检测。
|
||
//
|
||
// 公网服务器每天都会收到大量针对 .env / config.js / credentials.json 等
|
||
// 敏感文件的自动化探测。它们全部 404 时属于背景噪音，逐条打印访问日志只会淹没真正的告警。
|
||
// 这里接管 Beego 的访问日志过滤器：
|
||
// - 探针路径不再逐条打印；
|
||
// - 持续探测但均未命中时，每 10 分钟汇总提示一次「未发现泄露」；
|
||
// - 一旦探针路径返回 2xx（文件真实可访问），立即输出 [security][leak] 告警。
|
||
//
|
||
// 注意：该过滤器依赖 Beego 的 access log 流程，app.conf 中不要开启
|
||
// EnableStaticLogs=true，否则 Beego 会跳过 DefaultAccessLogFilter。
|
||
|
||
// filterconsole 是带探针过滤的 console 日志适配器。
|
||
// 在 main.go 里把 BConfig.Log.Outputs 指向它，既保留 Beego dev 模式原有的
|
||
// `[D] router.go` 访问日志样式，又能把探针/根路径 404 从日志里过滤掉。
|
||
func init() {
|
||
logs.Register("filterconsole", func() logs.Logger {
|
||
return &probeFilterLogger{inner: logs.NewConsole()}
|
||
})
|
||
}
|
||
|
||
type probeFilterLogger struct {
|
||
inner logs.Logger
|
||
}
|
||
|
||
func (l *probeFilterLogger) Init(config string) error { return l.inner.Init(config) }
|
||
|
||
func (l *probeFilterLogger) WriteMsg(lm *logs.LogMsg) error {
|
||
if shouldSuppressAccessLogMsg(lm.Msg) {
|
||
return nil
|
||
}
|
||
return l.inner.WriteMsg(lm)
|
||
}
|
||
|
||
func (l *probeFilterLogger) Destroy() { l.inner.Destroy() }
|
||
func (l *probeFilterLogger) Flush() { l.inner.Flush() }
|
||
|
||
func (l *probeFilterLogger) SetFormatter(f logs.LogFormatter) { l.inner.SetFormatter(f) }
|
||
|
||
var (
|
||
ansiColorRe = regexp.MustCompile(`\x1b\[[0-9;]*m`)
|
||
accessStatusRe = regexp.MustCompile(`\|\s*(\d{3})\s*\|`)
|
||
accessMethodPath = regexp.MustCompile(`\|\s*(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)\s+(\S+)`)
|
||
)
|
||
|
||
// shouldSuppressAccessLogMsg 解析 Beego dev 访问日志行，过滤探针与根路径 404。
|
||
func shouldSuppressAccessLogMsg(msg string) bool {
|
||
clean := ansiColorRe.ReplaceAllString(msg, "")
|
||
m := accessMethodPath.FindStringSubmatch(clean)
|
||
if len(m) < 3 {
|
||
return false
|
||
}
|
||
p := path.Clean(m[2])
|
||
status := 0
|
||
if sm := accessStatusRe.FindStringSubmatch(clean); len(sm) == 2 {
|
||
status, _ = strconv.Atoi(sm[1])
|
||
}
|
||
|
||
if p == "/" && status == 404 {
|
||
recordRootNoise()
|
||
return true
|
||
}
|
||
if IsScannerProbe(p) {
|
||
recordProbeNoise()
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// scannerProbeFiles 常见扫描器探测的完整文件名（小写）。
|
||
var scannerProbeFiles = map[string]struct{}{
|
||
".env": {},
|
||
"env": {},
|
||
"env.js": {},
|
||
"env.txt": {},
|
||
"config.js": {},
|
||
"config.json": {},
|
||
"config.yaml": {},
|
||
"config.yml": {},
|
||
"config.php": {},
|
||
"credentials": {},
|
||
"credentials.json": {},
|
||
"credentials.ini": {},
|
||
"secrets.json": {},
|
||
"lithic.json": {},
|
||
"sardine.json": {},
|
||
"database.sql": {},
|
||
"phpinfo": {},
|
||
"phpinfo.php": {},
|
||
"info.php": {},
|
||
"wp-config.php": {},
|
||
"wp-config.php.bak": {},
|
||
"wp-login.php": {},
|
||
"docker-compose.yml": {},
|
||
"docker-compose.yaml": {},
|
||
"composer.json": {},
|
||
"gitconfig": {},
|
||
"runtime-config.js": {},
|
||
"settings.json": {},
|
||
"index.js": {},
|
||
"aws.yml": {},
|
||
"aws.json": {},
|
||
}
|
||
|
||
// scannerProbeDirs 目录型探针（源代码/凭证/WordPress 目录）。
|
||
var scannerProbeDirs = []string{
|
||
"/.git",
|
||
"/.svn",
|
||
"/.aws",
|
||
"/.ssh",
|
||
"/vendor/composer",
|
||
"/wp-content",
|
||
"/wp-admin",
|
||
// Go 源码目录：日志里出现的 /middleware、/controllers、/routers 等
|
||
"/middleware",
|
||
"/routers",
|
||
"/controllers",
|
||
"/models",
|
||
"/services",
|
||
"/pkg",
|
||
"/scripts",
|
||
"/conf",
|
||
"/database",
|
||
"/docs",
|
||
"/cmd",
|
||
"/version",
|
||
}
|
||
|
||
// IsScannerProbe 判断路径是否属于常见的敏感文件扫描探针。
|
||
func IsScannerProbe(rawPath string) bool {
|
||
p := strings.ToLower(path.Clean("/" + strings.TrimPrefix(rawPath, "/")))
|
||
if p == "/" {
|
||
return false
|
||
}
|
||
base := path.Base(p)
|
||
if _, ok := scannerProbeFiles[base]; ok {
|
||
return true
|
||
}
|
||
// .env / .env.local / sendgrid.env / config.env 等变体
|
||
if base == ".env" || base == "env" ||
|
||
strings.HasPrefix(base, ".env.") || strings.HasSuffix(base, ".env") {
|
||
return true
|
||
}
|
||
for _, dir := range scannerProbeDirs {
|
||
if p == dir || strings.HasPrefix(p, dir+"/") {
|
||
return true
|
||
}
|
||
}
|
||
// 源码/配置文件后缀：/middleware/security_probe.go、/controllers/backend_reimburse.go 等
|
||
switch path.Ext(base) {
|
||
case ".go", ".vue", ".ts", ".sql", ".conf":
|
||
return true
|
||
}
|
||
// WordPress 常见备份文件，如 wp-config.php.bak
|
||
if strings.HasPrefix(base, "wp-") && strings.HasSuffix(base, ".bak") {
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
const probeSummaryInterval = 10 * time.Minute
|
||
|
||
var probeLogState = struct {
|
||
mu sync.Mutex
|
||
suppressed int
|
||
lastSummary time.Time
|
||
rootSuppressed int
|
||
lastRootSummary time.Time
|
||
leakLogged map[string]time.Time
|
||
}{
|
||
leakLogged: make(map[string]time.Time),
|
||
}
|
||
|
||
// probeAccessLogFilter 包装 Beego 默认访问日志过滤器。
|
||
// Filter 返回 true 时 Beego 跳过该请求的访问日志。
|
||
type probeAccessLogFilter struct {
|
||
fallback beego.FilterHandler
|
||
}
|
||
|
||
// NewSecurityProbeLogFilter 创建扫描探针访问日志过滤器。
|
||
func NewSecurityProbeLogFilter(fallback beego.FilterHandler) beego.FilterHandler {
|
||
return &probeAccessLogFilter{fallback: fallback}
|
||
}
|
||
|
||
func (f *probeAccessLogFilter) Filter(ctx *context.Context) bool {
|
||
p := path.Clean(ctx.Request.URL.Path)
|
||
status := ctx.ResponseWriter.Status
|
||
|
||
// 当前进程只提供后端 API / 认证中心，根路径没有绑定站点；
|
||
// 公网扫描器/爬虫大量请求 / 得到 404，单独聚合，不逐条刷日志。
|
||
if p == "/" && status == 404 {
|
||
recordRootNoise()
|
||
return true
|
||
}
|
||
|
||
if !IsScannerProbe(p) {
|
||
if f.fallback != nil {
|
||
return f.fallback.Filter(ctx)
|
||
}
|
||
return false
|
||
}
|
||
|
||
// 平台后台 SPA 对未知路径会兜底返回 index.html（200），
|
||
// 这不代表敏感文件可访问，不能当成泄露。
|
||
spaFallback, _ := ctx.Input.GetData("spa_fallback").(bool)
|
||
contentType := strings.TrimSpace(ctx.ResponseWriter.Header().Get("Content-Type"))
|
||
|
||
// 只有「明确 2xx + 有内容类型」才判定为可访问资源。
|
||
// status=0 或 Content-Type 为空多为 SPA 兜底/空响应，避免误报泄漏。
|
||
if status >= 200 && status < 300 && !spaFallback && contentType != "" {
|
||
recordProbeLeak(ctx, status, contentType)
|
||
} else {
|
||
recordProbeNoise()
|
||
}
|
||
|
||
// 探针请求不进入常规访问日志
|
||
return true
|
||
}
|
||
|
||
func recordProbeNoise() {
|
||
probeLogState.mu.Lock()
|
||
defer probeLogState.mu.Unlock()
|
||
|
||
probeLogState.suppressed++
|
||
if !probeLogState.lastSummary.IsZero() &&
|
||
time.Since(probeLogState.lastSummary) < probeSummaryInterval {
|
||
return
|
||
}
|
||
count := probeLogState.suppressed
|
||
probeLogState.suppressed = 0
|
||
probeLogState.lastSummary = time.Now()
|
||
log.Printf("[security] 已忽略 %d 次敏感文件探测请求（.env/config/credentials 等），未发现泄漏", count)
|
||
}
|
||
|
||
func recordRootNoise() {
|
||
probeLogState.mu.Lock()
|
||
defer probeLogState.mu.Unlock()
|
||
|
||
probeLogState.rootSuppressed++
|
||
if !probeLogState.lastRootSummary.IsZero() &&
|
||
time.Since(probeLogState.lastRootSummary) < probeSummaryInterval {
|
||
return
|
||
}
|
||
count := probeLogState.rootSuppressed
|
||
probeLogState.rootSuppressed = 0
|
||
probeLogState.lastRootSummary = time.Now()
|
||
log.Printf("[http] 已忽略 %d 次根路径 404（当前 Host 未绑定站点），如需排查请用正式域名访问", count)
|
||
}
|
||
|
||
func recordProbeLeak(ctx *context.Context, status int, contentType string) {
|
||
key := ctx.Request.Method + " " + ctx.Request.URL.Path
|
||
now := time.Now()
|
||
|
||
probeLogState.mu.Lock()
|
||
last := probeLogState.leakLogged[key]
|
||
if !last.IsZero() && now.Sub(last) < probeSummaryInterval {
|
||
probeLogState.mu.Unlock()
|
||
return
|
||
}
|
||
probeLogState.leakLogged[key] = now
|
||
probeLogState.mu.Unlock()
|
||
|
||
log.Printf(
|
||
"[security][leak] 敏感文件探测命中可访问资源: %s %s -> %d (Content-Type: %s)，疑似泄漏，请立即检查该路径是否应公开",
|
||
ctx.Request.Method, ctx.Request.URL.Path, status, contentType,
|
||
)
|
||
}
|