修复bug
This commit is contained in:
hero920103 committed
1 parent
1850ac640e
commit
892ba5886d
20 files changed
+1078
-178
No files matched your search
@@ -0,0 +1,283 @@
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"log"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/beego/beego/v2/core/logs"
|
||||
beego "github.com/beego/beego/v2/server/web"
|
||||
"github.com/beego/beego/v2/server/web/context"
|
||||
)
|
||||
|
||||
// security_probe.go 扫描器探针的日志降噪与泄露检测。
|
||||
//
|
||||
// 公网服务器每天都会收到大量针对 .env / config.js / credentials.json 等
|
||||
// 敏感文件的自动化探测。它们全部 404 时属于背景噪音，逐条打印访问日志只会淹没真正的告警。
|
||||
// 这里接管 Beego 的访问日志过滤器：
|
||||
// - 探针路径不再逐条打印；
|
||||
// - 持续探测但均未命中时，每 10 分钟汇总提示一次「未发现泄露」；
|
||||
// - 一旦探针路径返回 2xx（文件真实可访问），立即输出 [security][leak] 告警。
|
||||
//
|
||||
// 注意：该过滤器依赖 Beego 的 access log 流程，app.conf 中不要开启
|
||||
// EnableStaticLogs=true，否则 Beego 会跳过 DefaultAccessLogFilter。
|
||||
|
||||
// filterconsole 是带探针过滤的 console 日志适配器。
|
||||
// 在 main.go 里把 BConfig.Log.Outputs 指向它，既保留 Beego dev 模式原有的
|
||||
// `[D] router.go` 访问日志样式，又能把探针/根路径 404 从日志里过滤掉。
|
||||
func init() {
|
||||
logs.Register("filterconsole", func() logs.Logger {
|
||||
return &probeFilterLogger{inner: logs.NewConsole()}
|
||||
})
|
||||
}
|
||||
|
||||
type probeFilterLogger struct {
|
||||
inner logs.Logger
|
||||
}
|
||||
|
||||
func (l *probeFilterLogger) Init(config string) error { return l.inner.Init(config) }
|
||||
|
||||
func (l *probeFilterLogger) WriteMsg(lm *logs.LogMsg) error {
|
||||
if shouldSuppressAccessLogMsg(lm.Msg) {
|
||||
return nil
|
||||
}
|
||||
return l.inner.WriteMsg(lm)
|
||||
}
|
||||
|
||||
func (l *probeFilterLogger) Destroy() { l.inner.Destroy() }
|
||||
func (l *probeFilterLogger) Flush() { l.inner.Flush() }
|
||||
|
||||
func (l *probeFilterLogger) SetFormatter(f logs.LogFormatter) { l.inner.SetFormatter(f) }
|
||||
|
||||
var (
|
||||
ansiColorRe = regexp.MustCompile(`\x1b\[[0-9;]*m`)
|
||||
accessStatusRe = regexp.MustCompile(`\|\s*(\d{3})\s*\|`)
|
||||
accessMethodPath = regexp.MustCompile(`\|\s*(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)\s+(\S+)`)
|
||||
)
|
||||
|
||||
// shouldSuppressAccessLogMsg 解析 Beego dev 访问日志行，过滤探针与根路径 404。
|
||||
func shouldSuppressAccessLogMsg(msg string) bool {
|
||||
clean := ansiColorRe.ReplaceAllString(msg, "")
|
||||
m := accessMethodPath.FindStringSubmatch(clean)
|
||||
if len(m) < 3 {
|
||||
return false
|
||||
}
|
||||
p := path.Clean(m[2])
|
||||
status := 0
|
||||
if sm := accessStatusRe.FindStringSubmatch(clean); len(sm) == 2 {
|
||||
status, _ = strconv.Atoi(sm[1])
|
||||
}
|
||||
|
||||
if p == "/" && status == 404 {
|
||||
recordRootNoise()
|
||||
return true
|
||||
}
|
||||
if IsScannerProbe(p) {
|
||||
recordProbeNoise()
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// scannerProbeFiles 常见扫描器探测的完整文件名（小写）。
|
||||
var scannerProbeFiles = map[string]struct{}{
|
||||
".env": {},
|
||||
"env": {},
|
||||
"env.js": {},
|
||||
"env.txt": {},
|
||||
"config.js": {},
|
||||
"config.json": {},
|
||||
"config.yaml": {},
|
||||
"config.yml": {},
|
||||
"config.php": {},
|
||||
"credentials": {},
|
||||
"credentials.json": {},
|
||||
"credentials.ini": {},
|
||||
"secrets.json": {},
|
||||
"lithic.json": {},
|
||||
"sardine.json": {},
|
||||
"database.sql": {},
|
||||
"phpinfo": {},
|
||||
"phpinfo.php": {},
|
||||
"info.php": {},
|
||||
"wp-config.php": {},
|
||||
"wp-config.php.bak": {},
|
||||
"wp-login.php": {},
|
||||
"docker-compose.yml": {},
|
||||
"docker-compose.yaml": {},
|
||||
"composer.json": {},
|
||||
"gitconfig": {},
|
||||
"runtime-config.js": {},
|
||||
"settings.json": {},
|
||||
"index.js": {},
|
||||
"aws.yml": {},
|
||||
"aws.json": {},
|
||||
}
|
||||
|
||||
// scannerProbeDirs 目录型探针（源代码/凭证/WordPress 目录）。
|
||||
var scannerProbeDirs = []string{
|
||||
"/.git",
|
||||
"/.svn",
|
||||
"/.aws",
|
||||
"/.ssh",
|
||||
"/vendor/composer",
|
||||
"/wp-content",
|
||||
"/wp-admin",
|
||||
// Go 源码目录：日志里出现的 /middleware、/controllers、/routers 等
|
||||
"/middleware",
|
||||
"/routers",
|
||||
"/controllers",
|
||||
"/models",
|
||||
"/services",
|
||||
"/pkg",
|
||||
"/scripts",
|
||||
"/conf",
|
||||
"/database",
|
||||
"/docs",
|
||||
"/cmd",
|
||||
"/version",
|
||||
}
|
||||
|
||||
// IsScannerProbe 判断路径是否属于常见的敏感文件扫描探针。
|
||||
func IsScannerProbe(rawPath string) bool {
|
||||
p := strings.ToLower(path.Clean("/" + strings.TrimPrefix(rawPath, "/")))
|
||||
if p == "/" {
|
||||
return false
|
||||
}
|
||||
base := path.Base(p)
|
||||
if _, ok := scannerProbeFiles[base]; ok {
|
||||
return true
|
||||
}
|
||||
// .env / .env.local / sendgrid.env / config.env 等变体
|
||||
if base == ".env" || base == "env" ||
|
||||
strings.HasPrefix(base, ".env.") || strings.HasSuffix(base, ".env") {
|
||||
return true
|
||||
}
|
||||
for _, dir := range scannerProbeDirs {
|
||||
if p == dir || strings.HasPrefix(p, dir+"/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// 源码/配置文件后缀：/middleware/security_probe.go、/controllers/backend_reimburse.go 等
|
||||
switch path.Ext(base) {
|
||||
case ".go", ".vue", ".ts", ".sql", ".conf":
|
||||
return true
|
||||
}
|
||||
// WordPress 常见备份文件，如 wp-config.php.bak
|
||||
if strings.HasPrefix(base, "wp-") && strings.HasSuffix(base, ".bak") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
const probeSummaryInterval = 10 * time.Minute
|
||||
|
||||
var probeLogState = struct {
|
||||
mu sync.Mutex
|
||||
suppressed int
|
||||
lastSummary time.Time
|
||||
rootSuppressed int
|
||||
lastRootSummary time.Time
|
||||
leakLogged map[string]time.Time
|
||||
}{
|
||||
leakLogged: make(map[string]time.Time),
|
||||
}
|
||||
|
||||
// probeAccessLogFilter 包装 Beego 默认访问日志过滤器。
|
||||
// Filter 返回 true 时 Beego 跳过该请求的访问日志。
|
||||
type probeAccessLogFilter struct {
|
||||
fallback beego.FilterHandler
|
||||
}
|
||||
|
||||
// NewSecurityProbeLogFilter 创建扫描探针访问日志过滤器。
|
||||
func NewSecurityProbeLogFilter(fallback beego.FilterHandler) beego.FilterHandler {
|
||||
return &probeAccessLogFilter{fallback: fallback}
|
||||
}
|
||||
|
||||
func (f *probeAccessLogFilter) Filter(ctx *context.Context) bool {
|
||||
p := path.Clean(ctx.Request.URL.Path)
|
||||
status := ctx.ResponseWriter.Status
|
||||
|
||||
// 当前进程只提供后端 API / 认证中心，根路径没有绑定站点；
|
||||
// 公网扫描器/爬虫大量请求 / 得到 404，单独聚合，不逐条刷日志。
|
||||
if p == "/" && status == 404 {
|
||||
recordRootNoise()
|
||||
return true
|
||||
}
|
||||
|
||||
if !IsScannerProbe(p) {
|
||||
if f.fallback != nil {
|
||||
return f.fallback.Filter(ctx)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// 平台后台 SPA 对未知路径会兜底返回 index.html（200），
|
||||
// 这不代表敏感文件可访问，不能当成泄露。
|
||||
spaFallback, _ := ctx.Input.GetData("spa_fallback").(bool)
|
||||
contentType := strings.TrimSpace(ctx.ResponseWriter.Header().Get("Content-Type"))
|
||||
|
||||
// 只有「明确 2xx + 有内容类型」才判定为可访问资源。
|
||||
// status=0 或 Content-Type 为空多为 SPA 兜底/空响应，避免误报泄漏。
|
||||
if status >= 200 && status < 300 && !spaFallback && contentType != "" {
|
||||
recordProbeLeak(ctx, status, contentType)
|
||||
} else {
|
||||
recordProbeNoise()
|
||||
}
|
||||
|
||||
// 探针请求不进入常规访问日志
|
||||
return true
|
||||
}
|
||||
|
||||
func recordProbeNoise() {
|
||||
probeLogState.mu.Lock()
|
||||
defer probeLogState.mu.Unlock()
|
||||
|
||||
probeLogState.suppressed++
|
||||
if !probeLogState.lastSummary.IsZero() &&
|
||||
time.Since(probeLogState.lastSummary) < probeSummaryInterval {
|
||||
return
|
||||
}
|
||||
count := probeLogState.suppressed
|
||||
probeLogState.suppressed = 0
|
||||
probeLogState.lastSummary = time.Now()
|
||||
log.Printf("[security] 已忽略 %d 次敏感文件探测请求（.env/config/credentials 等），未发现泄漏", count)
|
||||
}
|
||||
|
||||
func recordRootNoise() {
|
||||
probeLogState.mu.Lock()
|
||||
defer probeLogState.mu.Unlock()
|
||||
|
||||
probeLogState.rootSuppressed++
|
||||
if !probeLogState.lastRootSummary.IsZero() &&
|
||||
time.Since(probeLogState.lastRootSummary) < probeSummaryInterval {
|
||||
return
|
||||
}
|
||||
count := probeLogState.rootSuppressed
|
||||
probeLogState.rootSuppressed = 0
|
||||
probeLogState.lastRootSummary = time.Now()
|
||||
log.Printf("[http] 已忽略 %d 次根路径 404（当前 Host 未绑定站点），如需排查请用正式域名访问", count)
|
||||
}
|
||||
|
||||
func recordProbeLeak(ctx *context.Context, status int, contentType string) {
|
||||
key := ctx.Request.Method + " " + ctx.Request.URL.Path
|
||||
now := time.Now()
|
||||
|
||||
probeLogState.mu.Lock()
|
||||
last := probeLogState.leakLogged[key]
|
||||
if !last.IsZero() && now.Sub(last) < probeSummaryInterval {
|
||||
probeLogState.mu.Unlock()
|
||||
return
|
||||
}
|
||||
probeLogState.leakLogged[key] = now
|
||||
probeLogState.mu.Unlock()
|
||||
|
||||
log.Printf(
|
||||
"[security][leak] 敏感文件探测命中可访问资源: %s %s -> %d (Content-Type: %s)，疑似泄漏，请立即检查该路径是否应公开",
|
||||
ctx.Request.Method, ctx.Request.URL.Path, status, contentType,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package middleware
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestShouldSuppressAccessLogMsg(t *testing.T) {
|
||||
probeLine := "| 127.0.0.1| 404 | 221.723µs| nomatch| GET /middleware/security_probe.go"
|
||||
if !shouldSuppressAccessLogMsg(probeLine) {
|
||||
t.Fatalf("probe access line should be suppressed")
|
||||
}
|
||||
|
||||
rootLine := "| 127.0.0.1| 404 | 219.710µs| nomatch| GET /"
|
||||
if !shouldSuppressAccessLogMsg(rootLine) {
|
||||
t.Fatalf("root 404 access line should be suppressed")
|
||||
}
|
||||
|
||||
normalLine := "| 127.0.0.1| 200 | 1.649ms| match| GET /auth/login r:/auth/login"
|
||||
if shouldSuppressAccessLogMsg(normalLine) {
|
||||
t.Fatalf("normal access line should not be suppressed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsScannerProbe(t *testing.T) {
|
||||
probes := []string{
|
||||
"/config.js",
|
||||
"/config.json",
|
||||
"/config.yaml",
|
||||
"/config.php",
|
||||
"/config/config.js",
|
||||
"/database.sql",
|
||||
"/.env",
|
||||
"/demo/.env",
|
||||
"/sendgrid.env",
|
||||
"/config.env",
|
||||
"/.env.lithic",
|
||||
"/credentials",
|
||||
"/credentials.json",
|
||||
"/credentials.ini",
|
||||
"/secrets.json",
|
||||
"/lithic.json",
|
||||
"/sardine.json",
|
||||
"/phpinfo",
|
||||
"/phpinfo.php",
|
||||
"/info.php",
|
||||
"/wp-config.php",
|
||||
"/wp-config.php.bak",
|
||||
"/wp-content/debug.log",
|
||||
"/wp-login.php",
|
||||
"/vendor/composer/installed.json",
|
||||
"/docker-compose.yml",
|
||||
"/composer.json",
|
||||
"/gitconfig",
|
||||
"/runtime-config.js",
|
||||
"/settings.json",
|
||||
"/index.js",
|
||||
"/config/aws.yml",
|
||||
"/v1/.env",
|
||||
"/demo/../.env",
|
||||
"/middleware",
|
||||
"/middleware/security_probe.go",
|
||||
"/controllers/backend_reimburse.go",
|
||||
"/routers",
|
||||
"/routers/spa.go",
|
||||
"/models/reimburse.go",
|
||||
"/services/approval/engine.go",
|
||||
"/scripts/uidmigrate/migrate_uid.go",
|
||||
"/conf/app.conf",
|
||||
}
|
||||
for _, p := range probes {
|
||||
if !IsScannerProbe(p) {
|
||||
t.Errorf("IsScannerProbe(%q) = false, want true", p)
|
||||
}
|
||||
}
|
||||
|
||||
normal := []string{
|
||||
"/",
|
||||
"/favicon.ico",
|
||||
"/backend/reimbursements",
|
||||
"/platform/menu/1",
|
||||
"/site/setting",
|
||||
"/site/news/1",
|
||||
"/uploads/logo.png",
|
||||
"/themes/default/index.html",
|
||||
"/static/js/app.js",
|
||||
"/news",
|
||||
"/page/about",
|
||||
}
|
||||
for _, p := range normal {
|
||||
if IsScannerProbe(p) {
|
||||
t.Errorf("IsScannerProbe(%q) = true, want false", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user