diff --git a/go.mod b/go.mod index fafa128..0a8a8c2 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/chainreactors/files v0.0.0-20240716182835-7884ee1e77f0 github.com/chainreactors/fingers v0.0.0-20240716172449-2fc3147b9c2a github.com/chainreactors/logs v0.0.0-20240207121836-c946f072f81f - github.com/chainreactors/parsers v0.0.0-20241013180542-88e2dc355c57 + github.com/chainreactors/parsers v0.0.0-20241016065831-bedaf68005f1 github.com/chainreactors/utils v0.0.0-20240805193040-ff3b97aa3c3f github.com/expr-lang/expr v1.16.9 github.com/gookit/config/v2 v2.2.5 diff --git a/go.sum b/go.sum index f10bbfb..4a76f75 100644 --- a/go.sum +++ b/go.sum @@ -103,6 +103,8 @@ github.com/chainreactors/parsers v0.0.0-20240910081704-fd57f462fc65 h1:subSvyczs github.com/chainreactors/parsers v0.0.0-20240910081704-fd57f462fc65/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA= github.com/chainreactors/parsers v0.0.0-20241013180542-88e2dc355c57 h1:KuijtekTNtSpQbKf2jqKp99gxnGQXffPeEF+EOHnXBE= github.com/chainreactors/parsers v0.0.0-20241013180542-88e2dc355c57/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA= +github.com/chainreactors/parsers v0.0.0-20241016065831-bedaf68005f1 h1:Ka/KBrqAgwiL07TwYjtqF2DQ3x0fCxw1XHG+GFqMKEc= +github.com/chainreactors/parsers v0.0.0-20241016065831-bedaf68005f1/go.mod h1:7rXdYz6jrdjF0WUH1ICcAXKIKKjKmJo2PU8u43V7jkA= github.com/chainreactors/utils v0.0.0-20240528085651-ba1b255482c1/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs= github.com/chainreactors/utils v0.0.0-20240704062557-662d623b74f4/go.mod h1:JA4eiQZm+7AsfjXBcIzIdVKBEhDCb16eNtWFCGTxlvs= github.com/chainreactors/utils v0.0.0-20240715080349-d2d0484c95ed/go.mod h1:LajXuvESQwP+qCMAvlcoSXppQCjuLlBrnQpu9XQ1HtU= diff --git a/internal/pool/brutepool.go b/internal/pool/brutepool.go index 3242329..2317767 100644 --- a/internal/pool/brutepool.go +++ b/internal/pool/brutepool.go @@ -562,17 +562,26 @@ func (pool *BrutePool) PreCompare(resp *ihttp.Response) error { return nil } +func (pool *BrutePool) checkHostname(u string) bool { + if v, err := url.Parse(u); err == nil { + if v.Host == "" { + return true + } + if v.Host == pool.url.Host { + return true + } else { + return false + } + } + return true +} + func (pool *BrutePool) BaseCompare(bl *pkg.Baseline) bool { if !bl.IsValid { return false } var status = -1 - // 30x状态码的特殊处理 - if bl.RedirectURL != "" && strings.HasSuffix(bl.RedirectURL, bl.Url.Path+"/") { - bl.Reason = pkg.ErrFuzzyRedirect.Error() - pool.putToFuzzy(bl) - return false - } + // 使用与baseline相同状态码, 需要在fuzzystatus中提前配置 base, ok := pool.baselines[bl.Status] // 挑选对应状态码的baseline进行compare if !ok { @@ -587,7 +596,15 @@ func (pool *BrutePool) BaseCompare(bl *pkg.Baseline) bool { } } - if ok { + // 30x状态码的特殊处理 + if bl.RedirectURL != "" { + if pool.checkHostname(bl.RedirectURL) && strings.HasSuffix(bl.RedirectURL, bl.Url.Path+"/") { + bl.Reason = pkg.ErrFuzzyRedirect.Error() + return false + } + } + + if ok && !bl.IsBaseline { if status = base.Compare(bl); status == 1 { bl.Reason = pkg.ErrCompareFailed.Error() return false @@ -619,6 +636,7 @@ func (pool *BrutePool) BaseCompare(bl *pkg.Baseline) bool { func (pool *BrutePool) addFuzzyBaseline(bl *pkg.Baseline) { if _, ok := pool.baselines[bl.Status]; !ok && (EnableAllFuzzy || iutils.IntsContains(pkg.FuzzyStatus, bl.Status)) { + bl.IsBaseline = true bl.Collect() pool.doCrawl(bl) // 非有效页面也可能存在一些特殊的url可以用来爬取 pool.baselines[bl.Status] = bl diff --git a/pkg/baseline.go b/pkg/baseline.go index df5fe23..aacf1a0 100644 --- a/pkg/baseline.go +++ b/pkg/baseline.go @@ -113,18 +113,19 @@ func NewInvalidBaseline(u, host string, resp *ihttp.Response, reason string) *Ba type Baseline struct { *parsers.SprayResult - Url *url.URL `json:"-"` - Dir bool `json:"-"` - Chunked bool `json:"-"` - Body BS `json:"-"` - Header BS `json:"-"` - Raw BS `json:"-"` - Response *http.Response `json:"-"` - Recu bool `json:"-"` - RecuDepth int `json:"-"` - URLs []string `json:"-"` - Collected bool `json:"-"` - Retry int `json:"-"` + Url *url.URL `json:"-"` + Dir bool `json:"-"` + Chunked bool `json:"-"` + Body BS `json:"-"` + Header BS `json:"-"` + Raw BS `json:"-"` + Response *http.Response `json:"-"` + Recu bool `json:"-"` + RecuDepth int `json:"-"` + URLs []string `json:"-"` + Collected bool `json:"-"` + Retry int `json:"-"` + IsBaseline bool `json:"-"` } func (bl *Baseline) IsDir() bool {