Merge pull request #65 from chainreactors/dev

merge v1.1.0

cmd/cmd.go

@@ -8,6 +8,7 @@ import (
 	"github.com/chainreactors/spray/internal"
 	"github.com/chainreactors/spray/internal/ihttp"
 	"github.com/chainreactors/spray/pkg"
+	"github.com/chainreactors/utils/iutils"
 	"github.com/jessevdk/go-flags"
 	"os"
 	"os/signal"
@@ -112,8 +113,23 @@ func Spray() {
 		return
 	}
 
+	if option.PrintPreset {
+		err = pkg.Load()
+		if err != nil {
+			iutils.Fatal(err.Error())
+		}
+
+		err = pkg.LoadFingers()
+		if err != nil {
+			iutils.Fatal(err.Error())
+		}
+		internal.PrintPreset()
+
+		return
+	}
+
 	if option.Format != "" {
-		internal.Format(option.Format, !option.NoColor)
+		internal.Format(option)
 		return
 	}
 
@@ -128,7 +144,7 @@ func Spray() {
 		logs.Log.Errorf(err.Error())
 		return
 	}
-	if option.ReadAll || runner.Crawl {
+	if option.ReadAll || runner.CrawlPlugin {
 		ihttp.DefaultMaxBodySize = -1
 	}
 

internal/format.go

@@ -5,37 +5,81 @@ import (
 	"encoding/json"
 	"github.com/chainreactors/logs"
 	"github.com/chainreactors/spray/pkg"
+	"github.com/chainreactors/words/mask"
 	"io"
+	"net/url"
 	"os"
+	"strings"
 )
 
-func Format(filename string, color bool) {
+func Format(opts Option) {
 	var content []byte
 	var err error
-	if filename == "stdin" {
+	if opts.Format == "stdin" {
 		content, err = io.ReadAll(os.Stdin)
 	} else {
-		content, err = os.ReadFile(filename)
+		content, err = os.ReadFile(opts.Format)
 	}
 
 	if err != nil {
 		return
 	}
-	var results []*pkg.Baseline
+	group := make(map[string][]*pkg.Baseline)
 	for _, line := range bytes.Split(bytes.TrimSpace(content), []byte("\n")) {
 		var result pkg.Baseline
 		err := json.Unmarshal(line, &result)
 		if err != nil {
 			logs.Log.Error(err.Error())
 			return
 		}
-		results = append(results, &result)
+		result.Url, err = url.Parse(result.UrlString)
+		if err != nil {
+			continue
+		}
+		group[result.Url.Host] = append(group[result.Url.Host], &result)
 	}
-	for _, result := range results {
-		if color {
-			logs.Log.Info(result.ColorString())
-		} else {
-			logs.Log.Info(result.String())
+
+	// 分组
+
+	for _, results := range group {
+		for _, result := range results {
+			if !opts.Fuzzy && result.IsFuzzy {
+				continue
+			}
+			if !opts.NoColor {
+				logs.Log.Console(result.ColorString() + "\n")
+			} else {
+				logs.Log.Console(result.String() + "\n")
+			}
 		}
 	}
 }
+
+func PrintPreset() {
+	logs.Log.Console("internal rules:\n")
+	for name, rule := range pkg.Rules {
+		logs.Log.Consolef("\t%s\t%d rules\n", name, len(strings.Split(rule, "\n")))
+	}
+
+	logs.Log.Console("\ninternal dicts:\n")
+	for name, dict := range pkg.Dicts {
+		logs.Log.Consolef("\t%s\t%d items\n", name, len(dict))
+	}
+
+	logs.Log.Console("\ninternal words keyword:\n")
+	for name, words := range mask.SpecialWords {
+		logs.Log.Consolef("\t%s\t%d words\n", name, len(words))
+	}
+
+	logs.Log.Console("\ninternal extractor:\n")
+	for name, _ := range pkg.ExtractRegexps {
+		logs.Log.Consolef("\t%s\n", name)
+	}
+
+	logs.Log.Console("\ninternal fingers:\n")
+	for name, engine := range pkg.FingerEngine.EnginesImpl {
+		logs.Log.Consolef("\t%s\t%d fingerprints \n", name, engine.Len())
+	}
+
+	logs.Log.Consolef("\nload %d active path\n", len(pkg.ActivePath))
+}

internal/ihttp/client.go

@@ -62,10 +62,8 @@ func NewClient(config *ClientConfig) *Client {
 		client = &Client{
 			standardClient: &http.Client{
 				Transport: &http.Transport{
-					//Proxy: Proxy,
-					//TLSHandshakeTimeout : delay * time.Second,
 					TLSClientConfig: &tls.Config{
-						Renegotiation:      tls.RenegotiateOnceAsClient,
+						Renegotiation:      tls.RenegotiateNever,
 						InsecureSkipVerify: true,
 					},
 					TLSHandshakeTimeout: config.Timeout,