From 94314f85f472a34e9d9a0e37ce8fc0ec6336863c Mon Sep 17 00:00:00 2001 From: wklken Date: Sat, 18 Jun 2022 15:19:26 +0800 Subject: [PATCH 1/2] feat(option): disable redirect option --- gorequest.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gorequest.go b/gorequest.go index e3d3e27..f47ba72 100644 --- a/gorequest.go +++ b/gorequest.go @@ -184,6 +184,13 @@ func (s *SuperAgent) DisableCompression() *SuperAgent { return s } +func (s *SuperAgent) DisableRedirect() *SuperAgent { + s.Client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + return s +} + // ClearSuperAgent clear SuperAgent data for another new request. func (s *SuperAgent) ClearSuperAgent() { if s.DoNotClearSuperAgent { From ed7db281cfd94d18b0e44419784d5e3250e6ad7c Mon Sep 17 00:00:00 2001 From: wklken Date: Sat, 18 Jun 2022 21:39:17 +0800 Subject: [PATCH 2/2] refactor(move): split into files --- cookie.go | 9 +++++++++ curl.go | 19 +++++++++++++++++- debug.go | 35 ++++++++++++++++++++++++++++++++ gorequest.go | 57 +++------------------------------------------------- mock.go | 10 +++++++++ redirect.go | 8 ++++++++ retry.go | 10 +++++++++ stats.go | 5 +++++ trace.go | 8 ++++++++ util.go | 27 +------------------------ 10 files changed, 107 insertions(+), 81 deletions(-) create mode 100644 debug.go create mode 100644 mock.go create mode 100644 trace.go diff --git a/cookie.go b/cookie.go index 6fd9d02..dfa6d67 100644 --- a/cookie.go +++ b/cookie.go @@ -13,3 +13,12 @@ func (s *SuperAgent) AddCookies(cookies []*http.Cookie) *SuperAgent { s.Cookies = append(s.Cookies, cookies...) return s } + +func shallowCopyCookies(old []*http.Cookie) []*http.Cookie { + if old == nil { + return nil + } + newData := make([]*http.Cookie, len(old)) + copy(newData, old) + return newData +} diff --git a/curl.go b/curl.go index e68cd51..996d964 100644 --- a/curl.go +++ b/curl.go @@ -1,6 +1,10 @@ package gorequest -import "moul.io/http2curl" +import ( + "net/http" + + "moul.io/http2curl" +) // SetCurlCommand enable the curlcommand mode which display a CURL command line. func (s *SuperAgent) SetCurlCommand(enable bool) *SuperAgent { @@ -11,6 +15,7 @@ func (s *SuperAgent) SetCurlCommand(enable bool) *SuperAgent { // AsCurlCommand returns a string representing the runnable `curl' command // version of the request. func (s *SuperAgent) AsCurlCommand() (string, error) { + // FIXME: why here MakeRequest again? req, err := s.MakeRequest() if err != nil { return "", err @@ -21,3 +26,15 @@ func (s *SuperAgent) AsCurlCommand() (string, error) { } return cmd.String(), nil } + +func (s *SuperAgent) logCurlCommand(req *http.Request) { + if s.CurlCommand { + curl, err := http2curl.GetCurlCommand(req) + s.logger.SetPrefix("[curl] ") + if err != nil { + s.logger.Println("Error:", err) + } else { + s.logger.Printf("CURL command line: %s", curl) + } + } +} diff --git a/debug.go b/debug.go new file mode 100644 index 0000000..e19bc14 --- /dev/null +++ b/debug.go @@ -0,0 +1,35 @@ +package gorequest + +import ( + "net/http" + "net/http/httputil" +) + +// SetDebug enable the debug mode which logs request/response detail. +func (s *SuperAgent) SetDebug(enable bool) *SuperAgent { + s.Debug = enable + return s +} + +func (s *SuperAgent) debuggingRequest(req *http.Request) { + if s.Debug { + dump, err := httputil.DumpRequest(req, true) + s.logger.SetPrefix("[http] ") + if err != nil { + s.logger.Println("Error:", err) + } else { + s.logger.Printf("HTTP Request: %s", BytesToString(dump)) + } + } +} + +func (s *SuperAgent) debuggingResponse(resp *http.Response) { + if s.Debug { + dump, err := httputil.DumpResponse(resp, true) + if nil != err { + s.logger.Println("Error:", err) + } else { + s.logger.Printf("HTTP Response: %s", BytesToString(dump)) + } + } +} diff --git a/gorequest.go b/gorequest.go index f47ba72..9b72db2 100644 --- a/gorequest.go +++ b/gorequest.go @@ -13,7 +13,6 @@ import ( "net/http" "net/http/cookiejar" "net/http/httptrace" - "net/http/httputil" "net/textproto" "net/url" "os" @@ -24,8 +23,6 @@ import ( "github.com/wklken/gorequest/internal/json" "golang.org/x/net/publicsuffix" - "gopkg.in/h2non/gock.v1" - "moul.io/http2curl" ) type ( @@ -154,24 +151,6 @@ func (s *SuperAgent) Context(ctx context.Context) *SuperAgent { return s } -func (s *SuperAgent) HttpTrace(trace *httptrace.ClientTrace) *SuperAgent { - s.trace = trace - return s -} - -// Mock will enable gock, http mocking for net/http -func (s *SuperAgent) Mock() *SuperAgent { - gock.InterceptClient(s.Client) - s.isMock = true - return s -} - -// SetDebug enable the debug mode which logs request/response detail. -func (s *SuperAgent) SetDebug(enable bool) *SuperAgent { - s.Debug = enable - return s -} - // SetDoNotClearSuperAgent enable the DoNotClear mode for not clearing super agent and reuse for the next request. func (s *SuperAgent) SetDoNotClearSuperAgent(enable bool) *SuperAgent { s.DoNotClearSuperAgent = enable @@ -184,13 +163,6 @@ func (s *SuperAgent) DisableCompression() *SuperAgent { return s } -func (s *SuperAgent) DisableRedirect() *SuperAgent { - s.Client.CheckRedirect = func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - } - return s -} - // ClearSuperAgent clear SuperAgent data for another new request. func (s *SuperAgent) ClearSuperAgent() { if s.DoNotClearSuperAgent { @@ -737,26 +709,10 @@ func (s *SuperAgent) getResponseBytes() (Response, []byte, []error) { } // Log details of this request - if s.Debug { - dump, err := httputil.DumpRequest(req, true) - s.logger.SetPrefix("[http] ") - if err != nil { - s.logger.Println("Error:", err) - } else { - s.logger.Printf("HTTP Request: %s", BytesToString(dump)) - } - } + s.debuggingRequest(req) // Display CURL command line - if s.CurlCommand { - curl, err := http2curl.GetCurlCommand(req) - s.logger.SetPrefix("[curl] ") - if err != nil { - s.logger.Println("Error:", err) - } else { - s.logger.Printf("CURL command line: %s", curl) - } - } + s.logCurlCommand(req) startTime := time.Now() // stats collect the requestBytes @@ -774,14 +730,7 @@ func (s *SuperAgent) getResponseBytes() (Response, []byte, []error) { s.Stats.RequestDuration = time.Since(startTime) // Log details of this response - if s.Debug { - dump, err := httputil.DumpResponse(resp, true) - if nil != err { - s.logger.Println("Error:", err) - } else { - s.logger.Printf("HTTP Response: %s", BytesToString(dump)) - } - } + s.debuggingResponse(resp) body, err := ioutil.ReadAll(resp.Body) // Reset resp.Body so it can be use again diff --git a/mock.go b/mock.go new file mode 100644 index 0000000..c630a10 --- /dev/null +++ b/mock.go @@ -0,0 +1,10 @@ +package gorequest + +import "gopkg.in/h2non/gock.v1" + +// Mock will enable gock, http mocking for net/http +func (s *SuperAgent) Mock() *SuperAgent { + gock.InterceptClient(s.Client) + s.isMock = true + return s +} diff --git a/redirect.go b/redirect.go index f4ddf60..253a17e 100644 --- a/redirect.go +++ b/redirect.go @@ -19,3 +19,11 @@ func (s *SuperAgent) RedirectPolicy(policy func(req Request, via []Request) erro } return s } + +// DisableRedirect will disable the redirect of status code 3xx. +func (s *SuperAgent) DisableRedirect() *SuperAgent { + s.Client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + return s +} diff --git a/retry.go b/retry.go index ee53639..e257dd7 100644 --- a/retry.go +++ b/retry.go @@ -56,3 +56,13 @@ func (s *SuperAgent) shouldRetry(resp Response, hasError bool) bool { } return false } + +// just need to change the array pointer? +func copyRetryable(old superAgentRetryable) superAgentRetryable { + newRetryable := old + newRetryable.RetryStatus = make([]int, len(old.RetryStatus)) + for i := range old.RetryStatus { + newRetryable.RetryStatus[i] = old.RetryStatus[i] + } + return newRetryable +} diff --git a/stats.go b/stats.go index 0337d2b..03b2285 100644 --- a/stats.go +++ b/stats.go @@ -8,3 +8,8 @@ type Stats struct { RequestDuration time.Duration } + +func copyStats(old Stats) Stats { + newStats := old + return newStats +} diff --git a/trace.go b/trace.go new file mode 100644 index 0000000..1d042fd --- /dev/null +++ b/trace.go @@ -0,0 +1,8 @@ +package gorequest + +import "net/http/httptrace" + +func (s *SuperAgent) HttpTrace(trace *httptrace.ClientTrace) *SuperAgent { + s.trace = trace + return s +} diff --git a/util.go b/util.go index 63d31eb..25cc91d 100644 --- a/util.go +++ b/util.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "mime/multipart" - "net/http" "net/textproto" "net/url" "reflect" @@ -25,26 +24,11 @@ func cloneMapArray(old map[string][]string) map[string][]string { return newMap } -// just need to change the array pointer? -func copyRetryable(old superAgentRetryable) superAgentRetryable { - newRetryable := old - newRetryable.RetryStatus = make([]int, len(old.RetryStatus)) - for i := range old.RetryStatus { - newRetryable.RetryStatus[i] = old.RetryStatus[i] - } - return newRetryable -} - -func copyStats(old Stats) Stats { - newStats := old - return newStats -} - func shallowCopyData(old map[string]interface{}) map[string]interface{} { if old == nil { return nil } - newData := make(map[string]interface{}) + newData := make(map[string]interface{}, len(old)) for k, val := range old { newData[k] = val } @@ -69,15 +53,6 @@ func shallowCopyFileArray(old []File) []File { return newData } -func shallowCopyCookies(old []*http.Cookie) []*http.Cookie { - if old == nil { - return nil - } - newData := make([]*http.Cookie, len(old)) - copy(newData, old) - return newData -} - func shallowCopyErrors(old []error) []error { if old == nil { return nil