diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..08406e6df --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,23 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior. + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..bbcbbe7d6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request.md b/.github/PULL_REQUEST_TEMPLATE/pull_request.md new file mode 100644 index 000000000..88409dc48 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/pull_request.md @@ -0,0 +1,5 @@ +## What does this PR do +Please link to the relevant issues. + +## Additional context +Add any other context here. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..4e4c1fabf --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,39 @@ + + +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + + +## [v0.2.0] - 2019-09-26 + +### Added +- Add proxy protocol to be compatible with F5 BigIP/Citrix ADC etc +- Add mod_access to write request/session log in customized format +- Add mod_key_log to wirte tls key log so that external programs(eg. wireshark) can decrypt TLS connections for trouble shooting +- Add security grade 'A+' in tls +- Add condition primitive: req_query_value_contain/req_header_value_contain/req_cookie_value_contain +- Documents optimization + +### Changed +- reverseproxy: flush response header immediately if flushInterval<0 + + +## [v0.1.0] - 2019-08-01 + +### Added +- Multiple protocols supported, including HTTP, HTTPS, SPDY, HTTP2, WebSocket, TLS, etc +- Content based routing, support user-defined routing rule in advanced domain-specific language +- Support multiple load balancing policies +- Flexible plugin framework to extend functionality. Based on the framework, developer can add new features rapidly +- Detailed built-in metrics available for service status monitor + + +[v0.2.0]: https://github.com/baidu/bfe/compare/v0.1.0...v0.2.0 +[v0.1.0]: https://github.com/baidu/bfe/releases/tag/v0.1.0 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..12d0c8587 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at bfe-osc@baidu.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md new file mode 100644 index 000000000..4ed1e4758 --- /dev/null +++ b/CONTRIBUTORS.md @@ -0,0 +1,15 @@ +* This is the list of people who have contributed code/doc to the BFE repository. +* Please keep the list sorted by **name**. + +| Name | Github Account | +| ---- | -------------- | +| Kaiyu Zheng | kaiyuzheng | +| Lu Guo | guolu60 | +| Miao Zhang | | +| Min Dai | daimin | +| Qingxin Yang | yangqingxin1993 | +| Sijie Yang | iyangsj | +| Wenjie Tian | WJTian | +| Wensi Yang | tianxinheihei | +| Xiaofei Yu | xiaofei0800 | +| Yang Liu | dut-yangliu | diff --git a/Makefile b/Makefile index 785dba6ec..b0fb10c25 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ GOGET := $(GO) get GOGEN := $(GO) generate # init bfe version -BFE_VERSION ?= $(shell git rev-parse --short HEAD || echo "GitNotFound") +BFE_VERSION ?= $(shell cat VERSION) # make, make all all: prepare compile package diff --git a/README.md b/README.md index f27418a30..3a5896f72 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,11 @@ # BFE + +[![GitHub](https://img.shields.io/github/license/baidu/bfe)](https://github.com/baidu/bfe/blob/develop/LICENSE) +[![Travis (.com)](https://img.shields.io/travis/com/baidu/bfe)](https://travis-ci.com/baidu/bfe) +[![Go Report Card](https://goreportcard.com/badge/github.com/baidu/bfe)](https://goreportcard.com/report/github.com/baidu/bfe) +[![GoDoc](https://godoc.org/github.com/baidu/bfe?status.svg)](https://godoc.org/github.com/baidu/bfe/bfe_module) +[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/3209/badge)](https://bestpractices.coreinfrastructure.org/projects/3209) + BFE is an open-source layer 7 load balancer derived from proprietary Baidu FrontEnd. ## Advantages @@ -29,12 +36,15 @@ BFE is an open-source layer 7 load balancer derived from proprietary Baidu Front - See the [CONTRIBUTING](CONTRIBUTING.md) file for details ## Authors -- Owners: zhangmiao02, yangsijie -- Committers: yangsijie +- Owners: [Miao Zhang](mailto:zhangmiao02@baidu.com), [Sijie Yang](mailto:yangsijie@baidu.com) +- Committers: [Sijie Yang](mailto:yangsijie@baidu.com) +- Contributors: [CONTRIBUTORS](CONTRIBUTORS.md) ## Discussion - Issue: https://github.com/baidu/bfe/issues +## Contact +- Email：bfe-osc@baidu.com + ## License BFE is under the Apache 2.0 license. See the [LICENSE](LICENSE) file for details. - diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..4e62d1458 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,9 @@ +# Security Policy + +## Reporting a Vulnerability + +Please do not open issues for anything you think might have a security implication. + +Security issues and bugs should be reported privately to bfe-security@baidu.com. +You should receive a response within 24 hours. If for some reason you do not, +please follow up via email to ensure we received your original message. diff --git a/VERSION b/VERSION new file mode 100644 index 000000000..0ea3a944b --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.2.0 diff --git a/bfe_balance/bal_gslb/bal_gslb.go b/bfe_balance/bal_gslb/bal_gslb.go index 2c250fe6d..1a17005fe 100644 --- a/bfe_balance/bal_gslb/bal_gslb.go +++ b/bfe_balance/bal_gslb/bal_gslb.go @@ -304,7 +304,7 @@ func (bal *BalanceGslb) Balance(req *bfe_basic.Request) (*bal_backend.BfeBackend var backend *bal_backend.BfeBackend var current *SubCluster var err error - balAlgor := bal_slb.WrrSmooth + var balAlgor int bal.lock.Lock() defer bal.lock.Unlock() diff --git a/bfe_basic/condition/build.go b/bfe_basic/condition/build.go index 41209b460..5bb31f39e 100644 --- a/bfe_basic/condition/build.go +++ b/bfe_basic/condition/build.go @@ -251,6 +251,13 @@ func buildPrimitive(node *parser.CallExpr) (Condition, error) { fetcher: &QueryValueFetcher{node.Args[0].Value}, matcher: NewRegMatcher(reg), }, nil + case "req_query_value_contain": + return &PrimitiveCond{ + name: node.Fun.Name, + node: node, + fetcher: &QueryValueFetcher{node.Args[0].Value}, + matcher: NewContainMatcher(node.Args[1].Value, node.Args[2].ToBool()), + }, nil case "req_cookie_key_in": return &PrimitiveCond{ name: node.Fun.Name, @@ -281,6 +288,13 @@ func buildPrimitive(node *parser.CallExpr) (Condition, error) { fetcher: &CookieValueFetcher{node.Args[0].Value}, matcher: NewSuffixInMatcher(node.Args[1].Value, node.Args[2].ToBool()), }, nil + case "req_cookie_value_contain": + return &PrimitiveCond{ + name: node.Fun.Name, + node: node, + fetcher: &CookieValueFetcher{node.Args[0].Value}, + matcher: NewContainMatcher(node.Args[1].Value, node.Args[2].ToBool()), + }, nil case "req_port_in": return &PrimitiveCond{ name: node.Fun.Name, @@ -349,6 +363,13 @@ func buildPrimitive(node *parser.CallExpr) (Condition, error) { fetcher: &HeaderValueFetcher{node.Args[0].Value}, matcher: NewRegMatcher(reg), }, nil + case "req_header_value_contain": + return &PrimitiveCond{ + name: node.Fun.Name, + node: node, + fetcher: &HeaderValueFetcher{node.Args[0].Value}, + matcher: NewContainMatcher(node.Args[1].Value, node.Args[2].ToBool()), + }, nil case "req_method_in": return &PrimitiveCond{ name: node.Fun.Name, diff --git a/bfe_basic/condition/parser/semant.go b/bfe_basic/condition/parser/semant.go index 5fe9626b2..55b590cdc 100644 --- a/bfe_basic/condition/parser/semant.go +++ b/bfe_basic/condition/parser/semant.go @@ -41,11 +41,13 @@ var funcProtos = map[string][]Token{ "req_query_value_prefix_in": {STRING, STRING, BOOL}, "req_query_value_suffix_in": {STRING, STRING, BOOL}, "req_query_value_regmatch": {STRING, STRING}, + "req_query_value_contain": {STRING, STRING, BOOL}, "req_url_regmatch": {STRING}, "req_cookie_key_in": {STRING}, "req_cookie_value_in": {STRING, STRING, BOOL}, "req_cookie_value_prefix_in": {STRING, STRING, BOOL}, "req_cookie_value_suffix_in": {STRING, STRING, BOOL}, + "req_cookie_value_contain": {STRING, STRING, BOOL}, "req_port_in": {STRING}, "req_tag_match": {STRING, STRING}, "req_ua_regmatch": {STRING}, @@ -54,6 +56,7 @@ var funcProtos = map[string][]Token{ "req_header_value_prefix_in": {STRING, STRING, BOOL}, "req_header_value_suffix_in": {STRING, STRING, BOOL}, "req_header_value_regmatch": {STRING, STRING}, + "req_header_value_contain": {STRING, STRING, BOOL}, "req_method_in": {STRING}, "req_cip_range": {STRING, STRING}, "req_vip_range": {STRING, STRING}, diff --git a/bfe_basic/condition/primitive.go b/bfe_basic/condition/primitive.go index ee03de807..b98531e58 100644 --- a/bfe_basic/condition/primitive.go +++ b/bfe_basic/condition/primitive.go @@ -720,3 +720,44 @@ func NewHostMatcher(patterns string) (*HostMatcher, error) { patterns: p, }, nil } + +type ContainMatcher struct { + patterns []string + foldCase bool +} + +func NewContainMatcher(patterns string, foldCase bool) *ContainMatcher { + p := strings.Split(patterns, "|") + + if foldCase { + p = toUpper(p) + } + + return &ContainMatcher{ + patterns: p, + foldCase: foldCase, + } +} + +func contain(v string, patterns []string) bool { + for _, pattern := range patterns { + if strings.Contains(v, pattern) { + return true + } + } + + return false +} + +func (cm *ContainMatcher) Match(v interface{}) bool { + vs, ok := v.(string) + if !ok { + return false + } + + if cm.foldCase { + vs = strings.ToUpper(vs) + } + + return contain(vs, cm.patterns) +} diff --git a/bfe_basic/condition/primitive_test.go b/bfe_basic/condition/primitive_test.go index 721abf9bc..b98774da7 100644 --- a/bfe_basic/condition/primitive_test.go +++ b/bfe_basic/condition/primitive_test.go @@ -185,3 +185,39 @@ func TestHostMatcher_2(t *testing.T) { t.Errorf("NewHostMatcher() return wrong error: %v", err) } } + +// test ContainMatcher, case-sensitive +func TestContainMatcher_1(t *testing.T) { + matcher := NewContainMatcher("yingwen|中文|%e4%b8%ad%e6%96%87|YINGWEN", false) + if !matcher.Match("yingwen") { + t.Fatalf("should match yingwen") + } + + if matcher.Match("Yingwen") { + t.Fatalf("should not match Yingwen") + } + + if !matcher.Match("hi，中文") { + t.Fatalf("should match hi，中文") + } + + if matcher.Match("文") { + t.Fatalf("should not match 文") + } + + if !matcher.Match("%e4%b8%ad%e6%96%87") { + t.Fatalf("should match %%e4%%b8%%ad%%e6%%96%%87") + } + + if !matcher.Match("YINGWEN") { + t.Fatalf("should match YINGWEN") + } +} + +// test for ContainMatcher, ignore case +func TestContainMatcher_2(t *testing.T) { + matcher := NewContainMatcher("yingwen", true) + if !matcher.Match("Yingwen") { + t.Fatalf("should match Yingwen") + } +} diff --git a/bfe_basic/request.go b/bfe_basic/request.go index 51f031feb..91a3a1639 100644 --- a/bfe_basic/request.go +++ b/bfe_basic/request.go @@ -113,8 +113,8 @@ func NewRequest(request *bfe_http.Request, conn net.Conn, stat *RequestStat, fReq.Context = make(map[interface{}]interface{}) fReq.Tags.TagTable = make(map[string][]string) - if conn != nil { - fReq.RemoteAddr = conn.RemoteAddr().(*net.TCPAddr) + if session != nil { + fReq.RemoteAddr = session.RemoteAddr } fReq.SvrDataConf = svrDataConf diff --git a/bfe_bufio/bufio_test.go b/bfe_bufio/bufio_test.go index 9e19838aa..e2da4a888 100644 --- a/bfe_bufio/bufio_test.go +++ b/bfe_bufio/bufio_test.go @@ -813,8 +813,8 @@ type errorWriterToTest struct { expected error } -func (r errorWriterToTest) Read(p []byte) (int, error) { - return len(p) * r.rn, r.rerr +func (w errorWriterToTest) Read(p []byte) (int, error) { + return len(p) * w.rn, w.rerr } func (w errorWriterToTest) Write(p []byte) (int, error) { @@ -879,8 +879,8 @@ func (r errorReaderFromTest) Read(p []byte) (int, error) { return len(p) * r.rn, r.rerr } -func (w errorReaderFromTest) Write(p []byte) (int, error) { - return len(p) * w.wn, w.werr +func (r errorReaderFromTest) Write(p []byte) (int, error) { + return len(p) * r.wn, r.werr } var errorReaderFromTests = []errorReaderFromTest{ diff --git a/bfe_config/bfe_conf/conf_basic.go b/bfe_config/bfe_conf/conf_basic.go index c7efba44f..231a48708 100644 --- a/bfe_config/bfe_conf/conf_basic.go +++ b/bfe_config/bfe_conf/conf_basic.go @@ -27,6 +27,12 @@ import ( "github.com/baidu/bfe/bfe_util" ) +const ( + BALANCER_BGW = "BGW" // layer4 balancer in baidu + BALANCER_PROXY = "PROXY" // layer4 balancer working in PROXY mode (eg. F5, Ctrix, ELB etc) + BALANCER_NONE = "NONE" // layer4 balancer not used +) + type ConfigBasic struct { HttpPort int // listen port for http HttpsPort int // listen port for https @@ -43,6 +49,7 @@ type ConfigBasic struct { GracefulShutdownTimeout int // graceful shutdown timeout, in seconds MaxHeaderBytes int // max header length in bytes in request MaxHeaderUriBytes int // max URI(in header) length in bytes in request + MaxProxyHeaderBytes int // max header length in bytes in Proxy protocol KeepAliveEnabled bool // if false, client connection is shutdown disregard of http headers Modules []string // modules to load @@ -117,8 +124,8 @@ func basicConfCheck(cfg *ConfigBasic) error { } // check Layer4LoadBalancer - if cfg.Layer4LoadBalancer != "BGW" && cfg.Layer4LoadBalancer != "" { - return fmt.Errorf("Layer4LoadBalancer[%s] not support", cfg.Layer4LoadBalancer) + if err := checkLayer4LoadBalancer(cfg); err != nil { + return err } // check TlsHandshakeTimeout @@ -175,6 +182,23 @@ func basicConfCheck(cfg *ConfigBasic) error { return nil } +func checkLayer4LoadBalancer(cfg *ConfigBasic) error { + if len(cfg.Layer4LoadBalancer) == 0 { + cfg.Layer4LoadBalancer = BALANCER_BGW // default BGW + } + + switch cfg.Layer4LoadBalancer { + case BALANCER_BGW: + return nil + case BALANCER_PROXY: + return nil + case BALANCER_NONE: + return nil + default: + return fmt.Errorf("Layer4LoadBalancer[%s] should be BGW/PROXY/NONE", cfg.Layer4LoadBalancer) + } +} + func dataFileConfCheck(cfg *ConfigBasic, confRoot string) error { // check HostRuleConf if cfg.HostRuleConf == "" { diff --git a/bfe_config/bfe_tls_conf/tls_rule_conf/tls_rule_conf_load.go b/bfe_config/bfe_tls_conf/tls_rule_conf/tls_rule_conf_load.go index aa1eaafb1..4b33faea1 100644 --- a/bfe_config/bfe_tls_conf/tls_rule_conf/tls_rule_conf_load.go +++ b/bfe_config/bfe_tls_conf/tls_rule_conf/tls_rule_conf_load.go @@ -129,7 +129,7 @@ func TlsRuleConfCheck(conf *TlsRuleConf) error { conf.Grade = strings.ToUpper(conf.Grade) if !checkGrade(conf) { - return fmt.Errorf("invalid tls grade: %s, currently only A,B,C supported", conf.Grade) + return fmt.Errorf("invalid tls grade: %s, currently only A+,A,B,C supported", conf.Grade) } if conf.ClientAuth && len(conf.ClientCAName) == 0 { @@ -181,7 +181,7 @@ func checkGrade(conf *TlsRuleConf) bool { } switch conf.Grade { - case bfe_tls.GradeA, bfe_tls.GradeB, bfe_tls.GradeC: + case bfe_tls.GRADE_APLUS, bfe_tls.GradeA, bfe_tls.GradeB, bfe_tls.GradeC: return true default: return false diff --git a/bfe_http/request_test.go b/bfe_http/request_test.go index 5426ff85e..3a440babb 100644 --- a/bfe_http/request_test.go +++ b/bfe_http/request_test.go @@ -134,7 +134,7 @@ func TestMultipartReader(t *testing.T) { } req.Header = Header{"Content-Type": {"text/plain"}} - multipart, err = req.MultipartReader() + multipart, _ = req.MultipartReader() if multipart != nil { t.Errorf("unexpected multipart for text/plain") } diff --git a/bfe_modules/bfe_modules.go b/bfe_modules/bfe_modules.go index 00a77ea9f..20e43bb53 100644 --- a/bfe_modules/bfe_modules.go +++ b/bfe_modules/bfe_modules.go @@ -18,8 +18,10 @@ package bfe_modules import ( "github.com/baidu/bfe/bfe_module" + "github.com/baidu/bfe/bfe_modules/mod_access" "github.com/baidu/bfe/bfe_modules/mod_block" "github.com/baidu/bfe/bfe_modules/mod_header" + "github.com/baidu/bfe/bfe_modules/mod_key_log" "github.com/baidu/bfe/bfe_modules/mod_logid" "github.com/baidu/bfe/bfe_modules/mod_redirect" "github.com/baidu/bfe/bfe_modules/mod_rewrite" @@ -49,6 +51,12 @@ var moduleList = []bfe_module.BfeModule{ // mod_header mod_header.NewModuleHeader(), + + // mod_key_log + mod_key_log.NewModuleKeyLog(), + + // mod_access + mod_access.NewModuleAccess(), } // init modules list diff --git a/bfe_modules/mod_access/conf_mod_access.go b/bfe_modules/mod_access/conf_mod_access.go new file mode 100644 index 000000000..b3c9abae0 --- /dev/null +++ b/bfe_modules/mod_access/conf_mod_access.go @@ -0,0 +1,204 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mod_access + +import ( + "fmt" +) + +import ( + "github.com/baidu/go-lib/log/log4go" + gcfg "gopkg.in/gcfg.v1" +) + +// ConfModAccess holds the config of access module. +type ConfModAccess struct { + Log struct { + LogPrefix string // log file prefix + LogDir string // log file dir + RotateWhen string // rotate time + BackupCount int // log file backup number + } + + Template struct { + RequestTemplate string // access log formate string + SessionTemplate string // session finish log formate string + } +} + +// ConfLoad loads config of access module from file. +func ConfLoad(filePath string) (*ConfModAccess, error) { + var err error + var cfg ConfModAccess + + err = gcfg.ReadFileInto(&cfg, filePath) + if err != nil { + return &cfg, err + } + + err = cfg.Check() + if err != nil { + return &cfg, err + } + + return &cfg, nil +} + +func (cfg *ConfModAccess) Check() error { + if cfg.Log.LogPrefix == "" { + return fmt.Errorf("ModAccess.LogPrefix is empty") + } + + if cfg.Log.LogDir == "" { + return fmt.Errorf("ModAccess.LogDir is empty") + } + + if !log4go.WhenIsValid(cfg.Log.RotateWhen) { + return fmt.Errorf("ModAccess.RotateWhen invalid: %s", cfg.Log.RotateWhen) + } + + if cfg.Log.BackupCount <= 0 { + return fmt.Errorf("ModAccess.BackupCount should > 0: %d", cfg.Log.BackupCount) + } + + if cfg.Template.RequestTemplate == "" { + return fmt.Errorf("ModAccess.RequestTemplate not set") + } + + if cfg.Template.SessionTemplate == "" { + return fmt.Errorf("ModAccess.SessionTemplate not set") + } + + return nil +} + +func checkLogFmt(item LogFmtItem, logFmtType string) error { + if logFmtType != Request && logFmtType != Session { + return fmt.Errorf("logFmtType should be Request or Session") + } + + domain, found := fmtItemDomainTable[item.Type] + if !found { + return fmt.Errorf("type : (%d, %s) not configured in domain table", + item.Type, item.Key) + } + + if domain != DomainAll && domain != logFmtType { + return fmt.Errorf("type : (%d, %s) should not in request finish log", + item.Type, item.Key) + } + + return nil +} + +func tokenTypeGet(templatePtr *string, offset int) (int, int, error) { + templateLen := len(*templatePtr) + + for key, logItemType := range fmtTable { + n := len(key) + if offset+n > templateLen { + continue + } + + if key == (*templatePtr)[offset:(offset+n)] { + return logItemType, offset + n - 1, nil + } + } + + return -1, -1, fmt.Errorf("no such log item format type : %s", *templatePtr) +} + +func parseBracketToken(templatePtr *string, offset int) (LogFmtItem, int, error) { + length := len(*templatePtr) + + var endOfBracket int + for endOfBracket = offset + 1; endOfBracket < length; endOfBracket++ { + if (*templatePtr)[endOfBracket] == '}' { + break + } + } + + if endOfBracket >= length { + return LogFmtItem{}, -1, fmt.Errorf("log format: { must be terminated by a }") + } + + if endOfBracket == (length - 1) { + return LogFmtItem{}, -1, fmt.Errorf("log format: } must followed a charactor") + } + + key := (*templatePtr)[offset+1 : endOfBracket] + + logItemType, end, err := tokenTypeGet(templatePtr, endOfBracket+1) + if err != nil { + return LogFmtItem{}, -1, err + } + + return LogFmtItem{key, logItemType}, end, nil +} + +func parseLogTemplate(logTemplate string) ([]LogFmtItem, error) { + reqFmts := []LogFmtItem{} + + start := 0 + templateLen := len(logTemplate) + var token string + + for i := 0; i < templateLen; i++ { + if logTemplate[i] != '$' { + continue + } + + if (i + 1) == templateLen { + return nil, fmt.Errorf("log format: $ must followed with a charactor") + } + + if start <= (i - 1) { + token = logTemplate[start:i] + item := LogFmtItem{token, FormatString} + reqFmts = append(reqFmts, item) + } + + if logTemplate[i+1] == '{' { + item, end, err := parseBracketToken(&logTemplate, i+1) + if err != nil { + return nil, err + } + reqFmts = append(reqFmts, item) + i = end + start = end + 1 + + } else { + logItemType, end, err := tokenTypeGet(&logTemplate, i+1) + if err != nil { + return nil, err + } + + token = logTemplate[(i + 1) : end+1] + item := LogFmtItem{token, logItemType} + reqFmts = append(reqFmts, item) + + i = end + start = end + 1 + } + } + + if start < templateLen { + token = logTemplate[start:templateLen] + item := LogFmtItem{token, FormatString} + reqFmts = append(reqFmts, item) + } + + return reqFmts, nil +} diff --git a/bfe_modules/mod_access/conf_mod_access_test.go b/bfe_modules/mod_access/conf_mod_access_test.go new file mode 100644 index 000000000..46d38d995 --- /dev/null +++ b/bfe_modules/mod_access/conf_mod_access_test.go @@ -0,0 +1,74 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mod_access + +import ( + "testing" +) + +func TestConfLoad(t *testing.T) { + config, err := ConfLoad("./testdata/mod_access/mod_access.conf") + if err != nil { + t.Errorf("BfeConfigLoad error: %v", err) + return + } + + if config.Log.LogPrefix != "access" { + t.Errorf("Log.Prefix should be access") + } +} + +func TestTokenTypeGet(t *testing.T) { + template := "123$status_code$res_header" + + logType, end, err := tokenTypeGet(&template, 4) + if err != nil { + t.Errorf("tokenTypeGet() error: %v", err) + } + if logType != fmtTable["status_code"] { + t.Errorf("logType error, logType: %d", logType) + } + if end != 14 { + t.Errorf("end error, end: %d", end) + } + + logType, end, err = tokenTypeGet(&template, 16) + if err != nil { + t.Errorf("tokenTypeGet() error: %v", err) + } + if logType != fmtTable["res_header"] { + t.Errorf("logType error, logType: %d", logType) + } + if end != 25 { + t.Errorf("end error, end: %d", end) + } +} + +func TestParseBracketToken(t *testing.T) { + template := "{CLIENTIP}res_cookie, log" + + item, end, err := parseBracketToken(&template, 0) + if err != nil { + t.Errorf("parseBracketToken() error: %v", err) + } + + if end != 19 { + t.Errorf("end error, end: %d", end) + } + + if item.Key != "CLIENTIP" || item.Type != fmtTable["res_cookie"] { + t.Errorf("item error, item: %v", item) + } +} diff --git a/bfe_modules/mod_access/log_items.go b/bfe_modules/mod_access/log_items.go new file mode 100644 index 000000000..ea5771a37 --- /dev/null +++ b/bfe_modules/mod_access/log_items.go @@ -0,0 +1,264 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mod_access + +const ( + FormatAllServeTime = iota + FormatBackend + FormatBodyLenIn + FormatBodyLenOut + FormatClientReadTime + FormatClusterDuration + FormatClusterName + FormatClusterServeTime + FormatConnectTime + FormatReqContentLen + FormatReqHeaderLen + FormatHost + FormatHTTP + FormatIsTrustIP + FormatLastBackendDuration + FormatLogID + FormatNthReqInSession + FormatResContentLen + FormatResHeaderLen + FormatStatusCode + FormatProduct + FormatProxyDelayTime + FormatReadReqDuration + FormatReadWriteSrvTime + FormatRedirect + FormatRemoteAddr + FormatReqCookie + FormatReqErrorCode + FormatReqHeader + FormatReqURI + FormatResCookie + FormatResHeader + FormatResDuration + FormatResProto + FormatResStatus + FormatRetryNum + FormatServerAddr + FormatSinceSessionTime + FormatSubclusterName + FormatTime + FormatURI + FormatVIP + FormatWriteServeTime + FormatString + + FormatSesClientIP + FormatSesEndTime + FormatSesErrorCode + FormatSesIsSecure + FormatSesKeepaliveNum + FormatSesOverHead + FormatSesReadTotal + FormatSesTLSClientRandom + FormatSesTLSServerRandom + FormatSesUse100 + FormatSesWriteTotal + FormatSesStartTime +) + +const ( + Request = "Request" + Session = "Session" + DomainAll = "DomainAll" +) + +type LogFmtItem struct { + Key string + Type int +} + +var ( + fmtTable = map[string]int{ + "time": FormatTime, + + "all_time": FormatAllServeTime, + "backend": FormatBackend, + "body_len_in": FormatBodyLenIn, + "body_len_out": FormatBodyLenOut, + "client_read_time": FormatClientReadTime, + "cluster_name": FormatClusterName, + "cluster_duration": FormatClusterDuration, + "cluster_time": FormatClusterServeTime, + "connect_time": FormatConnectTime, + "error": FormatReqErrorCode, + "host": FormatHost, + "http": FormatHTTP, + "is_trust_clientip": FormatIsTrustIP, + "last_backend_duration": FormatLastBackendDuration, + "log_id": FormatLogID, + "product": FormatProduct, + "proxy_delay": FormatProxyDelayTime, + "read_req_duration": FormatReadReqDuration, + "readwrite_serve_time": FormatReadWriteSrvTime, + "redirect": FormatRedirect, + "remote_addr": FormatRemoteAddr, + "req_content_len": FormatReqContentLen, + "req_cookie": FormatReqCookie, + "req_header": FormatReqHeader, + "req_nth": FormatNthReqInSession, + "req_uri": FormatReqURI, + "res_content_len": FormatResContentLen, + "res_cookie": FormatResCookie, + "res_header": FormatResHeader, + "res_proto": FormatResProto, + "response_duration": FormatResDuration, + "retry_num": FormatRetryNum, + "server_addr": FormatServerAddr, + "since_ses_start_time": FormatSinceSessionTime, + "status_code": FormatStatusCode, + "subcluster": FormatSubclusterName, + "uri": FormatURI, + "vip": FormatVIP, + "write_serve_time": FormatWriteServeTime, + + "ses_clientip": FormatSesClientIP, + "ses_end_time": FormatSesEndTime, + "ses_error": FormatSesErrorCode, + "ses_is_secure": FormatSesIsSecure, + "ses_overhead": FormatSesOverHead, + "ses_read_total": FormatSesReadTotal, + "ses_start_time": FormatSesStartTime, + "ses_tls_client_random": FormatSesTLSClientRandom, + "ses_tls_server_random": FormatSesTLSServerRandom, + "ses_use100": FormatSesUse100, + "ses_write_total": FormatSesWriteTotal, + "ses_keepalive_num": FormatSesKeepaliveNum, + } + + fmtItemDomainTable = map[int]string{ + FormatString: DomainAll, + FormatTime: DomainAll, + + FormatAllServeTime: Request, + FormatBackend: Request, + FormatBodyLenIn: Request, + FormatBodyLenOut: Request, + FormatClientReadTime: Request, + FormatClusterDuration: Request, + FormatClusterName: Request, + FormatClusterServeTime: Request, + FormatConnectTime: Request, + FormatReqContentLen: Request, + FormatHost: Request, + FormatHTTP: Request, + FormatIsTrustIP: Request, + FormatLastBackendDuration: Request, + FormatLogID: Request, + FormatNthReqInSession: Request, + FormatResContentLen: Request, + FormatStatusCode: Request, + FormatProduct: Request, + FormatProxyDelayTime: Request, + FormatReadReqDuration: Request, + FormatReadWriteSrvTime: Request, + FormatRedirect: Request, + FormatRemoteAddr: Request, + FormatReqCookie: Request, + FormatReqErrorCode: Request, + FormatReqHeaderLen: Request, + FormatReqHeader: Request, + FormatReqURI: Request, + FormatResCookie: Request, + FormatResDuration: Request, + FormatResHeader: Request, + FormatResHeaderLen: Request, + FormatResProto: Request, + FormatResStatus: Request, + FormatRetryNum: Request, + FormatServerAddr: Request, + FormatSinceSessionTime: Request, + FormatSubclusterName: Request, + FormatURI: Request, + FormatVIP: Request, + FormatWriteServeTime: Request, + + FormatSesClientIP: Session, + FormatSesEndTime: Session, + FormatSesErrorCode: Session, + FormatSesIsSecure: Session, + FormatSesOverHead: Session, + FormatSesReadTotal: Session, + FormatSesStartTime: Session, + FormatSesTLSClientRandom: Session, + FormatSesTLSServerRandom: Session, + FormatSesUse100: Session, + FormatSesWriteTotal: Session, + FormatSesKeepaliveNum: Session, + } + + fmtHandlerTable = map[int]interface{}{ + FormatAllServeTime: onLogFmtAllServeTime, + FormatBackend: onLogFmtBackend, + FormatBodyLenIn: onLogFmtBodyLenIn, + FormatBodyLenOut: onLogFmtBodyLenOut, + FormatClientReadTime: onLogFmtClientReadTime, + FormatClusterDuration: onLogFmtClusterDuration, + FormatClusterName: onLogFmtClusterName, + FormatClusterServeTime: onLogFmtClusterServeTime, + FormatConnectTime: onLogFmtConnectBackendTime, + FormatHTTP: onLogFmtHttp, + FormatIsTrustIP: onLogFmtIsTrustip, + FormatLastBackendDuration: onLogFmtLastBackendDuration, + FormatLogID: onLogFmtLogId, + FormatNthReqInSession: onLogFmtNthReqInSession, + FormatProduct: onLogFmtProduct, + FormatProxyDelayTime: onLogFmtProxyDelayTime, + FormatReadReqDuration: onLogFmtReadReqDuration, + FormatReadWriteSrvTime: onLogFmtReadWriteSrvTime, + FormatRedirect: onLogFmtRedirect, + FormatRemoteAddr: onLogFmtClientIp, + FormatReqCookie: onLogFmtReqCookie, + FormatReqContentLen: onLogFmtReqContentLen, + FormatReqErrorCode: onLogFmtErrorCode, + FormatReqHeader: onLogFmtRequestHeader, + FormatReqHeaderLen: onLogFmtReqHeaderLen, + FormatReqURI: onLogFmtRequestUri, + FormatResContentLen: onLogFmtResContentLen, + FormatResCookie: onLogFmtResCookie, + FormatResHeader: onLogFmtResponseHeader, + FormatResHeaderLen: onLogFmtResHeaderLen, + FormatResDuration: onLogFmtResDuration, + FormatResProto: onLogFmtResProto, + FormatResStatus: onLogFmtResStatus, + FormatRetryNum: onLogFmtRetryNum, + FormatServerAddr: onLogFmtServerAddr, + FormatSinceSessionTime: onLogFmtSinceSessionTime, + FormatStatusCode: onLogFmtStatusCode, + FormatSubclusterName: onLogFmtSubclusterName, + FormatURI: onLogFmtUri, + FormatVIP: onLogFmtVip, + FormatWriteServeTime: onLogFmtWriteSrvTime, + FormatHost: onLogFmtHost, + + FormatSesClientIP: onLogFmtSesClientIp, + FormatSesEndTime: onLogFmtSesEndTime, + FormatSesErrorCode: onLogFmtSesErrorCode, + FormatSesIsSecure: onLogFmtSesIsSecure, + FormatSesKeepaliveNum: onLogFmtSesKeepAliveNum, + FormatSesOverHead: onLogFmtSesOverhead, + FormatSesReadTotal: onLogFmtSesReadTotal, + FormatSesTLSClientRandom: onLogFmtSesTLSClientRandom, + FormatSesTLSServerRandom: onLogFmtSesTLSServerRandom, + FormatSesUse100: onLogFmtSesUse100, + FormatSesWriteTotal: onLogFmtSesWriteTotal, + FormatSesStartTime: onLogFmtSesStartTime, + } +) diff --git a/bfe_modules/mod_access/mod_access.go b/bfe_modules/mod_access/mod_access.go new file mode 100644 index 000000000..952a62bd6 --- /dev/null +++ b/bfe_modules/mod_access/mod_access.go @@ -0,0 +1,180 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mod_access + +import ( + "bytes" + "fmt" +) + +import ( + "github.com/baidu/bfe/bfe_basic" + "github.com/baidu/bfe/bfe_http" + "github.com/baidu/bfe/bfe_module" + "github.com/baidu/bfe/bfe_util/access_log" +) + +import ( + "github.com/baidu/go-lib/log/log4go" + "github.com/baidu/go-lib/web-monitor/web_monitor" +) + +type ModuleAccess struct { + name string + logger log4go.Logger + conf *ConfModAccess + + reqFmts []LogFmtItem + sessionFmts []LogFmtItem +} + +func NewModuleAccess() *ModuleAccess { + m := new(ModuleAccess) + m.name = "mod_access" + return m +} + +func (m *ModuleAccess) Name() string { + return m.name +} + +func (m *ModuleAccess) ParseConfig(conf *ConfModAccess) error { + var err error + + m.reqFmts, err = parseLogTemplate(conf.Template.RequestTemplate) + if err != nil { + return fmt.Errorf("%s.Init(): RequestTemplate %s", m.name, err.Error()) + } + + m.sessionFmts, err = parseLogTemplate(conf.Template.SessionTemplate) + if err != nil { + return fmt.Errorf("%s.Init(): SessionTemplate %s", m.name, err.Error()) + } + + return nil +} + +func (m *ModuleAccess) Init(cbs *bfe_module.BfeCallbacks, whs *web_monitor.WebHandlers, + cr string) error { + var err error + var conf *ConfModAccess + + confPath := bfe_module.ModConfPath(cr, m.name) + if conf, err = ConfLoad(confPath); err != nil { + return fmt.Errorf("%s: cond load err %s", m.name, err.Error()) + } + + return m.init(conf, cbs, whs) +} + +func (m *ModuleAccess) init(conf *ConfModAccess, cbs *bfe_module.BfeCallbacks, whs *web_monitor.WebHandlers) error { + var err error + + if err = m.ParseConfig(conf); err != nil { + return fmt.Errorf("%s.Init(): ParseConfig %s", m.name, err.Error()) + } + + m.conf = conf + + if err = m.CheckLogFormat(); err != nil { + return fmt.Errorf("%s.Init(): CheckLogFormat %s", m.name, err.Error()) + } + + m.logger, err = access_log.LoggerInit(conf.Log.LogPrefix, conf.Log.LogDir, + conf.Log.RotateWhen, conf.Log.BackupCount) + if err != nil { + return fmt.Errorf("%s.Init(): create logger", m.name) + } + + err = cbs.AddFilter(bfe_module.HANDLE_REQUEST_FINISH, m.requestLogHandler) + if err != nil { + return fmt.Errorf("%s.Init(): AddFilter(m. requestLogHandler): %s", m.name, err.Error()) + } + + err = cbs.AddFilter(bfe_module.HANDLE_FINISH, m.sessionLogHandler) + if err != nil { + return fmt.Errorf("%s.Init(): AddFilter(m.sessionLogHandler): %s", m.name, err.Error()) + } + + return nil +} + +func (m *ModuleAccess) CheckLogFormat() error { + for _, item := range m.reqFmts { + err := checkLogFmt(item, Request) + if err != nil { + return err + } + } + + for _, item := range m.sessionFmts { + err := checkLogFmt(item, Session) + if err != nil { + return err + } + } + + return nil +} + +func (m *ModuleAccess) requestLogHandler(req *bfe_basic.Request, res *bfe_http.Response) int { + byteStr := bytes.NewBuffer(nil) + + for _, item := range m.reqFmts { + switch item.Type { + case FormatString: + byteStr.WriteString(item.Key) + case FormatTime: + onLogFmtTime(m, byteStr) + default: + handler, found := fmtHandlerTable[item.Type] + if found { + h := handler.(func(*ModuleAccess, *LogFmtItem, *bytes.Buffer, + *bfe_basic.Request, *bfe_http.Response) error) + h(m, &item, byteStr, req, res) + } + } + } + + byteStr.WriteString("\n") + m.logger.Info(byteStr.Bytes()) + + return bfe_module.BFE_HANDLER_GOON +} + +func (m *ModuleAccess) sessionLogHandler(session *bfe_basic.Session) int { + byteStr := bytes.NewBuffer(nil) + + for _, item := range m.sessionFmts { + switch item.Type { + case FormatString: + byteStr.WriteString(item.Key) + case FormatTime: + onLogFmtTime(m, byteStr) + default: + handler, found := fmtHandlerTable[item.Type] + if found { + h := handler.(func(*ModuleAccess, *LogFmtItem, *bytes.Buffer, + *bfe_basic.Session) error) + h(m, &item, byteStr, session) + } + } + } + + byteStr.WriteString("\n") + m.logger.Info(byteStr.Bytes()) + + return bfe_module.BFE_HANDLER_GOON +} diff --git a/bfe_modules/mod_access/request_log.go b/bfe_modules/mod_access/request_log.go new file mode 100644 index 000000000..99551c0b7 --- /dev/null +++ b/bfe_modules/mod_access/request_log.go @@ -0,0 +1,706 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mod_access + +import ( + "bytes" + "errors" + "fmt" + "strings" + "time" +) + +import ( + "github.com/baidu/bfe/bfe_basic" + "github.com/baidu/bfe/bfe_http" +) + +func onLogFmtAllServeTime(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.Stat == nil { + return errors.New("req.Stat is nil") + } + + now := time.Now() + ms := now.Sub(req.Stat.ReadReqStart).Nanoseconds() / 1000000 + msg := fmt.Sprintf("%d", ms) + buff.WriteString(msg) + + return nil +} + +func onLogFmtBackend(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + msg := fmt.Sprintf("%s,%s,%s,%s", req.Backend.ClusterName, req.Backend.SubclusterName, + req.Backend.BackendAddr, req.Backend.BackendName) + buff.WriteString(msg) + + return nil +} + +func onLogFmtBodyLenIn(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.Stat == nil { + return errors.New("req.Stat is nil") + } + + msg := fmt.Sprintf("%d", req.Stat.BodyLenIn) + buff.WriteString(msg) + + return nil +} + +func onLogFmtBodyLenOut(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.Stat == nil { + return errors.New("req.Stat is nil") + } + + msg := fmt.Sprintf("%d", req.Stat.BodyLenOut) + buff.WriteString(msg) + + return nil +} + +func onLogFmtClientReadTime(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.Stat == nil { + return errors.New("req.Stat is nil") + } + + ms := req.Stat.ReadReqEnd.Sub(req.Stat.ReadReqStart).Nanoseconds() / 1000000 + msg := fmt.Sprintf("%d", ms) + buff.WriteString(msg) + + return nil +} + +func onLogFmtClusterDuration(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.Stat == nil { + return errors.New("req.Stat is nil") + } + + ms := req.Stat.ClusterEnd.Sub(req.Stat.ClusterStart).Nanoseconds() / 1000000 + msg := fmt.Sprintf("%d", ms) + buff.WriteString(msg) + + return nil +} + +func onLogFmtClusterName(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + msg := "-" + if req.Backend.ClusterName != "" { + msg = req.Backend.ClusterName + } + buff.WriteString(msg) + + return nil +} + +func onLogFmtClusterServeTime(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.Stat == nil { + return errors.New("req.Stat is nil") + } + + ms := req.Stat.ClusterEnd.Sub(req.Stat.ClusterStart).Nanoseconds() / 1000000 + msg := fmt.Sprintf("%d", ms) + buff.WriteString(msg) + + return nil +} + +func onLogFmtConnectBackendTime(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil || req.OutRequest == nil { + return errors.New("req is nil") + } + if req.OutRequest.State == nil { + return errors.New("req.OutRequest.State is nil") + } + + msg := "-" + stat := req.OutRequest.State + ms := stat.ConnectBackendEnd.Sub(stat.ConnectBackendStart).Nanoseconds() / 1000000 + if ms >= 0 { + msg = fmt.Sprintf("%d", ms) + } + + buff.WriteString(msg) + + return nil +} + +func onLogFmtReqContentLen(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.HttpRequest == nil { + return errors.New("req.HttpRequest is nil") + } + + msg := fmt.Sprintf("%d", req.HttpRequest.ContentLength) + buff.WriteString(msg) + + return nil +} + +func onLogFmtHost(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.HttpRequest == nil { + return errors.New("req.HttpRequest is nil") + } + + buff.WriteString(req.HttpRequest.Host) + + return nil +} + +func onLogFmtHttp(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.HttpRequest == nil { + return errors.New("req.HttpRequest is nil") + } + + msg := "-" + data, found := req.HttpRequest.Header[logItem.Key] + if found { + msg = strings.Join(data, ",") + } + buff.WriteString(msg) + + return nil +} + +func onLogFmtIsTrustip(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + msg := fmt.Sprintf("%v", req.Session.IsTrustIP) + buff.WriteString(msg) + + return nil +} + +func onLogFmtLastBackendDuration(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.Stat == nil { + return errors.New("req.Stat is nil") + } + + ms := req.Stat.BackendEnd.Sub(req.Stat.BackendStart).Nanoseconds() / 1000000 + msg := fmt.Sprintf("%d", ms) + buff.WriteString(msg) + + return nil +} + +func onLogFmtLogId(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + msg := fmt.Sprintf("%s", req.LogId) + buff.WriteString(msg) + + return nil +} + +func onLogFmtNthReqInSession(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + msg := "-" + if req.HttpRequest != nil && req.HttpRequest.State != nil { + msg = fmt.Sprintf("%d", req.HttpRequest.State.SerialNumber) + } + buff.WriteString(msg) + + return nil +} + +func onLogFmtResContentLen(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + msg := "-" + if req.Stat != nil { + msg = fmt.Sprintf("%d", req.Stat.BodyLenOut) + } + + buff.WriteString(msg) + + return nil +} + +func onLogFmtStatusCode(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + msg := "-" + if res != nil { + msg = fmt.Sprintf("%d", res.StatusCode) + } + buff.WriteString(msg) + + return nil +} + +func onLogFmtProduct(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + msg := "-" + if req.Route.Product != "" { + msg = req.Route.Product + } + + buff.WriteString(msg) + + return nil +} + +func onLogFmtProxyDelayTime(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.Stat == nil { + return errors.New("req.Stat is nil") + } + + msg := "-" + if !req.Stat.BackendFirst.IsZero() { + ms := req.Stat.BackendFirst.Sub(req.Stat.ReadReqEnd).Nanoseconds() / 1000000 + msg = fmt.Sprintf("%d", ms) + } + buff.WriteString(msg) + + return nil +} + +func onLogFmtReadReqDuration(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.Stat == nil { + return errors.New("req.Stat is nil") + } + + ms := req.Stat.ReadReqEnd.Sub(req.Stat.ReadReqStart).Nanoseconds() / 1000000 + msg := fmt.Sprintf("%d", ms) + buff.WriteString(msg) + + return nil +} + +func onLogFmtReadWriteSrvTime(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.Stat == nil { + return errors.New("req.Stat is nil") + } + + msg := "-" + if !req.Stat.BackendStart.IsZero() { + now := time.Now() + ms := now.Sub(req.Stat.BackendStart).Nanoseconds() / 1000000 + msg = fmt.Sprintf("%d", ms) + } + + buff.WriteString(msg) + + return nil +} + +func onLogFmtRedirect(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + msg := "-" + if req.Redirect.Url != "" { + msg = fmt.Sprintf("%s,%d", req.Redirect.Url, req.Redirect.Code) + } + buff.WriteString(msg) + + return nil +} + +func onLogFmtClientIp(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + msg := "-" + if req.RemoteAddr != nil { + msg = req.RemoteAddr.IP.String() + } + buff.WriteString(msg) + + return nil +} + +func onLogFmtReqCookie(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + msg := "-" + if co, ok := req.Cookie(logItem.Key); ok { + msg = co.Value + } + buff.WriteString(msg) + + return nil +} + +func onLogFmtErrorCode(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + msg := buildErrorMsg(req.ErrCode, req.ErrMsg) + buff.WriteString(msg) + + return nil +} + +func onLogFmtReqHeaderLen(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.Stat == nil { + return errors.New("req.Stat is nil") + } + + msg := fmt.Sprintf("%d", req.Stat.HeaderLenIn) + buff.WriteString(msg) + + return nil +} + +func onLogFmtRequestHeader(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.HttpRequest == nil { + return errors.New("req.HttpRequest is nil") + } + + msg := "-" + if data := req.HttpRequest.Header.Get(logItem.Key); data != "" { + msg = data + } + buff.WriteString(msg) + + return nil +} + +func onLogFmtRequestUri(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.HttpRequest == nil { + return errors.New("req.HttpRequest is nil") + } + + buff.WriteString(req.HttpRequest.RequestURI) + + return nil +} + +func onLogFmtResCookie(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + if res == nil { + buff.WriteString("-") + return nil + } + + msg := "-" + cookies := res.Cookies() + for _, co := range cookies { + if co.Name == logItem.Key { + msg = co.Value + break + } + } + + buff.WriteString(msg) + + return nil +} + +func onLogFmtResDuration(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.Stat == nil { + return errors.New("req.Stat is nil") + } + + ms := req.Stat.ResponseEnd.Sub(req.Stat.ResponseStart).Nanoseconds() / 1000000 + msg := fmt.Sprintf("%d", ms) + buff.WriteString(msg) + + return nil +} + +func onLogFmtResProto(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + msg := "-" + if res != nil { + msg = fmt.Sprintf("%s", res.Proto) + } + buff.WriteString(msg) + + return nil +} + +func onLogFmtResponseHeader(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + if res == nil { + buff.WriteString("-") + return nil + } + + msg := "-" + data, found := res.Header[logItem.Key] + if found { + msg = strings.Join(data, ",") + } + buff.WriteString(msg) + + return nil +} + +func onLogFmtResHeaderLen(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + if req.Stat == nil { + return errors.New("req.Stat is nil") + } + + msg := "-" + if res != nil { + msg = fmt.Sprintf("%d", req.Stat.HeaderLenOut) + } + + buff.WriteString(msg) + + return nil +} + +func onLogFmtResStatus(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + msg := "-" + if res != nil { + msg = fmt.Sprintf("%s", res.Status) + } + buff.WriteString(msg) + + return nil +} + +func onLogFmtRetryNum(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + msg := fmt.Sprintf("%d", req.RetryTime) + buff.WriteString(msg) + + return nil +} + +func onLogFmtServerAddr(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + msg := "-" + if req.Connection != nil { + msg = req.Connection.LocalAddr().String() + } + buff.WriteString(msg) + + return nil +} + +func onLogFmtSinceSessionTime(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + if req.Session == nil { + return errors.New("req.Session is nil") + } + + ms := time.Since(req.Session.StartTime).Nanoseconds() / 1000000 + msg := fmt.Sprintf("%d", ms) + buff.WriteString(msg) + + return nil +} + +func onLogFmtSubclusterName(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + buff.WriteString(req.Backend.SubclusterName) + + return nil +} + +func onLogFmtTime(m *ModuleAccess, buff *bytes.Buffer) error { + now := time.Now() + timeNowStr := fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", + now.Year(), now.Month(), now.Day(), + now.Hour(), now.Minute(), now.Second()) + buff.WriteString(timeNowStr) + + return nil +} + +func onLogFmtVip(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + + msg := "-" + if req.Session.Vip != nil { + if vip := req.Session.Vip.String(); len(vip) != 0 { + msg = vip + } + } + buff.WriteString(msg) + + return nil +} + +func onLogFmtUri(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.HttpRequest == nil { + return errors.New("req.HttpRequest is nil") + } + + buff.WriteString(req.HttpRequest.URL.String()) + + return nil +} + +func onLogFmtWriteSrvTime(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + req *bfe_basic.Request, res *bfe_http.Response) error { + if req == nil { + return errors.New("req is nil") + } + if req.Stat == nil { + return errors.New("req.Stat is nil") + } + + ms := req.Stat.BackendEnd.Sub(req.Stat.BackendStart).Nanoseconds() / 1000000 + msg := fmt.Sprintf("%d", ms) + buff.WriteString(msg) + + return nil +} diff --git a/bfe_modules/mod_access/session_log.go b/bfe_modules/mod_access/session_log.go new file mode 100644 index 000000000..fb303f40b --- /dev/null +++ b/bfe_modules/mod_access/session_log.go @@ -0,0 +1,186 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mod_access + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" +) + +import ( + "github.com/baidu/bfe/bfe_basic" +) + +func onLogFmtSesClientIp(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + session *bfe_basic.Session) error { + if session == nil { + return errors.New("session is nil") + } + + buff.WriteString(session.RemoteAddr.String()) + + return nil +} + +func onLogFmtSesEndTime(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + session *bfe_basic.Session) error { + if session == nil { + return errors.New("session is nil") + } + + buff.WriteString(session.EndTime.String()) + + return nil +} + +func buildErrorMsg(err error, errMsg string) string { + msg := "-" + if err != nil { + msg = err.Error() + if len(errMsg) != 0 { + msg += "," + errMsg + } + } + + return msg +} + +func onLogFmtSesErrorCode(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + session *bfe_basic.Session) error { + if session == nil { + return errors.New("session is nil") + } + + errCode, errMsg := session.GetError() + msg := buildErrorMsg(errCode, errMsg) + buff.WriteString(msg) + + return nil +} + +func onLogFmtSesIsSecure(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + session *bfe_basic.Session) error { + if session == nil { + return errors.New("session is nil") + } + + msg := fmt.Sprintf("%v", session.IsSecure) + buff.WriteString(msg) + + return nil +} + +func onLogFmtSesKeepAliveNum(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + session *bfe_basic.Session) error { + if session == nil { + return errors.New("session is nil") + } + + msg := fmt.Sprintf("%d", session.ReqNum) + buff.WriteString(msg) + + return nil +} + +func onLogFmtSesOverhead(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + session *bfe_basic.Session) error { + if session == nil { + return errors.New("session is nil") + } + + msg := fmt.Sprintf("%s", session.Overhead.String()) + buff.WriteString(msg) + + return nil +} + +func onLogFmtSesReadTotal(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + session *bfe_basic.Session) error { + if session == nil { + return errors.New("session is nil") + } + + msg := fmt.Sprintf("%d", session.ReadTotal) + buff.WriteString(msg) + + return nil +} + +func onLogFmtSesTLSClientRandom(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + session *bfe_basic.Session) error { + if session == nil { + return errors.New("session is nil") + } + + msg := "-" + if session.TlsState != nil { + msg = hex.EncodeToString(session.TlsState.ClientRandom) + } + buff.WriteString(msg) + + return nil +} + +func onLogFmtSesTLSServerRandom(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + session *bfe_basic.Session) error { + if session == nil { + return errors.New("session is nil") + } + + msg := "-" + if session.TlsState != nil { + msg = hex.EncodeToString(session.TlsState.ServerRandom) + } + buff.WriteString(msg) + + return nil +} + +func onLogFmtSesUse100(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + session *bfe_basic.Session) error { + if session == nil { + return errors.New("session is nil") + } + + msg := fmt.Sprintf("%v", session.Use100Continue) + buff.WriteString(msg) + + return nil +} + +func onLogFmtSesWriteTotal(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + session *bfe_basic.Session) error { + if session == nil { + return errors.New("session is nil") + } + + msg := fmt.Sprintf("%d", session.WriteTotal) + buff.WriteString(msg) + + return nil +} + +func onLogFmtSesStartTime(m *ModuleAccess, logItem *LogFmtItem, buff *bytes.Buffer, + session *bfe_basic.Session) error { + if session == nil { + return errors.New("session is nil") + } + + buff.WriteString(session.StartTime.String()) + + return nil +} diff --git a/bfe_modules/mod_access/testdata/mod_access/mod_access.conf b/bfe_modules/mod_access/testdata/mod_access/mod_access.conf new file mode 100644 index 000000000..050ff0e5e --- /dev/null +++ b/bfe_modules/mod_access/testdata/mod_access/mod_access.conf @@ -0,0 +1,9 @@ +[Log] +LogPrefix = access +LogDir = ./ +RotateWhen = NEXTHOUR +BackupCount = 2 + +[Template] +RequestTemplate = "REQUEST_LOG $time clientip: $remote_addr serverip: $server_addr host: $host product: $product user_agent: ${User-Agent}req_header status: $status_code error: $error" +SessionTemplate = "SESSION_LOG $time clientip: $ses_clientip start_time: $ses_start_time end_time: $ses_end_time overhead: $ses_overhead read_total: $ses_read_total write_total: $ses_write_total keepalive_num: $ses_keepalive_num error: $ses_error" \ No newline at end of file diff --git a/bfe_modules/mod_header/action_header_var.go b/bfe_modules/mod_header/action_header_var.go index b1b98386a..e0d1734be 100644 --- a/bfe_modules/mod_header/action_header_var.go +++ b/bfe_modules/mod_header/action_header_var.go @@ -19,6 +19,7 @@ import ( "encoding/asn1" "encoding/hex" "fmt" + "net" "os" "strconv" ) @@ -26,17 +27,27 @@ import ( import ( "github.com/baidu/bfe/bfe_basic" "github.com/baidu/bfe/bfe_tls" + "github.com/baidu/bfe/bfe_util" ) type HeaderValueHandler func(req *bfe_basic.Request) string +const ( + UNKNOWN = "unknown" +) + var VariableHandlers = map[string]HeaderValueHandler{ // for client "bfe_client_ip": getClientIp, "bfe_client_port": getClientPort, "bfe_request_host": getRequestHost, - "bfe_session_id": getSessionId, - "bfe_vip": getBfeVip, + + // for conn info + "bfe_session_id": getSessionId, + "bfe_cip": getClientIp, // client ip (alias for bfe_clientip) + "bfe_vip": getBfeVip, // virtual ip + "bfe_bip": getBfeBip, // balancer ip + "bfe_rip": getBfeRip, // bfe ip // for bfe "bfe_server_name": getBfeServerName, @@ -240,7 +251,46 @@ func getBfeVip(req *bfe_basic.Request) string { return req.Session.Vip.String() } - return "unknown" + return UNKNOWN +} + +func getAddressFetcher(conn net.Conn) bfe_util.AddressFetcher { + if c, ok := conn.(*bfe_tls.Conn); ok { + conn = c.GetNetConn() + } + if f, ok := conn.(bfe_util.AddressFetcher); ok { + return f + } + return nil +} + +func getBfeBip(req *bfe_basic.Request) string { + f := getAddressFetcher(req.Session.Connection) + if f == nil { + return UNKNOWN + } + + baddr := f.BalancerAddr() + if baddr == nil { + return UNKNOWN + } + bip, _, err := net.SplitHostPort(baddr.String()) + if err != nil { /* never come here */ + return UNKNOWN + } + + return bip +} + +func getBfeRip(req *bfe_basic.Request) string { + conn := req.Session.Connection + raddr := conn.LocalAddr() + rip, _, err := net.SplitHostPort(raddr.String()) + if err != nil { /* never come here */ + return UNKNOWN + } + + return rip } func getBfeBackendInfo(req *bfe_basic.Request) string { @@ -252,7 +302,7 @@ func getBfeBackendInfo(req *bfe_basic.Request) string { func getBfeServerName(req *bfe_basic.Request) string { hostname, err := os.Hostname() if err != nil { - return "unknown" + return UNKNOWN } return hostname diff --git a/bfe_modules/mod_key_log/conf_mod_key_log.go b/bfe_modules/mod_key_log/conf_mod_key_log.go new file mode 100644 index 000000000..f6dbfff4c --- /dev/null +++ b/bfe_modules/mod_key_log/conf_mod_key_log.go @@ -0,0 +1,75 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mod_key_log + +import ( + "fmt" +) + +import ( + "github.com/baidu/go-lib/log/log4go" + gcfg "gopkg.in/gcfg.v1" +) + +// ConfModKeyLog represents the basic config for mod_key_log. +type ConfModKeyLog struct { + Log struct { + LogPrefix string // log file prefix + LogDir string // log file dir + RotateWhen string // rotate time + BackupCount int // log file backup number + } +} + +// Check validates module config +func (cfg *ConfModKeyLog) Check() error { + if cfg.Log.LogPrefix == "" { + return fmt.Errorf("LogPrefix is empty") + } + + if cfg.Log.LogDir == "" { + return fmt.Errorf("LogDir is empty") + } + + if !log4go.WhenIsValid(cfg.Log.RotateWhen) { + return fmt.Errorf("RotateWhen invalid: %s", cfg.Log.RotateWhen) + } + + if cfg.Log.BackupCount <= 0 { + return fmt.Errorf("BackupCount should > 0: %d", cfg.Log.BackupCount) + } + + return nil +} + +// ConfLoad loads config from file +func ConfLoad(filePath string) (*ConfModKeyLog, error) { + var cfg ConfModKeyLog + var err error + + // read config from file + err = gcfg.ReadFileInto(&cfg, filePath) + if err != nil { + return nil, err + } + + // check config + err = cfg.Check() + if err != nil { + return nil, err + } + + return &cfg, nil +} diff --git a/bfe_modules/mod_key_log/mod_key_log.go b/bfe_modules/mod_key_log/mod_key_log.go new file mode 100644 index 000000000..2a462a65d --- /dev/null +++ b/bfe_modules/mod_key_log/mod_key_log.go @@ -0,0 +1,102 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mod_key_log + +import ( + "encoding/hex" + "fmt" +) + +import ( + "github.com/baidu/go-lib/log/log4go" + "github.com/baidu/go-lib/web-monitor/web_monitor" +) + +import ( + "github.com/baidu/bfe/bfe_basic" + "github.com/baidu/bfe/bfe_module" + "github.com/baidu/bfe/bfe_util/access_log" +) + +// ModuleKeyLog writes key logs in NSS key log format so that external +// programs(eg. wireshark) can decrypt TLS connections for trouble shooting. +// +// For more information about NSS key log format, see: +// https://developer.mozilla.org/en-US/docs/Mozilla/Projects/NSS/Key_Log_Format +type ModuleKeyLog struct { + name string // module name + conf *ConfModKeyLog // module config + logger log4go.Logger // key logger +} + +func NewModuleKeyLog() *ModuleKeyLog { + m := new(ModuleKeyLog) + m.name = "mod_key_log" + return m +} + +func (m *ModuleKeyLog) Name() string { + return m.name +} + +func (m *ModuleKeyLog) loadConf(confPath string) error { + conf, err := ConfLoad(confPath) + if err != nil { + return fmt.Errorf("%s: conf load err %s", m.name, err.Error()) + } + + m.conf = conf + return nil +} + +func (m *ModuleKeyLog) logTlsKey(session *bfe_basic.Session) int { + tlsState := session.TlsState + if tlsState == nil { + return bfe_module.BFE_HANDLER_GOON + } + + // key log format: