diff --git a/source/extensions/filters/network/metadata_exchange/BUILD b/source/extensions/filters/network/metadata_exchange/BUILD index 4c1f606da80..9d82f9f4b33 100644 --- a/source/extensions/filters/network/metadata_exchange/BUILD +++ b/source/extensions/filters/network/metadata_exchange/BUILD @@ -53,6 +53,7 @@ envoy_cc_library( "@envoy//envoy/stats:stats_macros", "@envoy//envoy/stream_info:filter_state_interface", "@envoy//source/common/http:utility_lib", + "@envoy//source/common/network:filter_state_dst_address_lib", "@envoy//source/common/network:utility_lib", "@envoy//source/common/protobuf", "@envoy//source/common/protobuf:utility_lib", diff --git a/source/extensions/filters/network/metadata_exchange/metadata_exchange.cc b/source/extensions/filters/network/metadata_exchange/metadata_exchange.cc index 7b7afec618f..96b5117943e 100644 --- a/source/extensions/filters/network/metadata_exchange/metadata_exchange.cc +++ b/source/extensions/filters/network/metadata_exchange/metadata_exchange.cc @@ -16,6 +16,7 @@ #include "source/extensions/filters/network/metadata_exchange/metadata_exchange.h" #include +#include #include #include "absl/base/internal/endian.h" @@ -25,6 +26,8 @@ #include "envoy/stats/scope.h" #include "source/common/buffer/buffer_impl.h" #include "source/common/protobuf/utility.h" +#include "source/common/network/utility.h" +#include "source/common/network/filter_state_dst_address.h" #include "source/extensions/filters/network/metadata_exchange/metadata_exchange_initial_header.h" namespace Envoy { @@ -327,11 +330,89 @@ std::string MetadataExchangeFilter::getMetadataId() { return local_info_.node(). void MetadataExchangeFilter::setMetadataNotFoundFilterState() { if (config_->metadata_provider_) { + Network::Address::InstanceConstSharedPtr upstream_peer; + const StreamInfo::StreamInfo& info = read_callbacks_->connection().streamInfo(); + if (info.upstreamInfo()) { + auto upstream_host = info.upstreamInfo().value().get().upstreamHost(); + if (upstream_host) { + const auto address = upstream_host->address(); + ENVOY_LOG(debug, "Trying to check upstream host info of host {}", address->asString()); + switch (address->type()) { + case Network::Address::Type::Ip: + upstream_peer = upstream_host->address(); + break; + case Network::Address::Type::EnvoyInternal: + if (upstream_host->metadata()) { + ENVOY_LOG(debug, "Trying to check filter metadata of host {}", + upstream_host->address()->asString()); + const auto& filter_metadata = upstream_host->metadata()->filter_metadata(); + const auto& it = filter_metadata.find("envoy.filters.listener.original_dst"); + if (it != filter_metadata.end()) { + const auto& destination_it = it->second.fields().find("local"); + if (destination_it != it->second.fields().end()) { + upstream_peer = Network::Utility::parseInternetAddressAndPortNoThrow( + destination_it->second.string_value(), /*v6only=*/false); + } + } + } + break; + default: + break; + } + } + } + // Get our metadata differently based on the direction of the filter + auto downstream_peer_address = [&]() -> Network::Address::InstanceConstSharedPtr { + if (upstream_peer) { + // Query upstream peer data and save it in metadata for stats + const auto metadata_object = config_->metadata_provider_->GetMetadata(upstream_peer); + if (metadata_object) { + ENVOY_LOG(debug, "Metadata found for upstream peer address {}", + upstream_peer->asString()); + const std::string fb = + Istio::Common::convertWorkloadMetadataToFlatNode(metadata_object.value()); + // Filter object captures schema by view, hence the global singleton for the + // prototype. + auto state = std::make_unique<::Envoy::Extensions::Filters::Common::Expr::CelState>( + MetadataExchangeConfig::nodeInfoPrototype()); + state->setValue(fb); + read_callbacks_->connection().streamInfo().filterState()->setData( + absl::StrCat(kMetadataPrefix, kUpstreamMetadataKey), std::move(state), + StreamInfo::FilterState::StateType::Mutable, + StreamInfo::FilterState::LifeSpan::Connection); + + CelStatePrototype prototype( + false, ::Envoy::Extensions::Filters::Common::Expr::CelStateType::String, + absl::string_view(), StreamInfo::FilterState::LifeSpan::Connection); + auto id_state = + std::make_unique<::Envoy::Extensions::Filters::Common::Expr::CelState>(prototype); + id_state->setValue("unknown"); + read_callbacks_->connection().streamInfo().filterState()->setData( + absl::StrCat(kMetadataPrefix, kUpstreamMetadataIdKey), std::move(id_state), + StreamInfo::FilterState::StateType::Mutable, + StreamInfo::FilterState::LifeSpan::Connection); + config_->stats().metadata_added_.inc(); + } + } + + // Regardless, return the downstream address for downstream metadata + return read_callbacks_->connection().connectionInfoProvider().remoteAddress(); + }; + + auto upstream_peer_address = [&]() -> Network::Address::InstanceConstSharedPtr { + if (upstream_peer) { + return upstream_peer; + } + ENVOY_LOG(debug, "Upstream peer address is null. Fall back to localAddress"); + return read_callbacks_->connection().connectionInfoProvider().localAddress(); + }; const Network::Address::InstanceConstSharedPtr peer_address = - read_callbacks_->connection().connectionInfoProvider().remoteAddress(); + config_->filter_direction_ == FilterDirection::Downstream ? downstream_peer_address() + : upstream_peer_address(); ENVOY_LOG(debug, "Look up metadata based on peer address {}", peer_address->asString()); const auto metadata_object = config_->metadata_provider_->GetMetadata(peer_address); if (metadata_object) { + ENVOY_LOG(debug, "Metadata found for peer address {}", peer_address->asString()); updatePeer(Istio::Common::convertWorkloadMetadataToFlatNode(metadata_object.value())); updatePeerId(config_->filter_direction_ == FilterDirection::Downstream ? kDownstreamMetadataIdKey @@ -339,6 +420,8 @@ void MetadataExchangeFilter::setMetadataNotFoundFilterState() { "unknown"); config_->stats().metadata_added_.inc(); return; + } else { + ENVOY_LOG(debug, "Metadata not found for peer address {}", peer_address->asString()); } } updatePeerId(kMetadataNotFoundValue, kMetadataNotFoundValue); diff --git a/test/envoye2e/basic_flow/basic_test.go b/test/envoye2e/basic_flow/basic_test.go index bf88cb7b3f1..6715c708698 100644 --- a/test/envoye2e/basic_flow/basic_test.go +++ b/test/envoye2e/basic_flow/basic_test.go @@ -55,7 +55,7 @@ func TestBasicTCPFlow(t *testing.T) { &driver.Update{ Node: "server", Version: "0", - Clusters: []string{driver.LoadTestData("testdata/cluster/tcp_server.yaml.tmpl")}, + Clusters: []string{driver.LoadTestData("testdata/cluster/tcp_server.yaml.tmpl")}, Listeners: []string{driver.LoadTestData("testdata/listener/tcp_server.yaml.tmpl")}, }, &driver.Envoy{Bootstrap: params.LoadTestData("testdata/bootstrap/client.yaml.tmpl")}, diff --git a/test/envoye2e/inventory.go b/test/envoye2e/inventory.go index 14b7c4894f2..354efce8e1b 100644 --- a/test/envoye2e/inventory.go +++ b/test/envoye2e/inventory.go @@ -40,6 +40,7 @@ func init() { "TestStatsEndpointLabels/#00", "TestStatsServerWaypointProxy", "TestStatsServerWaypointProxyCONNECT", + "TestTCPStatsServerWaypointProxyCONNECT", "TestStatsGrpc/#00", "TestStatsGrpcStream/#00", "TestStatsParallel/Default", diff --git a/test/envoye2e/stats_plugin/stats_test.go b/test/envoye2e/stats_plugin/stats_test.go index 67df263a934..712178f1532 100644 --- a/test/envoye2e/stats_plugin/stats_test.go +++ b/test/envoye2e/stats_plugin/stats_test.go @@ -728,6 +728,79 @@ func TestStatsServerWaypointProxyCONNECT(t *testing.T) { } } +func TestTCPStatsServerWaypointProxyCONNECT(t *testing.T) { + params := driver.NewTestParams(t, map[string]string{ + "EnableDelta": "true", + "EnableMetadataDiscovery": "true", + "DisableDirectResponse": "true", + "ConnectionCount": "10", + "StatsFilterServerConfig": driver.LoadTestJSON("testdata/stats/server_waypoint_proxy_config.yaml"), + }, envoye2e.ProxyE2ETests) + params.Vars["ServerClusterName"] = "internal_outbound" + params.Vars["ServerInternalAddress"] = "internal_inbound" + params.Vars["ServerMetadata"] = params.LoadTestData("testdata/server_waypoint_proxy_node_metadata.json.tmpl") + params.Vars["ServerNetworkFilters"] = driver.LoadTestData("testdata/filters/mx_waypoint_tcp.yaml.tmpl") + "\n" + + driver.LoadTestData("testdata/filters/stats_inbound.yaml.tmpl") + params.Vars["EnableTunnelEndpointMetadata"] = "true" + params.Vars["EnableOriginalDstPortOverride"] = "true" + + if err := (&driver.Scenario{ + Steps: []driver.Step{ + &driver.XDS{}, + &driver.Update{ + Node: "client", Version: "0", + Clusters: []string{ + driver.LoadTestData("testdata/cluster/internal_outbound.yaml.tmpl"), + driver.LoadTestData("testdata/cluster/original_dst.yaml.tmpl"), + }, + Listeners: []string{ + driver.LoadTestData("testdata/listener/tcp_client.yaml.tmpl"), + driver.LoadTestData("testdata/listener/internal_outbound.yaml.tmpl"), + }, + Secrets: []string{ + driver.LoadTestData("testdata/secret/client.yaml.tmpl"), + }, + }, + &driver.Update{ + Node: "server", Version: "0", + Clusters: []string{ + driver.LoadTestData("testdata/cluster/internal_inbound.yaml.tmpl"), + }, + Listeners: []string{ + driver.LoadTestData("testdata/listener/terminate_connect.yaml.tmpl"), + driver.LoadTestData("testdata/listener/tcp_waypoint_server.yaml.tmpl"), + }, + Secrets: []string{ + driver.LoadTestData("testdata/secret/server.yaml.tmpl"), + }, + }, + &driver.UpdateWorkloadMetadata{Workloads: []driver.WorkloadMetadata{{ + Address: "127.0.0.1", + Metadata: ProductPageMetadata, + }, { + Address: "127.0.0.3", + Metadata: BackendMetadata, + }}}, + &driver.Envoy{Bootstrap: params.LoadTestData("testdata/bootstrap/client.yaml.tmpl")}, + &driver.Envoy{Bootstrap: params.LoadTestData("testdata/bootstrap/server.yaml.tmpl")}, + &driver.Sleep{Duration: 1 * time.Second}, + &driver.TCPServer{Prefix: "hello"}, + &driver.Repeat{ + N: 10, + Step: &driver.TCPConnection{}, + }, + &driver.Stats{ + AdminPort: params.Ports.ServerAdmin, + Matchers: map[string]driver.StatMatcher{ + "istio_tcp_connections_opened_total": &driver.ExactStat{Metric: "testdata/metric/server_waypoint_proxy_connect_connections_opened_total.yaml.tmpl"}, + }, + }, + }, + }).Run(params); err != nil { + t.Fatal(err) + } +} + func TestStatsExpiry(t *testing.T) { params := driver.NewTestParams(t, map[string]string{ "RequestCount": "1", diff --git a/testdata/filters/mx_waypoint_tcp.yaml.tmpl b/testdata/filters/mx_waypoint_tcp.yaml.tmpl new file mode 100644 index 00000000000..f24979a5602 --- /dev/null +++ b/testdata/filters/mx_waypoint_tcp.yaml.tmpl @@ -0,0 +1,7 @@ +- name: tc_mx_inbound{{.N}} + typed_config: + "@type": type.googleapis.com/udpa.type.v1.TypedStruct + type_url: type.googleapis.com/envoy.tcp.metadataexchange.config.MetadataExchange + value: + protocol: "istio-peer-exchange" + enable_discovery: true diff --git a/testdata/listener/tcp_client.yaml.tmpl b/testdata/listener/tcp_client.yaml.tmpl index 3aac1fd1df7..ef63bfec52b 100644 --- a/testdata/listener/tcp_client.yaml.tmpl +++ b/testdata/listener/tcp_client.yaml.tmpl @@ -20,4 +20,8 @@ filter_chains: type_url: envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy value: stat_prefix: outbound_tcp + {{- if .Vars.ServerClusterName }} + cluster: {{ .Vars.ServerClusterName}} + {{- else }} cluster: outbound|9080|tcp|server.default.svc.cluster.local + {{- end }} diff --git a/testdata/listener/tcp_waypoint_server.yaml.tmpl b/testdata/listener/tcp_waypoint_server.yaml.tmpl new file mode 100644 index 00000000000..7d44d34cd8c --- /dev/null +++ b/testdata/listener/tcp_waypoint_server.yaml.tmpl @@ -0,0 +1,29 @@ +{{- if ne .Vars.ServerListeners "" }} +{{ .Vars.ServerListeners }} +{{- else }} +{{- if ne .Vars.ServerInternalAddress "" }} +name: {{ .Vars.ServerInternalAddress }} +{{- else }} +name: server +{{- end }} +traffic_direction: INBOUND +{{- if ne .Vars.ServerInternalAddress "" }} +internal_listener: {} +{{- else }} +address: + socket_address: + address: 127.0.0.2 + port_value: {{ .Ports.ServerPort }} +{{- end }} +filter_chains: +- filters: +{{ .Vars.ServerNetworkFilters | fill | indent 2 }} + - name: tcp_proxy + typed_config: + "@type": type.googleapis.com/udpa.type.v1.TypedStruct + type_url: envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + value: + stat_prefix: server_inbound_tcp + cluster: server-inbound-cluster +{{ .Vars.ServerListenerTLSContext | indent 2 }} +{{- end }} diff --git a/testdata/metric/server_waypoint_proxy_connect_connections_opened_total.yaml.tmpl b/testdata/metric/server_waypoint_proxy_connect_connections_opened_total.yaml.tmpl new file mode 100644 index 00000000000..0d0b6d7f676 --- /dev/null +++ b/testdata/metric/server_waypoint_proxy_connect_connections_opened_total.yaml.tmpl @@ -0,0 +1,52 @@ +name: istio_tcp_connections_opened_total +type: COUNTER +metric: +- counter: + value: 10 + label: + - name: reporter + value: waypoint + - name: source_workload + value: productpage-v1 + - name: source_canonical_service + value: unknown + - name: source_canonical_revision + value: latest + - name: source_workload_namespace + value: default + - name: source_principal + value: spiffe://cluster.local/ns/default/sa/client + - name: source_app + value: unknown + - name: source_version + value: unknown + - name: source_cluster + value: unknown + - name: destination_workload + value: ratings-v1 + - name: destination_workload_namespace + value: default + - name: destination_principal + value: spiffe://cluster.global/ns/default/sa/ratings + - name: destination_app + value: ratings + - name: destination_version + value: version-1 + - name: destination_service + value: server.default.svc.cluster.local + - name: destination_canonical_service + value: ratings + - name: destination_canonical_revision + value: version-1 + - name: destination_service_name + value: server + - name: destination_service_namespace + value: default + - name: destination_cluster + value: server-cluster + - name: request_protocol + value: tcp + - name: response_flags + value: "-" + - name: connection_security_policy + value: mutual_tls diff --git a/testdata/testdata.gen.go b/testdata/testdata.gen.go index 4c6208eeb3b..c78f3d74b2e 100644 --- a/testdata/testdata.gen.go +++ b/testdata/testdata.gen.go @@ -1,31 +1,35 @@ -// Code generated for package testdata by go-bindata DO NOT EDIT. (@generated) +// Code generated by go-bindata. DO NOT EDIT. // sources: -// bootstrap/client.yaml.tmpl -// bootstrap/client_cluster_metadata_precedence.yaml.tmpl -// bootstrap/otel_stats.yaml.tmpl -// bootstrap/server.yaml.tmpl -// bootstrap/stats.yaml.tmpl -// listener/client.yaml.tmpl -// listener/client_passthrough.yaml.tmpl -// listener/internal_outbound.yaml.tmpl -// listener/server.yaml.tmpl -// listener/tcp_client.yaml.tmpl -// listener/tcp_passthrough.yaml.tmpl -// listener/tcp_server.yaml.tmpl -// listener/terminate_connect.yaml.tmpl +// bootstrap/client.yaml.tmpl (2.319kB) +// bootstrap/client_cluster_metadata_precedence.yaml.tmpl (2.318kB) +// bootstrap/otel_stats.yaml.tmpl (293B) +// bootstrap/server.yaml.tmpl (3.279kB) +// bootstrap/stats.yaml.tmpl (2.983kB) +// listener/client.yaml.tmpl (1.149kB) +// listener/client_passthrough.yaml.tmpl (1.611kB) +// listener/internal_outbound.yaml.tmpl (539B) +// listener/server.yaml.tmpl (1.336kB) +// listener/tcp_client.yaml.tmpl (960B) +// listener/tcp_passthrough.yaml.tmpl (979B) +// listener/tcp_server.yaml.tmpl (878B) +// listener/tcp_waypoint_server.yaml.tmpl (802B) +// listener/terminate_connect.yaml.tmpl (4.304kB) + package testdata import ( + "crypto/sha256" "fmt" - "io/ioutil" "os" "path/filepath" "strings" "time" ) + type asset struct { - bytes []byte - info os.FileInfo + bytes []byte + info os.FileInfo + digest [sha256.Size]byte } type bindataFileInfo struct { @@ -35,32 +39,21 @@ type bindataFileInfo struct { modTime time.Time } -// Name return file name func (fi bindataFileInfo) Name() string { return fi.name } - -// Size return file size func (fi bindataFileInfo) Size() int64 { return fi.size } - -// Mode return file mode func (fi bindataFileInfo) Mode() os.FileMode { return fi.mode } - -// Mode return file modify time func (fi bindataFileInfo) ModTime() time.Time { return fi.modTime } - -// IsDir return file whether a directory func (fi bindataFileInfo) IsDir() bool { - return fi.mode&os.ModeDir != 0 + return false } - -// Sys return file is sys mode func (fi bindataFileInfo) Sys() interface{} { return nil } @@ -160,7 +153,7 @@ func bootstrapClientYamlTmpl() (*asset, error) { } info := bindataFileInfo{name: "bootstrap/client.yaml.tmpl", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x94, 0xa3, 0xc3, 0x85, 0xe3, 0xf4, 0x0, 0xd, 0xb7, 0x4, 0x13, 0x35, 0xf0, 0x12, 0xbc, 0x92, 0xf3, 0x53, 0xf2, 0xc1, 0x15, 0x28, 0x76, 0xd, 0xa6, 0x85, 0x16, 0x7a, 0x3a, 0xee, 0xd, 0x34}} return a, nil } @@ -259,7 +252,7 @@ func bootstrapClient_cluster_metadata_precedenceYamlTmpl() (*asset, error) { } info := bindataFileInfo{name: "bootstrap/client_cluster_metadata_precedence.yaml.tmpl", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xe2, 0x23, 0xc3, 0x9d, 0x1f, 0x13, 0x92, 0x17, 0xf0, 0x95, 0xf7, 0xd4, 0xaf, 0xe2, 0xcb, 0x46, 0x5, 0x97, 0xdd, 0xa3, 0x58, 0x14, 0xa1, 0x33, 0xdd, 0x1a, 0x81, 0x71, 0x72, 0xbe, 0xfe, 0x41}} return a, nil } @@ -288,7 +281,7 @@ func bootstrapOtel_statsYamlTmpl() (*asset, error) { } info := bindataFileInfo{name: "bootstrap/otel_stats.yaml.tmpl", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xa6, 0x4f, 0x8c, 0xf, 0xfe, 0xf2, 0x22, 0x8c, 0xd4, 0xd0, 0x8c, 0xc, 0x6, 0x46, 0xa3, 0xec, 0xf9, 0x50, 0x73, 0xee, 0x1a, 0x39, 0xcc, 0x49, 0xa9, 0xfb, 0x60, 0x83, 0x11, 0x67, 0x1, 0x79}} return a, nil } @@ -417,7 +410,7 @@ func bootstrapServerYamlTmpl() (*asset, error) { } info := bindataFileInfo{name: "bootstrap/server.yaml.tmpl", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x4e, 0xf5, 0x72, 0x35, 0x1b, 0xb2, 0x7d, 0xd6, 0xb4, 0x44, 0xdb, 0xf7, 0x9a, 0xdf, 0x28, 0xce, 0xe9, 0xe5, 0x92, 0xf8, 0xe2, 0x95, 0x8d, 0xc, 0xeb, 0x4f, 0x13, 0x61, 0x66, 0x8d, 0xa8, 0x6d}} return a, nil } @@ -507,7 +500,7 @@ func bootstrapStatsYamlTmpl() (*asset, error) { } info := bindataFileInfo{name: "bootstrap/stats.yaml.tmpl", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x62, 0x58, 0x44, 0xf7, 0xe2, 0x88, 0x48, 0x3c, 0x1c, 0x13, 0x4b, 0xf8, 0x45, 0x34, 0xd3, 0xe, 0x83, 0x30, 0xb8, 0x0, 0xf5, 0x15, 0xfc, 0x19, 0x28, 0x5e, 0x6e, 0xd5, 0x9f, 0x5c, 0x88, 0x4}} return a, nil } @@ -562,7 +555,7 @@ func listenerClientYamlTmpl() (*asset, error) { } info := bindataFileInfo{name: "listener/client.yaml.tmpl", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x1, 0xe2, 0xb9, 0x3a, 0x5d, 0xc4, 0x7d, 0xe6, 0x80, 0x76, 0x89, 0x64, 0x42, 0x32, 0x9c, 0xea, 0xcb, 0x4a, 0xca, 0x14, 0xaf, 0x84, 0x15, 0xfb, 0x87, 0x91, 0x39, 0x37, 0xd6, 0x26, 0x94, 0x6d}} return a, nil } @@ -625,7 +618,7 @@ func listenerClient_passthroughYamlTmpl() (*asset, error) { } info := bindataFileInfo{name: "listener/client_passthrough.yaml.tmpl", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x9e, 0x32, 0x92, 0xe4, 0x31, 0xdd, 0xde, 0x5e, 0x1a, 0x2e, 0x8, 0xa, 0x40, 0xe8, 0xfb, 0x79, 0xbb, 0xb6, 0xe7, 0xa2, 0x53, 0x11, 0x4f, 0x7e, 0xe7, 0x98, 0x7, 0x8b, 0x23, 0x55, 0x69, 0x4c}} return a, nil } @@ -658,7 +651,7 @@ func listenerInternal_outboundYamlTmpl() (*asset, error) { } info := bindataFileInfo{name: "listener/internal_outbound.yaml.tmpl", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x44, 0x13, 0x5a, 0xde, 0x24, 0x88, 0x35, 0x23, 0x9e, 0xc5, 0xc4, 0x93, 0xad, 0x70, 0x64, 0xf3, 0x80, 0x9d, 0x88, 0x5a, 0x73, 0x87, 0x76, 0x28, 0x9e, 0x20, 0x41, 0xb5, 0x8b, 0xfe, 0x95, 0x58}} return a, nil } @@ -720,7 +713,7 @@ func listenerServerYamlTmpl() (*asset, error) { } info := bindataFileInfo{name: "listener/server.yaml.tmpl", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xa2, 0x77, 0x1a, 0x40, 0xa1, 0xe2, 0xb5, 0x5, 0x34, 0x18, 0x91, 0x7c, 0xc3, 0xf6, 0xcd, 0x22, 0x86, 0xdd, 0x86, 0x5a, 0x4f, 0xd0, 0xe5, 0x9b, 0x8, 0x18, 0xb2, 0x5, 0x5a, 0xb9, 0x23, 0x84}} return a, nil } @@ -746,7 +739,11 @@ filter_chains: type_url: envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy value: stat_prefix: outbound_tcp + {{- if .Vars.ServerClusterName }} + cluster: {{ .Vars.ServerClusterName}} + {{- else }} cluster: outbound|9080|tcp|server.default.svc.cluster.local + {{- end }} `) func listenerTcp_clientYamlTmplBytes() ([]byte, error) { @@ -760,7 +757,7 @@ func listenerTcp_clientYamlTmpl() (*asset, error) { } info := bindataFileInfo{name: "listener/tcp_client.yaml.tmpl", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x4d, 0x34, 0x2f, 0xba, 0xb0, 0xc3, 0x24, 0xd2, 0x3, 0xd1, 0xf, 0xf2, 0xd, 0xe3, 0xdf, 0x4c, 0xf8, 0x70, 0x8a, 0x65, 0x98, 0x34, 0xd0, 0x30, 0xd8, 0x9, 0xa1, 0xe1, 0xae, 0x4d, 0xc8, 0xb4}} return a, nil } @@ -802,7 +799,7 @@ func listenerTcp_passthroughYamlTmpl() (*asset, error) { } info := bindataFileInfo{name: "listener/tcp_passthrough.yaml.tmpl", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x96, 0xf0, 0xe7, 0x31, 0xa2, 0x38, 0xe, 0xf3, 0xb8, 0x50, 0xa6, 0x1, 0x46, 0xdf, 0xf8, 0xa8, 0xaa, 0x9c, 0x90, 0xa, 0x76, 0xd0, 0x8, 0x95, 0x58, 0xb1, 0x28, 0x8b, 0x99, 0xe3, 0x6d, 0xaa}} return a, nil } @@ -843,7 +840,53 @@ func listenerTcp_serverYamlTmpl() (*asset, error) { } info := bindataFileInfo{name: "listener/tcp_server.yaml.tmpl", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xfc, 0x8f, 0x88, 0xbe, 0x5c, 0x69, 0x73, 0xfe, 0x78, 0x88, 0xf1, 0x48, 0xbf, 0xf8, 0x34, 0xb5, 0x6a, 0xd2, 0x48, 0x9f, 0xf4, 0x78, 0x24, 0x93, 0xd2, 0x62, 0xef, 0x40, 0xf1, 0x62, 0x9f, 0xe6}} + return a, nil +} + +var _listenerTcp_waypoint_serverYamlTmpl = []byte(`{{- if ne .Vars.ServerListeners "" }} +{{ .Vars.ServerListeners }} +{{- else }} +{{- if ne .Vars.ServerInternalAddress "" }} +name: {{ .Vars.ServerInternalAddress }} +{{- else }} +name: server +{{- end }} +traffic_direction: INBOUND +{{- if ne .Vars.ServerInternalAddress "" }} +internal_listener: {} +{{- else }} +address: + socket_address: + address: 127.0.0.2 + port_value: {{ .Ports.ServerPort }} +{{- end }} +filter_chains: +- filters: +{{ .Vars.ServerNetworkFilters | fill | indent 2 }} + - name: tcp_proxy + typed_config: + "@type": type.googleapis.com/udpa.type.v1.TypedStruct + type_url: envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + value: + stat_prefix: server_inbound_tcp + cluster: server-inbound-cluster +{{ .Vars.ServerListenerTLSContext | indent 2 }} +{{- end }} +`) + +func listenerTcp_waypoint_serverYamlTmplBytes() ([]byte, error) { + return _listenerTcp_waypoint_serverYamlTmpl, nil +} + +func listenerTcp_waypoint_serverYamlTmpl() (*asset, error) { + bytes, err := listenerTcp_waypoint_serverYamlTmplBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "listener/tcp_waypoint_server.yaml.tmpl", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xa1, 0xd5, 0x7f, 0x3b, 0x62, 0xfd, 0xa5, 0x57, 0x3d, 0xdd, 0x3d, 0xd7, 0x40, 0x28, 0x8e, 0xe9, 0xf9, 0x5a, 0x73, 0x3e, 0xf6, 0xed, 0xde, 0xbc, 0xd0, 0xe3, 0x0, 0x31, 0xe7, 0xb2, 0x82, 0xa7}} return a, nil } @@ -983,7 +1026,7 @@ func listenerTerminate_connectYamlTmpl() (*asset, error) { } info := bindataFileInfo{name: "listener/terminate_connect.yaml.tmpl", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xa1, 0x2b, 0xf0, 0x75, 0x1f, 0x63, 0x25, 0x2d, 0xee, 0xab, 0xc2, 0x58, 0x8c, 0xa, 0x50, 0xba, 0x99, 0x30, 0x46, 0xc1, 0x67, 0x42, 0xcb, 0x4, 0xaa, 0x91, 0xfa, 0x98, 0xb5, 0x16, 0xb7, 0x3f}} return a, nil } @@ -991,8 +1034,8 @@ func listenerTerminate_connectYamlTmpl() (*asset, error) { // It returns an error if the asset could not be found or // could not be loaded. func Asset(name string) ([]byte, error) { - cannonicalName := strings.Replace(name, "\\", "/", -1) - if f, ok := _bindata[cannonicalName]; ok { + canonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[canonicalName]; ok { a, err := f() if err != nil { return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err) @@ -1002,6 +1045,12 @@ func Asset(name string) ([]byte, error) { return nil, fmt.Errorf("Asset %s not found", name) } +// AssetString returns the asset contents as a string (instead of a []byte). +func AssetString(name string) (string, error) { + data, err := Asset(name) + return string(data), err +} + // MustAsset is like Asset but panics when Asset would return an error. // It simplifies safe initialization of global variables. func MustAsset(name string) []byte { @@ -1013,12 +1062,18 @@ func MustAsset(name string) []byte { return a } +// MustAssetString is like AssetString but panics when Asset would return an +// error. It simplifies safe initialization of global variables. +func MustAssetString(name string) string { + return string(MustAsset(name)) +} + // AssetInfo loads and returns the asset info for the given name. // It returns an error if the asset could not be found or // could not be loaded. func AssetInfo(name string) (os.FileInfo, error) { - cannonicalName := strings.Replace(name, "\\", "/", -1) - if f, ok := _bindata[cannonicalName]; ok { + canonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[canonicalName]; ok { a, err := f() if err != nil { return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err) @@ -1028,6 +1083,33 @@ func AssetInfo(name string) (os.FileInfo, error) { return nil, fmt.Errorf("AssetInfo %s not found", name) } +// AssetDigest returns the digest of the file with the given name. It returns an +// error if the asset could not be found or the digest could not be loaded. +func AssetDigest(name string) ([sha256.Size]byte, error) { + canonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[canonicalName]; ok { + a, err := f() + if err != nil { + return [sha256.Size]byte{}, fmt.Errorf("AssetDigest %s can't read by error: %v", name, err) + } + return a.digest, nil + } + return [sha256.Size]byte{}, fmt.Errorf("AssetDigest %s not found", name) +} + +// Digests returns a map of all known files and their checksums. +func Digests() (map[string][sha256.Size]byte, error) { + mp := make(map[string][sha256.Size]byte, len(_bindata)) + for name := range _bindata { + a, err := _bindata[name]() + if err != nil { + return nil, err + } + mp[name] = a.digest + } + return mp, nil +} + // AssetNames returns the names of the assets. func AssetNames() []string { names := make([]string, 0, len(_bindata)) @@ -1051,27 +1133,33 @@ var _bindata = map[string]func() (*asset, error){ "listener/tcp_client.yaml.tmpl": listenerTcp_clientYamlTmpl, "listener/tcp_passthrough.yaml.tmpl": listenerTcp_passthroughYamlTmpl, "listener/tcp_server.yaml.tmpl": listenerTcp_serverYamlTmpl, + "listener/tcp_waypoint_server.yaml.tmpl": listenerTcp_waypoint_serverYamlTmpl, "listener/terminate_connect.yaml.tmpl": listenerTerminate_connectYamlTmpl, } +// AssetDebug is true if the assets were built with the debug flag enabled. +const AssetDebug = false + // AssetDir returns the file names below a certain // directory embedded in the file by go-bindata. // For example if you run go-bindata on data/... and data contains the // following hierarchy: -// data/ -// foo.txt -// img/ -// a.png -// b.png -// then AssetDir("data") would return []string{"foo.txt", "img"} -// AssetDir("data/img") would return []string{"a.png", "b.png"} -// AssetDir("foo.txt") and AssetDir("notexist") would return an error +// +// data/ +// foo.txt +// img/ +// a.png +// b.png +// +// then AssetDir("data") would return []string{"foo.txt", "img"}, +// AssetDir("data/img") would return []string{"a.png", "b.png"}, +// AssetDir("foo.txt") and AssetDir("notexist") would return an error, and // AssetDir("") will return []string{"data"}. func AssetDir(name string) ([]string, error) { node := _bintree if len(name) != 0 { - cannonicalName := strings.Replace(name, "\\", "/", -1) - pathList := strings.Split(cannonicalName, "/") + canonicalName := strings.Replace(name, "\\", "/", -1) + pathList := strings.Split(canonicalName, "/") for _, p := range pathList { node = node.Children[p] if node == nil { @@ -1095,26 +1183,27 @@ type bintree struct { } var _bintree = &bintree{nil, map[string]*bintree{ - "bootstrap": &bintree{nil, map[string]*bintree{ - "client.yaml.tmpl": &bintree{bootstrapClientYamlTmpl, map[string]*bintree{}}, - "client_cluster_metadata_precedence.yaml.tmpl": &bintree{bootstrapClient_cluster_metadata_precedenceYamlTmpl, map[string]*bintree{}}, - "otel_stats.yaml.tmpl": &bintree{bootstrapOtel_statsYamlTmpl, map[string]*bintree{}}, - "server.yaml.tmpl": &bintree{bootstrapServerYamlTmpl, map[string]*bintree{}}, - "stats.yaml.tmpl": &bintree{bootstrapStatsYamlTmpl, map[string]*bintree{}}, + "bootstrap": {nil, map[string]*bintree{ + "client.yaml.tmpl": {bootstrapClientYamlTmpl, map[string]*bintree{}}, + "client_cluster_metadata_precedence.yaml.tmpl": {bootstrapClient_cluster_metadata_precedenceYamlTmpl, map[string]*bintree{}}, + "otel_stats.yaml.tmpl": {bootstrapOtel_statsYamlTmpl, map[string]*bintree{}}, + "server.yaml.tmpl": {bootstrapServerYamlTmpl, map[string]*bintree{}}, + "stats.yaml.tmpl": {bootstrapStatsYamlTmpl, map[string]*bintree{}}, }}, - "listener": &bintree{nil, map[string]*bintree{ - "client.yaml.tmpl": &bintree{listenerClientYamlTmpl, map[string]*bintree{}}, - "client_passthrough.yaml.tmpl": &bintree{listenerClient_passthroughYamlTmpl, map[string]*bintree{}}, - "internal_outbound.yaml.tmpl": &bintree{listenerInternal_outboundYamlTmpl, map[string]*bintree{}}, - "server.yaml.tmpl": &bintree{listenerServerYamlTmpl, map[string]*bintree{}}, - "tcp_client.yaml.tmpl": &bintree{listenerTcp_clientYamlTmpl, map[string]*bintree{}}, - "tcp_passthrough.yaml.tmpl": &bintree{listenerTcp_passthroughYamlTmpl, map[string]*bintree{}}, - "tcp_server.yaml.tmpl": &bintree{listenerTcp_serverYamlTmpl, map[string]*bintree{}}, - "terminate_connect.yaml.tmpl": &bintree{listenerTerminate_connectYamlTmpl, map[string]*bintree{}}, + "listener": {nil, map[string]*bintree{ + "client.yaml.tmpl": {listenerClientYamlTmpl, map[string]*bintree{}}, + "client_passthrough.yaml.tmpl": {listenerClient_passthroughYamlTmpl, map[string]*bintree{}}, + "internal_outbound.yaml.tmpl": {listenerInternal_outboundYamlTmpl, map[string]*bintree{}}, + "server.yaml.tmpl": {listenerServerYamlTmpl, map[string]*bintree{}}, + "tcp_client.yaml.tmpl": {listenerTcp_clientYamlTmpl, map[string]*bintree{}}, + "tcp_passthrough.yaml.tmpl": {listenerTcp_passthroughYamlTmpl, map[string]*bintree{}}, + "tcp_server.yaml.tmpl": {listenerTcp_serverYamlTmpl, map[string]*bintree{}}, + "tcp_waypoint_server.yaml.tmpl": {listenerTcp_waypoint_serverYamlTmpl, map[string]*bintree{}}, + "terminate_connect.yaml.tmpl": {listenerTerminate_connectYamlTmpl, map[string]*bintree{}}, }}, }} -// RestoreAsset restores an asset under the given directory +// RestoreAsset restores an asset under the given directory. func RestoreAsset(dir, name string) error { data, err := Asset(name) if err != nil { @@ -1128,18 +1217,14 @@ func RestoreAsset(dir, name string) error { if err != nil { return err } - err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) + err = os.WriteFile(_filePath(dir, name), data, info.Mode()) if err != nil { return err } - err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) - if err != nil { - return err - } - return nil + return os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) } -// RestoreAssets restores an asset under the given directory recursively +// RestoreAssets restores an asset under the given directory recursively. func RestoreAssets(dir, name string) error { children, err := AssetDir(name) // File @@ -1157,6 +1242,6 @@ func RestoreAssets(dir, name string) error { } func _filePath(dir, name string) string { - cannonicalName := strings.Replace(name, "\\", "/", -1) - return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) + canonicalName := strings.Replace(name, "\\", "/", -1) + return filepath.Join(append([]string{dir}, strings.Split(canonicalName, "/")...)...) } diff --git a/tools/gen_compilation_database.py b/tools/gen_compilation_database.py new file mode 100755 index 00000000000..a195ca98adf --- /dev/null +++ b/tools/gen_compilation_database.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +# Copyright Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import os +import shlex +import subprocess +from pathlib import Path + + +# This method is equivalent to https://github.com/grailbio/bazel-compilation-database/blob/master/generate.py +def generate_compilation_database(args): + # We need to download all remote outputs for generated source code. This option lives here to override those + # specified in bazelrc. + bazel_startup_options = shlex.split(os.environ.get("BAZEL_STARTUP_OPTION_LIST", "")) + bazel_options = shlex.split(os.environ.get("BAZEL_BUILD_OPTION_LIST", "")) + [ + "--config=compdb", + "--remote_download_outputs=all", + ] + + source_dir_targets = args.bazel_targets + + subprocess.check_call(["bazel", *bazel_startup_options, "build"] + bazel_options + [ + "--aspects=@bazel_compdb//:aspects.bzl%compilation_database_aspect", + "--output_groups=compdb_files,header_files" + ] + source_dir_targets) + + execroot = subprocess.check_output( + ["bazel", *bazel_startup_options, "info", *bazel_options, + "execution_root"]).decode().strip() + + db_entries = [] + for db in Path(execroot).glob('**/*.compile_commands.json'): + db_entries.extend(json.loads(db.read_text())) + + def replace_execroot_marker(db_entry): + if 'directory' in db_entry and db_entry['directory'] == '__EXEC_ROOT__': + db_entry['directory'] = execroot + if 'command' in db_entry: + db_entry['command'] = ( + db_entry['command'].replace('-isysroot __BAZEL_XCODE_SDKROOT__', '')) + return db_entry + + return list(map(replace_execroot_marker, db_entries)) + + +def is_header(filename): + for ext in (".h", ".hh", ".hpp", ".hxx"): + if filename.endswith(ext): + return True + return False + + +def is_compile_target(target, args): + filename = target["file"] + if is_header(filename): + if args.include_all: + return True + if not args.include_headers: + return False + + if filename.startswith("bazel-out/"): + if args.include_all: + return True + if not args.include_genfiles: + return False + + if filename.startswith("external/"): + if args.include_all: + return True + if not args.include_external: + return False + + return True + + +def modify_compile_command(target, args): + cc, options = target["command"].split(" ", 1) + + # Workaround for bazel added C++11 options, those doesn't affect build itself but + # clang-tidy will misinterpret them. + options = options.replace("-std=c++0x ", "") + options = options.replace("-std=c++11 ", "") + + if args.vscode: + # Visual Studio Code doesn't seem to like "-iquote". Replace it with + # old-style "-I". + options = options.replace("-iquote ", "-I ") + + if args.system_clang: + if cc.find("clang"): + cc = "clang++" + + if is_header(target["file"]): + options += " -Wno-pragma-once-outside-header -Wno-unused-const-variable" + options += " -Wno-unused-function" + # By treating external/envoy* as C++ files we are able to use this script from subrepos that + # depend on Envoy targets. + if not target["file"].startswith("external/") or target["file"].startswith( + "external/envoy"): + # *.h file is treated as C header by default while our headers files are all C++20. + options = "-x c++ -std=c++20 -fexceptions " + options + + target["command"] = " ".join([cc, options]) + return target + + +def fix_compilation_database(args, db): + db = [modify_compile_command(target, args) for target in db if is_compile_target(target, args)] + + with open("compile_commands.json", "w") as db_file: + json.dump(db, db_file, indent=2) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Generate JSON compilation database') + parser.add_argument('--include_external', action='store_true') + parser.add_argument('--include_genfiles', action='store_true') + parser.add_argument('--include_headers', action='store_true') + parser.add_argument('--vscode', action='store_true') + parser.add_argument('--include_all', action='store_true') + parser.add_argument( + '--system-clang', + action='store_true', + help= + 'Use `clang++` instead of the bazel wrapper for commands. This may help if `clangd` cannot find/run the tools.' + ) + parser.add_argument( + 'bazel_targets', nargs='*', default=[ + "//source/...", + ]) + args = parser.parse_args() + fix_compilation_database(args, generate_compilation_database(args)) diff --git a/tools/vscode/refresh_compdb.sh b/tools/vscode/refresh_compdb.sh new file mode 100755 index 00000000000..0c516d3d844 --- /dev/null +++ b/tools/vscode/refresh_compdb.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Copyright Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +opts=(--vscode) + +# Setting TEST_TMPDIR here so the compdb headers won't be overwritten by another bazel run +TEST_TMPDIR=${BUILD_DIR:-/tmp}/envoy-compdb tools/gen_compilation_database.py \ + "${opts[@]}" + +# Kill clangd to reload the compilation database +pkill clangd || :