add listener filter support in native plugin (#621)

Signed-off-by: spacewander <[email protected]>

api/pkg/plugins/plugins.go

@@ -99,7 +99,9 @@ func RegisterPlugin(name string, plugin Plugin) {
 			goPlugin.Factory(),
 			NewPluginConfigParser(goPlugin))
 	} else if _, ok := plugin.(NativePlugin); ok {
-		if order.Position != OrderPositionOuter && order.Position != OrderPositionInner {
+		switch order.Position {
+		case OrderPositionOuter, OrderPositionInner, OrderPositionListener, OrderPositionNetwork:
+		default:
 			panic(errInvalidNativePluginOrder)
 		}
 	} else {

controller/internal/istio/envoyfilter.go

@@ -23,6 +23,7 @@ import (
 	istiov1a3 "istio.io/client-go/pkg/apis/networking/v1alpha3"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	fmModel "mosn.io/htnn/api/pkg/filtermanager/model"
 	"mosn.io/htnn/api/pkg/plugins"
 	ctrlcfg "mosn.io/htnn/controller/internal/config"
 	"mosn.io/htnn/controller/internal/model"
@@ -207,69 +208,123 @@ func GenerateRouteFilter(host *model.VirtualHost, route string, config map[strin
 
 func GenerateLDSFilterViaECDS(key string, ldsName string, config map[string]interface{}) *istiov1a3.EnvoyFilter {
 	ef := &istiov1a3.EnvoyFilter{
-		Spec: istioapi.EnvoyFilter{
-			ConfigPatches: []*istioapi.EnvoyFilter_EnvoyConfigObjectPatch{
-				{
-					ApplyTo: istioapi.EnvoyFilter_HTTP_FILTER,
+		Spec: istioapi.EnvoyFilter{},
+	}
+
+	// Always create a Go filter for ECDS so we don't trigger LDS drain when adding/removing Go Plugins.
+	// We can't do this for native filters now.
+	// For TCP proxy use case, the extra golang-filter is harmless, because it only attaches to HCM.
+	cfg := config[model.ECDSGolangFilter]
+	if cfg == nil {
+		cfg = map[string]interface{}{}
+	}
+	ecdsName := key + "-" + model.GolangPluginsFilter
+	ef.Spec.ConfigPatches = append(ef.Spec.ConfigPatches,
+		&istioapi.EnvoyFilter_EnvoyConfigObjectPatch{
+			ApplyTo: istioapi.EnvoyFilter_HTTP_FILTER,
+			Match: &istioapi.EnvoyFilter_EnvoyConfigObjectMatch{
+				ObjectTypes: &istioapi.EnvoyFilter_EnvoyConfigObjectMatch_Listener{
+					Listener: &istioapi.EnvoyFilter_ListenerMatch{
+						Name: ldsName,
+						FilterChain: &istioapi.EnvoyFilter_ListenerMatch_FilterChainMatch{
+							Filter: &istioapi.EnvoyFilter_ListenerMatch_FilterMatch{
+								Name: "envoy.filters.network.http_connection_manager",
+								SubFilter: &istioapi.EnvoyFilter_ListenerMatch_SubFilterMatch{
+									Name: "htnn.filters.http.golang",
+								},
+							},
+						},
+					},
+				},
+			},
+			Patch: &istioapi.EnvoyFilter_Patch{
+				Operation: istioapi.EnvoyFilter_Patch_INSERT_BEFORE,
+				Value: MustNewStruct(map[string]interface{}{
+					"name": ecdsName,
+					"config_discovery": map[string]interface{}{
+						"apply_default_config_without_warming": true,
+						"default_config": map[string]interface{}{
+							"@type":        "type.googleapis.com/envoy.extensions.filters.http.golang.v3alpha.Config",
+							"library_id":   "fm",
+							"library_path": ctrlcfg.GoSoPath(),
+							"plugin_name":  "fm",
+						},
+						"config_source": map[string]interface{}{
+							"ads": map[string]interface{}{},
+						},
+						"type_urls": []interface{}{
+							"type.googleapis.com/envoy.extensions.filters.http.golang.v3alpha.Config",
+						},
+					},
+				}),
+			},
+		},
+		&istioapi.EnvoyFilter_EnvoyConfigObjectPatch{
+			ApplyTo: istioapi.EnvoyFilter_EXTENSION_CONFIG,
+			Patch: &istioapi.EnvoyFilter_Patch{
+				Operation: istioapi.EnvoyFilter_Patch_ADD,
+				Value: MustNewStruct(map[string]interface{}{
+					"name": ecdsName,
+					"typed_config": map[string]interface{}{
+						"@type":        "type.googleapis.com/envoy.extensions.filters.http.golang.v3alpha.Config",
+						"library_id":   "fm",
+						"library_path": ctrlcfg.GoSoPath(),
+						"plugin_name":  "fm",
+						"plugin_config": map[string]interface{}{
+							"@type": "type.googleapis.com/xds.type.v3.TypedStruct",
+							"value": cfg,
+						},
+					},
+				}),
+			},
+		},
+	)
+
+	if config[model.ECDSListenerFilter] != nil {
+		cfg, _ := config[model.ECDSListenerFilter].([]*fmModel.FilterConfig)
+		for _, filter := range cfg {
+			ecdsName := key + "-" + filter.Name
+			c, _ := filter.Config.(map[string]interface{})
+			typeURL, _ := c["@type"].(string)
+			ef.Spec.ConfigPatches = append(ef.Spec.ConfigPatches,
+				&istioapi.EnvoyFilter_EnvoyConfigObjectPatch{
+					ApplyTo: istioapi.EnvoyFilter_LISTENER_FILTER,
 					Match: &istioapi.EnvoyFilter_EnvoyConfigObjectMatch{
 						ObjectTypes: &istioapi.EnvoyFilter_EnvoyConfigObjectMatch_Listener{
 							Listener: &istioapi.EnvoyFilter_ListenerMatch{
 								Name: ldsName,
-								FilterChain: &istioapi.EnvoyFilter_ListenerMatch_FilterChainMatch{
-									Filter: &istioapi.EnvoyFilter_ListenerMatch_FilterMatch{
-										Name: "envoy.filters.network.http_connection_manager",
-										SubFilter: &istioapi.EnvoyFilter_ListenerMatch_SubFilterMatch{
-											Name: "htnn.filters.http.golang",
-										},
-									},
-								},
 							},
 						},
 					},
 					Patch: &istioapi.EnvoyFilter_Patch{
-						Operation: istioapi.EnvoyFilter_Patch_INSERT_BEFORE,
+						Operation: istioapi.EnvoyFilter_Patch_INSERT_FIRST,
 						Value: MustNewStruct(map[string]interface{}{
-							"name": key,
+							"name": ecdsName,
 							"config_discovery": map[string]interface{}{
-								"apply_default_config_without_warming": true,
-								"default_config": map[string]interface{}{
-									"@type":        "type.googleapis.com/envoy.extensions.filters.http.golang.v3alpha.Config",
-									"library_id":   "fm",
-									"library_path": ctrlcfg.GoSoPath(),
-									"plugin_name":  "fm",
-								},
 								"config_source": map[string]interface{}{
 									"ads": map[string]interface{}{},
 								},
 								"type_urls": []interface{}{
-									"type.googleapis.com/envoy.extensions.filters.http.golang.v3alpha.Config",
+									typeURL,
 								},
 							},
 						}),
 					},
 				},
-			},
-		},
-	}
-	ef.Spec.ConfigPatches = append(ef.Spec.ConfigPatches, &istioapi.EnvoyFilter_EnvoyConfigObjectPatch{
-		ApplyTo: istioapi.EnvoyFilter_EXTENSION_CONFIG,
-		Patch: &istioapi.EnvoyFilter_Patch{
-			Operation: istioapi.EnvoyFilter_Patch_ADD,
-			Value: MustNewStruct(map[string]interface{}{
-				"name": key,
-				"typed_config": map[string]interface{}{
-					"@type":        "type.googleapis.com/envoy.extensions.filters.http.golang.v3alpha.Config",
-					"library_id":   "fm",
-					"library_path": ctrlcfg.GoSoPath(),
-					"plugin_name":  "fm",
-					"plugin_config": map[string]interface{}{
-						"@type": "type.googleapis.com/xds.type.v3.TypedStruct",
-						"value": config,
+				&istioapi.EnvoyFilter_EnvoyConfigObjectPatch{
+					ApplyTo: istioapi.EnvoyFilter_EXTENSION_CONFIG,
+					Patch: &istioapi.EnvoyFilter_Patch{
+						Operation: istioapi.EnvoyFilter_Patch_ADD,
+						Value: MustNewStruct(map[string]interface{}{
+							"name":         ecdsName,
+							"typed_config": filter.Config,
+						}),
 					},
 				},
-			}),
-		},
-	})
+			)
+		}
+	}
+
 	return ef
 }
 

controller/internal/model/model.go

@@ -36,3 +36,10 @@ type VirtualHost struct {
 	Name             string
 	ECDSResourceName string
 }
+
+const (
+	ECDSGolangFilter   = "golang"
+	ECDSListenerFilter = "listener"
+
+	GolangPluginsFilter = "golang-filter"
+)

controller/internal/translation/final_state.go

@@ -85,10 +85,6 @@ type envoyFilterWrapper struct {
 	info *Info
 }
 
-func getECDSResourceName(workloadNamespace string, ldsName string) string {
-	return fmt.Sprintf("htnn-%s-%s-golang-filter", workloadNamespace, ldsName)
-}
-
 func toFinalState(_ *Ctx, state *mergedState) (*FinalState, error) {
 	efs := istio.DefaultEnvoyFilters()
 	for _, ef := range efs {

controller/internal/translation/merged_state.go

@@ -196,7 +196,7 @@ func translateFilterManagerConfigToPolicyInRDS(fmc *filtermanager.FilterManagerC
 
 		golangFilterName := "htnn.filters.http.golang"
 		if ctrlcfg.EnableLDSPluginViaECDS() {
-			golangFilterName = virtualHost.ECDSResourceName
+			golangFilterName = virtualHost.ECDSResourceName + "-" + model.GolangPluginsFilter
 		}
 		config[golangFilterName] = map[string]interface{}{
 			"@type": "type.googleapis.com/envoy.extensions.filters.http.golang.v3alpha.ConfigsPerRoute",
@@ -225,18 +225,18 @@ func translateFilterManagerConfigToPolicyInECDS(fmc *filtermanager.FilterManager
 	goFilterManager := &filtermanager.FilterManagerConfig{
 		Plugins: []*fmModel.FilterConfig{},
 	}
+	nativeFilters := map[string][]*fmModel.FilterConfig{
+		model.ECDSListenerFilter: {},
+	}
 
 	consumerNeeded := false
 	for _, plugin := range fmc.Plugins {
 		name := plugin.Name
 		p := plugins.LoadPlugin(name)
-		if p != nil {
-			// Native plugin is not supported
-			continue
+		if p == nil {
+			// For Go Plugins, only the type is registered
+			p = plugins.LoadPluginType(name)
 		}
-
-		// For Go Plugins, only the type is registered
-		p = plugins.LoadPluginType(name)
 		// As we don't reject configuration with unknown plugin to keep compatibility...
 		if p == nil {
 			continue
@@ -250,20 +250,45 @@ func translateFilterManagerConfigToPolicyInECDS(fmc *filtermanager.FilterManager
 		}
 		_ = json.Unmarshal(b, &cfg)
 
-		plugin.Config = cfg
-		goFilterManager.Plugins = append(goFilterManager.Plugins, plugin)
-		_, ok = p.(plugins.ConsumerPlugin)
-		if ok {
-			consumerNeeded = true
+		nativePlugin, ok := p.(plugins.NativePlugin)
+		if !ok {
+			plugin.Config = cfg
+			goFilterManager.Plugins = append(goFilterManager.Plugins, plugin)
+			_, ok = p.(plugins.ConsumerPlugin)
+			if ok {
+				consumerNeeded = true
+			}
+		} else {
+			order := nativePlugin.Order()
+			if order.Position == plugins.OrderPositionOuter || order.Position == plugins.OrderPositionInner {
+				// HTTP Native plugin is not supported
+				continue
+			}
+
+			url := nativePlugin.ConfigTypeURL()
+			conf := p.Config()
+			desc := conf.ProtoReflect().Descriptor()
+			fieldDescs := desc.Fields()
+			m, ok := cfg.(map[string]interface{})
+			if !ok {
+				panic(fmt.Sprintf("unexpected type: %s", reflect.TypeOf(cfg)))
+			}
+			stripUnknowFields(m, fieldDescs)
+
+			m["@type"] = url
+			plugin.Config = m
+			nativeFilters[model.ECDSListenerFilter] = append(nativeFilters[model.ECDSListenerFilter], plugin)
 		}
 	}
+
 	if consumerNeeded {
 		goFilterManager.Namespace = nsName.Namespace
 	}
 
 	if len(goFilterManager.Plugins) > 0 {
+		cfg := map[string]interface{}{}
 		if goFilterManager.Namespace != "" {
-			config["namespace"] = goFilterManager.Namespace
+			cfg["namespace"] = goFilterManager.Namespace
 		}
 		plugins := make([]interface{}, len(goFilterManager.Plugins))
 		for i, plugin := range goFilterManager.Plugins {
@@ -272,9 +297,13 @@ func translateFilterManagerConfigToPolicyInECDS(fmc *filtermanager.FilterManager
 				"config": plugin.Config,
 			}
 		}
-		config["plugins"] = plugins
+		cfg["plugins"] = plugins
+		config[model.ECDSGolangFilter] = cfg
 	}
 
+	for category, filters := range nativeFilters {
+		config[category] = filters
+	}
 	return config
 }
 

controller/internal/translation/testdata/translation/l4_native_plugin_only.in.yml

@@ -0,0 +1,33 @@
+features:
+  enableLDSPluginViaECDS: true
+istioGateway:
+- apiVersion: networking.istio.io/v1beta1
+  kind: Gateway
+  metadata:
+    name: gateway
+    namespace: default
+  spec:
+    selector:
+      istio: ingressgateway
+    servers:
+    - hosts:
+      - example.com
+      port:
+        name: http
+        number: 80
+        protocol: HTTP
+filterPolicy:
+  gateway:
+  - apiVersion: htnn.mosn.io/v1
+    kind: FilterPolicy
+    metadata:
+      name: policy
+      namespace: default
+    spec:
+      targetRef:
+        group: networking.istio.io
+        kind: Gateway
+        name: gateway
+      filters:
+        tlsInspector:
+          config: {}