Skip to content

Commit

Permalink
initial gateway reconcile logic
Browse files Browse the repository at this point in the history
Add a new GatewayConfig model type to represent all Gateway-defined
configuration, and a corresponding GatewayReconciler interface for
taking this model and generating Pomerium configuration.

Update DataBrokerReconciler to implement this new interface, with
initial logic for converting from Gateway-defined routes to Pomerium
routes.
  • Loading branch information
kenjenkins committed Nov 7, 2024
1 parent a5fb66d commit 01e87a3
Show file tree
Hide file tree
Showing 9 changed files with 379 additions and 67 deletions.
37 changes: 19 additions & 18 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,15 @@ require (
google.golang.org/grpc v1.67.1
google.golang.org/protobuf v1.35.1
gopkg.in/yaml.v3 v3.0.1
k8s.io/api v0.30.0
k8s.io/apiextensions-apiserver v0.30.0
k8s.io/apimachinery v0.30.0
k8s.io/apiserver v0.30.0
k8s.io/client-go v0.30.0
sigs.k8s.io/controller-runtime v0.18.0
k8s.io/api v0.31.1
k8s.io/apiextensions-apiserver v0.31.1
k8s.io/apimachinery v0.31.1
k8s.io/apiserver v0.31.1
k8s.io/client-go v0.31.1
sigs.k8s.io/controller-runtime v0.19.0
sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20240501194100-d9f479e0225c
sigs.k8s.io/controller-tools v0.15.0
sigs.k8s.io/controller-tools v0.16.3
sigs.k8s.io/gateway-api v1.2.0
)

require (
Expand Down Expand Up @@ -89,13 +90,13 @@ require (
github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect
github.com/coreos/go-oidc/v3 v3.11.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/emicklei/go-restful/v3 v3.12.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
github.com/fatih/color v1.16.0 // indirect
github.com/fatih/color v1.17.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/fxamacker/cbor/v2 v2.6.0 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/go-chi/chi/v5 v5.1.0 // indirect
github.com/go-ini/ini v1.67.0 // indirect
Expand All @@ -105,9 +106,9 @@ require (
github.com/go-logfmt/logfmt v0.6.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/jsonreference v0.21.0 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/gobuffalo/flect v1.0.2 // indirect
Expand All @@ -131,7 +132,7 @@ require (
github.com/hashicorp/go-set/v3 v3.0.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
Expand Down Expand Up @@ -228,10 +229,10 @@ require (
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
k8s.io/component-base v0.30.0 // indirect
k8s.io/klog/v2 v2.120.1 // indirect
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
k8s.io/component-base v0.31.1 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/kube-openapi v0.0.0-20240423202451-8948a665c108 // indirect
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
namespacelabs.dev/go-filenotify v0.0.0-20220511192020-53ea11be7eaa // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
Expand Down
91 changes: 46 additions & 45 deletions go.sum

Large diffs are not rendered by default.

36 changes: 36 additions & 0 deletions model/gateway_config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Package model contains common data structures between the controller and pomerium config reconciler
package model

import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
gateway_v1 "sigs.k8s.io/gateway-api/apis/v1"
)

// GatewayConfig represents the entirety of the Gateway-defined configuration.
type GatewayConfig struct {
Routes []GatewayHTTPRouteConfig
Certificates []*corev1.Secret
}

// GatewayHTTPRouteConfig represents a single Gateway-defined route together
// with all objects needed to translate it into Pomerium routes.
type GatewayHTTPRouteConfig struct {
*gateway_v1.HTTPRoute

// Hostnames this route should match. This may differ from the list of Hostnames in the
// HTTPRoute Spec depending on the Gateway configuration. "All" is represented as "*".
Hostnames []gateway_v1.Hostname

// ValidBackendRefs determines which BackendRefs are allowed to be used for route "To" URLs.
ValidBackendRefs BackendRefChecker

// Services is a map of all known services in the cluster.
Services map[types.NamespacedName]*corev1.Service
}

// BackendRefChecker is used to determine which BackendRefs are valid.
type BackendRefChecker interface {
Valid(obj client.Object, r *gateway_v1.BackendRef) bool
}
79 changes: 79 additions & 0 deletions pomerium/gateway/backendrefs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package gateway

Check failure on line 1 in pomerium/gateway/backendrefs.go

View workflow job for this annotation

GitHub Actions / lint

package-comments: should have a package comment (revive)

import (
"fmt"
"log"
"net/http"

"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
gateway_v1 "sigs.k8s.io/gateway-api/apis/v1"

"github.com/pomerium/ingress-controller/model"
pb "github.com/pomerium/pomerium/pkg/grpc/config"
)

// applyBackendRefs translates backendRefs to a weighted set of Pomerium "To" URLs.
// [applyFilters] must be called prior to this method.
func applyBackendRefs(
route *pb.Route,
gc *model.GatewayHTTPRouteConfig,
backendRefs []gateway_v1.HTTPBackendRef,
) {
for i := range backendRefs {
if !gc.ValidBackendRefs.Valid(gc.HTTPRoute, &backendRefs[i].BackendRef) {
log.Printf("backendRef %v not valid", &backendRefs[i].BackendRef) // XXX
continue
}
if u, w := backendRefToToURLAndWeight(gc, &backendRefs[i]); w > 0 {
route.To = append(route.To, u)
route.LoadBalancingWeights = append(route.LoadBalancingWeights, w)
}
}

// From the spec: "If all entries in BackendRefs are invalid, and there are also no filters
// specified in this route rule, all traffic which matches this rule MUST receive a 500 status
// code."
if route.Redirect == nil && len(route.To) == 0 {
route.Response = &pb.RouteDirectResponse{
Status: http.StatusInternalServerError,
Body: "no valid backend",
}
}
}

func backendRefToToURLAndWeight(
gc *model.GatewayHTTPRouteConfig,
br *gateway_v1.HTTPBackendRef,
) (string, uint32) {
// Note: currently the only supported backendRef kind is "Service".
namespace := gc.Namespace
if br.Namespace != nil {
namespace = string(*br.Namespace)
}

port := int32(*br.Port)

// For a headless service we need the targetPort instead.
// For now this supports only port numbers, not named ports, but this is enough to pass the
// HTTPRouteServiceTypes conformance test cases.
svc := gc.Services[types.NamespacedName{Namespace: namespace, Name: string(br.Name)}]
if svc != nil && svc.Spec.ClusterIP == "None" {
for i := range svc.Spec.Ports {
p := &svc.Spec.Ports[i]
if p.Port == port && p.TargetPort.Type == intstr.Int {
port = p.TargetPort.IntVal
break
}
}
}

u := fmt.Sprintf("http://%s.%s.svc.cluster.local:%d", br.Name, namespace, port)

weight := uint32(1)
if br.Weight != nil {
weight = uint32(*br.Weight)
}

return u, weight
}
56 changes: 56 additions & 0 deletions pomerium/gateway/filters.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
package gateway

Check failure on line 1 in pomerium/gateway/filters.go

View workflow job for this annotation

GitHub Actions / lint

ST1000: at least one file in a package should have a package comment (stylecheck)

import (
gateway_v1 "sigs.k8s.io/gateway-api/apis/v1"

pb "github.com/pomerium/pomerium/pkg/grpc/config"
)

func applyFilters(route *pb.Route, filters []gateway_v1.HTTPRouteFilter) {
for i := range filters {
applyFilter(route, &filters[i])
}
}

func applyFilter(route *pb.Route, filter *gateway_v1.HTTPRouteFilter) {
switch filter.Type {
case gateway_v1.HTTPRouteFilterRequestHeaderModifier:
applyRequestHeaderFilter(route, filter.RequestHeaderModifier)
case gateway_v1.HTTPRouteFilterRequestRedirect:
applyRedirectFilter(route, filter.RequestRedirect)
}
}

func applyRequestHeaderFilter(route *pb.Route, filter *gateway_v1.HTTPHeaderFilter) {
// Note: "append" is not supported yet.
route.SetRequestHeaders = makeHeadersMap(filter.Set)
route.RemoveRequestHeaders = filter.Remove
}

func makeHeadersMap(headers []gateway_v1.HTTPHeader) map[string]string {
if len(headers) == 0 {
return nil
}

m := make(map[string]string)
for i := range headers {
m[string(headers[i].Name)] = headers[i].Value
}
return m
}

func applyRedirectFilter(route *pb.Route, filter *gateway_v1.HTTPRequestRedirectFilter) {
rr := pb.RouteRedirect{
SchemeRedirect: filter.Scheme,
HostRedirect: (*string)(filter.Hostname),
}
if filter.StatusCode != nil {
code := int32(*filter.StatusCode)
rr.ResponseCode = &code
}
if filter.Port != nil {
port := uint32(*filter.Port)
rr.PortRedirect = &port
}
route.Redirect = &rr
}
30 changes: 30 additions & 0 deletions pomerium/gateway/matches.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package gateway

Check failure on line 1 in pomerium/gateway/matches.go

View workflow job for this annotation

GitHub Actions / lint

ST1000: at least one file in a package should have a package comment (stylecheck)

import (
gateway_v1 "sigs.k8s.io/gateway-api/apis/v1"

pb "github.com/pomerium/pomerium/pkg/grpc/config"
)

func applyMatch(route *pb.Route, match *gateway_v1.HTTPRouteMatch) (ok bool) {
if len(match.Headers) > 0 || len(match.QueryParams) > 0 || match.Method != nil {
return false // these features are not supported yet
}
applyPathMatch(route, match.Path)
return true
}

func applyPathMatch(route *pb.Route, match *gateway_v1.HTTPPathMatch) {
if match == nil || match.Type == nil || match.Value == nil {
return
}

switch *match.Type {
case gateway_v1.PathMatchExact:
route.Path = *match.Value
case gateway_v1.PathMatchPathPrefix:
route.Prefix = *match.Value
case gateway_v1.PathMatchRegularExpression:
route.Regex = *match.Value
}
}
77 changes: 77 additions & 0 deletions pomerium/gateway/translate.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
package gateway

Check failure on line 1 in pomerium/gateway/translate.go

View workflow job for this annotation

GitHub Actions / lint

ST1000: at least one file in a package should have a package comment (stylecheck)

import (
"net/url"

"github.com/pomerium/pomerium/config"
pb "github.com/pomerium/pomerium/pkg/grpc/config"
"google.golang.org/protobuf/proto"

"github.com/pomerium/ingress-controller/model"
)

func GatewayRoutes(gc *model.GatewayHTTPRouteConfig) []*pb.Route {

Check failure on line 13 in pomerium/gateway/translate.go

View workflow job for this annotation

GitHub Actions / lint

exported: exported function GatewayRoutes should have comment or be unexported (revive)
// A single HTTPRoute may need to be represented using many Pomerium routes:
// - An HTTPRoute may have multiple hostnames.
// - An HTTPRoute may have multiple HTTPRouteRules.
// - An HTTPRouteRule may have multiple HTTPRouteMatches.
// First we'll expand all HTTPRouteRules into "template" Pomerium routes, and then we'll
// repeat each "template" route once per hostname.
trs := templateRoutes(gc)

prs := make([]*pb.Route, 0, len(gc.Hostnames)*len(trs))
for _, h := range gc.Hostnames {
from := (&url.URL{
Scheme: "https",
Host: string(h),
}).String()
for _, tr := range trs {
r := proto.Clone(tr).(*pb.Route)
r.From = from

// Skip any routes that fail to validate.
coreRoute, err := config.NewPolicyFromProto(r)
if err != nil || coreRoute.Validate() != nil {
continue
}

prs = append(prs, r)
}
}

return prs
}

// templateRoutes converts an HTTPRoute into zero or more Pomerium routes, ignoring hostname.
func templateRoutes(gc *model.GatewayHTTPRouteConfig) []*pb.Route {
var prs []*pb.Route

rules := gc.Spec.Rules
for i := range rules {
rule := &rules[i]
pr := &pb.Route{}

// From the spec (near https://gateway-api.sigs.k8s.io/reference/spec/#gateway.networking.k8s.io%2fv1.HTTPRoute):
// "Implementations MUST ignore any port value specified in the HTTP Host header while
// performing a match and (absent of any applicable header modification configuration) MUST
// forward this header unmodified to the backend."
pr.PreserveHostHeader = true

applyFilters(pr, rule.Filters)
applyBackendRefs(pr, gc, rule.BackendRefs)

if len(rule.Matches) == 0 {
prs = append(prs, pr)
continue
}

for j := range rule.Matches {
cloned := proto.Clone(pr).(*pb.Route)
if applyMatch(cloned, &rule.Matches[j]) {
prs = append(prs, cloned)
}
}
}

return prs
}
6 changes: 6 additions & 0 deletions pomerium/reconcile.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,12 @@ type IngressReconciler interface {
Delete(ctx context.Context, namespacedName types.NamespacedName) (changes bool, err error)
}

// GatewayReconciler updates Pomerium configuration based on Gateway-defined resources.
type GatewayReconciler interface {
// GatewaySetConfig updates the entire Gateway-defined route configuration.
SetGatewayConfig(ctx context.Context, config *model.GatewayConfig) (changes bool, err error)
}

// ConfigReconciler only updates global parameters and does not deal with individual routes
type ConfigReconciler interface {
// SetConfig updates just the shared config settings
Expand Down
Loading

0 comments on commit 01e87a3

Please sign in to comment.