Skip to content

Commit

Permalink
Rework install and bootstrap
Browse files Browse the repository at this point in the history
  • Loading branch information
@ibuildthecloud
ibuildthecloud committed Aug 22, 2020
1 parent a79914f commit cfbe1dc
Show file tree
Hide file tree
Showing 25 changed files with 452 additions and 865 deletions.
2 changes: 1 addition & 1 deletion generate.go
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//go:generate go run pkg/codegen/cleanup/main.go
//go:generate go run pkg/codegen/main.go
//go:generate go run main.go install manager --crds-only -o ./chart/crds/crds.yaml
//go:generate go run ./pkg/codegen crds ./build/charts/fleet-crd/templates/crds.yaml

package main
2 changes: 1 addition & 1 deletion modules/agent/pkg/controllers/bundledeployment/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func Register(ctx context.Context,
func (h *handler) garbageCollect() {
for {
if err := h.deployManager.Cleanup(); err != nil {
logrus.Errorf("failed to cleanup orphaned releases")
logrus.Errorf("failed to cleanup orphaned releases: %v", err)
}
select {
case <-h.ctx.Done():
Expand Down
95 changes: 69 additions & 26 deletions modules/agent/pkg/register/register.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,29 +5,32 @@ import (
"fmt"
"time"

fleet2 "github.com/rancher/fleet/pkg/generated/controllers/fleet.cattle.io"

"github.com/rancher/wrangler/pkg/ratelimit"

fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
"github.com/rancher/fleet/pkg/config"
fleetcontrollers "github.com/rancher/fleet/pkg/generated/controllers/fleet.cattle.io"
"github.com/rancher/fleet/pkg/registration"
"github.com/rancher/wrangler/pkg/generated/controllers/core"
corev1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
corecontrollers "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
"github.com/rancher/wrangler/pkg/randomtoken"
"github.com/rancher/wrangler/pkg/ratelimit"
"github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
)

const (
CredName = "fleet-agent"
BootstrapCredName = "fleet-agent-bootstrap"
Kubeconfig = "kubeconfig"
Token = "token"
APIServerURL = "apiServerURL"
APIServerCA = "apiServerCA"
SystemNamespace = "systemNamespace"
DeploymentNamespace = "deploymentNamespace"
ClusterNamespace = "clusterNamespace"
ClusterName = "clusterName"
Expand Down Expand Up @@ -64,16 +67,16 @@ func tryRegister(ctx context.Context, namespace, clusterID string, config *rest.

secret, err := k8s.Core().V1().Secret().Get(namespace, CredName, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
return nil, fmt.Errorf("no credential found")
} else if err != nil {
return nil, err
}

if secret.Annotations[fleet.BootstrapToken] == "true" {
secret, err = k8s.Core().V1().Secret().Get(namespace, BootstrapCredName, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("looking up secret %s/%s: %w", namespace, BootstrapCredName, err)
}
secret, err = createClusterSecret(ctx, clusterID, k8s.Core().V1(), secret)
if err != nil {
return nil, err
}
} else if err != nil {
return nil, err
}

clientConfig, err := clientcmd.NewClientConfigFromBytes(secret.Data[Kubeconfig])
Expand All @@ -88,11 +91,8 @@ func tryRegister(ctx context.Context, namespace, clusterID string, config *rest.
}, nil
}

func createClusterSecret(ctx context.Context, clusterID string, k8s corev1.Interface, secret *v1.Secret) (*v1.Secret, error) {
clientConfig, err := clientcmd.NewClientConfigFromBytes(secret.Data[Kubeconfig])
if err != nil {
return nil, err
}
func createClusterSecret(ctx context.Context, clusterID string, k8s corecontrollers.Interface, secret *corev1.Secret) (*corev1.Secret, error) {
clientConfig := createClientConfigFromSecret(secret)

ns, _, err := clientConfig.Namespace()
if err != nil {
Expand All @@ -114,7 +114,7 @@ func createClusterSecret(ctx context.Context, clusterID string, k8s corev1.Inter
return nil, err
}

fc, err := fleet2.NewFactoryFromConfig(kc)
fc, err := fleetcontrollers.NewFactoryFromConfig(kc)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -171,7 +171,13 @@ func createClusterSecret(ctx context.Context, clusterID string, k8s corev1.Inter
newToken := newSecret.Data[Token]
clusterNamespace := newSecret.Data[ClusterNamespace]
clusterName := newSecret.Data[ClusterName]
systemNamespace := string(newSecret.Data[SystemNamespace])
deploymentNamespace := newSecret.Data[DeploymentNamespace]

if systemNamespace != secret.Namespace {
return nil, fmt.Errorf("fleet-agent must be installed in the namespace %s", systemNamespace)
}

newKubeconfig, err := updateClientConfig(clientConfig, string(newToken), string(deploymentNamespace))
if err != nil {
return nil, err
Expand All @@ -181,17 +187,54 @@ func createClusterSecret(ctx context.Context, clusterID string, k8s corev1.Inter
return nil, err
}

updatedSecret := secret.DeepCopy()
updatedSecret.Data[Kubeconfig] = newKubeconfig
updatedSecret.Data[DeploymentNamespace] = deploymentNamespace
updatedSecret.Data[ClusterNamespace] = clusterNamespace
updatedSecret.Data[ClusterName] = clusterName
delete(updatedSecret.Annotations, fleet.BootstrapToken)
updatedSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: CredName,
Namespace: secret.Namespace,
},
Data: map[string][]byte{
Kubeconfig: newKubeconfig,
DeploymentNamespace: deploymentNamespace,
ClusterNamespace: clusterNamespace,
ClusterName: clusterName,
},
}

return k8s.Secret().Update(updatedSecret)
return k8s.Secret().Create(updatedSecret)
}
}

func createClientConfigFromSecret(secret *corev1.Secret) clientcmd.ClientConfig {
apiServerURL := string(secret.Data[APIServerURL])
apiServerCA := secret.Data[APIServerCA]
namespace := string(secret.Data[ClusterNamespace])
token := string(secret.Data[Token])

cfg := clientcmdapi.Config{
Clusters: map[string]*clientcmdapi.Cluster{
"cluster": {
Server: apiServerURL,
CertificateAuthorityData: apiServerCA,
},
},
AuthInfos: map[string]*clientcmdapi.AuthInfo{
"user": {
Token: token,
},
},
Contexts: map[string]*clientcmdapi.Context{
"default": {
Cluster: "cluster",
AuthInfo: "user",
Namespace: namespace,
},
},
CurrentContext: "default",
}

return clientcmd.NewDefaultClientConfig(cfg, &clientcmd.ConfigOverrides{})
}

func testClientConfig(ctx context.Context, cfg []byte) error {
cc, err := clientcmd.NewClientConfigFromBytes(cfg)
if err != nil {
Expand All @@ -208,7 +251,7 @@ func testClientConfig(ctx context.Context, cfg []byte) error {
return err
}

fc, err := fleet2.NewFactoryFromConfig(rest)
fc, err := fleetcontrollers.NewFactoryFromConfig(rest)
if err != nil {
return err
}
Expand Down
104 changes: 69 additions & 35 deletions modules/cli/agentmanifest/agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,36 +3,35 @@ package agentmanifest
import (
"context"
"crypto/tls"
fmt "fmt"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"strings"
"time"

"k8s.io/client-go/kubernetes"

"github.com/rancher/fleet/modules/cli/agentconfig"

"k8s.io/apimachinery/pkg/runtime"

"github.com/rancher/fleet/pkg/agent"

"github.com/pkg/errors"
"github.com/rancher/fleet/modules/cli/agentconfig"
"github.com/rancher/fleet/modules/cli/pkg/client"
"github.com/rancher/fleet/pkg/agent"
fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
"github.com/rancher/fleet/pkg/config"
fleetcontrollers "github.com/rancher/fleet/pkg/generated/controllers/fleet.cattle.io/v1alpha1"
"github.com/rancher/wrangler/pkg/kubeconfig"
"github.com/rancher/wrangler/pkg/yaml"
coreV1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
)

var (
ErrNoHostInConfig = errors.New("failed to find cluster server parameter")
)

type Options struct {
CA []byte
Host string
Expand All @@ -48,7 +47,7 @@ func AgentToken(ctx context.Context, controllerNamespace, kubeConfigFile string,
return nil, err
}

kubeConfig, err := getKubeConfig(kubeConfigFile, client.Namespace, token, opts.Host, opts.CA, opts.NoCA)
kubeConfig, err := getKubeConfig(kubeConfigFile, client.Namespace, token["token"], opts.Host, opts.CA, opts.NoCA)
if err != nil {
return nil, err
}
Expand All @@ -59,7 +58,7 @@ func AgentToken(ctx context.Context, controllerNamespace, kubeConfigFile string,
}
}

return objects(controllerNamespace, kubeConfig), nil
return objects(controllerNamespace, token), nil
}

func insecurePing(host string) {
Expand Down Expand Up @@ -149,7 +148,7 @@ func checkHost(host string) error {
return nil
}

func getKubeConfig(kubeConfig string, namespace, token, host string, ca []byte, noCA bool) (string, error) {
func getKubeConfig(kubeConfig string, namespace string, token []byte, host string, ca []byte, noCA bool) (string, error) {
cc := kubeconfig.GetNonInteractiveClientConfig(kubeConfig)
cfg, err := cc.RawConfig()
if err != nil {
Expand Down Expand Up @@ -185,7 +184,7 @@ func getKubeConfig(kubeConfig string, namespace, token, host string, ca []byte,
},
AuthInfos: map[string]*clientcmdapi.AuthInfo{
"user": {
Token: token,
Token: string(token),
},
},
Contexts: map[string]*clientcmdapi.Context{
Expand Down Expand Up @@ -221,52 +220,55 @@ func getHost(host string, cfg clientcmdapi.Config) (string, bool, error) {
return host, false, nil
}

cluster, err := getCluster(cfg)
host, err := GetHostFromConfig(cfg)
if err != nil {
return "", false, err
}

return cluster.Server, true, nil
return host, true, nil
}

func getCA(ca []byte, cfg clientcmdapi.Config) ([]byte, error) {
if len(ca) > 0 {
return ca, nil
}

cluster, err := getCluster(cfg)
return GetCAFromConfig(cfg)
}

func getToken(ctx context.Context, tokenName string, client *client.Client) (map[string][]byte, error) {
secretName, err := waitForSecretName(ctx, tokenName, client)
if err != nil {
return nil, err
}

if len(cluster.CertificateAuthorityData) > 0 {
return cluster.CertificateAuthorityData, nil
secret, err := client.Core.Secret().Get(client.Namespace, secretName, metav1.GetOptions{})
if err != nil {
return nil, err
}

if cluster.CertificateAuthority != "" {
return ioutil.ReadFile(cluster.CertificateAuthority)
values := secret.Data["values"]
if len(values) == 0 {
return nil, fmt.Errorf("failed to find \"values\" on secret %s/%s", client.Namespace, secretName)
}

return nil, nil
}

func getToken(ctx context.Context, tokenName string, client *client.Client) (string, error) {
secretName, err := waitForSecretName(ctx, tokenName, client)
if err != nil {
return "", err
data := map[string]interface{}{}
if err := yaml.Unmarshal(values, &data); err != nil {
return nil, err
}

secret, err := client.Core.Secret().Get(client.Namespace, secretName, metav1.GetOptions{})
if err != nil {
return "", err
if _, ok := data["token"]; !ok {
return nil, fmt.Errorf("failed to find token in values")
}

token := secret.Data[coreV1.ServiceAccountTokenKey]
if len(token) == 0 {
return "", fmt.Errorf("failed to find token on secret %s/%s", client.Namespace, secretName)
byteData := map[string][]byte{}
for k, v := range data {
if s, ok := v.(string); ok {
byteData[k] = []byte(s)
}
}

return string(token), nil
return byteData, nil
}

func waitForSecretName(ctx context.Context, tokenName string, client *client.Client) (string, error) {
Expand Down Expand Up @@ -318,3 +320,35 @@ func startWatch(namespace string, sa fleetcontrollers.ClusterRegistrationTokenCl
}
return sa.Watch(namespace, metav1.ListOptions{ResourceVersion: secrets.ResourceVersion})
}

func GetCAFromConfig(rawConfig clientcmdapi.Config) ([]byte, error) {
cluster, ok := rawConfig.Clusters[rawConfig.CurrentContext]
if !ok {
for _, v := range rawConfig.Clusters {
cluster = v
break
}
}

if cluster != nil {
if len(cluster.CertificateAuthorityData) > 0 {
return cluster.CertificateAuthorityData, nil
}
return ioutil.ReadFile(cluster.CertificateAuthority)
}

return nil, nil
}

func GetHostFromConfig(rawConfig clientcmdapi.Config) (string, error) {
cluster, ok := rawConfig.Clusters[rawConfig.CurrentContext]
if ok {
return cluster.Server, nil
}

for _, v := range rawConfig.Clusters {
return v.Server, nil
}

return "", ErrNoHostInConfig
}
Loading

0 comments on commit cfbe1dc

Please sign in to comment.