diff --git a/bundle/manifests/numaresources-operator.clusterserviceversion.yaml b/bundle/manifests/numaresources-operator.clusterserviceversion.yaml index 5c0a4e748..ce4de29ba 100644 --- a/bundle/manifests/numaresources-operator.clusterserviceversion.yaml +++ b/bundle/manifests/numaresources-operator.clusterserviceversion.yaml @@ -62,7 +62,7 @@ metadata: } ] capabilities: Basic Install - createdAt: "2024-12-18T20:58:55Z" + createdAt: "2024-12-19T08:31:49Z" olm.skipRange: '>=4.18.0 <4.19.0' operatorframework.io/cluster-monitoring: "true" operators.operatorframework.io/builder: operator-sdk-v1.36.1 @@ -359,6 +359,7 @@ spec: resources: - configmaps - serviceaccounts + - services verbs: - '*' - apiGroups: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 2f0ef938e..3425f69b5 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -9,6 +9,7 @@ rules: resources: - configmaps - serviceaccounts + - services verbs: - '*' - apiGroups: diff --git a/controllers/numaresourcesoperator_controller.go b/controllers/numaresourcesoperator_controller.go index a5516f342..6120bee4d 100644 --- a/controllers/numaresourcesoperator_controller.go +++ b/controllers/numaresourcesoperator_controller.go @@ -61,8 +61,11 @@ import ( "github.com/openshift-kni/numaresources-operator/pkg/hash" "github.com/openshift-kni/numaresources-operator/pkg/images" "github.com/openshift-kni/numaresources-operator/pkg/loglevel" + rtemetricsmanifests "github.com/openshift-kni/numaresources-operator/pkg/metrics/manifests/monitor" "github.com/openshift-kni/numaresources-operator/pkg/objectnames" apistate "github.com/openshift-kni/numaresources-operator/pkg/objectstate/api" + "github.com/openshift-kni/numaresources-operator/pkg/objectstate/compare" + "github.com/openshift-kni/numaresources-operator/pkg/objectstate/merge" rtestate "github.com/openshift-kni/numaresources-operator/pkg/objectstate/rte" rteupdate "github.com/openshift-kni/numaresources-operator/pkg/objectupdate/rte" "github.com/openshift-kni/numaresources-operator/pkg/status" @@ -83,15 +86,16 @@ type poolDaemonSet struct { // NUMAResourcesOperatorReconciler reconciles a NUMAResourcesOperator object type NUMAResourcesOperatorReconciler struct { client.Client - Scheme *runtime.Scheme - Platform platform.Platform - APIManifests apimanifests.Manifests - RTEManifests rtemanifests.Manifests - Namespace string - Images images.Data - ImagePullPolicy corev1.PullPolicy - Recorder record.EventRecorder - ForwardMCPConds bool + Scheme *runtime.Scheme + Platform platform.Platform + APIManifests apimanifests.Manifests + RTEManifests rtemanifests.Manifests + RTEMetricsManifests rtemetricsmanifests.Manifests + Namespace string + Images images.Data + ImagePullPolicy corev1.PullPolicy + Recorder record.EventRecorder + ForwardMCPConds bool } // TODO: narrow down @@ -118,6 +122,7 @@ type NUMAResourcesOperatorReconciler struct { //+kubebuilder:rbac:groups=nodetopology.openshift.io,resources=numaresourcesoperators,verbs=* //+kubebuilder:rbac:groups=nodetopology.openshift.io,resources=numaresourcesoperators/status,verbs=get;update;patch //+kubebuilder:rbac:groups=nodetopology.openshift.io,resources=numaresourcesoperators/finalizers,verbs=update +//+kubebuilder:rbac:groups="",resources=services,verbs=* // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -578,6 +583,30 @@ func (r *NUMAResourcesOperatorReconciler) syncNUMAResourcesOperatorResources(ctx return nil, fmt.Errorf("failed to apply (%s) %s/%s: %w", objState.Desired.GetObjectKind().GroupVersionKind(), objState.Desired.GetNamespace(), objState.Desired.GetName(), err) } } + + for _, obj := range r.RTEMetricsManifests.ToObjects() { + // Check if the object already exists + existingObj := obj.DeepCopyObject().(client.Object) + err := r.Client.Get(ctx, client.ObjectKeyFromObject(obj), existingObj) + if err != nil && !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("failed to get %s/%s: %w", obj.GetNamespace(), obj.GetName(), err) + } + if apierrors.IsNotFound(err) { + err := controllerutil.SetControllerReference(instance, obj, r.Scheme) + if err != nil { + return nil, fmt.Errorf("failed to set controller reference to %s %s: %w", obj.GetNamespace(), obj.GetName(), err) + } + err = r.Client.Create(ctx, obj) + if err != nil { + return nil, fmt.Errorf("failed to create %s/%s: %w", obj.GetNamespace(), obj.GetName(), err) + } + } else { + if err := updateIfNeeded(ctx, existingObj, obj, r.Client); err != nil { + return nil, err + } + } + } + if len(dsPoolPairs) < len(trees) { klog.Warningf("daemonset and tree size mismatch: expected %d got in daemonsets %d", len(trees), len(dsPoolPairs)) } @@ -780,3 +809,21 @@ func getTreesByNodeGroup(ctx context.Context, cli client.Client, nodeGroups []nr return nil, fmt.Errorf("unsupported platform") } } + +func updateIfNeeded(ctx context.Context, existingObj, desiredObj client.Object, cli client.Client) error { + merged, err := merge.MetadataForUpdate(existingObj, desiredObj) + if err != nil { + return fmt.Errorf("could not merge object %s with existing: %w", desiredObj.GetName(), err) + } + isEqual, err := compare.Object(existingObj, merged) + if err != nil { + return fmt.Errorf("could not compare object %s with existing: %w", desiredObj.GetName(), err) + } + if !isEqual { + err = cli.Update(ctx, desiredObj) + if err != nil { + return fmt.Errorf("failed to update %s/%s: %w", desiredObj.GetNamespace(), desiredObj.GetName(), err) + } + } + return nil +} diff --git a/controllers/numaresourcesoperator_controller_test.go b/controllers/numaresourcesoperator_controller_test.go index ed26ac8eb..d0045c05e 100644 --- a/controllers/numaresourcesoperator_controller_test.go +++ b/controllers/numaresourcesoperator_controller_test.go @@ -49,6 +49,7 @@ import ( "github.com/openshift-kni/numaresources-operator/internal/api/annotations" testobjs "github.com/openshift-kni/numaresources-operator/internal/objects" "github.com/openshift-kni/numaresources-operator/pkg/images" + rtemetricsmanifests "github.com/openshift-kni/numaresources-operator/pkg/metrics/manifests/monitor" "github.com/openshift-kni/numaresources-operator/pkg/objectnames" "github.com/openshift-kni/numaresources-operator/pkg/objectstate/rte" "github.com/openshift-kni/numaresources-operator/pkg/status" @@ -70,16 +71,20 @@ func NewFakeNUMAResourcesOperatorReconciler(plat platform.Platform, platVersion if err != nil { return nil, err } - + rtemetricsmanifests, err := rtemetricsmanifests.GetManifests(testNamespace) + if err != nil { + return nil, err + } recorder := record.NewFakeRecorder(bufferSize) return &NUMAResourcesOperatorReconciler{ - Client: fakeClient, - Scheme: scheme.Scheme, - Platform: plat, - APIManifests: apiManifests, - RTEManifests: rteManifests, - Namespace: testNamespace, + Client: fakeClient, + Scheme: scheme.Scheme, + Platform: plat, + APIManifests: apiManifests, + RTEManifests: rteManifests, + RTEMetricsManifests: rtemetricsmanifests, + Namespace: testNamespace, Images: images.Data{ Builtin: testImageSpec, }, @@ -1475,6 +1480,17 @@ var _ = Describe("Test NUMAResourcesOperator Reconcile", func() { Namespace: testNamespace, } Expect(reconciler.Client.Get(context.TODO(), mcp2DSKey, ds)).ToNot(HaveOccurred()) + + By("Check All RTE metrics components are created") + for _, obj := range reconciler.RTEMetricsManifests.ToObjects() { + objectKey := client.ObjectKeyFromObject(obj) + switch obj.(type) { + case *corev1.Service: + service := &corev1.Service{} + Expect(reconciler.Client.Get(context.TODO(), objectKey, service)).ToNot(HaveOccurred()) + default: + } + } }) When("daemonsets are ready", func() { var dsDesiredNumberScheduled int32 diff --git a/main.go b/main.go index 7047f1086..46309b990 100644 --- a/main.go +++ b/main.go @@ -59,6 +59,7 @@ import ( intkloglevel "github.com/openshift-kni/numaresources-operator/internal/kloglevel" "github.com/openshift-kni/numaresources-operator/pkg/hash" "github.com/openshift-kni/numaresources-operator/pkg/images" + rtemetricsmanifests "github.com/openshift-kni/numaresources-operator/pkg/metrics/manifests/monitor" "github.com/openshift-kni/numaresources-operator/pkg/numaresourcesscheduler/controlplane" schedmanifests "github.com/openshift-kni/numaresources-operator/pkg/numaresourcesscheduler/manifests/sched" rteupdate "github.com/openshift-kni/numaresources-operator/pkg/objectupdate/rte" @@ -262,18 +263,24 @@ func main() { klog.ErrorS(err, "unable to render RTE manifests", "controller", "NUMAResourcesOperator") os.Exit(1) } + rteMetricsManifests, err := rtemetricsmanifests.GetManifests(namespace) + if err != nil { + klog.ErrorS(err, "unable to load the RTE metrics manifests") + os.Exit(1) + } if err = (&controllers.NUMAResourcesOperatorReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor("numaresources-controller"), - APIManifests: apiManifests, - RTEManifests: rteManifestsRendered, - Platform: clusterPlatform, - Images: imgs, - ImagePullPolicy: pullPolicy, - Namespace: namespace, - ForwardMCPConds: params.enableMCPCondsForward, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("numaresources-controller"), + APIManifests: apiManifests, + RTEManifests: rteManifestsRendered, + RTEMetricsManifests: rteMetricsManifests, + Platform: clusterPlatform, + Images: imgs, + ImagePullPolicy: pullPolicy, + Namespace: namespace, + ForwardMCPConds: params.enableMCPCondsForward, }).SetupWithManager(mgr); err != nil { klog.ErrorS(err, "unable to create controller", "controller", "NUMAResourcesOperator") os.Exit(1) diff --git a/pkg/metrics/manifests/manifests.go b/pkg/metrics/manifests/manifests.go new file mode 100644 index 000000000..dabeae496 --- /dev/null +++ b/pkg/metrics/manifests/manifests.go @@ -0,0 +1,64 @@ +/* + * Copyright 2024 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package manifests + +import ( + "embed" + "fmt" + "path/filepath" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" +) + +//go:embed yaml +var src embed.FS + +func Service(namespace string) (*corev1.Service, error) { + obj, err := loadObject(filepath.Join("yaml", "service.yaml")) + if err != nil { + return nil, err + } + + service, ok := obj.(*corev1.Service) + if !ok { + return nil, fmt.Errorf("unexpected type, got %t", obj) + } + + if namespace != "" { + service.Namespace = namespace + } + return service, nil +} + +func deserializeObjectFromData(data []byte) (runtime.Object, error) { + decode := scheme.Codecs.UniversalDeserializer().Decode + obj, _, err := decode(data, nil, nil) + if err != nil { + return nil, err + } + return obj, nil +} + +func loadObject(path string) (runtime.Object, error) { + data, err := src.ReadFile(path) + if err != nil { + return nil, err + } + return deserializeObjectFromData(data) +} diff --git a/pkg/metrics/manifests/manifests_test.go b/pkg/metrics/manifests/manifests_test.go new file mode 100644 index 000000000..9c5b1d61a --- /dev/null +++ b/pkg/metrics/manifests/manifests_test.go @@ -0,0 +1,27 @@ +/* + * Copyright 2024 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package manifests + +import ( + "testing" +) + +func TestLoad(t *testing.T) { + if obj, err := Service(""); obj == nil || err != nil { + t.Errorf("Service() failed: err=%v", err) + } +} diff --git a/pkg/metrics/manifests/monitor/monitor.go b/pkg/metrics/manifests/monitor/monitor.go new file mode 100644 index 000000000..99e828305 --- /dev/null +++ b/pkg/metrics/manifests/monitor/monitor.go @@ -0,0 +1,52 @@ +/* + * Copyright 2024 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sched + +import ( + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openshift-kni/numaresources-operator/pkg/metrics/manifests" +) + +type Manifests struct { + Service *corev1.Service +} + +func (mf Manifests) ToObjects() []client.Object { + return []client.Object{ + mf.Service, + } +} + +func (mf Manifests) Clone() Manifests { + return Manifests{ + Service: mf.Service.DeepCopy(), + } +} + +func GetManifests(namespace string) (Manifests, error) { + var err error + mf := Manifests{} + + mf.Service, err = manifests.Service(namespace) + if err != nil { + return mf, err + } + + return mf, nil +} diff --git a/pkg/metrics/manifests/monitor/monitor_test.go b/pkg/metrics/manifests/monitor/monitor_test.go new file mode 100644 index 000000000..09d23f623 --- /dev/null +++ b/pkg/metrics/manifests/monitor/monitor_test.go @@ -0,0 +1,53 @@ +/* + * Copyright 2024 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sched + +import ( + "reflect" + "testing" +) + +func TestGetManifests(t *testing.T) { + mf, err := GetManifests("") + if err != nil { + t.Fatalf("GetManifests() failed: err=%v", err) + } + + for _, obj := range mf.ToObjects() { + if obj == nil { + t.Fatalf("GetManifests(): loaded nil manifest") + } + } +} + +func TestCloneManifests(t *testing.T) { + mf, err := GetManifests("") + if err != nil { + t.Fatalf("GetManifests() failed: err=%v", err) + } + + mf2 := mf.Clone() + if !reflect.DeepEqual(mf, mf2) { + t.Fatalf("Clone() returned manifests failing DeepEqual") + } + + for _, obj := range mf2.ToObjects() { + if obj == nil { + t.Fatalf("Clone(): produced nil manifest") + } + } +} diff --git a/pkg/metrics/manifests/yaml/service.yaml b/pkg/metrics/manifests/yaml/service.yaml new file mode 100644 index 000000000..fbf7ee10b --- /dev/null +++ b/pkg/metrics/manifests/yaml/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + service.beta.openshift.io/serving-cert-secret-name: rte-metrics-service-cert + labels: + name: resource-topology + name: numaresources-rte-metrics-service +spec: + ports: + - name: metrics-port + port: 2112 + protocol: TCP + targetPort: metrics-port + selector: + name: resource-topology diff --git a/pkg/objectupdate/rte/rte.go b/pkg/objectupdate/rte/rte.go index e56e24b5b..25c66a578 100644 --- a/pkg/objectupdate/rte/rte.go +++ b/pkg/objectupdate/rte/rte.go @@ -140,6 +140,7 @@ func DaemonSetArgs(ds *appsv1.DaemonSet, conf nropv1.NodeGroupConfig) error { if flags == nil { return fmt.Errorf("cannot modify the arguments for container %s", cnt.Name) } + flags.SetOption("--metrics-mode", "httptls") infoRefreshPauseEnabled := isInfoRefreshPauseEnabled(&conf) klog.V(2).InfoS("DaemonSet update: InfoRefreshPause status", "daemonset", ds.Name, "enabled", infoRefreshPauseEnabled) @@ -208,6 +209,11 @@ func AddVolumeMountMemory(podSpec *corev1.PodSpec, cnt *corev1.Container, mountN Name: mountName, MountPath: dirName, }, + corev1.VolumeMount{ + MountPath: "/etc/secrets/rte/", + Name: "rte-metrics-service-cert", + ReadOnly: true, + }, ) podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ @@ -219,6 +225,14 @@ func AddVolumeMountMemory(podSpec *corev1.PodSpec, cnt *corev1.Container, mountN }, }, }, + corev1.Volume{ + Name: "rte-metrics-service-cert", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "rte-metrics-service-cert", + }, + }, + }, ) } diff --git a/pkg/objectupdate/rte/rte_test.go b/pkg/objectupdate/rte/rte_test.go index 3c4b51374..fa407ce19 100644 --- a/pkg/objectupdate/rte/rte_test.go +++ b/pkg/objectupdate/rte/rte_test.go @@ -84,7 +84,7 @@ func TestUpdateDaemonSetArgs(t *testing.T) { name: "defaults", conf: nropv1.DefaultNodeGroupConfig(), expectedArgs: []string{ - "--pods-fingerprint", "--pods-fingerprint-status-file=/run/pfpstatus/dump.json", "--pods-fingerprint-method=with-exclusive-resources", "--refresh-node-resources", "--add-nrt-owner=false", "--sleep-interval=10s", + "--pods-fingerprint", "--pods-fingerprint-status-file=/run/pfpstatus/dump.json", "--pods-fingerprint-method=with-exclusive-resources", "--refresh-node-resources", "--add-nrt-owner=false", "--sleep-interval=10s", "--metrics-mode=httptls", }, }, { @@ -95,7 +95,7 @@ func TestUpdateDaemonSetArgs(t *testing.T) { }, }, expectedArgs: []string{ - "--pods-fingerprint", "--pods-fingerprint-status-file=/run/pfpstatus/dump.json", "--pods-fingerprint-method=with-exclusive-resources", "--refresh-node-resources", "--add-nrt-owner=false", "--sleep-interval=32s", + "--pods-fingerprint", "--pods-fingerprint-status-file=/run/pfpstatus/dump.json", "--pods-fingerprint-method=with-exclusive-resources", "--refresh-node-resources", "--add-nrt-owner=false", "--sleep-interval=32s", "--metrics-mode=httptls", }, }, { @@ -104,7 +104,7 @@ func TestUpdateDaemonSetArgs(t *testing.T) { PodsFingerprinting: &pfpEnabled, }, expectedArgs: []string{ - "--pods-fingerprint", "--pods-fingerprint-status-file=/run/pfpstatus/dump.json", "--pods-fingerprint-method=all", "--refresh-node-resources", "--add-nrt-owner=false", "--sleep-interval=10s", + "--pods-fingerprint", "--pods-fingerprint-status-file=/run/pfpstatus/dump.json", "--pods-fingerprint-method=all", "--refresh-node-resources", "--add-nrt-owner=false", "--sleep-interval=10s", "--metrics-mode=httptls", }, }, { @@ -113,7 +113,7 @@ func TestUpdateDaemonSetArgs(t *testing.T) { PodsFingerprinting: &pfpDisabled, }, expectedArgs: []string{ - "--refresh-node-resources", "--add-nrt-owner=false", "--sleep-interval=10s", + "--refresh-node-resources", "--add-nrt-owner=false", "--sleep-interval=10s", "--metrics-mode=httptls", }, }, { @@ -122,7 +122,7 @@ func TestUpdateDaemonSetArgs(t *testing.T) { InfoRefreshMode: &refreshEvents, }, expectedArgs: []string{ - "--pods-fingerprint", "--pods-fingerprint-status-file=/run/pfpstatus/dump.json", "--pods-fingerprint-method=with-exclusive-resources", "--refresh-node-resources", "--add-nrt-owner=false", "--notify-file=/run/rte/notify", + "--pods-fingerprint", "--pods-fingerprint-status-file=/run/pfpstatus/dump.json", "--pods-fingerprint-method=with-exclusive-resources", "--refresh-node-resources", "--add-nrt-owner=false", "--notify-file=/run/rte/notify", "--metrics-mode=httptls", }, }, { @@ -131,7 +131,7 @@ func TestUpdateDaemonSetArgs(t *testing.T) { InfoRefreshMode: &refreshPeriodic, }, expectedArgs: []string{ - "--pods-fingerprint", "--pods-fingerprint-status-file=/run/pfpstatus/dump.json", "--pods-fingerprint-method=with-exclusive-resources", "--refresh-node-resources", "--add-nrt-owner=false", "--sleep-interval=10s", + "--pods-fingerprint", "--pods-fingerprint-status-file=/run/pfpstatus/dump.json", "--pods-fingerprint-method=with-exclusive-resources", "--refresh-node-resources", "--add-nrt-owner=false", "--sleep-interval=10s", "--metrics-mode=httptls", }, }, { @@ -140,7 +140,7 @@ func TestUpdateDaemonSetArgs(t *testing.T) { InfoRefreshPause: &infoRefreshPauseEnabled, }, expectedArgs: []string{ - "--pods-fingerprint", "--pods-fingerprint-status-file=/run/pfpstatus/dump.json", "--pods-fingerprint-method=with-exclusive-resources", "--no-publish", "--refresh-node-resources", "--add-nrt-owner=false", "--sleep-interval=10s", + "--pods-fingerprint", "--pods-fingerprint-status-file=/run/pfpstatus/dump.json", "--pods-fingerprint-method=with-exclusive-resources", "--no-publish", "--refresh-node-resources", "--add-nrt-owner=false", "--sleep-interval=10s", "--metrics-mode=httptls", }, }, { @@ -149,7 +149,7 @@ func TestUpdateDaemonSetArgs(t *testing.T) { InfoRefreshPause: &infoRefreshPauseDisabled, }, expectedArgs: []string{ - "--pods-fingerprint", "--pods-fingerprint-status-file=/run/pfpstatus/dump.json", "--pods-fingerprint-method=with-exclusive-resources", "--refresh-node-resources", "--add-nrt-owner=false", "--sleep-interval=10s", + "--pods-fingerprint", "--pods-fingerprint-status-file=/run/pfpstatus/dump.json", "--pods-fingerprint-method=with-exclusive-resources", "--refresh-node-resources", "--add-nrt-owner=false", "--sleep-interval=10s", "--metrics-mode=httptls", }, }, }