From 168ed71abb7ba31d4ee52f7209be6bb436876bea Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 22 Nov 2024 13:13:15 +0100 Subject: [PATCH 01/42] NO-ISSUE: Update oci e2e test --- ...oci_generic_create_ci_machine_playbook.yml | 1 - .../defaults/main.yml | 5 + .../tasks/custom_manifests.yml | 12 - .../setup_oci_for_test_infra/tasks/main.yml | 14 +- .../templates/assisted-additional-config.j2 | 16 +- .../templates/machineconfig-ccm.yml.j2 | 83 --- .../templates/machineconfig-csi.yml.j2 | 37 -- .../templates/oci-ccm.yml.j2 | 285 ---------- .../templates/oci-csi.yml.j2 | 504 ------------------ .../setup_oci_for_test_infra/vars/main.yml | 6 - ansible_files/vars/standalone_oci_sample.yml | 8 +- .../oci-ci-machine/01_networking.tf | 178 ++----- .../{03_compute.tf => 02_compute.tf} | 11 +- .../02_network_security_groups.tf | 133 ----- terraform_files/oci-ci-machine/output.tf | 11 - 15 files changed, 75 insertions(+), 1229 deletions(-) create mode 100644 ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml delete mode 100644 ansible_files/roles/oci/setup_oci_for_test_infra/tasks/custom_manifests.yml delete mode 100644 ansible_files/roles/oci/setup_oci_for_test_infra/templates/machineconfig-ccm.yml.j2 delete mode 100644 ansible_files/roles/oci/setup_oci_for_test_infra/templates/machineconfig-csi.yml.j2 delete mode 100644 ansible_files/roles/oci/setup_oci_for_test_infra/templates/oci-ccm.yml.j2 delete mode 100644 ansible_files/roles/oci/setup_oci_for_test_infra/templates/oci-csi.yml.j2 delete mode 100644 ansible_files/roles/oci/setup_oci_for_test_infra/vars/main.yml rename terraform_files/oci-ci-machine/{03_compute.tf => 02_compute.tf} (90%) delete mode 100644 terraform_files/oci-ci-machine/02_network_security_groups.tf diff --git a/ansible_files/oci_generic_create_ci_machine_playbook.yml b/ansible_files/oci_generic_create_ci_machine_playbook.yml index 694313b51dd..1af13b92765 100644 --- a/ansible_files/oci_generic_create_ci_machine_playbook.yml +++ b/ansible_files/oci_generic_create_ci_machine_playbook.yml @@ -45,4 +45,3 @@ - name: oci/setup_oci_for_test_infra vars: ci_machine: "{{ groups['primary'][0] }}" - infra: "{{ deployed_tf.outputs.infra.value }}" diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml b/ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml new file mode 100644 index 00000000000..939234c503c --- /dev/null +++ b/ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml @@ -0,0 +1,5 @@ +oci_compute_shape: "VM.Standard.E4.Flex" +oci_control_plane_shape: "VM.Standard.E4.Flex" + +oci_infrastructure_zip_url: "https://github.com/oracle-quickstart/oci-openshift/archive/refs/heads/beta-v1.zip" +oci_infrastructure_zip_file: "{{ hostvars[ci_machine].ansible_user_dir }}/infrastructure.zip" diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/custom_manifests.yml b/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/custom_manifests.yml deleted file mode 100644 index d9e37240951..00000000000 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/custom_manifests.yml +++ /dev/null @@ -1,12 +0,0 @@ -- name: Create file custom manifests directories - ansible.builtin.file: - path: "{{ custom_manifests_dir }}/manifests" - state: directory - mode: '0755' - -- name: Render custom manifests on disk - ansible.builtin.template: - src: "{{ item }}.j2" - dest: "{{ custom_manifests_dir }}/manifests/{{ item }}" - mode: '0644' - loop: "{{ oci_manifests }}" diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/main.yml b/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/main.yml index d52c71e45c7..e7ac1f012a8 100644 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/main.yml +++ b/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/main.yml @@ -1,7 +1,6 @@ - name: Directory where OCI private key will be stored ansible.builtin.set_fact: oci_private_key_path_ci_machine: "{{ hostvars[ci_machine].ansible_user_dir }}/.oci/{{ oci_private_key_path | basename }}" - custom_manifests_dir: "{{ hostvars[ci_machine].ansible_user_dir }}/custom_manifests" - name: Create directory where OCI private key will be stored on CI machine ansible.builtin.file: @@ -17,14 +16,17 @@ mode: '0600' delegate_to: "{{ ci_machine }}" +- name: Download terraform template to create the infrastructure on OCI + ansible.builtin.get_url: + url: "{{ oci_infrastructure_zip_url }}" + dest: "{{ oci_infrastructure_zip_file }}" + mode: '0600' + delegate_to: "{{ ci_machine }}" + - name: Export OCI configuration in assisted-additional-config to shared_dir ansible.builtin.template: src: "assisted-additional-config.j2" dest: "{{ shared_dir }}/assisted-additional-config" - mode: 0644 + mode: '0644' when: shared_dir is defined -- name: Create OCI custom manifests - ansible.builtin.import_tasks: - file: custom_manifests.yml - delegate_to: "{{ ci_machine }}" diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 index 222705450be..076b0104b7a 100644 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 +++ b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 @@ -9,20 +9,12 @@ export OCI_PRIVATE_KEY_PATH="{{ oci_private_key_path_ci_machine }}" export OCI_PUBLIC_KEY_FINGERPRINT="{{ oci_fingerprint }}" export OCI_TENANCY="{{ oci_tenancy_id }}" export OCI_REGION="{{ oci_region }}" -export OCI_VCN="{{ infra.oci_vcn_id }}" -export OCI_PUBLIC_SUBNET="{{ infra.oci_public_subnet_id }}" -export OCI_PRIVATE_SUBNET="{{ infra.oci_private_subnet_id }}" -# CI machine to access nodes (connect on them through SSH) -# Nodes to access CI machine (assisted-service API through HTTP/HTTPS) -export OCI_EXTRA_NODE_NSG_IDS="{{ infra.oci_ci_machine_access_nsg_id }},{{ infra.oci_cluster_ci_nsg_id }}" +export OCI_INFRASTRUCTURE_ZIP_FILE="{{ oci_infrastructure_zip_file }}" +export OCI_COMPUTE_SHAPE="{{ oci_compute_shape }}" +export OCI_CONTROL_PLANE_SHAPE="{{ oci_control_plane_shape }}" -# CI machine to access load balancer (cluster access through HTTP/HTTPS/MCS/API) -export OCI_EXTRA_LOAD_BALANCER_NSG_IDS="{{ infra.oci_load_balancer_ci_nsg_id }}" - -export BASE_DOMAIN="{{ oci_dns_zone }}" - -export CUSTOM_MANIFESTS_FILES="{{ custom_manifests_dir }}" +export BASE_DOMAIN="{{ unique_id }}.{{ oci_dns_zone }}" export HOST_INSTALLER_ARGS='{"args": ["--append-karg", "console=ttyS0"]}' export KERNEL_ARGUMENTS='[{"operation": "append", "value": "console=ttyS0"}]' diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/machineconfig-ccm.yml.j2 b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/machineconfig-ccm.yml.j2 deleted file mode 100644 index 6428309494b..00000000000 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/machineconfig-ccm.yml.j2 +++ /dev/null @@ -1,83 +0,0 @@ - -# 99_openshift-machineconfig_00-master-kubelet-providerid.yaml -# Generated by Butane; do not edit -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: master - name: 00-master-oci-kubelet-providerid -spec: - config: - ignition: - version: 3.2.0 - storage: - files: - - contents: - compression: gzip - source: data:;base64,H4sIAAAAAAAC/1yPUYvaQBCA3/dXTFMf2odkrbSFWlNQE2moJEW9exGRdTO5DBd3w+4YvBP/+yF6IPf0zcPMxzefP8kdGblTvhYeGUKE0EJLLVaKGiHyIkmnRT6LJbKW/sUz7ssb5fNhhw1y5NF1pDEq5aAfWk1h62xHJToqI21NJQRVsL64g97p3XgOYPMbuEYjAABQ1xaC3DI4bBulyTwBHsnzZbi/um4fiaEvKhJZvlyN82m6zZK490UfXAPhXwjGB66to1fFZM0QJqgcOiic0g0GEPo51MztUMpvP39Fgx/foxulbbXsBpKMZ2U0Siq/CqEVw58P8aNRWszEenn9fSNS05GzZo+G4+DfwySdp6vt/0XxmCXpIktiq2koZe90F3wOxMXxFgAA///yWfIkhAEAAA== - mode: 493 - path: /usr/local/bin/oci-kubelet-providerid - systemd: - units: - - contents: | - [Unit] - Description=Fetch kubelet provider id from OCI Metadata - - # Wait for NetworkManager to report it's online - After=NetworkManager-wait-online.service - # Run before kubelet - Before=kubelet.service - - [Service] - ExecStart=/usr/local/bin/oci-kubelet-providerid - Type=oneshot - - [Install] - WantedBy=network-online.target - enabled: true - name: oci-kubelet-providerid.service - ---- - -# 99_openshift-machineconfig_00-worker-kubelet-providerid.yaml -# Generated by Butane; do not edit -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: worker - name: 00-worker-oci-kubelet-providerid -spec: - config: - ignition: - version: 3.2.0 - storage: - files: - - contents: - compression: gzip - source: data:;base64,H4sIAAAAAAAC/1yPUYvaQBCA3/dXTFMf2odkrbSFWlNQE2moJEW9exGRdTO5DBd3w+4YvBP/+yF6IPf0zcPMxzefP8kdGblTvhYeGUKE0EJLLVaKGiHyIkmnRT6LJbKW/sUz7ssb5fNhhw1y5NF1pDEq5aAfWk1h62xHJToqI21NJQRVsL64g97p3XgOYPMbuEYjAABQ1xaC3DI4bBulyTwBHsnzZbi/um4fiaEvKhJZvlyN82m6zZK490UfXAPhXwjGB66to1fFZM0QJqgcOiic0g0GEPo51MztUMpvP39Fgx/foxulbbXsBpKMZ2U0Siq/CqEVw58P8aNRWszEenn9fSNS05GzZo+G4+DfwySdp6vt/0XxmCXpIktiq2koZe90F3wOxMXxFgAA///yWfIkhAEAAA== - mode: 493 - path: /usr/local/bin/oci-kubelet-providerid - systemd: - units: - - contents: | - [Unit] - Description=Fetch kubelet provider id from OCI Metadata - - # Wait for NetworkManager to report it's online - After=NetworkManager-wait-online.service - # Run before kubelet - Before=kubelet.service - - [Service] - ExecStart=/usr/local/bin/oci-kubelet-providerid - Type=oneshot - - [Install] - WantedBy=network-online.target - enabled: true - name: oci-kubelet-providerid.service - ---- - diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/machineconfig-csi.yml.j2 b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/machineconfig-csi.yml.j2 deleted file mode 100644 index 75176fe5f1a..00000000000 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/machineconfig-csi.yml.j2 +++ /dev/null @@ -1,37 +0,0 @@ - -# 99_openshift-machineconfig_00-master-iscsi-service.yaml -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: master - name: 99-master-iscsid -spec: - config: - ignition: - version: 3.1.0 - systemd: - units: - - enabled: true - name: iscsid.service - ---- - -# 99_openshift-machineconfig_00-worker-iscsi-service.yaml -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: worker - name: 99-worker-iscsid -spec: - config: - ignition: - version: 3.1.0 - systemd: - units: - - enabled: true - name: iscsid.service - ---- - diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/oci-ccm.yml.j2 b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/oci-ccm.yml.j2 deleted file mode 100644 index e561847eae5..00000000000 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/oci-ccm.yml.j2 +++ /dev/null @@ -1,285 +0,0 @@ - -# oci-ccm-00-namespace.yaml -apiVersion: v1 -kind: Namespace -metadata: - name: oci-cloud-controller-manager - annotations: - workload.openshift.io/allowed: management - labels: - "pod-security.kubernetes.io/enforce": "privileged" - "pod-security.kubernetes.io/audit": "privileged" - "pod-security.kubernetes.io/warn": "privileged" - "security.openshift.io/scc.podSecurityLabelSync": "false" - "openshift.io/run-level": "0" - "pod-security.kubernetes.io/enforce-version": "v1.24" - ---- - -# oci-ccm-01-service-account.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cloud-controller-manager - namespace: oci-cloud-controller-manager - ---- - -# oci-ccm-02-cluster-role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: system:cloud-controller-manager - labels: - kubernetes.io/cluster-service: "true" -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - '*' - -- apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - -- apiGroups: - - "" - resources: - - services - verbs: - - list - - watch - - patch - -- apiGroups: - - "" - resources: - - services/status - verbs: - - patch - - get - - update - -- apiGroups: - - "" - resources: - - configmaps - resourceNames: - - "extension-apiserver-authentication" - verbs: - - get - -- apiGroups: - - "" - resources: - - events - verbs: - - list - - watch - - create - - patch - - update - -# For leader election -- apiGroups: - - "" - resources: - - endpoints - verbs: - - create - -- apiGroups: - - "" - resources: - - endpoints - resourceNames: - - "cloud-controller-manager" - verbs: - - get - - list - - watch - - update - -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create - -- apiGroups: - - "coordination.k8s.io" - resources: - - leases - verbs: - - get - - create - - update - - delete - - patch - - watch - -- apiGroups: - - "" - resources: - - configmaps - resourceNames: - - "cloud-controller-manager" - verbs: - - get - - update - -- apiGroups: - - "" - resources: - - configmaps - resourceNames: - - "extension-apiserver-authentication" - verbs: - - get - - list - - watch - -- apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - -# For the PVL -- apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - list - - watch - - patch - ---- - -# oci-ccm-03-cluster-role-binding.yaml -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: oci-cloud-controller-manager -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:cloud-controller-manager -subjects: -- kind: ServiceAccount - name: cloud-controller-manager - namespace: oci-cloud-controller-manager - ---- - -# oci-ccm-04-cloud-controller-manager-config.yaml -apiVersion: v1 -kind: Secret -metadata: - creationTimestamp: null - name: oci-cloud-controller-manager - namespace: oci-cloud-controller-manager -stringData: - cloud-provider.yaml: | - auth: - region: {{ oci_region }} - useInstancePrincipals: true - compartment: {{ oci_compartment_id }} - vcn: {{ infra.oci_vcn_id }} - loadBalancer: - subnet1: {{ infra.oci_public_subnet_id }} - securityListManagementMode: None - rateLimiter: - rateLimitQPSRead: 20.0 - rateLimitBucketRead: 5 - rateLimitQPSWrite: 20.0 - rateLimitBucketWrite: 5 - ---- - -# oci-ccm-05-daemon-set.yaml -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: oci-cloud-controller-manager - namespace: oci-cloud-controller-manager - labels: - k8s-app: oci-cloud-controller-manager -spec: - selector: - matchLabels: - component: oci-cloud-controller-manager - tier: control-plane - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - component: oci-cloud-controller-manager - tier: control-plane - spec: - serviceAccountName: cloud-controller-manager - hostNetwork: true - nodeSelector: - node-role.kubernetes.io/control-plane: "" - priorityClassName: system-cluster-critical - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - effect: NoSchedule - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - - key: node.kubernetes.io/not-ready - operator: Exists - effect: NoSchedule - volumes: - - name: cfg - secret: - secretName: oci-cloud-controller-manager - - name: kubernetes - hostPath: - path: /etc/kubernetes - containers: - - name: oci-cloud-controller-manager - image: ghcr.io/oracle/cloud-provider-oci:v1.25.0 - command: - - /bin/bash - - -c - - | - #!/bin/bash - set -o allexport - if [[ -f /etc/kubernetes/apiserver-url.env ]]; then - source /etc/kubernetes/apiserver-url.env - fi - exec /usr/local/bin/oci-cloud-controller-manager --cloud-config=/etc/oci/cloud-provider.yaml --cloud-provider=oci --leader-elect-resource-lock=configmapsleases --v=2 - volumeMounts: - - name: cfg - mountPath: /etc/oci - readOnly: true - - name: kubernetes - mountPath: /etc/kubernetes - readOnly: true - ---- - diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/oci-csi.yml.j2 b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/oci-csi.yml.j2 deleted file mode 100644 index fe6a114c25e..00000000000 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/oci-csi.yml.j2 +++ /dev/null @@ -1,504 +0,0 @@ - -# oci-csi-00-namespace.yaml ---- -apiVersion: v1 -kind: Namespace -metadata: - name: oci-csi - annotations: - workload.openshift.io/allowed: management - labels: - "pod-security.kubernetes.io/enforce": "privileged" - "pod-security.kubernetes.io/audit": "privileged" - "pod-security.kubernetes.io/warn": "privileged" - "security.openshift.io/scc.podSecurityLabelSync": "false" - "openshift.io/run-level": "0" - "pod-security.kubernetes.io/enforce-version": "v1.24" - ---- - -# oci-csi-01-config.yaml -apiVersion: v1 -kind: Secret -metadata: - creationTimestamp: null - name: oci-volume-provisioner - namespace: oci-csi -stringData: - config.yaml: | - auth: - region: {{ oci_region }} - useInstancePrincipals: true - compartment: {{ oci_compartment_id }} - vcn: {{ infra.oci_vcn_id }} - loadBalancer: - subnet1: {{ infra.oci_public_subnet_id }} - securityListManagementMode: None - rateLimiter: - rateLimitQPSRead: 20.0 - rateLimitBucketRead: 5 - rateLimitQPSWrite: 20.0 - rateLimitBucketWrite: 5 - ---- - -# oci-csi-02-controller-driver.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - annotations: - deprecated.daemonset.template.generation: "1" - generation: 1 - name: csi-oci-controller - namespace: oci-csi -spec: - revisionHistoryLimit: 10 - selector: - matchLabels: - app: csi-oci-controller - template: - metadata: - creationTimestamp: null - labels: - app: csi-oci-controller - role: csi-oci - spec: - nodeSelector: - node-role.kubernetes.io/control-plane: "" - containers: - - name: csi-volume-provisioner - image: k8s.gcr.io/sig-storage/csi-provisioner:v3.2.1 - args: - - --csi-address=/var/run/shared-tmpfs/csi.sock - - --volume-name-prefix=csi - - --feature-gates=Topology=true - - --timeout=120s - - --leader-election - - --leader-election-namespace=oci-csi - volumeMounts: - - name: config - mountPath: /etc/oci/ - readOnly: true - - mountPath: /var/run/shared-tmpfs - name: shared-tmpfs - - name: csi-fss-volume-provisioner - image: k8s.gcr.io/sig-storage/csi-provisioner:v3.2.1 - args: - - --csi-address=/var/run/shared-tmpfs/csi-fss.sock - - --volume-name-prefix=csi-fss - - --feature-gates=Topology=true - - --timeout=120s - - --leader-election - - --leader-election-namespace=oci-csi - volumeMounts: - - name: config - mountPath: /etc/oci/ - readOnly: true - - mountPath: /var/run/shared-tmpfs - name: shared-tmpfs - - name: csi-attacher - image: k8s.gcr.io/sig-storage/csi-attacher:v4.2.0 - args: - - --csi-address=/var/run/shared-tmpfs/csi.sock - - --timeout=120s - - --leader-election=true - - --leader-election-namespace=oci-csi - volumeMounts: - - name: config - mountPath: /etc/oci/ - readOnly: true - - mountPath: /var/run/shared-tmpfs - name: shared-tmpfs - - name: csi-resizer - image: k8s.gcr.io/sig-storage/csi-resizer:v1.7.0 - args: - - --csi-address=/var/run/shared-tmpfs/csi.sock - - --leader-election - imagePullPolicy: "IfNotPresent" - volumeMounts: - - mountPath: /var/run/shared-tmpfs - name: shared-tmpfs - - name: oci-csi-controller-driver - args: - - --endpoint=unix://var/run/shared-tmpfs/csi.sock - - --fss-csi-endpoint=unix://var/run/shared-tmpfs/csi-fss.sock - command: - - /usr/local/bin/oci-csi-controller-driver - image: ghcr.io/oracle/cloud-provider-oci:v1.25.0 - imagePullPolicy: IfNotPresent - volumeMounts: - - name: config - mountPath: /etc/oci/ - readOnly: true - - name: kubernetes - mountPath: /etc/kubernetes - readOnly: true - - mountPath: /var/run/shared-tmpfs - name: shared-tmpfs - volumes: - - name: config - secret: - secretName: oci-volume-provisioner - - name: kubernetes - hostPath: - path: /etc/kubernetes - - name: shared-tmpfs - emptyDir: {} - dnsPolicy: ClusterFirst - hostNetwork: true - imagePullSecrets: - - name: image-pull-secret - restartPolicy: Always - schedulerName: default-scheduler - serviceAccount: csi-oci-node-sa - serviceAccountName: csi-oci-node-sa - terminationGracePeriodSeconds: 30 - tolerations: - - operator: Exists - ---- - -# oci-csi-03-fss-driver.yaml -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: fss.csi.oraclecloud.com -spec: - attachRequired: false - podInfoOnMount: false - ---- - -# oci-csi-04-bv-driver.yaml -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: blockvolume.csi.oraclecloud.com -spec: - fsGroupPolicy: File - ---- - -# oci-csi-05-iscsiadm.yaml -kind: ConfigMap -apiVersion: v1 -metadata: - name: oci-csi-iscsiadm - namespace: oci-csi -data: - iscsiadm: | - #!/bin/sh - if [ -x /host/sbin/iscsiadm ]; then - chroot /host /sbin/iscsiadm "$@" - elif [ -x /host/usr/local/sbin/iscsiadm ]; then - chroot /host /usr/local/sbin/iscsiadm "$@" - elif [ -x /host/bin/iscsiadm ]; then - chroot /host /bin/iscsiadm "$@" - elif [ -x /host/usr/local/bin/iscsiadm ]; then - chroot /host /usr/local/bin/iscsiadm "$@" - else - chroot /host iscsiadm "$@" - fi - ---- - -# oci-csi-06-fss-csi.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: oci-fss-csi - namespace: oci-csi -data: - mount: |- - #!/bin/sh - if [ -x /sbin/mount ]; then - chroot /host mount "$@" - elif [ -x /usr/local/sbin/mount ]; then - chroot /host mount "$@" - elif [ -x /usr/sbin/mount ]; then - chroot /host mount "$@" - elif [ -x /usr/local/bin/mount ]; then - chroot /host mount "$@" - else - chroot /host mount "$@" - fi - umount: |- - #!/bin/sh - if [ -x /sbin/umount ]; then - chroot /host umount "$@" - elif [ -x /usr/local/sbin/umount ]; then - chroot /host umount "$@" - elif [ -x /usr/sbin/umount ]; then - chroot /host umount "$@" - elif [ -x /usr/local/bin/umount ]; then - chroot /host umount "$@" - else - chroot /host umount "$@" - fi - umount.oci-fss: |- - #!/bin/sh - if [ -x /sbin/umount-oci-fss ]; then - chroot /host umount.oci-fss "$@" - elif [ -x /usr/local/sbin/umount-oci-fss ]; then - chroot /host umount.oci-fss "$@" - elif [ -x /usr/sbin/umount-oci-fss ]; then - chroot /host umount.oci-fss "$@" - elif [ -x /usr/local/bin/umount-oci-fss ]; then - chroot /host umount.oci-fss "$@" - else - chroot /host umount.oci-fss "$@" - fi - ---- - -# oci-csi-07-node-driver.yaml -apiVersion: apps/v1 -kind: DaemonSet -metadata: - annotations: - deprecated.daemonset.template.generation: "1" - generation: 1 - name: csi-oci-node - namespace: oci-csi -spec: - revisionHistoryLimit: 10 - selector: - matchLabels: - app: csi-oci-node - template: - metadata: - creationTimestamp: null - labels: - app: csi-oci-node - role: csi-oci - spec: - containers: - - name: oci-csi-node-driver - args: - - --v=2 - - --endpoint=unix:///csi/csi.sock - - --nodeid=$(KUBE_NODE_NAME) - - --loglevel=debug - - --fss-endpoint=unix:///fss/csi.sock - command: - - /usr/local/bin/oci-csi-node-driver - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: PATH - value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/bin:/host/sbin - image: ghcr.io/oracle/cloud-provider-oci:v1.25.0 - securityContext: - privileged: true - volumeMounts: - - mountPath: /csi - name: plugin-dir - - mountPath: /fss - name: fss-plugin-dir - - mountPath: /var/lib/kubelet - mountPropagation: Bidirectional - name: pods-mount-dir - - mountPath: /dev - name: device-dir - - mountPath: /host - name: host-root - - mountPath: /sbin/iscsiadm - name: chroot-iscsiadm - subPath: iscsiadm - - mountPath: /host/var/lib/kubelet - mountPropagation: Bidirectional - name: encrypt-pods-mount-dir - - mountPath: /sbin/umount.oci-fss - name: fss-driver-mounts - subPath: umount.oci-fss - - mountPath: /sbin/umount - name: fss-driver-mounts - subPath: umount - - mountPath: /sbin/mount - name: fss-driver-mounts - subPath: mount - - name: csi-node-registrar - args: - - --csi-address=/csi/csi.sock - - --kubelet-registration-path=/var/lib/kubelet/plugins/blockvolume.csi.oraclecloud.com/csi.sock - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.1 - securityContext: - privileged: true - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - rm -rf /registration/blockvolume.csi.oraclecloud.com /registration/blockvolume.csi.oraclecloud.com-reg.sock - volumeMounts: - - mountPath: /csi - name: plugin-dir - - mountPath: /registration - name: registration-dir - - name: csi-node-registrar-fss - args: - - --csi-address=/fss/csi.sock - - --kubelet-registration-path=/var/lib/kubelet/plugins/fss.csi.oraclecloud.com/csi.sock - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0 - securityContext: - privileged: true - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - rm -rf /registration/fss.csi.oraclecloud.com /registration/fss.csi.oraclecloud.com-reg.sock - volumeMounts: - - mountPath: /fss - name: fss-plugin-dir - - mountPath: /registration - name: registration-dir - dnsPolicy: ClusterFirst - hostNetwork: true - restartPolicy: Always - schedulerName: default-scheduler - serviceAccount: csi-oci-node-sa - serviceAccountName: csi-oci-node-sa - terminationGracePeriodSeconds: 30 - tolerations: - - operator: Exists - volumes: - - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: DirectoryOrCreate - name: registration-dir - - hostPath: - path: /var/lib/kubelet/plugins/blockvolume.csi.oraclecloud.com - type: DirectoryOrCreate - name: plugin-dir - - hostPath: - path: /var/lib/kubelet/plugins/fss.csi.oraclecloud.com - type: DirectoryOrCreate - name: fss-plugin-dir - - hostPath: - path: /var/lib/kubelet - type: Directory - name: pods-mount-dir - - hostPath: - path: /var/lib/kubelet - type: Directory - name: encrypt-pods-mount-dir - - hostPath: - path: /dev - type: "" - name: device-dir - - hostPath: - path: / - type: Directory - name: host-root - - configMap: - name: oci-csi-iscsiadm - defaultMode: 0755 - name: chroot-iscsiadm - - configMap: - name: oci-fss-csi - defaultMode: 0755 - name: fss-driver-mounts - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate - ---- - -# oci-csi-08-node-rbac-sa.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-oci-node-sa - namespace: oci-csi - ---- - -# oci-csi-09-node-rbac-cr.yaml -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-oci - namespace: oci-csi -rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["volume.oci.oracle.com"] - resources: ["blockscsiinfos"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update", "create"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses", "volumeattachments", "volumeattachments/status", "csinodes"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "create", "update"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["patch"] - ---- - -# oci-csi-10-node-rbac-crb.yaml -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-oci-binding -subjects: - - kind: ServiceAccount - name: csi-oci-node-sa - namespace: oci-csi -roleRef: - kind: ClusterRole - name: csi-oci - apiGroup: rbac.authorization.k8s.io - ---- - -# oci-csi-11-storage-class-bv.yaml -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: oci-bv -provisioner: blockvolume.csi.oraclecloud.com -volumeBindingMode: WaitForFirstConsumer -allowVolumeExpansion: true -reclaimPolicy: Delete - ---- - -# oci-csi-12-storage-class-bv-enc.yaml -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: oci-bv-encrypted -provisioner: blockvolume.csi.oraclecloud.com -parameters: - attachment-type: "paravirtualized" -reclaimPolicy: Delete -volumeBindingMode: WaitForFirstConsumer -allowVolumeExpansion: true - ---- - diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/vars/main.yml b/ansible_files/roles/oci/setup_oci_for_test_infra/vars/main.yml deleted file mode 100644 index 020a0fc874d..00000000000 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/vars/main.yml +++ /dev/null @@ -1,6 +0,0 @@ -# taken from https://github.com/oracle-quickstart/oci-openshift -oci_manifests: - - oci-ccm.yml - - oci-csi.yml - - machineconfig-ccm.yml - - machineconfig-csi.yml diff --git a/ansible_files/vars/standalone_oci_sample.yml b/ansible_files/vars/standalone_oci_sample.yml index 2ee76a755cc..90660a213d9 100644 --- a/ansible_files/vars/standalone_oci_sample.yml +++ b/ansible_files/vars/standalone_oci_sample.yml @@ -9,7 +9,7 @@ oci_user_id: "{{ lookup('env', 'OCI_USER_ID') }}" oci_fingerprint: "{{ lookup('env', 'OCI_FINGERPRINT') }}" oci_region: "{{ lookup('env', 'OCI_REGION') }}" oci_private_key_path: "{{ lookup('env', 'OCI_PRIVATE_KEY_PATH') }}" -oci_compartment_id: "{{ lookup('env', 'oci_compartment_id') }}" -oci_dns_zone: "{{ lookup('env', 'oci_dns_zone') }}" -oci_private_ssh_key_path: "{{ lookup('env', 'USER') }}/.ssh/id_rsa" -oci_public_ssh_key_path: "{{ lookup('env', 'USER') }}/.ssh/id_rsa.pub" +oci_compartment_id: "{{ lookup('env', 'OCI_COMPARTMENT_ID') }}" +oci_dns_zone: "{{ lookup('env', 'OCI_DNS_ZONE') }}" +oci_private_ssh_key_path: "{{ lookup('env', 'HOME') }}/.ssh/id_rsa" +oci_public_ssh_key_path: "{{ lookup('env', 'HOME') }}/.ssh/id_rsa.pub" diff --git a/terraform_files/oci-ci-machine/01_networking.tf b/terraform_files/oci-ci-machine/01_networking.tf index 54b03a15370..e589699c21e 100644 --- a/terraform_files/oci-ci-machine/01_networking.tf +++ b/terraform_files/oci-ci-machine/01_networking.tf @@ -1,99 +1,42 @@ -module "vcn" { - source = "oracle-terraform-modules/vcn/oci" - version = "3.5.4" - # insert the 5 required variables here +locals { + all_protocols = "all" + anywhere = "0.0.0.0/0" +} - # Required Inputs +resource "oci_core_vcn" "ci_machine_vcn" { + cidr_blocks = [ + "10.0.0.0/16", + ] compartment_id = var.oci_compartment_id - - internet_gateway_route_rules = null - local_peering_gateways = null - nat_gateway_route_rules = null - - # Optional Inputs - vcn_name = "vcn-ci-${var.unique_id}" - vcn_dns_label = "v${substr(var.unique_id, -14, -1)}" # dns label is limited to 15 chacracters - vcn_cidrs = ["10.0.0.0/16"] - - create_internet_gateway = true - create_nat_gateway = true + display_name = "vcn-ci-${var.unique_id}" + dns_label = "v${substr(var.unique_id, -14, -1)}" # dns label is limited to 15 chacracters } -resource "oci_core_security_list" "private_security_list" { - - # Required +resource "oci_core_internet_gateway" "internet_gateway" { compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - - # Optional - display_name = "security-list-for-private-subnet" - - egress_security_rules { - stateless = false - destination = "0.0.0.0/0" - destination_type = "CIDR_BLOCK" - protocol = "all" - } + display_name = "InternetGateway" + vcn_id = oci_core_vcn.ci_machine_vcn.id +} - ingress_security_rules { - stateless = false - source = "10.0.0.0/16" - source_type = "CIDR_BLOCK" - # Get protocol numbers from https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml TCP is 6 - protocol = "6" - tcp_options { - min = 22 - max = 22 - } - } - ingress_security_rules { - stateless = false - source = "0.0.0.0/0" - source_type = "CIDR_BLOCK" - # Get protocol numbers from https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml ICMP is 1 - protocol = "1" +resource "oci_core_route_table" "public_routes" { + compartment_id = var.oci_compartment_id + vcn_id = oci_core_vcn.ci_machine_vcn.id + display_name = "public" - # For ICMP type and code see: https://www.iana.org/assignments/icmp-parameters/icmp-parameters.xhtml - icmp_options { - type = 3 - code = 4 - } - } - ingress_security_rules { - stateless = false - source = "10.0.0.0/16" - source_type = "CIDR_BLOCK" - # Get protocol numbers from https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml ICMP is 1 - protocol = "1" - - # For ICMP type and code see: https://www.iana.org/assignments/icmp-parameters/icmp-parameters.xhtml - icmp_options { - type = 3 - } + route_rules { + destination = local.anywhere + destination_type = "CIDR_BLOCK" + network_entity_id = oci_core_internet_gateway.internet_gateway.id } } -resource "oci_core_security_list" "public_security_list" { - - # Required +resource "oci_core_security_list" "public" { compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - - # Optional - display_name = "security-list-for-public-subnet" - - egress_security_rules { - stateless = false - destination = "0.0.0.0/0" - destination_type = "CIDR_BLOCK" - protocol = "all" - } + display_name = "public" + vcn_id = oci_core_vcn.ci_machine_vcn.id ingress_security_rules { - stateless = false - source = "0.0.0.0/0" - source_type = "CIDR_BLOCK" - # Get protocol numbers from https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml TCP is 6 + source = local.anywhere protocol = "6" tcp_options { min = 22 @@ -101,58 +44,39 @@ resource "oci_core_security_list" "public_security_list" { } } ingress_security_rules { - stateless = false - source = "0.0.0.0/0" - source_type = "CIDR_BLOCK" - # Get protocol numbers from https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml ICMP is 1 - protocol = "1" - - # For ICMP type and code see: https://www.iana.org/assignments/icmp-parameters/icmp-parameters.xhtml - icmp_options { - type = 3 - code = 4 + source = local.anywhere + protocol = "6" + tcp_options { + min = 8080 + max = 8080 } } ingress_security_rules { - stateless = false - source = "10.0.0.0/16" - source_type = "CIDR_BLOCK" - # Get protocol numbers from https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml ICMP is 1 - protocol = "1" - - # For ICMP type and code see: https://www.iana.org/assignments/icmp-parameters/icmp-parameters.xhtml - icmp_options { - type = 3 + source = local.anywhere + protocol = "6" + tcp_options { + min = 8090 + max = 8090 } } + egress_security_rules { + destination = local.anywhere + protocol = local.all_protocols + } } -resource "oci_core_subnet" "vcn_private_subnet" { - - # Required +resource "oci_core_subnet" "public" { + cidr_block = "10.0.0.0/24" + display_name = "public" compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - cidr_block = "10.0.1.0/24" - dns_label = "private" + vcn_id = oci_core_vcn.ci_machine_vcn.id + route_table_id = oci_core_route_table.public_routes.id - # Optional - # Caution: For the route table id, use module.vcn.nat_route_id. - # Do not use module.vcn.nat_gateway_id, because it is the OCID for the gateway and not the route table. - route_table_id = module.vcn.nat_route_id - security_list_ids = [oci_core_security_list.private_security_list.id] - display_name = "private-subnet" -} + security_list_ids = [ + oci_core_security_list.public.id, + ] -resource "oci_core_subnet" "vcn_public_subnet" { - - # Required - compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - cidr_block = "10.0.0.0/24" - dns_label = "public" - - # Optional - route_table_id = module.vcn.ig_route_id - security_list_ids = [oci_core_security_list.public_security_list.id] - display_name = "public-subnet" + dns_label = "public" + prohibit_public_ip_on_vnic = false } + diff --git a/terraform_files/oci-ci-machine/03_compute.tf b/terraform_files/oci-ci-machine/02_compute.tf similarity index 90% rename from terraform_files/oci-ci-machine/03_compute.tf rename to terraform_files/oci-ci-machine/02_compute.tf index 2f0cdf59cc7..367dad61ea4 100644 --- a/terraform_files/oci-ci-machine/03_compute.tf +++ b/terraform_files/oci-ci-machine/02_compute.tf @@ -11,8 +11,8 @@ data "oci_marketplace_listings" "os_listings" { compartment_id = var.oci_compartment_id filter { name = "name" - values = ["Rocky Linux 9\\.\\d+ Supported by CIQ - Free \\(x86_64\\)"] - regex = true + values = ["Rocky Linux 9 x86 Supported by CIQ"] + regex = false } } @@ -100,12 +100,7 @@ resource "oci_core_instance" "ci_instance" { assign_public_ip = true assign_private_dns_record = true hostname_label = "ci-instance" - subnet_id = oci_core_subnet.vcn_public_subnet.id - nsg_ids = [ - oci_core_network_security_group.nsg_ci_machine.id, - oci_core_network_security_group.nsg_load_balancer_ci_access.id, # allow access to load balancer - oci_core_network_security_group.nsg_cluster_ci_access.id # allow access to cluster (SSH) - ] + subnet_id = oci_core_subnet.public.id } metadata = { user_data = data.cloudinit_config.config.rendered diff --git a/terraform_files/oci-ci-machine/02_network_security_groups.tf b/terraform_files/oci-ci-machine/02_network_security_groups.tf deleted file mode 100644 index 8117b2b6ae5..00000000000 --- a/terraform_files/oci-ci-machine/02_network_security_groups.tf +++ /dev/null @@ -1,133 +0,0 @@ -# Create security groups for the future cluster -# CI machine should be able to reach: -# - LB on public IP -# - cluster nodes in private subnet (SSH) -# cluster nodes should be able to reach: -# - CI machine (assisted-service/image-service) -# Prow should be able to reach: -# - CI machine on SSH - -# cluster NSG is hold by all clusters nodes -resource "oci_core_network_security_group" "nsg_cluster_ci" { - #Required - compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - - #Optional - display_name = "cluster-ci" -} - -# cluster-access is hold by LB and CI machine -resource "oci_core_network_security_group" "nsg_cluster_ci_access" { - #Required - compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - - #Optional - display_name = "cluster-ci-access" -} - -# all instances holding cluster-access NSG can reach cluster -resource "oci_core_network_security_group_security_rule" "rule_allow_from_nsg_cluster_ci_access" { - network_security_group_id = oci_core_network_security_group.nsg_cluster_ci.id - direction = "INGRESS" - source_type = "NETWORK_SECURITY_GROUP" - source = oci_core_network_security_group.nsg_cluster_ci_access.id - protocol = "all" -} - -# ci-machine is hold by CI machine -resource "oci_core_network_security_group" "nsg_ci_machine" { - #Required - compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - - #Optional - display_name = "ci-machine" -} - -# ci-machine-access is hold bu cluster nodes -resource "oci_core_network_security_group" "nsg_ci_machine_access" { - #Required - compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - - #Optional - display_name = "ci-machine-access" -} - -# all instances holding ci-machine-access NSG can reach ci-machine -resource "oci_core_network_security_group_security_rule" "rule_allow_from_nsg_ci_machine_access" { - network_security_group_id = oci_core_network_security_group.nsg_ci_machine.id - direction = "INGRESS" - source_type = "NETWORK_SECURITY_GROUP" - source = oci_core_network_security_group.nsg_ci_machine_access.id - protocol = "all" -} - -# Allow Prow to connect on CI machine -resource "oci_core_network_security_group_security_rule" "rule_allow_from_prow_to_ci_machine" { - network_security_group_id = oci_core_network_security_group.nsg_ci_machine.id - direction = "INGRESS" - source = "0.0.0.0/0" - protocol = "6" - tcp_options { - destination_port_range { - min = 22 - max = 22 - } - } -} - -# load-balancer is hold by LB -resource "oci_core_network_security_group" "nsg_load_balancer_ci" { - #Required - compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - - #Optional - display_name = "load-balancer-ci" -} - -# load-balancer-access is hold by cluster nodes and CI machine -resource "oci_core_network_security_group" "nsg_load_balancer_ci_access" { - #Required - compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - - #Optional - display_name = "load-balancer-ci-access" -} - -# all instances holding load-balancer-access NSG can reach load-balancer -resource "oci_core_network_security_group_security_rule" "rule_allow_from_nsg_load_balancer_access" { - network_security_group_id = oci_core_network_security_group.nsg_load_balancer_ci.id - direction = "INGRESS" - source_type = "NETWORK_SECURITY_GROUP" - source = oci_core_network_security_group.nsg_load_balancer_ci_access.id - protocol = "all" -} - -locals { - nat_ip = one([for attr in module.vcn.nat_gateway_all_attributes : attr.nat_ip]) -} - -# ci-machine reach load-balancer with its public IP -resource "oci_core_network_security_group_security_rule" "rule_allow_from_public_ci_machine" { - network_security_group_id = oci_core_network_security_group.nsg_load_balancer_ci.id - description = "Allow traffic from ci-machine" - direction = "INGRESS" - source_type = "CIDR_BLOCK" - source = "${oci_core_instance.ci_instance.public_ip}/32" - protocol = "all" -} - -# all private instances behind NAT can reach load-balancer -resource "oci_core_network_security_group_security_rule" "rule_allow_from_public_nat_gateway" { - network_security_group_id = oci_core_network_security_group.nsg_load_balancer_ci.id - description = "Allow traffic from NAT gateway" - direction = "INGRESS" - source_type = "CIDR_BLOCK" - source = "${local.nat_ip}/32" - protocol = "all" -} diff --git a/terraform_files/oci-ci-machine/output.tf b/terraform_files/oci-ci-machine/output.tf index 9c67e092535..9499c02bd94 100644 --- a/terraform_files/oci-ci-machine/output.tf +++ b/terraform_files/oci-ci-machine/output.tf @@ -6,14 +6,3 @@ output "ci_machine_inventory" { "user" : "root", } } - -output "infra" { - value = { - "oci_vcn_id" : module.vcn.vcn_id, - "oci_private_subnet_id" : oci_core_subnet.vcn_private_subnet.id, - "oci_public_subnet_id" : oci_core_subnet.vcn_public_subnet.id, - "oci_ci_machine_access_nsg_id" : oci_core_network_security_group.nsg_ci_machine_access.id, - "oci_cluster_ci_nsg_id" : oci_core_network_security_group.nsg_cluster_ci.id, - "oci_load_balancer_ci_nsg_id" : oci_core_network_security_group.nsg_load_balancer_ci.id - } -} From 878cb9e8b7e3f0d2264cf6c7f925bff37cf67d15 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 22 Nov 2024 14:44:24 +0100 Subject: [PATCH 02/42] wait --- ansible_files/oci_generic_create_ci_machine_playbook.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ansible_files/oci_generic_create_ci_machine_playbook.yml b/ansible_files/oci_generic_create_ci_machine_playbook.yml index 1af13b92765..5e97d0ced33 100644 --- a/ansible_files/oci_generic_create_ci_machine_playbook.yml +++ b/ansible_files/oci_generic_create_ci_machine_playbook.yml @@ -45,3 +45,7 @@ - name: oci/setup_oci_for_test_infra vars: ci_machine: "{{ groups['primary'][0] }}" + tasks: + - name: Wait 1h + ansible.builtin.pause: + minutes: 60 From 6c50890d8eb1d0ad3fb307b9c9f361ffbab721a5 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 22 Nov 2024 15:35:48 +0100 Subject: [PATCH 03/42] remove pause --- ansible_files/oci_generic_create_ci_machine_playbook.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ansible_files/oci_generic_create_ci_machine_playbook.yml b/ansible_files/oci_generic_create_ci_machine_playbook.yml index 5e97d0ced33..1af13b92765 100644 --- a/ansible_files/oci_generic_create_ci_machine_playbook.yml +++ b/ansible_files/oci_generic_create_ci_machine_playbook.yml @@ -45,7 +45,3 @@ - name: oci/setup_oci_for_test_infra vars: ci_machine: "{{ groups['primary'][0] }}" - tasks: - - name: Wait 1h - ansible.builtin.pause: - minutes: 60 From 3bf839218d1890970c6165be7e42de7f43003e1f Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 22 Nov 2024 16:38:26 +0100 Subject: [PATCH 04/42] upgrade all packages and reboot --- .../roles/oci/create_infra/tasks/main.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/ansible_files/roles/oci/create_infra/tasks/main.yml b/ansible_files/roles/oci/create_infra/tasks/main.yml index 726d78d040a..35a7864ab2b 100644 --- a/ansible_files/roles/oci/create_infra/tasks/main.yml +++ b/ansible_files/roles/oci/create_infra/tasks/main.yml @@ -2,7 +2,7 @@ ansible.builtin.template: src: "terraform.tfvars.j2" dest: "{{ oci_tf_vars_file }}" - mode: 0644 + mode: '0644' - name: "Deploy Terraform Instance" community.general.terraform: @@ -17,7 +17,7 @@ ansible.builtin.copy: src: "{{ oci_terraform_workdir }}/terraform.tfstate" dest: "{{ oci_tf_state_file }}" - mode: 0644 + mode: '0644' when: oci_tf_state_file is defined - name: "Add ci_machine to inventory" @@ -36,3 +36,13 @@ delay: 30 register: result until: result is succeeded + +- name: Update all packages on host + ansible.builtin.dnf: + name: "*" + state: latest + delegate_to: "{{ deployed_tf.outputs.ci_machine_inventory.value.display_name }}" + +- name: Reboot machine + ansible.builtin.reboot: + delegate_to: "{{ deployed_tf.outputs.ci_machine_inventory.value.display_name }}" From 56b67b906add22d7675380ef78998064fe8d34dd Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 22 Nov 2024 17:40:45 +0100 Subject: [PATCH 05/42] create shared dir with tes-infra to store OCI artifacts --- .../roles/oci/setup_oci_for_test_infra/defaults/main.yml | 2 +- .../roles/oci/setup_oci_for_test_infra/tasks/main.yml | 7 +++++++ skipper.yaml | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml b/ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml index 939234c503c..dc4583b6522 100644 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml +++ b/ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml @@ -2,4 +2,4 @@ oci_compute_shape: "VM.Standard.E4.Flex" oci_control_plane_shape: "VM.Standard.E4.Flex" oci_infrastructure_zip_url: "https://github.com/oracle-quickstart/oci-openshift/archive/refs/heads/beta-v1.zip" -oci_infrastructure_zip_file: "{{ hostvars[ci_machine].ansible_user_dir }}/infrastructure.zip" +oci_infrastructure_zip_file: "{{ hostvars[ci_machine].ansible_user_dir }}/oci/infrastructure.zip" diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/main.yml b/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/main.yml index e7ac1f012a8..a6918d2142a 100644 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/main.yml +++ b/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/main.yml @@ -16,6 +16,13 @@ mode: '0600' delegate_to: "{{ ci_machine }}" +- name: Create directory where OCI infrastructure archive will be stored on CI machine + ansible.builtin.file: + path: "{{ oci_infrastructure_zip_file | dirname }}" + state: directory + mode: '0700' + delegate_to: "{{ ci_machine }}" + - name: Download terraform template to create the infrastructure on OCI ansible.builtin.get_url: url: "{{ oci_infrastructure_zip_url }}" diff --git a/skipper.yaml b/skipper.yaml index 8cabc9859bb..2af2e8c2743 100644 --- a/skipper.yaml +++ b/skipper.yaml @@ -13,7 +13,7 @@ volumes: - $HOME/.minikube/:$HOME/.minikube/ - $HOME/.ssh/:$HOME/.ssh/ - $HOME/.oci/:$HOME/.oci/ - - $HOME/custom_manifests/:$HOME/custom_manifests/ + - $HOME/oci/:$HOME/oci/ # logs - /var/log:/var/log From 6f1f19f40d841e79d0c96ab9f43b73ac9ff9d66c Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Mon, 25 Nov 2024 10:25:10 +0100 Subject: [PATCH 06/42] update terraform URL --- .../roles/oci/setup_oci_for_test_infra/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml b/ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml index dc4583b6522..7e13f08c629 100644 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml +++ b/ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml @@ -1,5 +1,5 @@ oci_compute_shape: "VM.Standard.E4.Flex" oci_control_plane_shape: "VM.Standard.E4.Flex" -oci_infrastructure_zip_url: "https://github.com/oracle-quickstart/oci-openshift/archive/refs/heads/beta-v1.zip" +oci_infrastructure_zip_url: "https://github.com/dfoster-oracle/oci-openshift/releases/download/v1.0.0-preview/create-cluster-v0.1.0.zip" oci_infrastructure_zip_file: "{{ hostvars[ci_machine].ansible_user_dir }}/oci/infrastructure.zip" From b8f5e29cf859e42f8c67c5de88f956e20a3b6649 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Mon, 25 Nov 2024 15:23:14 +0100 Subject: [PATCH 07/42] set service url --- ansible_files/roles/oci/setup_oci_for_test_infra/tasks/main.yml | 1 + .../templates/assisted-additional-config.j2 | 1 + 2 files changed, 2 insertions(+) diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/main.yml b/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/main.yml index a6918d2142a..9db3a89d927 100644 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/main.yml +++ b/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/main.yml @@ -1,6 +1,7 @@ - name: Directory where OCI private key will be stored ansible.builtin.set_fact: oci_private_key_path_ci_machine: "{{ hostvars[ci_machine].ansible_user_dir }}/.oci/{{ oci_private_key_path | basename }}" + oci_ci_machine_public_ip: "{{ hostvars[ci_machine].ansible_host }}" - name: Create directory where OCI private key will be stored on CI machine ansible.builtin.file: diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 index 076b0104b7a..922f70e327c 100644 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 +++ b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 @@ -15,6 +15,7 @@ export OCI_COMPUTE_SHAPE="{{ oci_compute_shape }}" export OCI_CONTROL_PLANE_SHAPE="{{ oci_control_plane_shape }}" export BASE_DOMAIN="{{ unique_id }}.{{ oci_dns_zone }}" +export SERVICE_URL="{{ oci_ci_machine_public_ip }}" export HOST_INSTALLER_ARGS='{"args": ["--append-karg", "console=ttyS0"]}' export KERNEL_ARGUMENTS='[{"operation": "append", "value": "console=ttyS0"}]' From 415d13ec819dac171af95bc2e604851f7a2d9c43 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Mon, 25 Nov 2024 16:44:43 +0100 Subject: [PATCH 08/42] infra is at the root of zip --- .../controllers/node_controllers/oci_api_controller.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index 164cac63e44..518734cd3ce 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -258,7 +258,6 @@ def _create_stack( template_config = { "config_source_type": "ZIP_UPLOAD", "zip_file_base64_encoded": self._base64_zip_file(terraform_zip_path), - "working_directory": "infrastructure", } template_config_create = oci.resource_manager.models.CreateZipUploadConfigSourceDetails(**template_config) From d25624394456246bbc210c761129101f08b95947 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 29 Nov 2024 16:49:58 +0100 Subject: [PATCH 09/42] test --- .../node_controllers/oci_api_controller.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index 518734cd3ce..a4abd98cef3 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -10,6 +10,7 @@ import libvirt import oci import waiting +import requests from assisted_test_infra.test_infra import BaseClusterConfig from assisted_test_infra.test_infra.controllers.node_controllers.disk import Disk @@ -183,14 +184,21 @@ def _create_pre_authenticated( bucket_name=bucket_name, create_preauthenticated_request_details=pre_authenticated_req, ) - self._cleanup_resources.append( - CleanupResource( - self._object_storage_client.delete_preauthenticated_request, namespace, bucket_name, obj.data.id - ) - ) +# self._cleanup_resources.append( +# CleanupResource( +# self._object_storage_client.delete_preauthenticated_request, namespace, bucket_name, obj.data.id +# ) +# ) assert obj.status == 200 - return obj.data.full_path + par = obj.data.full_path + + r = requests.head(par) + print(r) + assert r.status_code == 200 + print(f"content-length: {int(r.headers["Content-Length"])} file size: {os.path.getsize(file_path)}") + + return par def _terraform_variables( self, From e53d022c9b83a1e0dab32097e4bdd008f0aec66c Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Mon, 2 Dec 2024 16:51:01 +0100 Subject: [PATCH 10/42] use log instead of print --- .../controllers/node_controllers/oci_api_controller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index a4abd98cef3..f5d00ce6adc 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -194,9 +194,9 @@ def _create_pre_authenticated( par = obj.data.full_path r = requests.head(par) - print(r) + log.info("r = requests.head(par) -> {r}") assert r.status_code == 200 - print(f"content-length: {int(r.headers["Content-Length"])} file size: {os.path.getsize(file_path)}") + log.info(f"r = requests.head(par) -> content-length: {int(r.headers["Content-Length"])} file size: {os.path.getsize(file_path)}") return par From 780e2e9d6f217e1d90ad34dd9b91d89816406f6a Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Tue, 3 Dec 2024 13:41:03 +0100 Subject: [PATCH 11/42] increase timeout --- .../controllers/node_controllers/oci_api_controller.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index f5d00ce6adc..174219f2c5c 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -194,9 +194,9 @@ def _create_pre_authenticated( par = obj.data.full_path r = requests.head(par) - log.info("r = requests.head(par) -> {r}") + log.info(f"r = requests.head(par) -> {r}") assert r.status_code == 200 - log.info(f"r = requests.head(par) -> content-length: {int(r.headers["Content-Length"])} file size: {os.path.getsize(file_path)}") + assert int(r.headers["Content-Length"]) == os.path.getsize(file_path) return par @@ -305,7 +305,7 @@ def _create_stack( return obj.data.id def _apply_job_from_stack( - self, stack_id: str, display_name: str, timeout_seconds: int = 3200, interval_wait: int = 60 + self, stack_id: str, display_name: str, timeout_seconds: int = 3600, interval_wait: int = 60 ) -> str: """Apply job will run the stack terraform code and create the resources. From a32b733fbea84584508d03e31d62004673da285f Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Tue, 3 Dec 2024 14:59:08 +0100 Subject: [PATCH 12/42] add OCI custom manifests to cluster --- .../node_controllers/oci_api_controller.py | 26 +++++++------------ .../test_infra/helper_classes/cluster.py | 3 +-- .../test_infra/utils/manifests.py | 15 ++++++----- 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index 174219f2c5c..82c2f539f8b 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -18,6 +18,7 @@ from assisted_test_infra.test_infra.controllers.node_controllers.node_controller import NodeController from assisted_test_infra.test_infra.helper_classes.config import BaseNodesConfig from assisted_test_infra.test_infra.helper_classes.config.base_oci_config import BaseOciConfig +from assisted_test_infra.test_infra.utils.manifests import Manifest from service_client import log @@ -85,18 +86,11 @@ class OciApiController(NodeController): def __init__(self, config: BaseNodesConfig, cluster_config: BaseClusterConfig): super().__init__(config, cluster_config) - self._cloud_provider = None + self.cloud_provider = None + self.custom_manifests = None self._oci_compartment_oicd = self._config.oci_compartment_oicd self._initialize_oci_clients() - @property - def cloud_provider(self): - # Called from test_cases , modify manifests - return self._cloud_provider - - @cloud_provider.setter - def cloud_provider(self, cloud_provider): - self._cloud_provider = cloud_provider def _initialize_oci_clients(self): """Initialize oci clients. @@ -364,13 +358,9 @@ def _apply_job_from_stack( items = self._resource_manager_client.list_job_outputs(job.data.id).data.items for item in items: if item.output_name == "dynamic_custom_manifest": - file_path = ( - f"/tmp/oci-{self._entity_config.cluster_id}/terraform-output-{self._entity_config.cluster_id}.yml" - ) - with open(file_path, "w") as f: - f.write(item.output_value) - return item.output_value - raise RuntimeError(f"Missing dynamic_custom_manifest for stack {stack_id}") + manifest = Manifest(folder="manifests", file_name="oci_custom_manifests.yaml", content=item.output_value) + self._entity_config.custom_manifests.append(manifest) + raise RuntimeError(f"Missing oci_ccm_config for stack {stack_id}") @staticmethod def _waiter_status(client_callback: Callable, name: str, status: str, **callback_kwargs) -> None: @@ -547,8 +537,12 @@ def prepare_nodes(self) -> None: self._config.oci_infrastructure_zip_file, terraform_variables, ) +<<<<<<< HEAD terraform_output = self._apply_job_from_stack(stack_id, f"apply-job-{self._entity_config.cluster_id}") self.cloud_provider = terraform_output +======= + self._apply_job_from_stack(stack_id, random_name("job-")) +>>>>>>> bc450ea0 (add OCI custom manifests to cluster) def is_active(self, node_name) -> bool: pass diff --git a/src/assisted_test_infra/test_infra/helper_classes/cluster.py b/src/assisted_test_infra/test_infra/helper_classes/cluster.py index 8edb3f148c6..14bb1a0e542 100644 --- a/src/assisted_test_infra/test_infra/helper_classes/cluster.py +++ b/src/assisted_test_infra/test_infra/helper_classes/cluster.py @@ -958,8 +958,7 @@ def prepare_nodes(self, is_static_ip: bool = False, **kwargs): def create_custom_manifests(self): log.info(f"Adding {len(self._config.custom_manifests)} custom manifests") for local_manifest in self._config.custom_manifests: - with open(local_manifest.local_path, "rb") as f: - encoded_content = base64.b64encode(f.read()).decode("utf-8", "ignore") + encoded_content = base64.b64encode(local_manifest.content).decode("utf-8", "ignore") manifest = self.create_custom_manifest(local_manifest.folder, local_manifest.file_name, encoded_content) diff --git a/src/assisted_test_infra/test_infra/utils/manifests.py b/src/assisted_test_infra/test_infra/utils/manifests.py index 2b9ae8b22b6..b6ac5ef89ac 100644 --- a/src/assisted_test_infra/test_infra/utils/manifests.py +++ b/src/assisted_test_infra/test_infra/utils/manifests.py @@ -10,7 +10,7 @@ class Manifest: folder: str file_name: str - local_path: Path + content: str def is_folder_allowed(self) -> bool: return self.folder in self.__ALLOWED_FOLDERS @@ -20,16 +20,17 @@ def get_allowed_folders(self) -> List[str]: @classmethod def get_manifests(cls, path: Path) -> List["Manifest"]: - manifests_files = [] - + manifest_files = [] if path.is_dir(): for file_type in ("yaml", "yml", "json"): - manifests_files.extend(list(path.rglob(f"*.{file_type}"))) + manifest_files.extend(list(path.rglob(f"*.{file_type}"))) else: - manifests_files.append(path) + manifest_files.append(path) manifests = [] - for file in manifests_files: - manifests.append(Manifest(folder=file.parent.name, file_name=file.name, local_path=file)) + for manifest in manifest_files: + with open(manifest, "rb") as f: + content = f.read() + manifests.append(Manifest(folder=manifest.parent.name, file_name=manifest.name, content=content)) return manifests From 82f2aac4190549254addec0c5de4a692bccd3d87 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 6 Dec 2024 11:49:20 +0100 Subject: [PATCH 13/42] add cleanup back --- .../node_controllers/oci_api_controller.py | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index 82c2f539f8b..4b9814bdda9 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -9,8 +9,8 @@ import libvirt import oci -import waiting import requests +import waiting from assisted_test_infra.test_infra import BaseClusterConfig from assisted_test_infra.test_infra.controllers.node_controllers.disk import Disk @@ -91,7 +91,6 @@ def __init__(self, config: BaseNodesConfig, cluster_config: BaseClusterConfig): self._oci_compartment_oicd = self._config.oci_compartment_oicd self._initialize_oci_clients() - def _initialize_oci_clients(self): """Initialize oci clients. @@ -178,11 +177,11 @@ def _create_pre_authenticated( bucket_name=bucket_name, create_preauthenticated_request_details=pre_authenticated_req, ) -# self._cleanup_resources.append( -# CleanupResource( -# self._object_storage_client.delete_preauthenticated_request, namespace, bucket_name, obj.data.id -# ) -# ) + self._cleanup_resources.append( + CleanupResource( + self._object_storage_client.delete_preauthenticated_request, namespace, bucket_name, obj.data.id + ) + ) assert obj.status == 200 par = obj.data.full_path @@ -358,8 +357,9 @@ def _apply_job_from_stack( items = self._resource_manager_client.list_job_outputs(job.data.id).data.items for item in items: if item.output_name == "dynamic_custom_manifest": - manifest = Manifest(folder="manifests", file_name="oci_custom_manifests.yaml", content=item.output_value) - self._entity_config.custom_manifests.append(manifest) + self._entity_config.custom_manifests.append( + Manifest(folder="manifests", file_name="oci_custom_manifests.yaml", content=item.output_value) + ) raise RuntimeError(f"Missing oci_ccm_config for stack {stack_id}") @staticmethod @@ -537,12 +537,8 @@ def prepare_nodes(self) -> None: self._config.oci_infrastructure_zip_file, terraform_variables, ) -<<<<<<< HEAD terraform_output = self._apply_job_from_stack(stack_id, f"apply-job-{self._entity_config.cluster_id}") self.cloud_provider = terraform_output -======= - self._apply_job_from_stack(stack_id, random_name("job-")) ->>>>>>> bc450ea0 (add OCI custom manifests to cluster) def is_active(self, node_name) -> bool: pass From b29bdf05d8928e6ff20960123f97a5c355fc0aea Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 6 Dec 2024 12:05:23 +0100 Subject: [PATCH 14/42] exclude preauth request from cleanup --- ansible_files/roles/oci/cleanup_resources/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible_files/roles/oci/cleanup_resources/defaults/main.yml b/ansible_files/roles/oci/cleanup_resources/defaults/main.yml index 5743d423c3a..0f0148a0e95 100644 --- a/ansible_files/roles/oci/cleanup_resources/defaults/main.yml +++ b/ansible_files/roles/oci/cleanup_resources/defaults/main.yml @@ -10,4 +10,5 @@ excluded_types: - oci_network_load_balancer_backend - oci_network_load_balancer_backend_set - oci_network_load_balancer_listener + - oci_objectstorage_preauthrequest expired_after_hours: 7 From 840a8e5d20c17f5f2075a874837d98807d25d9c1 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 6 Dec 2024 14:37:56 +0100 Subject: [PATCH 15/42] specify right service and image base url --- scripts/deploy_assisted_service.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deploy_assisted_service.sh b/scripts/deploy_assisted_service.sh index 23de5b82407..c5b5f5d40b2 100755 --- a/scripts/deploy_assisted_service.sh +++ b/scripts/deploy_assisted_service.sh @@ -175,7 +175,7 @@ else skipper run src/update_assisted_service_cm.py - (cd assisted-service/ && skipper --env-file ../skipper.env run "make deploy-all" ${SKIPPER_PARAMS} $ENABLE_KUBE_API_CMD TARGET=$DEPLOY_TARGET DEPLOY_TAG=${DEPLOY_TAG} DEPLOY_MANIFEST_PATH=${DEPLOY_MANIFEST_PATH} DEPLOY_MANIFEST_TAG=${DEPLOY_MANIFEST_TAG} NAMESPACE=${NAMESPACE} AUTH_TYPE=${AUTH_TYPE} ${DEBUG_DEPLOY_AI_PARAMS:-} IP=$(get_main_ip)) + (cd assisted-service/ && skipper --env-file ../skipper.env run "make deploy-all" ${SKIPPER_PARAMS} $ENABLE_KUBE_API_CMD TARGET=$DEPLOY_TARGET DEPLOY_TAG=${DEPLOY_TAG} DEPLOY_MANIFEST_PATH=${DEPLOY_MANIFEST_PATH} DEPLOY_MANIFEST_TAG=${DEPLOY_MANIFEST_TAG} NAMESPACE=${NAMESPACE} AUTH_TYPE=${AUTH_TYPE} ${DEBUG_DEPLOY_AI_PARAMS:-} IP=${SERVICE_URL}) add_firewalld_port $SERVICE_PORT From 06b10f8b6d0bf811c89a6317ade52631cbfb2bd8 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 6 Dec 2024 18:13:53 +0100 Subject: [PATCH 16/42] update success condition after stack is applied --- .../node_controllers/oci_api_controller.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index 4b9814bdda9..58459cb891c 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -354,13 +354,23 @@ def _apply_job_from_stack( log.info(f"Exception raised during apply_job_from_stack {e}: destroying") raise # on success, we return the jobs output - list + success = False items = self._resource_manager_client.list_job_outputs(job.data.id).data.items for item in items: +<<<<<<< HEAD if item.output_name == "dynamic_custom_manifest": +======= + if item.output_name == "oci_ccm_config": + self.cloud_provider = item.output_value + success = True + elif item.output_name == "dynamic_custom_manifest": +>>>>>>> a6df15cc (update success condition after stack is applied) self._entity_config.custom_manifests.append( Manifest(folder="manifests", file_name="oci_custom_manifests.yaml", content=item.output_value) ) - raise RuntimeError(f"Missing oci_ccm_config for stack {stack_id}") + success = True + if not success: + raise RuntimeError(f"Missing oci_ccm_config for stack {stack_id}") @staticmethod def _waiter_status(client_callback: Callable, name: str, status: str, **callback_kwargs) -> None: From bddf8ea03dd21b920913873d4241438e830a6162 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Mon, 9 Dec 2024 16:01:06 +0100 Subject: [PATCH 17/42] oci returns no nodes --- .../controllers/node_controllers/oci_api_controller.py | 4 ++-- src/assisted_test_infra/test_infra/helper_classes/nodes.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index 58459cb891c..17b941296d8 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -299,7 +299,7 @@ def _create_stack( def _apply_job_from_stack( self, stack_id: str, display_name: str, timeout_seconds: int = 3600, interval_wait: int = 60 - ) -> str: + ) -> None: """Apply job will run the stack terraform code and create the resources. On failure - raise Exception and cleanup resources @@ -397,7 +397,7 @@ def terraform_vm_resource_type(self) -> str: return "oci_core_instance" def list_nodes(self) -> List[Node]: - pass + return [] def list_disks(self, node_name: str) -> List[Disk]: pass diff --git a/src/assisted_test_infra/test_infra/helper_classes/nodes.py b/src/assisted_test_infra/test_infra/helper_classes/nodes.py index 22bf53a59e1..5a0ff34c01f 100644 --- a/src/assisted_test_infra/test_infra/helper_classes/nodes.py +++ b/src/assisted_test_infra/test_infra/helper_classes/nodes.py @@ -9,6 +9,7 @@ import consts from assisted_test_infra.test_infra.controllers.node_controllers import Node +from assisted_test_infra.test_infra.controllers.node_controllers.nutanix_controller import NutanixController from assisted_test_infra.test_infra.controllers.node_controllers.node_controller import NodeController from assisted_test_infra.test_infra.tools import run_concurrently from service_client.logger import SuppressAndLog, log @@ -153,7 +154,7 @@ def run_for_all_nodes(self, func_name, *args): def run_for_given_nodes(self, nodes, func_name, *args): log.info("Running <%s> on nodes: %s", func_name, [node.name for node in nodes]) - if self.controller._config.tf_platform == consts.Platforms.NUTANIX: + if isinstance(self.controller, NutanixController): # nutanix doesn't allow concurrent requests res = [] for node in nodes: From 60b2cac3a4c5c377711b77c4eab8bb763f654498 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Mon, 9 Dec 2024 16:03:52 +0100 Subject: [PATCH 18/42] don't change that --- src/assisted_test_infra/test_infra/helper_classes/nodes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/assisted_test_infra/test_infra/helper_classes/nodes.py b/src/assisted_test_infra/test_infra/helper_classes/nodes.py index 5a0ff34c01f..22bf53a59e1 100644 --- a/src/assisted_test_infra/test_infra/helper_classes/nodes.py +++ b/src/assisted_test_infra/test_infra/helper_classes/nodes.py @@ -9,7 +9,6 @@ import consts from assisted_test_infra.test_infra.controllers.node_controllers import Node -from assisted_test_infra.test_infra.controllers.node_controllers.nutanix_controller import NutanixController from assisted_test_infra.test_infra.controllers.node_controllers.node_controller import NodeController from assisted_test_infra.test_infra.tools import run_concurrently from service_client.logger import SuppressAndLog, log @@ -154,7 +153,7 @@ def run_for_all_nodes(self, func_name, *args): def run_for_given_nodes(self, nodes, func_name, *args): log.info("Running <%s> on nodes: %s", func_name, [node.name for node in nodes]) - if isinstance(self.controller, NutanixController): + if self.controller._config.tf_platform == consts.Platforms.NUTANIX: # nutanix doesn't allow concurrent requests res = [] for node in nodes: From c839bc44015697aa6d134aed889d7dea1a1dd280 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Tue, 10 Dec 2024 14:08:00 +0100 Subject: [PATCH 19/42] list and extract nics from OCI --- .../node_controllers/oci_api_controller.py | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index 17b941296d8..ec9a78ace2c 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -103,6 +103,7 @@ def _initialize_oci_clients(self): self._object_storage_client = oci.object_storage.ObjectStorageClient(self._config.get_provider_config()) self._compute_client = oci.core.ComputeClient(self._config.get_provider_config()) self._volume_client = oci.core.BlockstorageClient(self._config.get_provider_config()) + self._virtual_network_client = oci.core.VirtualNetworkClient(self._config.get_provider_config()) # resource manager for stack creation and job self._resource_manager_client = oci.resource_manager.ResourceManagerClient( self._config.get_provider_config() @@ -388,6 +389,13 @@ def is_status(): waiting_for="Resource to be created", ) + @property + def _instances(self) -> List[oci.core.models.Instance]: + response = oci.pagination.list_call_get_all_results( + self._compute_client.list_instances, self._oci_compartment_oicd + ) + return [instance for instance in response.data if self._entity_config.entity_name in instance.display_name] + @property def terraform_vm_name_key(self) -> str: return "display_name" @@ -397,7 +405,7 @@ def terraform_vm_resource_type(self) -> str: return "oci_core_instance" def list_nodes(self) -> List[Node]: - return [] + return [Node(instance.display_name, self) for instance in self._instances] def list_disks(self, node_name: str) -> List[Disk]: pass @@ -565,8 +573,37 @@ def set_per_device_boot_order(self, node_name, key: Callable[[Disk], int]) -> No """ pass + def _get_vnic_attachments(self, instance: oci.core.models.Instance) -> List[oci.core.models.VnicAttachment]: + response = oci.pagination.list_call_get_all_results( + self._compute_client.list_vnic_attachments, self._oci_compartment_oicd, instance_id=instance.id + ) + return response.data + + def _get_vnics(self, instance: oci.core.models.Instance) -> List[oci.core.models.Vnic]: + vnic_attachments = self._get_vnic_attachments(instance.id) + reponses = [ + self._virtual_network_client.get_vnic(vnic_id=vnic_attachment.vnic_id) + for vnic_attachment in vnic_attachments + ] + return [response.data for response in responses] + def get_node_ips_and_macs(self, node_name) -> Tuple[List[str], List[str]]: - pass + instance = next(instance for instance in self._instances if node_name == instance.display_name) + vnic_attachments = self._get_vnic_attachments(instance.id) + vnics = [self.get_vnic(vnic_attachment.vnic_id) for vnic_attachment in vnic_attachments] + ips = [] + macs = [] + for vnic in vnics: + if vnic.private_ip: + ips.append(vnic.private_ip) + if vnic.public_ip: + ips.append(vnic.public_ip) + if vnic.ipv6_addresses: + ips.extend(vnic.ipv6_addresses) + + macs.append(vnic.mac_address) + + return (ips, macs) def set_single_node_ip(self, ip) -> None: pass From b47bfb14b317bb92c7ae4a19124b2166ad571670 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Tue, 10 Dec 2024 16:23:29 +0100 Subject: [PATCH 20/42] fix --- .../controllers/node_controllers/oci_api_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index ec9a78ace2c..c16b5bfc061 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -394,7 +394,7 @@ def _instances(self) -> List[oci.core.models.Instance]: response = oci.pagination.list_call_get_all_results( self._compute_client.list_instances, self._oci_compartment_oicd ) - return [instance for instance in response.data if self._entity_config.entity_name in instance.display_name] + return [instance for instance in response.data if str(self._entity_config.entity_name) in instance.display_name] @property def terraform_vm_name_key(self) -> str: From 8d2420b646a50943063df6b8c917bdd6810d2204 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Wed, 11 Dec 2024 13:33:34 +0100 Subject: [PATCH 21/42] fix --- .../controllers/node_controllers/oci_api_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index c16b5bfc061..91c6637ee6a 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -580,7 +580,7 @@ def _get_vnic_attachments(self, instance: oci.core.models.Instance) -> List[oci. return response.data def _get_vnics(self, instance: oci.core.models.Instance) -> List[oci.core.models.Vnic]: - vnic_attachments = self._get_vnic_attachments(instance.id) + vnic_attachments = self._get_vnic_attachments(instance) reponses = [ self._virtual_network_client.get_vnic(vnic_id=vnic_attachment.vnic_id) for vnic_attachment in vnic_attachments From cc3f642be8e5c620d1a6648315fc1a05a7961905 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Wed, 11 Dec 2024 15:09:52 +0100 Subject: [PATCH 22/42] fix --- .../controllers/node_controllers/oci_api_controller.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index 91c6637ee6a..08488d5dfef 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -589,8 +589,7 @@ def _get_vnics(self, instance: oci.core.models.Instance) -> List[oci.core.models def get_node_ips_and_macs(self, node_name) -> Tuple[List[str], List[str]]: instance = next(instance for instance in self._instances if node_name == instance.display_name) - vnic_attachments = self._get_vnic_attachments(instance.id) - vnics = [self.get_vnic(vnic_attachment.vnic_id) for vnic_attachment in vnic_attachments] + vnics = self.get_vnics(instance) ips = [] macs = [] for vnic in vnics: From ede220ec59adedfdd5680958e51a44f6750b9b9e Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Wed, 11 Dec 2024 17:27:18 +0100 Subject: [PATCH 23/42] fix --- .../controllers/node_controllers/oci_api_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index 08488d5dfef..08c63ce6ba0 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -589,7 +589,7 @@ def _get_vnics(self, instance: oci.core.models.Instance) -> List[oci.core.models def get_node_ips_and_macs(self, node_name) -> Tuple[List[str], List[str]]: instance = next(instance for instance in self._instances if node_name == instance.display_name) - vnics = self.get_vnics(instance) + vnics = self._get_vnics(instance) ips = [] macs = [] for vnic in vnics: From 144830f2c44afacb734f4a3724c75e94117c9d4b Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Wed, 11 Dec 2024 17:29:18 +0100 Subject: [PATCH 24/42] don't delete resources --- ansible_files/oci_generic_destroy_ci_machine_playbook.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible_files/oci_generic_destroy_ci_machine_playbook.yml b/ansible_files/oci_generic_destroy_ci_machine_playbook.yml index 8a95b11d3ba..8393f7a65a0 100644 --- a/ansible_files/oci_generic_destroy_ci_machine_playbook.yml +++ b/ansible_files/oci_generic_destroy_ci_machine_playbook.yml @@ -1,3 +1,4 @@ +- meta: end_play - name: Destroy OCI instrastructure provisionned by test-infra hosts: primary tasks: From b2fb661fa879a7c97e60c21ddcf4c026623dd8ae Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Thu, 12 Dec 2024 17:16:10 +0100 Subject: [PATCH 25/42] add role property to node object --- .../controllers/node_controllers/node.py | 31 ++++++++++++++++--- .../node_controllers/oci_api_controller.py | 22 +++++++++++-- .../test_infra/helper_classes/base_cluster.py | 20 +++--------- 3 files changed, 51 insertions(+), 22 deletions(-) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/node.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/node.py index 145b1371d7f..a8dac067a93 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/node.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/node.py @@ -11,7 +11,14 @@ class Node: - def __init__(self, name, node_controller, private_ssh_key_path: Optional[Path] = None, username="core"): + def __init__( + self, + name, + node_controller, + private_ssh_key_path: Optional[Path] = None, + username="core", + role: Optional[str] = None, + ): self.name = name self.private_ssh_key_path = private_ssh_key_path self.username = username @@ -20,6 +27,7 @@ def __init__(self, name, node_controller, private_ssh_key_path: Optional[Path] = self.original_ram_kib = self.get_ram_kib() self._ips = [] self._macs = [] + self._role = role def __str__(self): return self.name @@ -28,11 +36,24 @@ def __str__(self): def is_active(self): return self.node_controller.is_active(self.name) - def is_master_in_name(self): - return consts.NodeRoles.MASTER in self.name + def is_master_in_name(self) -> bool: + return self.role == consts.NodeRoles.MASTER - def is_worker_in_name(self): - return consts.NodeRoles.WORKER in self.name + def is_worker_in_name(self) -> bool: + return self.role == consts.NodeRoles.WORKER + + @property + def role(self) -> str: + if self._role: + return self._role + + if consts.NodeRoles.MASTER in self.name: + return consts.NodeRoles.MASTER + + if consts.NodeRoles.WORKER in self.name: + return consts.NodeRoles.WORKER + + return consts.NodeRoles.AUTO_ASSIGN def _set_ips_and_macs(self): self._ips, self._macs = self.node_controller.get_node_ips_and_macs(self.name) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index 08c63ce6ba0..1940a27db2c 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -20,6 +20,7 @@ from assisted_test_infra.test_infra.helper_classes.config.base_oci_config import BaseOciConfig from assisted_test_infra.test_infra.utils.manifests import Manifest from service_client import log +import consts def random_name(prefix="", length=8): @@ -404,8 +405,25 @@ def terraform_vm_name_key(self) -> str: def terraform_vm_resource_type(self) -> str: return "oci_core_instance" + @staticmethod + def _get_instance_role(instance: oci.core.models.Instance) -> str: + namespace_key = "openshift-{}".format(self._entity_config.entity_name) + namespace = instance.defined_tags.get(tag_namespace_key) + assert namespace, "expected namespace {} to exist in defined tags {}".format( + namespace_key, instance.defined_tags + ) + role = namespace.get("instance-role") + assert role, "expected key instance-role to exist in namespace {} in defined tags {}".format( + namespace_key, instance.defined_tags + ) + + if role == "control_plane": + return consts.NodeRoles.MASTER + + return consts.NodeRoles.WORKER + def list_nodes(self) -> List[Node]: - return [Node(instance.display_name, self) for instance in self._instances] + return [Node(instance.display_name, self, role=_get_instance_role(instance)) for instance in self._instances] def list_disks(self, node_name: str) -> List[Disk]: pass @@ -581,7 +599,7 @@ def _get_vnic_attachments(self, instance: oci.core.models.Instance) -> List[oci. def _get_vnics(self, instance: oci.core.models.Instance) -> List[oci.core.models.Vnic]: vnic_attachments = self._get_vnic_attachments(instance) - reponses = [ + responses = [ self._virtual_network_client.get_vnic(vnic_id=vnic_attachment.vnic_id) for vnic_attachment in vnic_attachments ] diff --git a/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py b/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py index 278cbe2a131..954a5b57dc0 100644 --- a/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py +++ b/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py @@ -153,13 +153,12 @@ def set_hostnames_and_roles(self): nodes = self.nodes.get_nodes(refresh=True) for host in hosts: - name = self.find_matching_node_name(host, nodes) - assert name is not None, ( + node = self.find_matching_node_name(host, nodes) + assert node is not None, ( f"Failed to find matching node for host with mac address {host.macs()}" f" nodes: {[(n.name, n.ips, n.macs) for n in nodes]}" ) - role = consts.NodeRoles.MASTER if consts.NodeRoles.MASTER in name else consts.NodeRoles.WORKER - self._infra_env.update_host(host_id=host.get_id(), host_role=role, host_name=name) + self._infra_env.update_host(host_id=host.get_id(), host_role=node.role, host_name=name) def set_installer_args(self): hosts = self.to_cluster_hosts(self.api_client.get_cluster_hosts(self.id)) @@ -170,21 +169,12 @@ def set_installer_args(self): def to_cluster_hosts(hosts: list[dict[str, Any]]) -> list[ClusterHost]: return [ClusterHost(models.Host(**h)) for h in hosts] - def find_matching_node_name(self, host: ClusterHost, nodes: list[Node]) -> Union[str, None]: + def find_matching_node(self, host: ClusterHost, nodes: list[Node]) -> Optional[Node]: # Looking for node matches the given host by its mac address (which is unique) for node in nodes: for mac in node.macs: if mac.lower() in host.macs(): - return node.name - - # IPv6 static ips - if self._infra_env_config.is_static_ip: - mappings = static_network.get_name_to_mac_addresses_mapping(self.nodes.controller.tf_folder) - for mac in host.macs(): - for name, macs in mappings.items(): - if mac in macs: - return name - + return node return None @JunitTestCase() From ac41fa3de12a5d6a075a2a87534885ac8c4b39cb Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Thu, 12 Dec 2024 17:19:40 +0100 Subject: [PATCH 26/42] fix --- .../controllers/node_controllers/oci_api_controller.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index 1940a27db2c..f3eb587dcd8 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -423,7 +423,9 @@ def _get_instance_role(instance: oci.core.models.Instance) -> str: return consts.NodeRoles.WORKER def list_nodes(self) -> List[Node]: - return [Node(instance.display_name, self, role=_get_instance_role(instance)) for instance in self._instances] + return [ + Node(instance.display_name, self, role=self._get_instance_role(instance)) for instance in self._instances + ] def list_disks(self, node_name: str) -> List[Disk]: pass From 75e77a9403c23d946a561415974deac1d89dd998 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Thu, 12 Dec 2024 17:21:59 +0100 Subject: [PATCH 27/42] fix --- .../controllers/node_controllers/oci_api_controller.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index f3eb587dcd8..abe760acf53 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -405,8 +405,7 @@ def terraform_vm_name_key(self) -> str: def terraform_vm_resource_type(self) -> str: return "oci_core_instance" - @staticmethod - def _get_instance_role(instance: oci.core.models.Instance) -> str: + def _get_instance_role(self, instance: oci.core.models.Instance) -> str: namespace_key = "openshift-{}".format(self._entity_config.entity_name) namespace = instance.defined_tags.get(tag_namespace_key) assert namespace, "expected namespace {} to exist in defined tags {}".format( From cfad37e72c8c818a5db2846cacf13c61b1cdb050 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Thu, 12 Dec 2024 17:23:41 +0100 Subject: [PATCH 28/42] fix --- .../controllers/node_controllers/oci_api_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index abe760acf53..4d43240ed87 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -407,7 +407,7 @@ def terraform_vm_resource_type(self) -> str: def _get_instance_role(self, instance: oci.core.models.Instance) -> str: namespace_key = "openshift-{}".format(self._entity_config.entity_name) - namespace = instance.defined_tags.get(tag_namespace_key) + namespace = instance.defined_tags.get(namespace_key) assert namespace, "expected namespace {} to exist in defined tags {}".format( namespace_key, instance.defined_tags ) From c38339a4181d380a1e4cad065f75e3385e974a34 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Thu, 12 Dec 2024 17:30:28 +0100 Subject: [PATCH 29/42] fix --- .../test_infra/helper_classes/base_cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py b/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py index 954a5b57dc0..5495b39d18c 100644 --- a/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py +++ b/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py @@ -153,7 +153,7 @@ def set_hostnames_and_roles(self): nodes = self.nodes.get_nodes(refresh=True) for host in hosts: - node = self.find_matching_node_name(host, nodes) + node = self.find_matching_node(host, nodes) assert node is not None, ( f"Failed to find matching node for host with mac address {host.macs()}" f" nodes: {[(n.name, n.ips, n.macs) for n in nodes]}" From f63b9765473835522e025f1917e8bbd6244681f0 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Thu, 12 Dec 2024 17:34:01 +0100 Subject: [PATCH 30/42] fix --- .../test_infra/helper_classes/base_cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py b/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py index 5495b39d18c..a185bd2439f 100644 --- a/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py +++ b/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py @@ -158,7 +158,7 @@ def set_hostnames_and_roles(self): f"Failed to find matching node for host with mac address {host.macs()}" f" nodes: {[(n.name, n.ips, n.macs) for n in nodes]}" ) - self._infra_env.update_host(host_id=host.get_id(), host_role=node.role, host_name=name) + self._infra_env.update_host(host_id=host.get_id(), host_role=node.role, host_name=node.name) def set_installer_args(self): hosts = self.to_cluster_hosts(self.api_client.get_cluster_hosts(self.id)) From 3ab5bc98f543287c4ecd33d4ccbfc7c1dc469ef8 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 13 Dec 2024 11:38:27 +0100 Subject: [PATCH 31/42] fix --- src/assisted_test_infra/test_infra/helper_classes/cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/assisted_test_infra/test_infra/helper_classes/cluster.py b/src/assisted_test_infra/test_infra/helper_classes/cluster.py index 14bb1a0e542..81e9b8d54e8 100644 --- a/src/assisted_test_infra/test_infra/helper_classes/cluster.py +++ b/src/assisted_test_infra/test_infra/helper_classes/cluster.py @@ -958,7 +958,7 @@ def prepare_nodes(self, is_static_ip: bool = False, **kwargs): def create_custom_manifests(self): log.info(f"Adding {len(self._config.custom_manifests)} custom manifests") for local_manifest in self._config.custom_manifests: - encoded_content = base64.b64encode(local_manifest.content).decode("utf-8", "ignore") + encoded_content = base64.b64encode(local_manifest.content.encode("uff-8")).decode("utf-8", "ignore") manifest = self.create_custom_manifest(local_manifest.folder, local_manifest.file_name, encoded_content) From 26a234dd6f7567529197ea91076408fe31599b29 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 13 Dec 2024 11:40:41 +0100 Subject: [PATCH 32/42] fix --- src/assisted_test_infra/test_infra/helper_classes/cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/assisted_test_infra/test_infra/helper_classes/cluster.py b/src/assisted_test_infra/test_infra/helper_classes/cluster.py index 81e9b8d54e8..0f883c30629 100644 --- a/src/assisted_test_infra/test_infra/helper_classes/cluster.py +++ b/src/assisted_test_infra/test_infra/helper_classes/cluster.py @@ -958,7 +958,7 @@ def prepare_nodes(self, is_static_ip: bool = False, **kwargs): def create_custom_manifests(self): log.info(f"Adding {len(self._config.custom_manifests)} custom manifests") for local_manifest in self._config.custom_manifests: - encoded_content = base64.b64encode(local_manifest.content.encode("uff-8")).decode("utf-8", "ignore") + encoded_content = base64.b64encode(local_manifest.content.encode("utf-8")).decode("utf-8", "ignore") manifest = self.create_custom_manifest(local_manifest.folder, local_manifest.file_name, encoded_content) From 748506c08d968730e90d76af10dde8a9d4b47a7e Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 13 Dec 2024 14:24:01 +0100 Subject: [PATCH 33/42] fix --- ansible_files/oci_generic_destroy_ci_machine_playbook.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ansible_files/oci_generic_destroy_ci_machine_playbook.yml b/ansible_files/oci_generic_destroy_ci_machine_playbook.yml index 8393f7a65a0..8a95b11d3ba 100644 --- a/ansible_files/oci_generic_destroy_ci_machine_playbook.yml +++ b/ansible_files/oci_generic_destroy_ci_machine_playbook.yml @@ -1,4 +1,3 @@ -- meta: end_play - name: Destroy OCI instrastructure provisionned by test-infra hosts: primary tasks: From f825841063bcca57b5d65e57e11c6fd447f438ff Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Thu, 30 Jan 2025 16:12:49 +0100 Subject: [PATCH 34/42] remove old terraform oci controller --- .../test_infra/controllers/__init__.py | 2 - .../controllers/node_controllers/__init__.py | 2 - .../node_controllers/oci_controller.py | 141 ------------------ 3 files changed, 145 deletions(-) delete mode 100644 src/assisted_test_infra/test_infra/controllers/node_controllers/oci_controller.py diff --git a/src/assisted_test_infra/test_infra/controllers/__init__.py b/src/assisted_test_infra/test_infra/controllers/__init__.py index d16753d739e..d07cb2a3d81 100644 --- a/src/assisted_test_infra/test_infra/controllers/__init__.py +++ b/src/assisted_test_infra/test_infra/controllers/__init__.py @@ -8,7 +8,6 @@ from .node_controllers.node import Node from .node_controllers.nutanix_controller import NutanixController from .node_controllers.oci_api_controller import OciApiController -from .node_controllers.oci_controller import OciController from .node_controllers.redfish_controller import RedfishController from .node_controllers.terraform_controller import TerraformController from .node_controllers.vsphere_controller import VSphereController @@ -28,7 +27,6 @@ "TangController", "TerraformController", "LibvirtController", - "OciController", "OciApiController", "VSphereController", "NutanixController", diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/__init__.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/__init__.py index e49f19292dc..ac479935069 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/__init__.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/__init__.py @@ -4,7 +4,6 @@ from .node import Node from .node_controller import NodeController from .oci_api_controller import OciApiController -from .oci_controller import OciController from .redfish_controller import RedfishController from .terraform_controller import TerraformController from .vsphere_controller import VSphereController @@ -17,7 +16,6 @@ "Disk", "Node", "LibvirtController", - "OciController", "OciApiController", "ZVMController", "RedfishController", diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_controller.py deleted file mode 100644 index 1ab32735c60..00000000000 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_controller.py +++ /dev/null @@ -1,141 +0,0 @@ -from enum import Enum -from typing import List, Tuple, Union - -import oci -from oci.core import VirtualNetworkClient -from oci.core.models import Instance - -import consts -from assisted_test_infra.test_infra import BaseClusterConfig -from assisted_test_infra.test_infra.controllers.node_controllers.tf_controller import TFController -from assisted_test_infra.test_infra.helper_classes.config import BaseNodesConfig -from assisted_test_infra.test_infra.helper_classes.config.base_oci_config import BaseOciConfig -from service_client import log - - -class OciInstanceState(Enum): - RUNNING = "RUNNING" - STARTING = "STARTING" - STOPPING = "STOPPING" - STOPPED = "STOPPED" - TERMINATING = "TERMINATING" - TERMINATED = "TERMINATED" - RESETTING = "RESETTING" - RESTARTING = "RESTARTING" - - -class OciInstanceAction(Enum): - START = "START" - STOP = "STOP" - SOFTRESET = "SOFTRESET" - - -class OciController(TFController): - _config: BaseOciConfig - - def __init__(self, config: BaseNodesConfig, cluster_config: BaseClusterConfig): - super().__init__(config, cluster_config) - self._virtual_network_client: VirtualNetworkClient = None - - def get_all_vars(self): - tfvars = super(OciController, self).get_all_vars() - tfvars["unique_id"] = self._entity_config.entity_name.suffix - tfvars["master_memory_gib"] = self._config.master_memory / consts.MiB_UNITS - tfvars["worker_memory_gib"] = self._config.worker_memory / consts.MiB_UNITS - tfvars["master_disk_size_gib"] = self._config.master_disk / consts.GB - - # Minimal disk size on OCI is 50GB - if self._config.worker_disk == consts.resources.DEFAULT_WORKER_DISK: - self._config.worker_disk = 50 * consts.GB - - tfvars["worker_disk_size_gib"] = self._config.worker_disk / consts.GB - return tfvars - - @property - def terraform_vm_name_key(self): - return "display_name" - - @property - def terraform_vm_resource_type(self) -> str: - return "oci_core_instance" - - def _get_provider_client(self) -> object: - oci_config = self._config.get_provider_config() - - # Raise exception if failed - oci.config.validate_config(oci_config) - oci_client = oci.core.ComputeClient(oci_config) - oci_client.list_instances(self._config.oci_compartment_oicd) - self._virtual_network_client = oci.core.VirtualNetworkClient(self._config.get_provider_config()) - - return oci_client - - def _get_provider_vm(self, tf_vm_name: str) -> Union[Instance, None]: - vm_attributes = self._get_vm(tf_vm_name)["attributes"] - - oci_instances = self._provider_client.list_instances(self._config.oci_compartment_oicd).data - for instance in oci_instances: - if instance.id == vm_attributes["id"]: - return instance - - raise ValueError(f"Can't find node with name: {tf_vm_name}") - - def start_node(self, node_name: str, check_ips: bool) -> None: - """ - :raises ValueError if node_name does not exist - """ - instance = self._get_provider_vm(node_name) - if instance.lifecycle_state != OciInstanceState.RUNNING.value: - log.info(f"Powering on OCI instance {node_name}") - self._instance_action(instance, OciInstanceAction.START) - else: - log.warning( - f"Attempted to power on node {node_name}, " - f"but the instance is already on - lifecycle_state={instance.lifecycle_state}" - ) - - def shutdown_node(self, node_name: str) -> None: - instance = self._get_provider_vm(node_name) - if instance.lifecycle_state != OciInstanceState.STOPPED.value: - log.info(f"Powering off OCI instance {node_name}") - self._instance_action(instance, OciInstanceAction.STOP) - - else: - log.warning( - f"Attempted to power off node {node_name}, " - f"but the instance is already off - lifecycle_state={instance.lifecycle_state}" - ) - - def _instance_action(self, instance: Instance, action: OciInstanceAction): - response = self._provider_client.instance_action(instance_id=instance.id, action=action.value) - assert response.status == 200, f"Failed to {action.value.lower()} {instance.display_name} OCI instance" - - def restart_node(self, node_name: str) -> None: - log.info(f"Restarting OCI instance {node_name}") - instance = self._get_provider_vm(node_name) - self._instance_action(instance, OciInstanceAction.SOFTRESET) - - def is_active(self, node_name: str) -> bool: - instance = self._get_provider_vm(node_name) - return instance.lifecycle_state == OciInstanceState.RUNNING.value - - def get_cpu_cores(self, node_name: str) -> int: - return self._get_vm(node_name)["attributes"]["shape_config"][0]["ocpus"] - - def get_ram_kib(self, node_name: str) -> int: - return self._get_vm(node_name)["attributes"]["shape_config"][0]["memory_in_gbs"] * 1024 * 1024 - - def get_node_ips_and_macs(self, node_name) -> Tuple[List[str], List[str]]: - vm_attributes = self._get_vm(node_name)["attributes"] - instance_id = vm_attributes["id"] - - vnics = self._provider_client.list_vnic_attachments( - self._config.oci_compartment_oicd, instance_id=instance_id - ).data - mac_addresses = [self._virtual_network_client.get_vnic(vnic.vnic_id).data.mac_address for vnic in vnics] - ip_addresses = [self._virtual_network_client.get_vnic(vnic.vnic_id).data.private_ip for vnic in vnics] - - return ip_addresses, mac_addresses - - def set_dns(self, api_ip: str, ingress_ip: str) -> None: - return From 54d97b886c8a81d899c5a0a401d00bb3d2cc826c Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Tue, 4 Feb 2025 13:38:35 +0100 Subject: [PATCH 35/42] destroy OCI nodes --- Makefile | 3 --- ansible_files/oci_generic_destroy_ci_machine_playbook.yml | 2 +- .../controllers/node_controllers/oci_api_controller.py | 7 ------- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 9bbe9cb6b48..f9ac0e16063 100644 --- a/Makefile +++ b/Makefile @@ -234,9 +234,6 @@ destroy_nutanix: destroy_vsphere: PLATFORM=vsphere make destroy_terraform_controller -destroy_oci: - PLATFORM=oci make destroy_terraform_controller - ####### # Run # ####### diff --git a/ansible_files/oci_generic_destroy_ci_machine_playbook.yml b/ansible_files/oci_generic_destroy_ci_machine_playbook.yml index 8a95b11d3ba..cf94317f4be 100644 --- a/ansible_files/oci_generic_destroy_ci_machine_playbook.yml +++ b/ansible_files/oci_generic_destroy_ci_machine_playbook.yml @@ -4,7 +4,7 @@ - name: Destroy OCI instrastructure provisionned by test-infra ansible.builtin.shell: | source /root/config.sh - make destroy_oci + make destroy_nodes_oci args: chdir: /home/assisted retries: 5 diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index 4d43240ed87..d0609bc3318 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -359,14 +359,7 @@ def _apply_job_from_stack( success = False items = self._resource_manager_client.list_job_outputs(job.data.id).data.items for item in items: -<<<<<<< HEAD if item.output_name == "dynamic_custom_manifest": -======= - if item.output_name == "oci_ccm_config": - self.cloud_provider = item.output_value - success = True - elif item.output_name == "dynamic_custom_manifest": ->>>>>>> a6df15cc (update success condition after stack is applied) self._entity_config.custom_manifests.append( Manifest(folder="manifests", file_name="oci_custom_manifests.yaml", content=item.output_value) ) From 1898d9b05b413eb3192153658a39d191a3e32ff4 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Tue, 4 Feb 2025 17:08:09 +0100 Subject: [PATCH 36/42] use oracle linux using other OSes is painful because of the agreement system --- ...ci_generic_destroy_ci_machine_playbook.yml | 2 +- terraform_files/oci-ci-machine/00_main.tf | 2 +- terraform_files/oci-ci-machine/02_compute.tf | 60 +------------------ terraform_files/oci-ci-machine/variables.tf | 8 +++ 4 files changed, 13 insertions(+), 59 deletions(-) diff --git a/ansible_files/oci_generic_destroy_ci_machine_playbook.yml b/ansible_files/oci_generic_destroy_ci_machine_playbook.yml index cf94317f4be..80b1786debf 100644 --- a/ansible_files/oci_generic_destroy_ci_machine_playbook.yml +++ b/ansible_files/oci_generic_destroy_ci_machine_playbook.yml @@ -24,4 +24,4 @@ vars: oci_terraform_workdir: "{{ [playbook_dir, '..', 'terraform_files', 'oci-ci-machine'] | path_join | realpath }}" roles: - - name: oci/destroy_infra + - name: oci/destroy_infra \ No newline at end of file diff --git a/terraform_files/oci-ci-machine/00_main.tf b/terraform_files/oci-ci-machine/00_main.tf index cbf2eb4fb8e..1ae9e824290 100644 --- a/terraform_files/oci-ci-machine/00_main.tf +++ b/terraform_files/oci-ci-machine/00_main.tf @@ -2,7 +2,7 @@ terraform { required_providers { oci = { source = "oracle/oci" - version = "5.45.0" + version = "6.23.0" } cloudinit = { source = "hashicorp/cloudinit" diff --git a/terraform_files/oci-ci-machine/02_compute.tf b/terraform_files/oci-ci-machine/02_compute.tf index 367dad61ea4..004e10cfc56 100644 --- a/terraform_files/oci-ci-machine/02_compute.tf +++ b/terraform_files/oci-ci-machine/02_compute.tf @@ -2,56 +2,6 @@ data "oci_identity_availability_domains" "ads" { compartment_id = var.oci_compartment_id } -# Fetch Rocky Linux 9.x OS image from OCI marketpalce -# See https://blogs.oracle.com/cloud-infrastructure/post/using-terraform-for-marketplace-images -data "oci_marketplace_listings" "os_listings" { - category = ["Operating Systems"] - pricing = ["FREE"] - package_type = "IMAGE" - compartment_id = var.oci_compartment_id - filter { - name = "name" - values = ["Rocky Linux 9 x86 Supported by CIQ"] - regex = false - } -} - -data "oci_marketplace_listing" "os_listing" { - listing_id = data.oci_marketplace_listings.os_listings.listings[0].id - compartment_id = var.oci_compartment_id -} - -data "oci_marketplace_listing_package" "os_listing_package" { - listing_id = data.oci_marketplace_listing.os_listing.id - package_version = data.oci_marketplace_listing.os_listing.default_package_version - compartment_id = var.oci_compartment_id -} - -data "oci_core_app_catalog_listing_resource_version" "os_catalog_listing" { - listing_id = data.oci_marketplace_listing_package.os_listing_package.app_catalog_listing_id - resource_version = data.oci_marketplace_listing_package.os_listing_package.app_catalog_listing_resource_version -} - -data "oci_marketplace_listing_package_agreements" "os_listing_package_agreements" { - listing_id = data.oci_marketplace_listing.os_listing.id - package_version = data.oci_marketplace_listing.os_listing.default_package_version - compartment_id = var.oci_compartment_id -} - -# Sign agreement to use Rocky Linux from the OCI marketpalce -resource "oci_marketplace_accepted_agreement" "os_accepted_agreement" { - agreement_id = oci_marketplace_listing_package_agreement.os_listing_package_agreement.agreement_id - compartment_id = var.oci_compartment_id - listing_id = data.oci_marketplace_listing.os_listing.id - package_version = data.oci_marketplace_listing.os_listing.default_package_version - signature = oci_marketplace_listing_package_agreement.os_listing_package_agreement.signature -} -resource "oci_marketplace_listing_package_agreement" "os_listing_package_agreement" { - agreement_id = data.oci_marketplace_listing_package_agreements.os_listing_package_agreements.agreements[0].id - listing_id = data.oci_marketplace_listing.os_listing.id - package_version = data.oci_marketplace_listing.os_listing.default_package_version -} - # Use cloud init to configure root user data "cloudinit_config" "config" { part { @@ -74,20 +24,16 @@ resource "oci_core_instance" "ci_instance" { # Required availability_domain = data.oci_identity_availability_domains.ads.availability_domains[0].name compartment_id = var.oci_compartment_id - shape = "VM.Standard3.Flex" + shape = "VM.Standard.E5.Flex" shape_config { memory_in_gbs = 16 ocpus = 4 } - platform_config { - type = "INTEL_VM" - are_virtual_instructions_enabled = true - } - source_details { - source_id = data.oci_core_app_catalog_listing_resource_version.os_catalog_listing.listing_resource_id + # source_id = data.oci_core_app_catalog_listing_resource_version.os_catalog_listing.listing_resource_id + source_id = var.operating_system_source_id source_type = "image" boot_volume_size_in_gbs = 500 boot_volume_vpus_per_gb = 30 diff --git a/terraform_files/oci-ci-machine/variables.tf b/terraform_files/oci-ci-machine/variables.tf index 0df1071bb02..1aedfbae511 100644 --- a/terraform_files/oci-ci-machine/variables.tf +++ b/terraform_files/oci-ci-machine/variables.tf @@ -13,6 +13,14 @@ variable "public_ssh_key_path" { description = "Path to public key" } +# Check the folowing URL to update the default value: +# https://docs.oracle.com/en-us/iaas/images/oracle-linux-9x/index.htm +variable "operating_system_source_id" { + type = string + default = "ocid1.image.oc1.us-sanjose-1.aaaaaaaaizqrsucjjxtqwnbhlxmmjtkj4qeebqhipp2d6m4rovhsdjjxbmzq" + description = "Base OS image ID being used to provision the CI machine" +} + /////////// // OCI variables /////////// From e0a44ac9a1cb38b541fc92dfec6322181b69eb49 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Wed, 5 Feb 2025 11:24:21 +0100 Subject: [PATCH 37/42] pin rocky linux image --- terraform_files/oci-ci-machine/variables.tf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/terraform_files/oci-ci-machine/variables.tf b/terraform_files/oci-ci-machine/variables.tf index 1aedfbae511..dae49019cbf 100644 --- a/terraform_files/oci-ci-machine/variables.tf +++ b/terraform_files/oci-ci-machine/variables.tf @@ -13,11 +13,14 @@ variable "public_ssh_key_path" { description = "Path to public key" } -# Check the folowing URL to update the default value: -# https://docs.oracle.com/en-us/iaas/images/oracle-linux-9x/index.htm +# Rocky Linux image ID +# +# Best way to find it is to spawn an instance with Rocky Linux and then copy +# the image ID that is in use. It is best to pin it, as the agreement system in +# OCI is painful to get right programmatically. variable "operating_system_source_id" { type = string - default = "ocid1.image.oc1.us-sanjose-1.aaaaaaaaizqrsucjjxtqwnbhlxmmjtkj4qeebqhipp2d6m4rovhsdjjxbmzq" + default = "ocid1.image.oc1..aaaaaaaauo3kyxlty6himw7uecc4xdzshdnc43qf4q2uyvy32gi3t3ixg5pa" description = "Base OS image ID being used to provision the CI machine" } From 42dcabb75516e79a78dfa9d76f0bfc0c412c71a1 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Thu, 6 Feb 2025 10:19:34 +0100 Subject: [PATCH 38/42] use latest infra --- .../roles/oci/setup_oci_for_test_infra/defaults/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml b/ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml index 7e13f08c629..637ab2eebed 100644 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml +++ b/ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml @@ -1,5 +1,5 @@ -oci_compute_shape: "VM.Standard.E4.Flex" -oci_control_plane_shape: "VM.Standard.E4.Flex" +oci_compute_shape: "VM.Standard.E5.Flex" +oci_control_plane_shape: "VM.Standard.E5.Flex" -oci_infrastructure_zip_url: "https://github.com/dfoster-oracle/oci-openshift/releases/download/v1.0.0-preview/create-cluster-v0.1.0.zip" +oci_infrastructure_zip_url: "https://github.com/oracle-quickstart/oci-openshift/releases/download/v1.0.0/create-cluster-v1.0.0.zip" oci_infrastructure_zip_file: "{{ hostvars[ci_machine].ansible_user_dir }}/oci/infrastructure.zip" From 04bc8ee433d493e42f70d780ac9b2257e9e50d10 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Thu, 6 Feb 2025 10:19:57 +0100 Subject: [PATCH 39/42] set cluster name is order to be able to destroy it after --- .../templates/assisted-additional-config.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 index 922f70e327c..53a1154a62b 100644 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 +++ b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 @@ -1,3 +1,5 @@ +export CLUSTER_NAME=test-infra-oci-{{ lookup('community.general.random_string') }} + export PLATFORM=external export TF_PLATFORM=oci export EXTERNAL_PLATFORM_NAME=oci From cec730315df2c2c731cf7a0272e5f1f150035688 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Thu, 6 Feb 2025 11:44:25 +0100 Subject: [PATCH 40/42] no special chars in cluster name --- .../templates/assisted-additional-config.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 index 53a1154a62b..83c303464ac 100644 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 +++ b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 @@ -1,4 +1,4 @@ -export CLUSTER_NAME=test-infra-oci-{{ lookup('community.general.random_string') }} +export CLUSTER_NAME=test-infra-oci-{{ lookup('community.general.random_string', special=false) }} export PLATFORM=external export TF_PLATFORM=oci From e1de90a033826371635f0de011ac82dd1292de70 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Thu, 6 Feb 2025 13:50:40 +0100 Subject: [PATCH 41/42] upper case characters are not allowed in cluster name --- .../templates/assisted-additional-config.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 index 83c303464ac..d6ef67d183a 100644 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 +++ b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 @@ -1,4 +1,4 @@ -export CLUSTER_NAME=test-infra-oci-{{ lookup('community.general.random_string', special=false) }} +export CLUSTER_NAME=test-infra-oci-{{ lookup('community.general.random_string', special=false, upper=false) }} export PLATFORM=external export TF_PLATFORM=oci From 02a8eac36828751cde6255049332b2beae66b876 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Thu, 6 Feb 2025 17:35:23 +0100 Subject: [PATCH 42/42] re-create cluster from cluster name --- .../test_infra/helper_classes/base_cluster.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py b/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py index a185bd2439f..f1a0b49a870 100644 --- a/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py +++ b/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py @@ -38,6 +38,11 @@ def __init__( @property def id(self) -> str: + if not self._config.cluster_id: + clusters = self.api_client.clusters_list() + found = [c for c in clusters if c["name"] == self._config.cluster_name] + self._config.cluster_id = found[0]["id"] if found else None + return self._config.cluster_id def get_details(self) -> Union[models.infra_env.InfraEnv, models.cluster.Cluster]: