diff --git a/Makefile b/Makefile index 9bbe9cb6b48..f9ac0e16063 100644 --- a/Makefile +++ b/Makefile @@ -234,9 +234,6 @@ destroy_nutanix: destroy_vsphere: PLATFORM=vsphere make destroy_terraform_controller -destroy_oci: - PLATFORM=oci make destroy_terraform_controller - ####### # Run # ####### diff --git a/ansible_files/oci_generic_create_ci_machine_playbook.yml b/ansible_files/oci_generic_create_ci_machine_playbook.yml index 694313b51dd..1af13b92765 100644 --- a/ansible_files/oci_generic_create_ci_machine_playbook.yml +++ b/ansible_files/oci_generic_create_ci_machine_playbook.yml @@ -45,4 +45,3 @@ - name: oci/setup_oci_for_test_infra vars: ci_machine: "{{ groups['primary'][0] }}" - infra: "{{ deployed_tf.outputs.infra.value }}" diff --git a/ansible_files/oci_generic_destroy_ci_machine_playbook.yml b/ansible_files/oci_generic_destroy_ci_machine_playbook.yml index 8a95b11d3ba..80b1786debf 100644 --- a/ansible_files/oci_generic_destroy_ci_machine_playbook.yml +++ b/ansible_files/oci_generic_destroy_ci_machine_playbook.yml @@ -4,7 +4,7 @@ - name: Destroy OCI instrastructure provisionned by test-infra ansible.builtin.shell: | source /root/config.sh - make destroy_oci + make destroy_nodes_oci args: chdir: /home/assisted retries: 5 @@ -24,4 +24,4 @@ vars: oci_terraform_workdir: "{{ [playbook_dir, '..', 'terraform_files', 'oci-ci-machine'] | path_join | realpath }}" roles: - - name: oci/destroy_infra + - name: oci/destroy_infra \ No newline at end of file diff --git a/ansible_files/roles/oci/cleanup_resources/defaults/main.yml b/ansible_files/roles/oci/cleanup_resources/defaults/main.yml index 5743d423c3a..0f0148a0e95 100644 --- a/ansible_files/roles/oci/cleanup_resources/defaults/main.yml +++ b/ansible_files/roles/oci/cleanup_resources/defaults/main.yml @@ -10,4 +10,5 @@ excluded_types: - oci_network_load_balancer_backend - oci_network_load_balancer_backend_set - oci_network_load_balancer_listener + - oci_objectstorage_preauthrequest expired_after_hours: 7 diff --git a/ansible_files/roles/oci/create_infra/tasks/main.yml b/ansible_files/roles/oci/create_infra/tasks/main.yml index 726d78d040a..35a7864ab2b 100644 --- a/ansible_files/roles/oci/create_infra/tasks/main.yml +++ b/ansible_files/roles/oci/create_infra/tasks/main.yml @@ -2,7 +2,7 @@ ansible.builtin.template: src: "terraform.tfvars.j2" dest: "{{ oci_tf_vars_file }}" - mode: 0644 + mode: '0644' - name: "Deploy Terraform Instance" community.general.terraform: @@ -17,7 +17,7 @@ ansible.builtin.copy: src: "{{ oci_terraform_workdir }}/terraform.tfstate" dest: "{{ oci_tf_state_file }}" - mode: 0644 + mode: '0644' when: oci_tf_state_file is defined - name: "Add ci_machine to inventory" @@ -36,3 +36,13 @@ delay: 30 register: result until: result is succeeded + +- name: Update all packages on host + ansible.builtin.dnf: + name: "*" + state: latest + delegate_to: "{{ deployed_tf.outputs.ci_machine_inventory.value.display_name }}" + +- name: Reboot machine + ansible.builtin.reboot: + delegate_to: "{{ deployed_tf.outputs.ci_machine_inventory.value.display_name }}" diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml b/ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml new file mode 100644 index 00000000000..637ab2eebed --- /dev/null +++ b/ansible_files/roles/oci/setup_oci_for_test_infra/defaults/main.yml @@ -0,0 +1,5 @@ +oci_compute_shape: "VM.Standard.E5.Flex" +oci_control_plane_shape: "VM.Standard.E5.Flex" + +oci_infrastructure_zip_url: "https://github.com/oracle-quickstart/oci-openshift/releases/download/v1.0.0/create-cluster-v1.0.0.zip" +oci_infrastructure_zip_file: "{{ hostvars[ci_machine].ansible_user_dir }}/oci/infrastructure.zip" diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/custom_manifests.yml b/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/custom_manifests.yml deleted file mode 100644 index d9e37240951..00000000000 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/custom_manifests.yml +++ /dev/null @@ -1,12 +0,0 @@ -- name: Create file custom manifests directories - ansible.builtin.file: - path: "{{ custom_manifests_dir }}/manifests" - state: directory - mode: '0755' - -- name: Render custom manifests on disk - ansible.builtin.template: - src: "{{ item }}.j2" - dest: "{{ custom_manifests_dir }}/manifests/{{ item }}" - mode: '0644' - loop: "{{ oci_manifests }}" diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/main.yml b/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/main.yml index d52c71e45c7..9db3a89d927 100644 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/main.yml +++ b/ansible_files/roles/oci/setup_oci_for_test_infra/tasks/main.yml @@ -1,7 +1,7 @@ - name: Directory where OCI private key will be stored ansible.builtin.set_fact: oci_private_key_path_ci_machine: "{{ hostvars[ci_machine].ansible_user_dir }}/.oci/{{ oci_private_key_path | basename }}" - custom_manifests_dir: "{{ hostvars[ci_machine].ansible_user_dir }}/custom_manifests" + oci_ci_machine_public_ip: "{{ hostvars[ci_machine].ansible_host }}" - name: Create directory where OCI private key will be stored on CI machine ansible.builtin.file: @@ -17,14 +17,24 @@ mode: '0600' delegate_to: "{{ ci_machine }}" +- name: Create directory where OCI infrastructure archive will be stored on CI machine + ansible.builtin.file: + path: "{{ oci_infrastructure_zip_file | dirname }}" + state: directory + mode: '0700' + delegate_to: "{{ ci_machine }}" + +- name: Download terraform template to create the infrastructure on OCI + ansible.builtin.get_url: + url: "{{ oci_infrastructure_zip_url }}" + dest: "{{ oci_infrastructure_zip_file }}" + mode: '0600' + delegate_to: "{{ ci_machine }}" + - name: Export OCI configuration in assisted-additional-config to shared_dir ansible.builtin.template: src: "assisted-additional-config.j2" dest: "{{ shared_dir }}/assisted-additional-config" - mode: 0644 + mode: '0644' when: shared_dir is defined -- name: Create OCI custom manifests - ansible.builtin.import_tasks: - file: custom_manifests.yml - delegate_to: "{{ ci_machine }}" diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 index 222705450be..d6ef67d183a 100644 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 +++ b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/assisted-additional-config.j2 @@ -1,3 +1,5 @@ +export CLUSTER_NAME=test-infra-oci-{{ lookup('community.general.random_string', special=false, upper=false) }} + export PLATFORM=external export TF_PLATFORM=oci export EXTERNAL_PLATFORM_NAME=oci @@ -9,20 +11,13 @@ export OCI_PRIVATE_KEY_PATH="{{ oci_private_key_path_ci_machine }}" export OCI_PUBLIC_KEY_FINGERPRINT="{{ oci_fingerprint }}" export OCI_TENANCY="{{ oci_tenancy_id }}" export OCI_REGION="{{ oci_region }}" -export OCI_VCN="{{ infra.oci_vcn_id }}" -export OCI_PUBLIC_SUBNET="{{ infra.oci_public_subnet_id }}" -export OCI_PRIVATE_SUBNET="{{ infra.oci_private_subnet_id }}" - -# CI machine to access nodes (connect on them through SSH) -# Nodes to access CI machine (assisted-service API through HTTP/HTTPS) -export OCI_EXTRA_NODE_NSG_IDS="{{ infra.oci_ci_machine_access_nsg_id }},{{ infra.oci_cluster_ci_nsg_id }}" - -# CI machine to access load balancer (cluster access through HTTP/HTTPS/MCS/API) -export OCI_EXTRA_LOAD_BALANCER_NSG_IDS="{{ infra.oci_load_balancer_ci_nsg_id }}" -export BASE_DOMAIN="{{ oci_dns_zone }}" +export OCI_INFRASTRUCTURE_ZIP_FILE="{{ oci_infrastructure_zip_file }}" +export OCI_COMPUTE_SHAPE="{{ oci_compute_shape }}" +export OCI_CONTROL_PLANE_SHAPE="{{ oci_control_plane_shape }}" -export CUSTOM_MANIFESTS_FILES="{{ custom_manifests_dir }}" +export BASE_DOMAIN="{{ unique_id }}.{{ oci_dns_zone }}" +export SERVICE_URL="{{ oci_ci_machine_public_ip }}" export HOST_INSTALLER_ARGS='{"args": ["--append-karg", "console=ttyS0"]}' export KERNEL_ARGUMENTS='[{"operation": "append", "value": "console=ttyS0"}]' diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/machineconfig-ccm.yml.j2 b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/machineconfig-ccm.yml.j2 deleted file mode 100644 index 6428309494b..00000000000 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/machineconfig-ccm.yml.j2 +++ /dev/null @@ -1,83 +0,0 @@ - -# 99_openshift-machineconfig_00-master-kubelet-providerid.yaml -# Generated by Butane; do not edit -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: master - name: 00-master-oci-kubelet-providerid -spec: - config: - ignition: - version: 3.2.0 - storage: - files: - - contents: - compression: gzip - source: data:;base64,H4sIAAAAAAAC/1yPUYvaQBCA3/dXTFMf2odkrbSFWlNQE2moJEW9exGRdTO5DBd3w+4YvBP/+yF6IPf0zcPMxzefP8kdGblTvhYeGUKE0EJLLVaKGiHyIkmnRT6LJbKW/sUz7ssb5fNhhw1y5NF1pDEq5aAfWk1h62xHJToqI21NJQRVsL64g97p3XgOYPMbuEYjAABQ1xaC3DI4bBulyTwBHsnzZbi/um4fiaEvKhJZvlyN82m6zZK490UfXAPhXwjGB66to1fFZM0QJqgcOiic0g0GEPo51MztUMpvP39Fgx/foxulbbXsBpKMZ2U0Siq/CqEVw58P8aNRWszEenn9fSNS05GzZo+G4+DfwySdp6vt/0XxmCXpIktiq2koZe90F3wOxMXxFgAA///yWfIkhAEAAA== - mode: 493 - path: /usr/local/bin/oci-kubelet-providerid - systemd: - units: - - contents: | - [Unit] - Description=Fetch kubelet provider id from OCI Metadata - - # Wait for NetworkManager to report it's online - After=NetworkManager-wait-online.service - # Run before kubelet - Before=kubelet.service - - [Service] - ExecStart=/usr/local/bin/oci-kubelet-providerid - Type=oneshot - - [Install] - WantedBy=network-online.target - enabled: true - name: oci-kubelet-providerid.service - ---- - -# 99_openshift-machineconfig_00-worker-kubelet-providerid.yaml -# Generated by Butane; do not edit -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: worker - name: 00-worker-oci-kubelet-providerid -spec: - config: - ignition: - version: 3.2.0 - storage: - files: - - contents: - compression: gzip - source: data:;base64,H4sIAAAAAAAC/1yPUYvaQBCA3/dXTFMf2odkrbSFWlNQE2moJEW9exGRdTO5DBd3w+4YvBP/+yF6IPf0zcPMxzefP8kdGblTvhYeGUKE0EJLLVaKGiHyIkmnRT6LJbKW/sUz7ssb5fNhhw1y5NF1pDEq5aAfWk1h62xHJToqI21NJQRVsL64g97p3XgOYPMbuEYjAABQ1xaC3DI4bBulyTwBHsnzZbi/um4fiaEvKhJZvlyN82m6zZK490UfXAPhXwjGB66to1fFZM0QJqgcOiic0g0GEPo51MztUMpvP39Fgx/foxulbbXsBpKMZ2U0Siq/CqEVw58P8aNRWszEenn9fSNS05GzZo+G4+DfwySdp6vt/0XxmCXpIktiq2koZe90F3wOxMXxFgAA///yWfIkhAEAAA== - mode: 493 - path: /usr/local/bin/oci-kubelet-providerid - systemd: - units: - - contents: | - [Unit] - Description=Fetch kubelet provider id from OCI Metadata - - # Wait for NetworkManager to report it's online - After=NetworkManager-wait-online.service - # Run before kubelet - Before=kubelet.service - - [Service] - ExecStart=/usr/local/bin/oci-kubelet-providerid - Type=oneshot - - [Install] - WantedBy=network-online.target - enabled: true - name: oci-kubelet-providerid.service - ---- - diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/machineconfig-csi.yml.j2 b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/machineconfig-csi.yml.j2 deleted file mode 100644 index 75176fe5f1a..00000000000 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/machineconfig-csi.yml.j2 +++ /dev/null @@ -1,37 +0,0 @@ - -# 99_openshift-machineconfig_00-master-iscsi-service.yaml -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: master - name: 99-master-iscsid -spec: - config: - ignition: - version: 3.1.0 - systemd: - units: - - enabled: true - name: iscsid.service - ---- - -# 99_openshift-machineconfig_00-worker-iscsi-service.yaml -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: worker - name: 99-worker-iscsid -spec: - config: - ignition: - version: 3.1.0 - systemd: - units: - - enabled: true - name: iscsid.service - ---- - diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/oci-ccm.yml.j2 b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/oci-ccm.yml.j2 deleted file mode 100644 index e561847eae5..00000000000 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/oci-ccm.yml.j2 +++ /dev/null @@ -1,285 +0,0 @@ - -# oci-ccm-00-namespace.yaml -apiVersion: v1 -kind: Namespace -metadata: - name: oci-cloud-controller-manager - annotations: - workload.openshift.io/allowed: management - labels: - "pod-security.kubernetes.io/enforce": "privileged" - "pod-security.kubernetes.io/audit": "privileged" - "pod-security.kubernetes.io/warn": "privileged" - "security.openshift.io/scc.podSecurityLabelSync": "false" - "openshift.io/run-level": "0" - "pod-security.kubernetes.io/enforce-version": "v1.24" - ---- - -# oci-ccm-01-service-account.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cloud-controller-manager - namespace: oci-cloud-controller-manager - ---- - -# oci-ccm-02-cluster-role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: system:cloud-controller-manager - labels: - kubernetes.io/cluster-service: "true" -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - '*' - -- apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - -- apiGroups: - - "" - resources: - - services - verbs: - - list - - watch - - patch - -- apiGroups: - - "" - resources: - - services/status - verbs: - - patch - - get - - update - -- apiGroups: - - "" - resources: - - configmaps - resourceNames: - - "extension-apiserver-authentication" - verbs: - - get - -- apiGroups: - - "" - resources: - - events - verbs: - - list - - watch - - create - - patch - - update - -# For leader election -- apiGroups: - - "" - resources: - - endpoints - verbs: - - create - -- apiGroups: - - "" - resources: - - endpoints - resourceNames: - - "cloud-controller-manager" - verbs: - - get - - list - - watch - - update - -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create - -- apiGroups: - - "coordination.k8s.io" - resources: - - leases - verbs: - - get - - create - - update - - delete - - patch - - watch - -- apiGroups: - - "" - resources: - - configmaps - resourceNames: - - "cloud-controller-manager" - verbs: - - get - - update - -- apiGroups: - - "" - resources: - - configmaps - resourceNames: - - "extension-apiserver-authentication" - verbs: - - get - - list - - watch - -- apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - -# For the PVL -- apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - list - - watch - - patch - ---- - -# oci-ccm-03-cluster-role-binding.yaml -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: oci-cloud-controller-manager -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:cloud-controller-manager -subjects: -- kind: ServiceAccount - name: cloud-controller-manager - namespace: oci-cloud-controller-manager - ---- - -# oci-ccm-04-cloud-controller-manager-config.yaml -apiVersion: v1 -kind: Secret -metadata: - creationTimestamp: null - name: oci-cloud-controller-manager - namespace: oci-cloud-controller-manager -stringData: - cloud-provider.yaml: | - auth: - region: {{ oci_region }} - useInstancePrincipals: true - compartment: {{ oci_compartment_id }} - vcn: {{ infra.oci_vcn_id }} - loadBalancer: - subnet1: {{ infra.oci_public_subnet_id }} - securityListManagementMode: None - rateLimiter: - rateLimitQPSRead: 20.0 - rateLimitBucketRead: 5 - rateLimitQPSWrite: 20.0 - rateLimitBucketWrite: 5 - ---- - -# oci-ccm-05-daemon-set.yaml -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: oci-cloud-controller-manager - namespace: oci-cloud-controller-manager - labels: - k8s-app: oci-cloud-controller-manager -spec: - selector: - matchLabels: - component: oci-cloud-controller-manager - tier: control-plane - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - component: oci-cloud-controller-manager - tier: control-plane - spec: - serviceAccountName: cloud-controller-manager - hostNetwork: true - nodeSelector: - node-role.kubernetes.io/control-plane: "" - priorityClassName: system-cluster-critical - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - effect: NoSchedule - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - - key: node.kubernetes.io/not-ready - operator: Exists - effect: NoSchedule - volumes: - - name: cfg - secret: - secretName: oci-cloud-controller-manager - - name: kubernetes - hostPath: - path: /etc/kubernetes - containers: - - name: oci-cloud-controller-manager - image: ghcr.io/oracle/cloud-provider-oci:v1.25.0 - command: - - /bin/bash - - -c - - | - #!/bin/bash - set -o allexport - if [[ -f /etc/kubernetes/apiserver-url.env ]]; then - source /etc/kubernetes/apiserver-url.env - fi - exec /usr/local/bin/oci-cloud-controller-manager --cloud-config=/etc/oci/cloud-provider.yaml --cloud-provider=oci --leader-elect-resource-lock=configmapsleases --v=2 - volumeMounts: - - name: cfg - mountPath: /etc/oci - readOnly: true - - name: kubernetes - mountPath: /etc/kubernetes - readOnly: true - ---- - diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/oci-csi.yml.j2 b/ansible_files/roles/oci/setup_oci_for_test_infra/templates/oci-csi.yml.j2 deleted file mode 100644 index fe6a114c25e..00000000000 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/templates/oci-csi.yml.j2 +++ /dev/null @@ -1,504 +0,0 @@ - -# oci-csi-00-namespace.yaml ---- -apiVersion: v1 -kind: Namespace -metadata: - name: oci-csi - annotations: - workload.openshift.io/allowed: management - labels: - "pod-security.kubernetes.io/enforce": "privileged" - "pod-security.kubernetes.io/audit": "privileged" - "pod-security.kubernetes.io/warn": "privileged" - "security.openshift.io/scc.podSecurityLabelSync": "false" - "openshift.io/run-level": "0" - "pod-security.kubernetes.io/enforce-version": "v1.24" - ---- - -# oci-csi-01-config.yaml -apiVersion: v1 -kind: Secret -metadata: - creationTimestamp: null - name: oci-volume-provisioner - namespace: oci-csi -stringData: - config.yaml: | - auth: - region: {{ oci_region }} - useInstancePrincipals: true - compartment: {{ oci_compartment_id }} - vcn: {{ infra.oci_vcn_id }} - loadBalancer: - subnet1: {{ infra.oci_public_subnet_id }} - securityListManagementMode: None - rateLimiter: - rateLimitQPSRead: 20.0 - rateLimitBucketRead: 5 - rateLimitQPSWrite: 20.0 - rateLimitBucketWrite: 5 - ---- - -# oci-csi-02-controller-driver.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - annotations: - deprecated.daemonset.template.generation: "1" - generation: 1 - name: csi-oci-controller - namespace: oci-csi -spec: - revisionHistoryLimit: 10 - selector: - matchLabels: - app: csi-oci-controller - template: - metadata: - creationTimestamp: null - labels: - app: csi-oci-controller - role: csi-oci - spec: - nodeSelector: - node-role.kubernetes.io/control-plane: "" - containers: - - name: csi-volume-provisioner - image: k8s.gcr.io/sig-storage/csi-provisioner:v3.2.1 - args: - - --csi-address=/var/run/shared-tmpfs/csi.sock - - --volume-name-prefix=csi - - --feature-gates=Topology=true - - --timeout=120s - - --leader-election - - --leader-election-namespace=oci-csi - volumeMounts: - - name: config - mountPath: /etc/oci/ - readOnly: true - - mountPath: /var/run/shared-tmpfs - name: shared-tmpfs - - name: csi-fss-volume-provisioner - image: k8s.gcr.io/sig-storage/csi-provisioner:v3.2.1 - args: - - --csi-address=/var/run/shared-tmpfs/csi-fss.sock - - --volume-name-prefix=csi-fss - - --feature-gates=Topology=true - - --timeout=120s - - --leader-election - - --leader-election-namespace=oci-csi - volumeMounts: - - name: config - mountPath: /etc/oci/ - readOnly: true - - mountPath: /var/run/shared-tmpfs - name: shared-tmpfs - - name: csi-attacher - image: k8s.gcr.io/sig-storage/csi-attacher:v4.2.0 - args: - - --csi-address=/var/run/shared-tmpfs/csi.sock - - --timeout=120s - - --leader-election=true - - --leader-election-namespace=oci-csi - volumeMounts: - - name: config - mountPath: /etc/oci/ - readOnly: true - - mountPath: /var/run/shared-tmpfs - name: shared-tmpfs - - name: csi-resizer - image: k8s.gcr.io/sig-storage/csi-resizer:v1.7.0 - args: - - --csi-address=/var/run/shared-tmpfs/csi.sock - - --leader-election - imagePullPolicy: "IfNotPresent" - volumeMounts: - - mountPath: /var/run/shared-tmpfs - name: shared-tmpfs - - name: oci-csi-controller-driver - args: - - --endpoint=unix://var/run/shared-tmpfs/csi.sock - - --fss-csi-endpoint=unix://var/run/shared-tmpfs/csi-fss.sock - command: - - /usr/local/bin/oci-csi-controller-driver - image: ghcr.io/oracle/cloud-provider-oci:v1.25.0 - imagePullPolicy: IfNotPresent - volumeMounts: - - name: config - mountPath: /etc/oci/ - readOnly: true - - name: kubernetes - mountPath: /etc/kubernetes - readOnly: true - - mountPath: /var/run/shared-tmpfs - name: shared-tmpfs - volumes: - - name: config - secret: - secretName: oci-volume-provisioner - - name: kubernetes - hostPath: - path: /etc/kubernetes - - name: shared-tmpfs - emptyDir: {} - dnsPolicy: ClusterFirst - hostNetwork: true - imagePullSecrets: - - name: image-pull-secret - restartPolicy: Always - schedulerName: default-scheduler - serviceAccount: csi-oci-node-sa - serviceAccountName: csi-oci-node-sa - terminationGracePeriodSeconds: 30 - tolerations: - - operator: Exists - ---- - -# oci-csi-03-fss-driver.yaml -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: fss.csi.oraclecloud.com -spec: - attachRequired: false - podInfoOnMount: false - ---- - -# oci-csi-04-bv-driver.yaml -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: blockvolume.csi.oraclecloud.com -spec: - fsGroupPolicy: File - ---- - -# oci-csi-05-iscsiadm.yaml -kind: ConfigMap -apiVersion: v1 -metadata: - name: oci-csi-iscsiadm - namespace: oci-csi -data: - iscsiadm: | - #!/bin/sh - if [ -x /host/sbin/iscsiadm ]; then - chroot /host /sbin/iscsiadm "$@" - elif [ -x /host/usr/local/sbin/iscsiadm ]; then - chroot /host /usr/local/sbin/iscsiadm "$@" - elif [ -x /host/bin/iscsiadm ]; then - chroot /host /bin/iscsiadm "$@" - elif [ -x /host/usr/local/bin/iscsiadm ]; then - chroot /host /usr/local/bin/iscsiadm "$@" - else - chroot /host iscsiadm "$@" - fi - ---- - -# oci-csi-06-fss-csi.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: oci-fss-csi - namespace: oci-csi -data: - mount: |- - #!/bin/sh - if [ -x /sbin/mount ]; then - chroot /host mount "$@" - elif [ -x /usr/local/sbin/mount ]; then - chroot /host mount "$@" - elif [ -x /usr/sbin/mount ]; then - chroot /host mount "$@" - elif [ -x /usr/local/bin/mount ]; then - chroot /host mount "$@" - else - chroot /host mount "$@" - fi - umount: |- - #!/bin/sh - if [ -x /sbin/umount ]; then - chroot /host umount "$@" - elif [ -x /usr/local/sbin/umount ]; then - chroot /host umount "$@" - elif [ -x /usr/sbin/umount ]; then - chroot /host umount "$@" - elif [ -x /usr/local/bin/umount ]; then - chroot /host umount "$@" - else - chroot /host umount "$@" - fi - umount.oci-fss: |- - #!/bin/sh - if [ -x /sbin/umount-oci-fss ]; then - chroot /host umount.oci-fss "$@" - elif [ -x /usr/local/sbin/umount-oci-fss ]; then - chroot /host umount.oci-fss "$@" - elif [ -x /usr/sbin/umount-oci-fss ]; then - chroot /host umount.oci-fss "$@" - elif [ -x /usr/local/bin/umount-oci-fss ]; then - chroot /host umount.oci-fss "$@" - else - chroot /host umount.oci-fss "$@" - fi - ---- - -# oci-csi-07-node-driver.yaml -apiVersion: apps/v1 -kind: DaemonSet -metadata: - annotations: - deprecated.daemonset.template.generation: "1" - generation: 1 - name: csi-oci-node - namespace: oci-csi -spec: - revisionHistoryLimit: 10 - selector: - matchLabels: - app: csi-oci-node - template: - metadata: - creationTimestamp: null - labels: - app: csi-oci-node - role: csi-oci - spec: - containers: - - name: oci-csi-node-driver - args: - - --v=2 - - --endpoint=unix:///csi/csi.sock - - --nodeid=$(KUBE_NODE_NAME) - - --loglevel=debug - - --fss-endpoint=unix:///fss/csi.sock - command: - - /usr/local/bin/oci-csi-node-driver - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: PATH - value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/bin:/host/sbin - image: ghcr.io/oracle/cloud-provider-oci:v1.25.0 - securityContext: - privileged: true - volumeMounts: - - mountPath: /csi - name: plugin-dir - - mountPath: /fss - name: fss-plugin-dir - - mountPath: /var/lib/kubelet - mountPropagation: Bidirectional - name: pods-mount-dir - - mountPath: /dev - name: device-dir - - mountPath: /host - name: host-root - - mountPath: /sbin/iscsiadm - name: chroot-iscsiadm - subPath: iscsiadm - - mountPath: /host/var/lib/kubelet - mountPropagation: Bidirectional - name: encrypt-pods-mount-dir - - mountPath: /sbin/umount.oci-fss - name: fss-driver-mounts - subPath: umount.oci-fss - - mountPath: /sbin/umount - name: fss-driver-mounts - subPath: umount - - mountPath: /sbin/mount - name: fss-driver-mounts - subPath: mount - - name: csi-node-registrar - args: - - --csi-address=/csi/csi.sock - - --kubelet-registration-path=/var/lib/kubelet/plugins/blockvolume.csi.oraclecloud.com/csi.sock - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.1 - securityContext: - privileged: true - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - rm -rf /registration/blockvolume.csi.oraclecloud.com /registration/blockvolume.csi.oraclecloud.com-reg.sock - volumeMounts: - - mountPath: /csi - name: plugin-dir - - mountPath: /registration - name: registration-dir - - name: csi-node-registrar-fss - args: - - --csi-address=/fss/csi.sock - - --kubelet-registration-path=/var/lib/kubelet/plugins/fss.csi.oraclecloud.com/csi.sock - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0 - securityContext: - privileged: true - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - rm -rf /registration/fss.csi.oraclecloud.com /registration/fss.csi.oraclecloud.com-reg.sock - volumeMounts: - - mountPath: /fss - name: fss-plugin-dir - - mountPath: /registration - name: registration-dir - dnsPolicy: ClusterFirst - hostNetwork: true - restartPolicy: Always - schedulerName: default-scheduler - serviceAccount: csi-oci-node-sa - serviceAccountName: csi-oci-node-sa - terminationGracePeriodSeconds: 30 - tolerations: - - operator: Exists - volumes: - - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: DirectoryOrCreate - name: registration-dir - - hostPath: - path: /var/lib/kubelet/plugins/blockvolume.csi.oraclecloud.com - type: DirectoryOrCreate - name: plugin-dir - - hostPath: - path: /var/lib/kubelet/plugins/fss.csi.oraclecloud.com - type: DirectoryOrCreate - name: fss-plugin-dir - - hostPath: - path: /var/lib/kubelet - type: Directory - name: pods-mount-dir - - hostPath: - path: /var/lib/kubelet - type: Directory - name: encrypt-pods-mount-dir - - hostPath: - path: /dev - type: "" - name: device-dir - - hostPath: - path: / - type: Directory - name: host-root - - configMap: - name: oci-csi-iscsiadm - defaultMode: 0755 - name: chroot-iscsiadm - - configMap: - name: oci-fss-csi - defaultMode: 0755 - name: fss-driver-mounts - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate - ---- - -# oci-csi-08-node-rbac-sa.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-oci-node-sa - namespace: oci-csi - ---- - -# oci-csi-09-node-rbac-cr.yaml -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-oci - namespace: oci-csi -rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["volume.oci.oracle.com"] - resources: ["blockscsiinfos"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update", "create"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses", "volumeattachments", "volumeattachments/status", "csinodes"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "create", "update"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["patch"] - ---- - -# oci-csi-10-node-rbac-crb.yaml -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-oci-binding -subjects: - - kind: ServiceAccount - name: csi-oci-node-sa - namespace: oci-csi -roleRef: - kind: ClusterRole - name: csi-oci - apiGroup: rbac.authorization.k8s.io - ---- - -# oci-csi-11-storage-class-bv.yaml -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: oci-bv -provisioner: blockvolume.csi.oraclecloud.com -volumeBindingMode: WaitForFirstConsumer -allowVolumeExpansion: true -reclaimPolicy: Delete - ---- - -# oci-csi-12-storage-class-bv-enc.yaml -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: oci-bv-encrypted -provisioner: blockvolume.csi.oraclecloud.com -parameters: - attachment-type: "paravirtualized" -reclaimPolicy: Delete -volumeBindingMode: WaitForFirstConsumer -allowVolumeExpansion: true - ---- - diff --git a/ansible_files/roles/oci/setup_oci_for_test_infra/vars/main.yml b/ansible_files/roles/oci/setup_oci_for_test_infra/vars/main.yml deleted file mode 100644 index 020a0fc874d..00000000000 --- a/ansible_files/roles/oci/setup_oci_for_test_infra/vars/main.yml +++ /dev/null @@ -1,6 +0,0 @@ -# taken from https://github.com/oracle-quickstart/oci-openshift -oci_manifests: - - oci-ccm.yml - - oci-csi.yml - - machineconfig-ccm.yml - - machineconfig-csi.yml diff --git a/ansible_files/vars/standalone_oci_sample.yml b/ansible_files/vars/standalone_oci_sample.yml index 2ee76a755cc..90660a213d9 100644 --- a/ansible_files/vars/standalone_oci_sample.yml +++ b/ansible_files/vars/standalone_oci_sample.yml @@ -9,7 +9,7 @@ oci_user_id: "{{ lookup('env', 'OCI_USER_ID') }}" oci_fingerprint: "{{ lookup('env', 'OCI_FINGERPRINT') }}" oci_region: "{{ lookup('env', 'OCI_REGION') }}" oci_private_key_path: "{{ lookup('env', 'OCI_PRIVATE_KEY_PATH') }}" -oci_compartment_id: "{{ lookup('env', 'oci_compartment_id') }}" -oci_dns_zone: "{{ lookup('env', 'oci_dns_zone') }}" -oci_private_ssh_key_path: "{{ lookup('env', 'USER') }}/.ssh/id_rsa" -oci_public_ssh_key_path: "{{ lookup('env', 'USER') }}/.ssh/id_rsa.pub" +oci_compartment_id: "{{ lookup('env', 'OCI_COMPARTMENT_ID') }}" +oci_dns_zone: "{{ lookup('env', 'OCI_DNS_ZONE') }}" +oci_private_ssh_key_path: "{{ lookup('env', 'HOME') }}/.ssh/id_rsa" +oci_public_ssh_key_path: "{{ lookup('env', 'HOME') }}/.ssh/id_rsa.pub" diff --git a/scripts/deploy_assisted_service.sh b/scripts/deploy_assisted_service.sh index 23de5b82407..c5b5f5d40b2 100755 --- a/scripts/deploy_assisted_service.sh +++ b/scripts/deploy_assisted_service.sh @@ -175,7 +175,7 @@ else skipper run src/update_assisted_service_cm.py - (cd assisted-service/ && skipper --env-file ../skipper.env run "make deploy-all" ${SKIPPER_PARAMS} $ENABLE_KUBE_API_CMD TARGET=$DEPLOY_TARGET DEPLOY_TAG=${DEPLOY_TAG} DEPLOY_MANIFEST_PATH=${DEPLOY_MANIFEST_PATH} DEPLOY_MANIFEST_TAG=${DEPLOY_MANIFEST_TAG} NAMESPACE=${NAMESPACE} AUTH_TYPE=${AUTH_TYPE} ${DEBUG_DEPLOY_AI_PARAMS:-} IP=$(get_main_ip)) + (cd assisted-service/ && skipper --env-file ../skipper.env run "make deploy-all" ${SKIPPER_PARAMS} $ENABLE_KUBE_API_CMD TARGET=$DEPLOY_TARGET DEPLOY_TAG=${DEPLOY_TAG} DEPLOY_MANIFEST_PATH=${DEPLOY_MANIFEST_PATH} DEPLOY_MANIFEST_TAG=${DEPLOY_MANIFEST_TAG} NAMESPACE=${NAMESPACE} AUTH_TYPE=${AUTH_TYPE} ${DEBUG_DEPLOY_AI_PARAMS:-} IP=${SERVICE_URL}) add_firewalld_port $SERVICE_PORT diff --git a/skipper.yaml b/skipper.yaml index 8cabc9859bb..2af2e8c2743 100644 --- a/skipper.yaml +++ b/skipper.yaml @@ -13,7 +13,7 @@ volumes: - $HOME/.minikube/:$HOME/.minikube/ - $HOME/.ssh/:$HOME/.ssh/ - $HOME/.oci/:$HOME/.oci/ - - $HOME/custom_manifests/:$HOME/custom_manifests/ + - $HOME/oci/:$HOME/oci/ # logs - /var/log:/var/log diff --git a/src/assisted_test_infra/test_infra/controllers/__init__.py b/src/assisted_test_infra/test_infra/controllers/__init__.py index d16753d739e..d07cb2a3d81 100644 --- a/src/assisted_test_infra/test_infra/controllers/__init__.py +++ b/src/assisted_test_infra/test_infra/controllers/__init__.py @@ -8,7 +8,6 @@ from .node_controllers.node import Node from .node_controllers.nutanix_controller import NutanixController from .node_controllers.oci_api_controller import OciApiController -from .node_controllers.oci_controller import OciController from .node_controllers.redfish_controller import RedfishController from .node_controllers.terraform_controller import TerraformController from .node_controllers.vsphere_controller import VSphereController @@ -28,7 +27,6 @@ "TangController", "TerraformController", "LibvirtController", - "OciController", "OciApiController", "VSphereController", "NutanixController", diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/__init__.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/__init__.py index e49f19292dc..ac479935069 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/__init__.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/__init__.py @@ -4,7 +4,6 @@ from .node import Node from .node_controller import NodeController from .oci_api_controller import OciApiController -from .oci_controller import OciController from .redfish_controller import RedfishController from .terraform_controller import TerraformController from .vsphere_controller import VSphereController @@ -17,7 +16,6 @@ "Disk", "Node", "LibvirtController", - "OciController", "OciApiController", "ZVMController", "RedfishController", diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/node.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/node.py index 145b1371d7f..a8dac067a93 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/node.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/node.py @@ -11,7 +11,14 @@ class Node: - def __init__(self, name, node_controller, private_ssh_key_path: Optional[Path] = None, username="core"): + def __init__( + self, + name, + node_controller, + private_ssh_key_path: Optional[Path] = None, + username="core", + role: Optional[str] = None, + ): self.name = name self.private_ssh_key_path = private_ssh_key_path self.username = username @@ -20,6 +27,7 @@ def __init__(self, name, node_controller, private_ssh_key_path: Optional[Path] = self.original_ram_kib = self.get_ram_kib() self._ips = [] self._macs = [] + self._role = role def __str__(self): return self.name @@ -28,11 +36,24 @@ def __str__(self): def is_active(self): return self.node_controller.is_active(self.name) - def is_master_in_name(self): - return consts.NodeRoles.MASTER in self.name + def is_master_in_name(self) -> bool: + return self.role == consts.NodeRoles.MASTER - def is_worker_in_name(self): - return consts.NodeRoles.WORKER in self.name + def is_worker_in_name(self) -> bool: + return self.role == consts.NodeRoles.WORKER + + @property + def role(self) -> str: + if self._role: + return self._role + + if consts.NodeRoles.MASTER in self.name: + return consts.NodeRoles.MASTER + + if consts.NodeRoles.WORKER in self.name: + return consts.NodeRoles.WORKER + + return consts.NodeRoles.AUTO_ASSIGN def _set_ips_and_macs(self): self._ips, self._macs = self.node_controller.get_node_ips_and_macs(self.name) diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py index 164cac63e44..d0609bc3318 100644 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py +++ b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_api_controller.py @@ -9,6 +9,7 @@ import libvirt import oci +import requests import waiting from assisted_test_infra.test_infra import BaseClusterConfig @@ -17,7 +18,9 @@ from assisted_test_infra.test_infra.controllers.node_controllers.node_controller import NodeController from assisted_test_infra.test_infra.helper_classes.config import BaseNodesConfig from assisted_test_infra.test_infra.helper_classes.config.base_oci_config import BaseOciConfig +from assisted_test_infra.test_infra.utils.manifests import Manifest from service_client import log +import consts def random_name(prefix="", length=8): @@ -84,19 +87,11 @@ class OciApiController(NodeController): def __init__(self, config: BaseNodesConfig, cluster_config: BaseClusterConfig): super().__init__(config, cluster_config) - self._cloud_provider = None + self.cloud_provider = None + self.custom_manifests = None self._oci_compartment_oicd = self._config.oci_compartment_oicd self._initialize_oci_clients() - @property - def cloud_provider(self): - # Called from test_cases , modify manifests - return self._cloud_provider - - @cloud_provider.setter - def cloud_provider(self, cloud_provider): - self._cloud_provider = cloud_provider - def _initialize_oci_clients(self): """Initialize oci clients. @@ -109,6 +104,7 @@ def _initialize_oci_clients(self): self._object_storage_client = oci.object_storage.ObjectStorageClient(self._config.get_provider_config()) self._compute_client = oci.core.ComputeClient(self._config.get_provider_config()) self._volume_client = oci.core.BlockstorageClient(self._config.get_provider_config()) + self._virtual_network_client = oci.core.VirtualNetworkClient(self._config.get_provider_config()) # resource manager for stack creation and job self._resource_manager_client = oci.resource_manager.ResourceManagerClient( self._config.get_provider_config() @@ -190,7 +186,14 @@ def _create_pre_authenticated( ) assert obj.status == 200 - return obj.data.full_path + par = obj.data.full_path + + r = requests.head(par) + log.info(f"r = requests.head(par) -> {r}") + assert r.status_code == 200 + assert int(r.headers["Content-Length"]) == os.path.getsize(file_path) + + return par def _terraform_variables( self, @@ -258,7 +261,6 @@ def _create_stack( template_config = { "config_source_type": "ZIP_UPLOAD", "zip_file_base64_encoded": self._base64_zip_file(terraform_zip_path), - "working_directory": "infrastructure", } template_config_create = oci.resource_manager.models.CreateZipUploadConfigSourceDetails(**template_config) @@ -298,8 +300,8 @@ def _create_stack( return obj.data.id def _apply_job_from_stack( - self, stack_id: str, display_name: str, timeout_seconds: int = 3200, interval_wait: int = 60 - ) -> str: + self, stack_id: str, display_name: str, timeout_seconds: int = 3600, interval_wait: int = 60 + ) -> None: """Apply job will run the stack terraform code and create the resources. On failure - raise Exception and cleanup resources @@ -354,16 +356,16 @@ def _apply_job_from_stack( log.info(f"Exception raised during apply_job_from_stack {e}: destroying") raise # on success, we return the jobs output - list + success = False items = self._resource_manager_client.list_job_outputs(job.data.id).data.items for item in items: if item.output_name == "dynamic_custom_manifest": - file_path = ( - f"/tmp/oci-{self._entity_config.cluster_id}/terraform-output-{self._entity_config.cluster_id}.yml" + self._entity_config.custom_manifests.append( + Manifest(folder="manifests", file_name="oci_custom_manifests.yaml", content=item.output_value) ) - with open(file_path, "w") as f: - f.write(item.output_value) - return item.output_value - raise RuntimeError(f"Missing dynamic_custom_manifest for stack {stack_id}") + success = True + if not success: + raise RuntimeError(f"Missing oci_ccm_config for stack {stack_id}") @staticmethod def _waiter_status(client_callback: Callable, name: str, status: str, **callback_kwargs) -> None: @@ -381,6 +383,13 @@ def is_status(): waiting_for="Resource to be created", ) + @property + def _instances(self) -> List[oci.core.models.Instance]: + response = oci.pagination.list_call_get_all_results( + self._compute_client.list_instances, self._oci_compartment_oicd + ) + return [instance for instance in response.data if str(self._entity_config.entity_name) in instance.display_name] + @property def terraform_vm_name_key(self) -> str: return "display_name" @@ -389,8 +398,26 @@ def terraform_vm_name_key(self) -> str: def terraform_vm_resource_type(self) -> str: return "oci_core_instance" + def _get_instance_role(self, instance: oci.core.models.Instance) -> str: + namespace_key = "openshift-{}".format(self._entity_config.entity_name) + namespace = instance.defined_tags.get(namespace_key) + assert namespace, "expected namespace {} to exist in defined tags {}".format( + namespace_key, instance.defined_tags + ) + role = namespace.get("instance-role") + assert role, "expected key instance-role to exist in namespace {} in defined tags {}".format( + namespace_key, instance.defined_tags + ) + + if role == "control_plane": + return consts.NodeRoles.MASTER + + return consts.NodeRoles.WORKER + def list_nodes(self) -> List[Node]: - pass + return [ + Node(instance.display_name, self, role=self._get_instance_role(instance)) for instance in self._instances + ] def list_disks(self, node_name: str) -> List[Disk]: pass @@ -558,8 +585,36 @@ def set_per_device_boot_order(self, node_name, key: Callable[[Disk], int]) -> No """ pass + def _get_vnic_attachments(self, instance: oci.core.models.Instance) -> List[oci.core.models.VnicAttachment]: + response = oci.pagination.list_call_get_all_results( + self._compute_client.list_vnic_attachments, self._oci_compartment_oicd, instance_id=instance.id + ) + return response.data + + def _get_vnics(self, instance: oci.core.models.Instance) -> List[oci.core.models.Vnic]: + vnic_attachments = self._get_vnic_attachments(instance) + responses = [ + self._virtual_network_client.get_vnic(vnic_id=vnic_attachment.vnic_id) + for vnic_attachment in vnic_attachments + ] + return [response.data for response in responses] + def get_node_ips_and_macs(self, node_name) -> Tuple[List[str], List[str]]: - pass + instance = next(instance for instance in self._instances if node_name == instance.display_name) + vnics = self._get_vnics(instance) + ips = [] + macs = [] + for vnic in vnics: + if vnic.private_ip: + ips.append(vnic.private_ip) + if vnic.public_ip: + ips.append(vnic.public_ip) + if vnic.ipv6_addresses: + ips.extend(vnic.ipv6_addresses) + + macs.append(vnic.mac_address) + + return (ips, macs) def set_single_node_ip(self, ip) -> None: pass diff --git a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_controller.py b/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_controller.py deleted file mode 100644 index 1ab32735c60..00000000000 --- a/src/assisted_test_infra/test_infra/controllers/node_controllers/oci_controller.py +++ /dev/null @@ -1,141 +0,0 @@ -from enum import Enum -from typing import List, Tuple, Union - -import oci -from oci.core import VirtualNetworkClient -from oci.core.models import Instance - -import consts -from assisted_test_infra.test_infra import BaseClusterConfig -from assisted_test_infra.test_infra.controllers.node_controllers.tf_controller import TFController -from assisted_test_infra.test_infra.helper_classes.config import BaseNodesConfig -from assisted_test_infra.test_infra.helper_classes.config.base_oci_config import BaseOciConfig -from service_client import log - - -class OciInstanceState(Enum): - RUNNING = "RUNNING" - STARTING = "STARTING" - STOPPING = "STOPPING" - STOPPED = "STOPPED" - TERMINATING = "TERMINATING" - TERMINATED = "TERMINATED" - RESETTING = "RESETTING" - RESTARTING = "RESTARTING" - - -class OciInstanceAction(Enum): - START = "START" - STOP = "STOP" - SOFTRESET = "SOFTRESET" - - -class OciController(TFController): - _config: BaseOciConfig - - def __init__(self, config: BaseNodesConfig, cluster_config: BaseClusterConfig): - super().__init__(config, cluster_config) - self._virtual_network_client: VirtualNetworkClient = None - - def get_all_vars(self): - tfvars = super(OciController, self).get_all_vars() - tfvars["unique_id"] = self._entity_config.entity_name.suffix - tfvars["master_memory_gib"] = self._config.master_memory / consts.MiB_UNITS - tfvars["worker_memory_gib"] = self._config.worker_memory / consts.MiB_UNITS - tfvars["master_disk_size_gib"] = self._config.master_disk / consts.GB - - # Minimal disk size on OCI is 50GB - if self._config.worker_disk == consts.resources.DEFAULT_WORKER_DISK: - self._config.worker_disk = 50 * consts.GB - - tfvars["worker_disk_size_gib"] = self._config.worker_disk / consts.GB - return tfvars - - @property - def terraform_vm_name_key(self): - return "display_name" - - @property - def terraform_vm_resource_type(self) -> str: - return "oci_core_instance" - - def _get_provider_client(self) -> object: - oci_config = self._config.get_provider_config() - - # Raise exception if failed - oci.config.validate_config(oci_config) - oci_client = oci.core.ComputeClient(oci_config) - oci_client.list_instances(self._config.oci_compartment_oicd) - self._virtual_network_client = oci.core.VirtualNetworkClient(self._config.get_provider_config()) - - return oci_client - - def _get_provider_vm(self, tf_vm_name: str) -> Union[Instance, None]: - vm_attributes = self._get_vm(tf_vm_name)["attributes"] - - oci_instances = self._provider_client.list_instances(self._config.oci_compartment_oicd).data - for instance in oci_instances: - if instance.id == vm_attributes["id"]: - return instance - - raise ValueError(f"Can't find node with name: {tf_vm_name}") - - def start_node(self, node_name: str, check_ips: bool) -> None: - """ - :raises ValueError if node_name does not exist - """ - instance = self._get_provider_vm(node_name) - if instance.lifecycle_state != OciInstanceState.RUNNING.value: - log.info(f"Powering on OCI instance {node_name}") - self._instance_action(instance, OciInstanceAction.START) - else: - log.warning( - f"Attempted to power on node {node_name}, " - f"but the instance is already on - lifecycle_state={instance.lifecycle_state}" - ) - - def shutdown_node(self, node_name: str) -> None: - instance = self._get_provider_vm(node_name) - if instance.lifecycle_state != OciInstanceState.STOPPED.value: - log.info(f"Powering off OCI instance {node_name}") - self._instance_action(instance, OciInstanceAction.STOP) - - else: - log.warning( - f"Attempted to power off node {node_name}, " - f"but the instance is already off - lifecycle_state={instance.lifecycle_state}" - ) - - def _instance_action(self, instance: Instance, action: OciInstanceAction): - response = self._provider_client.instance_action(instance_id=instance.id, action=action.value) - assert response.status == 200, f"Failed to {action.value.lower()} {instance.display_name} OCI instance" - - def restart_node(self, node_name: str) -> None: - log.info(f"Restarting OCI instance {node_name}") - instance = self._get_provider_vm(node_name) - self._instance_action(instance, OciInstanceAction.SOFTRESET) - - def is_active(self, node_name: str) -> bool: - instance = self._get_provider_vm(node_name) - return instance.lifecycle_state == OciInstanceState.RUNNING.value - - def get_cpu_cores(self, node_name: str) -> int: - return self._get_vm(node_name)["attributes"]["shape_config"][0]["ocpus"] - - def get_ram_kib(self, node_name: str) -> int: - return self._get_vm(node_name)["attributes"]["shape_config"][0]["memory_in_gbs"] * 1024 * 1024 - - def get_node_ips_and_macs(self, node_name) -> Tuple[List[str], List[str]]: - vm_attributes = self._get_vm(node_name)["attributes"] - instance_id = vm_attributes["id"] - - vnics = self._provider_client.list_vnic_attachments( - self._config.oci_compartment_oicd, instance_id=instance_id - ).data - mac_addresses = [self._virtual_network_client.get_vnic(vnic.vnic_id).data.mac_address for vnic in vnics] - ip_addresses = [self._virtual_network_client.get_vnic(vnic.vnic_id).data.private_ip for vnic in vnics] - - return ip_addresses, mac_addresses - - def set_dns(self, api_ip: str, ingress_ip: str) -> None: - return diff --git a/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py b/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py index 278cbe2a131..f1a0b49a870 100644 --- a/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py +++ b/src/assisted_test_infra/test_infra/helper_classes/base_cluster.py @@ -38,6 +38,11 @@ def __init__( @property def id(self) -> str: + if not self._config.cluster_id: + clusters = self.api_client.clusters_list() + found = [c for c in clusters if c["name"] == self._config.cluster_name] + self._config.cluster_id = found[0]["id"] if found else None + return self._config.cluster_id def get_details(self) -> Union[models.infra_env.InfraEnv, models.cluster.Cluster]: @@ -153,13 +158,12 @@ def set_hostnames_and_roles(self): nodes = self.nodes.get_nodes(refresh=True) for host in hosts: - name = self.find_matching_node_name(host, nodes) - assert name is not None, ( + node = self.find_matching_node(host, nodes) + assert node is not None, ( f"Failed to find matching node for host with mac address {host.macs()}" f" nodes: {[(n.name, n.ips, n.macs) for n in nodes]}" ) - role = consts.NodeRoles.MASTER if consts.NodeRoles.MASTER in name else consts.NodeRoles.WORKER - self._infra_env.update_host(host_id=host.get_id(), host_role=role, host_name=name) + self._infra_env.update_host(host_id=host.get_id(), host_role=node.role, host_name=node.name) def set_installer_args(self): hosts = self.to_cluster_hosts(self.api_client.get_cluster_hosts(self.id)) @@ -170,21 +174,12 @@ def set_installer_args(self): def to_cluster_hosts(hosts: list[dict[str, Any]]) -> list[ClusterHost]: return [ClusterHost(models.Host(**h)) for h in hosts] - def find_matching_node_name(self, host: ClusterHost, nodes: list[Node]) -> Union[str, None]: + def find_matching_node(self, host: ClusterHost, nodes: list[Node]) -> Optional[Node]: # Looking for node matches the given host by its mac address (which is unique) for node in nodes: for mac in node.macs: if mac.lower() in host.macs(): - return node.name - - # IPv6 static ips - if self._infra_env_config.is_static_ip: - mappings = static_network.get_name_to_mac_addresses_mapping(self.nodes.controller.tf_folder) - for mac in host.macs(): - for name, macs in mappings.items(): - if mac in macs: - return name - + return node return None @JunitTestCase() diff --git a/src/assisted_test_infra/test_infra/helper_classes/cluster.py b/src/assisted_test_infra/test_infra/helper_classes/cluster.py index 8edb3f148c6..0f883c30629 100644 --- a/src/assisted_test_infra/test_infra/helper_classes/cluster.py +++ b/src/assisted_test_infra/test_infra/helper_classes/cluster.py @@ -958,8 +958,7 @@ def prepare_nodes(self, is_static_ip: bool = False, **kwargs): def create_custom_manifests(self): log.info(f"Adding {len(self._config.custom_manifests)} custom manifests") for local_manifest in self._config.custom_manifests: - with open(local_manifest.local_path, "rb") as f: - encoded_content = base64.b64encode(f.read()).decode("utf-8", "ignore") + encoded_content = base64.b64encode(local_manifest.content.encode("utf-8")).decode("utf-8", "ignore") manifest = self.create_custom_manifest(local_manifest.folder, local_manifest.file_name, encoded_content) diff --git a/src/assisted_test_infra/test_infra/utils/manifests.py b/src/assisted_test_infra/test_infra/utils/manifests.py index 2b9ae8b22b6..b6ac5ef89ac 100644 --- a/src/assisted_test_infra/test_infra/utils/manifests.py +++ b/src/assisted_test_infra/test_infra/utils/manifests.py @@ -10,7 +10,7 @@ class Manifest: folder: str file_name: str - local_path: Path + content: str def is_folder_allowed(self) -> bool: return self.folder in self.__ALLOWED_FOLDERS @@ -20,16 +20,17 @@ def get_allowed_folders(self) -> List[str]: @classmethod def get_manifests(cls, path: Path) -> List["Manifest"]: - manifests_files = [] - + manifest_files = [] if path.is_dir(): for file_type in ("yaml", "yml", "json"): - manifests_files.extend(list(path.rglob(f"*.{file_type}"))) + manifest_files.extend(list(path.rglob(f"*.{file_type}"))) else: - manifests_files.append(path) + manifest_files.append(path) manifests = [] - for file in manifests_files: - manifests.append(Manifest(folder=file.parent.name, file_name=file.name, local_path=file)) + for manifest in manifest_files: + with open(manifest, "rb") as f: + content = f.read() + manifests.append(Manifest(folder=manifest.parent.name, file_name=manifest.name, content=content)) return manifests diff --git a/terraform_files/oci-ci-machine/00_main.tf b/terraform_files/oci-ci-machine/00_main.tf index cbf2eb4fb8e..1ae9e824290 100644 --- a/terraform_files/oci-ci-machine/00_main.tf +++ b/terraform_files/oci-ci-machine/00_main.tf @@ -2,7 +2,7 @@ terraform { required_providers { oci = { source = "oracle/oci" - version = "5.45.0" + version = "6.23.0" } cloudinit = { source = "hashicorp/cloudinit" diff --git a/terraform_files/oci-ci-machine/01_networking.tf b/terraform_files/oci-ci-machine/01_networking.tf index 54b03a15370..e589699c21e 100644 --- a/terraform_files/oci-ci-machine/01_networking.tf +++ b/terraform_files/oci-ci-machine/01_networking.tf @@ -1,99 +1,42 @@ -module "vcn" { - source = "oracle-terraform-modules/vcn/oci" - version = "3.5.4" - # insert the 5 required variables here +locals { + all_protocols = "all" + anywhere = "0.0.0.0/0" +} - # Required Inputs +resource "oci_core_vcn" "ci_machine_vcn" { + cidr_blocks = [ + "10.0.0.0/16", + ] compartment_id = var.oci_compartment_id - - internet_gateway_route_rules = null - local_peering_gateways = null - nat_gateway_route_rules = null - - # Optional Inputs - vcn_name = "vcn-ci-${var.unique_id}" - vcn_dns_label = "v${substr(var.unique_id, -14, -1)}" # dns label is limited to 15 chacracters - vcn_cidrs = ["10.0.0.0/16"] - - create_internet_gateway = true - create_nat_gateway = true + display_name = "vcn-ci-${var.unique_id}" + dns_label = "v${substr(var.unique_id, -14, -1)}" # dns label is limited to 15 chacracters } -resource "oci_core_security_list" "private_security_list" { - - # Required +resource "oci_core_internet_gateway" "internet_gateway" { compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - - # Optional - display_name = "security-list-for-private-subnet" - - egress_security_rules { - stateless = false - destination = "0.0.0.0/0" - destination_type = "CIDR_BLOCK" - protocol = "all" - } + display_name = "InternetGateway" + vcn_id = oci_core_vcn.ci_machine_vcn.id +} - ingress_security_rules { - stateless = false - source = "10.0.0.0/16" - source_type = "CIDR_BLOCK" - # Get protocol numbers from https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml TCP is 6 - protocol = "6" - tcp_options { - min = 22 - max = 22 - } - } - ingress_security_rules { - stateless = false - source = "0.0.0.0/0" - source_type = "CIDR_BLOCK" - # Get protocol numbers from https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml ICMP is 1 - protocol = "1" +resource "oci_core_route_table" "public_routes" { + compartment_id = var.oci_compartment_id + vcn_id = oci_core_vcn.ci_machine_vcn.id + display_name = "public" - # For ICMP type and code see: https://www.iana.org/assignments/icmp-parameters/icmp-parameters.xhtml - icmp_options { - type = 3 - code = 4 - } - } - ingress_security_rules { - stateless = false - source = "10.0.0.0/16" - source_type = "CIDR_BLOCK" - # Get protocol numbers from https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml ICMP is 1 - protocol = "1" - - # For ICMP type and code see: https://www.iana.org/assignments/icmp-parameters/icmp-parameters.xhtml - icmp_options { - type = 3 - } + route_rules { + destination = local.anywhere + destination_type = "CIDR_BLOCK" + network_entity_id = oci_core_internet_gateway.internet_gateway.id } } -resource "oci_core_security_list" "public_security_list" { - - # Required +resource "oci_core_security_list" "public" { compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - - # Optional - display_name = "security-list-for-public-subnet" - - egress_security_rules { - stateless = false - destination = "0.0.0.0/0" - destination_type = "CIDR_BLOCK" - protocol = "all" - } + display_name = "public" + vcn_id = oci_core_vcn.ci_machine_vcn.id ingress_security_rules { - stateless = false - source = "0.0.0.0/0" - source_type = "CIDR_BLOCK" - # Get protocol numbers from https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml TCP is 6 + source = local.anywhere protocol = "6" tcp_options { min = 22 @@ -101,58 +44,39 @@ resource "oci_core_security_list" "public_security_list" { } } ingress_security_rules { - stateless = false - source = "0.0.0.0/0" - source_type = "CIDR_BLOCK" - # Get protocol numbers from https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml ICMP is 1 - protocol = "1" - - # For ICMP type and code see: https://www.iana.org/assignments/icmp-parameters/icmp-parameters.xhtml - icmp_options { - type = 3 - code = 4 + source = local.anywhere + protocol = "6" + tcp_options { + min = 8080 + max = 8080 } } ingress_security_rules { - stateless = false - source = "10.0.0.0/16" - source_type = "CIDR_BLOCK" - # Get protocol numbers from https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml ICMP is 1 - protocol = "1" - - # For ICMP type and code see: https://www.iana.org/assignments/icmp-parameters/icmp-parameters.xhtml - icmp_options { - type = 3 + source = local.anywhere + protocol = "6" + tcp_options { + min = 8090 + max = 8090 } } + egress_security_rules { + destination = local.anywhere + protocol = local.all_protocols + } } -resource "oci_core_subnet" "vcn_private_subnet" { - - # Required +resource "oci_core_subnet" "public" { + cidr_block = "10.0.0.0/24" + display_name = "public" compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - cidr_block = "10.0.1.0/24" - dns_label = "private" + vcn_id = oci_core_vcn.ci_machine_vcn.id + route_table_id = oci_core_route_table.public_routes.id - # Optional - # Caution: For the route table id, use module.vcn.nat_route_id. - # Do not use module.vcn.nat_gateway_id, because it is the OCID for the gateway and not the route table. - route_table_id = module.vcn.nat_route_id - security_list_ids = [oci_core_security_list.private_security_list.id] - display_name = "private-subnet" -} + security_list_ids = [ + oci_core_security_list.public.id, + ] -resource "oci_core_subnet" "vcn_public_subnet" { - - # Required - compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - cidr_block = "10.0.0.0/24" - dns_label = "public" - - # Optional - route_table_id = module.vcn.ig_route_id - security_list_ids = [oci_core_security_list.public_security_list.id] - display_name = "public-subnet" + dns_label = "public" + prohibit_public_ip_on_vnic = false } + diff --git a/terraform_files/oci-ci-machine/02_compute.tf b/terraform_files/oci-ci-machine/02_compute.tf new file mode 100644 index 00000000000..004e10cfc56 --- /dev/null +++ b/terraform_files/oci-ci-machine/02_compute.tf @@ -0,0 +1,76 @@ +data "oci_identity_availability_domains" "ads" { + compartment_id = var.oci_compartment_id +} + +# Use cloud init to configure root user +data "cloudinit_config" "config" { + part { + content_type = "text/cloud-config" + + content = yamlencode({ + "users" : [ + { + "name" : "root", + "ssh-authorized-keys" : [ + file(var.public_ssh_key_path) + ] + } + ] + }) + } +} + +resource "oci_core_instance" "ci_instance" { + # Required + availability_domain = data.oci_identity_availability_domains.ads.availability_domains[0].name + compartment_id = var.oci_compartment_id + shape = "VM.Standard.E5.Flex" + + shape_config { + memory_in_gbs = 16 + ocpus = 4 + } + + source_details { + # source_id = data.oci_core_app_catalog_listing_resource_version.os_catalog_listing.listing_resource_id + source_id = var.operating_system_source_id + source_type = "image" + boot_volume_size_in_gbs = 500 + boot_volume_vpus_per_gb = 30 + } + + # Optional + display_name = "ci-instance-${var.unique_id}" + + create_vnic_details { + assign_public_ip = true + assign_private_dns_record = true + hostname_label = "ci-instance" + subnet_id = oci_core_subnet.public.id + } + metadata = { + user_data = data.cloudinit_config.config.rendered + } + preserve_boot_volume = false + + # wait an ssh connection and wait for cloud-init to complete + connection { + type = "ssh" + user = "root" + host = self.public_ip + timeout = "5m" + private_key = file(var.private_ssh_key_path) + } + + provisioner "remote-exec" { + inline = [ + # Wait for cloud-init to complete. + "cloud-init status --wait || true" + ] + } + lifecycle { + ignore_changes = [ + source_details[0].source_id, + ] + } +} diff --git a/terraform_files/oci-ci-machine/02_network_security_groups.tf b/terraform_files/oci-ci-machine/02_network_security_groups.tf deleted file mode 100644 index 8117b2b6ae5..00000000000 --- a/terraform_files/oci-ci-machine/02_network_security_groups.tf +++ /dev/null @@ -1,133 +0,0 @@ -# Create security groups for the future cluster -# CI machine should be able to reach: -# - LB on public IP -# - cluster nodes in private subnet (SSH) -# cluster nodes should be able to reach: -# - CI machine (assisted-service/image-service) -# Prow should be able to reach: -# - CI machine on SSH - -# cluster NSG is hold by all clusters nodes -resource "oci_core_network_security_group" "nsg_cluster_ci" { - #Required - compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - - #Optional - display_name = "cluster-ci" -} - -# cluster-access is hold by LB and CI machine -resource "oci_core_network_security_group" "nsg_cluster_ci_access" { - #Required - compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - - #Optional - display_name = "cluster-ci-access" -} - -# all instances holding cluster-access NSG can reach cluster -resource "oci_core_network_security_group_security_rule" "rule_allow_from_nsg_cluster_ci_access" { - network_security_group_id = oci_core_network_security_group.nsg_cluster_ci.id - direction = "INGRESS" - source_type = "NETWORK_SECURITY_GROUP" - source = oci_core_network_security_group.nsg_cluster_ci_access.id - protocol = "all" -} - -# ci-machine is hold by CI machine -resource "oci_core_network_security_group" "nsg_ci_machine" { - #Required - compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - - #Optional - display_name = "ci-machine" -} - -# ci-machine-access is hold bu cluster nodes -resource "oci_core_network_security_group" "nsg_ci_machine_access" { - #Required - compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - - #Optional - display_name = "ci-machine-access" -} - -# all instances holding ci-machine-access NSG can reach ci-machine -resource "oci_core_network_security_group_security_rule" "rule_allow_from_nsg_ci_machine_access" { - network_security_group_id = oci_core_network_security_group.nsg_ci_machine.id - direction = "INGRESS" - source_type = "NETWORK_SECURITY_GROUP" - source = oci_core_network_security_group.nsg_ci_machine_access.id - protocol = "all" -} - -# Allow Prow to connect on CI machine -resource "oci_core_network_security_group_security_rule" "rule_allow_from_prow_to_ci_machine" { - network_security_group_id = oci_core_network_security_group.nsg_ci_machine.id - direction = "INGRESS" - source = "0.0.0.0/0" - protocol = "6" - tcp_options { - destination_port_range { - min = 22 - max = 22 - } - } -} - -# load-balancer is hold by LB -resource "oci_core_network_security_group" "nsg_load_balancer_ci" { - #Required - compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - - #Optional - display_name = "load-balancer-ci" -} - -# load-balancer-access is hold by cluster nodes and CI machine -resource "oci_core_network_security_group" "nsg_load_balancer_ci_access" { - #Required - compartment_id = var.oci_compartment_id - vcn_id = module.vcn.vcn_id - - #Optional - display_name = "load-balancer-ci-access" -} - -# all instances holding load-balancer-access NSG can reach load-balancer -resource "oci_core_network_security_group_security_rule" "rule_allow_from_nsg_load_balancer_access" { - network_security_group_id = oci_core_network_security_group.nsg_load_balancer_ci.id - direction = "INGRESS" - source_type = "NETWORK_SECURITY_GROUP" - source = oci_core_network_security_group.nsg_load_balancer_ci_access.id - protocol = "all" -} - -locals { - nat_ip = one([for attr in module.vcn.nat_gateway_all_attributes : attr.nat_ip]) -} - -# ci-machine reach load-balancer with its public IP -resource "oci_core_network_security_group_security_rule" "rule_allow_from_public_ci_machine" { - network_security_group_id = oci_core_network_security_group.nsg_load_balancer_ci.id - description = "Allow traffic from ci-machine" - direction = "INGRESS" - source_type = "CIDR_BLOCK" - source = "${oci_core_instance.ci_instance.public_ip}/32" - protocol = "all" -} - -# all private instances behind NAT can reach load-balancer -resource "oci_core_network_security_group_security_rule" "rule_allow_from_public_nat_gateway" { - network_security_group_id = oci_core_network_security_group.nsg_load_balancer_ci.id - description = "Allow traffic from NAT gateway" - direction = "INGRESS" - source_type = "CIDR_BLOCK" - source = "${local.nat_ip}/32" - protocol = "all" -} diff --git a/terraform_files/oci-ci-machine/03_compute.tf b/terraform_files/oci-ci-machine/03_compute.tf deleted file mode 100644 index 2f0cdf59cc7..00000000000 --- a/terraform_files/oci-ci-machine/03_compute.tf +++ /dev/null @@ -1,135 +0,0 @@ -data "oci_identity_availability_domains" "ads" { - compartment_id = var.oci_compartment_id -} - -# Fetch Rocky Linux 9.x OS image from OCI marketpalce -# See https://blogs.oracle.com/cloud-infrastructure/post/using-terraform-for-marketplace-images -data "oci_marketplace_listings" "os_listings" { - category = ["Operating Systems"] - pricing = ["FREE"] - package_type = "IMAGE" - compartment_id = var.oci_compartment_id - filter { - name = "name" - values = ["Rocky Linux 9\\.\\d+ Supported by CIQ - Free \\(x86_64\\)"] - regex = true - } -} - -data "oci_marketplace_listing" "os_listing" { - listing_id = data.oci_marketplace_listings.os_listings.listings[0].id - compartment_id = var.oci_compartment_id -} - -data "oci_marketplace_listing_package" "os_listing_package" { - listing_id = data.oci_marketplace_listing.os_listing.id - package_version = data.oci_marketplace_listing.os_listing.default_package_version - compartment_id = var.oci_compartment_id -} - -data "oci_core_app_catalog_listing_resource_version" "os_catalog_listing" { - listing_id = data.oci_marketplace_listing_package.os_listing_package.app_catalog_listing_id - resource_version = data.oci_marketplace_listing_package.os_listing_package.app_catalog_listing_resource_version -} - -data "oci_marketplace_listing_package_agreements" "os_listing_package_agreements" { - listing_id = data.oci_marketplace_listing.os_listing.id - package_version = data.oci_marketplace_listing.os_listing.default_package_version - compartment_id = var.oci_compartment_id -} - -# Sign agreement to use Rocky Linux from the OCI marketpalce -resource "oci_marketplace_accepted_agreement" "os_accepted_agreement" { - agreement_id = oci_marketplace_listing_package_agreement.os_listing_package_agreement.agreement_id - compartment_id = var.oci_compartment_id - listing_id = data.oci_marketplace_listing.os_listing.id - package_version = data.oci_marketplace_listing.os_listing.default_package_version - signature = oci_marketplace_listing_package_agreement.os_listing_package_agreement.signature -} -resource "oci_marketplace_listing_package_agreement" "os_listing_package_agreement" { - agreement_id = data.oci_marketplace_listing_package_agreements.os_listing_package_agreements.agreements[0].id - listing_id = data.oci_marketplace_listing.os_listing.id - package_version = data.oci_marketplace_listing.os_listing.default_package_version -} - -# Use cloud init to configure root user -data "cloudinit_config" "config" { - part { - content_type = "text/cloud-config" - - content = yamlencode({ - "users" : [ - { - "name" : "root", - "ssh-authorized-keys" : [ - file(var.public_ssh_key_path) - ] - } - ] - }) - } -} - -resource "oci_core_instance" "ci_instance" { - # Required - availability_domain = data.oci_identity_availability_domains.ads.availability_domains[0].name - compartment_id = var.oci_compartment_id - shape = "VM.Standard3.Flex" - - shape_config { - memory_in_gbs = 16 - ocpus = 4 - } - - platform_config { - type = "INTEL_VM" - are_virtual_instructions_enabled = true - } - - source_details { - source_id = data.oci_core_app_catalog_listing_resource_version.os_catalog_listing.listing_resource_id - source_type = "image" - boot_volume_size_in_gbs = 500 - boot_volume_vpus_per_gb = 30 - } - - # Optional - display_name = "ci-instance-${var.unique_id}" - - create_vnic_details { - assign_public_ip = true - assign_private_dns_record = true - hostname_label = "ci-instance" - subnet_id = oci_core_subnet.vcn_public_subnet.id - nsg_ids = [ - oci_core_network_security_group.nsg_ci_machine.id, - oci_core_network_security_group.nsg_load_balancer_ci_access.id, # allow access to load balancer - oci_core_network_security_group.nsg_cluster_ci_access.id # allow access to cluster (SSH) - ] - } - metadata = { - user_data = data.cloudinit_config.config.rendered - } - preserve_boot_volume = false - - # wait an ssh connection and wait for cloud-init to complete - connection { - type = "ssh" - user = "root" - host = self.public_ip - timeout = "5m" - private_key = file(var.private_ssh_key_path) - } - - provisioner "remote-exec" { - inline = [ - # Wait for cloud-init to complete. - "cloud-init status --wait || true" - ] - } - lifecycle { - ignore_changes = [ - source_details[0].source_id, - ] - } -} diff --git a/terraform_files/oci-ci-machine/output.tf b/terraform_files/oci-ci-machine/output.tf index 9c67e092535..9499c02bd94 100644 --- a/terraform_files/oci-ci-machine/output.tf +++ b/terraform_files/oci-ci-machine/output.tf @@ -6,14 +6,3 @@ output "ci_machine_inventory" { "user" : "root", } } - -output "infra" { - value = { - "oci_vcn_id" : module.vcn.vcn_id, - "oci_private_subnet_id" : oci_core_subnet.vcn_private_subnet.id, - "oci_public_subnet_id" : oci_core_subnet.vcn_public_subnet.id, - "oci_ci_machine_access_nsg_id" : oci_core_network_security_group.nsg_ci_machine_access.id, - "oci_cluster_ci_nsg_id" : oci_core_network_security_group.nsg_cluster_ci.id, - "oci_load_balancer_ci_nsg_id" : oci_core_network_security_group.nsg_load_balancer_ci.id - } -} diff --git a/terraform_files/oci-ci-machine/variables.tf b/terraform_files/oci-ci-machine/variables.tf index 0df1071bb02..dae49019cbf 100644 --- a/terraform_files/oci-ci-machine/variables.tf +++ b/terraform_files/oci-ci-machine/variables.tf @@ -13,6 +13,17 @@ variable "public_ssh_key_path" { description = "Path to public key" } +# Rocky Linux image ID +# +# Best way to find it is to spawn an instance with Rocky Linux and then copy +# the image ID that is in use. It is best to pin it, as the agreement system in +# OCI is painful to get right programmatically. +variable "operating_system_source_id" { + type = string + default = "ocid1.image.oc1..aaaaaaaauo3kyxlty6himw7uecc4xdzshdnc43qf4q2uyvy32gi3t3ixg5pa" + description = "Base OS image ID being used to provision the CI machine" +} + /////////// // OCI variables ///////////