Skip to content

Commit

Permalink
Hot Fix: fix ocp upgrade (#972)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ebattat
ebattat authored Feb 6, 2025
1 parent 761b7cb commit 33a2150
Show file tree
Hide file tree
Showing 3 changed files with 35 additions and 50 deletions.
50 changes: 28 additions & 22 deletions benchmark_runner/common/clouds/BareMetal/bare_metal_operations.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,10 @@ class BareMetalOperations:
SHORT_TIMEOUT = 600

@typechecked
def __init__(self, user: str):
def __init__(self, user: str = None):
self._environment_variables_dict = environment_variables.environment_variables_dict
self._user = user
self._ocp_env_flavor = self._environment_variables_dict.get('ocp_env_flavor', '')
self._create_pod_ci_cmd = self._environment_variables_dict.get('create_pod_ci_cmd', '')
self._provision_kubeadmin_password_path = self._environment_variables_dict.get('provision_kubeadmin_password_path', '')
self._provision_kubeconfig_path = self._environment_variables_dict.get('provision_kubeconfig_path', '')
self._provision_installer_path = self._environment_variables_dict.get('provision_installer_path', '')
self._provision_installer_cmd = self._environment_variables_dict.get('provision_installer_cmd', '')
self._provision_installer_log = self._environment_variables_dict.get('provision_installer_log', '')
self._install_ocp_version = self._environment_variables_dict.get('install_ocp_version', '')
self._cluster_type = self._environment_variables_dict.get('cluster_type', '')
self._expected_nodes = self._environment_variables_dict.get('expected_nodes', '')
Expand All @@ -41,22 +35,30 @@ def __init__(self, user: str):
self._expected_nodes = ast.literal_eval(self._expected_nodes)
self._ocp_version_build = self._environment_variables_dict.get('ocp_version_build', '')
self._num_odf_disks = int(self._environment_variables_dict.get('num_odf_disk', 1))
self._provision_ip = self._environment_variables_dict.get('provision_ip', '')
self._provision_port = int(self._environment_variables_dict.get('provision_port', ''))
self._provision_timeout = int(self._environment_variables_dict.get('provision_timeout', ''))
self._container_private_key_path = self._environment_variables_dict.get('container_private_key_path', '')
self._connection_data = ConnectionData(host_name=self._provision_ip,
user_name=self._user,
port=self._provision_port,
timeout=self._provision_timeout,
ssh_key=self._container_private_key_path)
self._remote_ssh = RemoteSsh(self._connection_data)
if self._environment_variables_dict.get('github_repository_short', ''):
self._github_operations = GitHubOperations()
self._ssh = SSH()
self._cli = self._environment_variables_dict.get('cli', '')
self._upgrade_ocp_version = self._environment_variables_dict.get('upgrade_ocp_version', '')
self._upgrade_channel = self._environment_variables_dict.get('upgrade_channel', '')
self._timeout = int(self._environment_variables_dict.get('timeout', ''))
if user:
self._user = user
self._provision_kubeadmin_password_path = self._environment_variables_dict.get('provision_kubeadmin_password_path', '')
self._provision_kubeconfig_path = self._environment_variables_dict.get('provision_kubeconfig_path', '')
self._provision_installer_path = self._environment_variables_dict.get('provision_installer_path', '')
self._provision_installer_cmd = self._environment_variables_dict.get('provision_installer_cmd', '')
self._provision_installer_log = self._environment_variables_dict.get('provision_installer_log', '')
self._provision_ip = self._environment_variables_dict.get('provision_ip', '')
self._provision_port = int(self._environment_variables_dict.get('provision_port', ''))
self._provision_timeout = int(self._environment_variables_dict.get('provision_timeout', ''))
self._container_private_key_path = self._environment_variables_dict.get('container_private_key_path', '')
self._connection_data = ConnectionData(host_name=self._provision_ip,
user_name=self._user,
port=self._provision_port,
timeout=self._provision_timeout,
ssh_key=self._container_private_key_path)
self._remote_ssh = RemoteSsh(self._connection_data)
if self._environment_variables_dict.get('github_repository_short', ''):
self._github_operations = GitHubOperations()

def _get_kubeadmin_password(self):
"""
Expand Down Expand Up @@ -103,7 +105,7 @@ def _wait_for_upgrade_complete(self, oc: OC, sleep_time: int = SHORT_TIMEOUT):
current_wait_time = 0
logger.info(f'Waiting until the upgrade to version {self._upgrade_ocp_version} starts...')
oc.wait_for_ocp_upgrade_start(upgrade_version=self._upgrade_ocp_version)
while self._provision_timeout <= 0 or current_wait_time <= self._provision_timeout and oc.upgrade_in_progress():
while self._timeout <= 0 or current_wait_time <= self._timeout and oc.upgrade_in_progress():
logger.info(f'Waiting till OCP upgrade complete, waiting {int(current_wait_time / 60)} minutes')
# sleep for x seconds
time.sleep(sleep_time)
Expand Down Expand Up @@ -302,12 +304,16 @@ def verify_upgrade_complete(self, oc: OC):
raise OCPUpgradeFailed(status=oc.get_cluster_status())

@logger_time_stamp
def oc_login(self):
def oc_login(self, kubeadmin_password: str):
"""
This method login to the cluster with new credentials
:param kubeadmin_password
:return:
"""
oc = OC(kubeadmin_password=self._get_kubeadmin_password())
if kubeadmin_password:
oc = OC(kubeadmin_password=kubeadmin_password)
else:
oc = OC(kubeadmin_password=self._get_kubeadmin_password())
oc.login()
return oc

Expand Down
8 changes: 4 additions & 4 deletions benchmark_runner/main/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
ci_status = environment_variables_dict.get('ci_status', '')
install_ocp_version = environment_variables_dict.get('install_ocp_version', '')
upgrade_ocp_version = environment_variables_dict.get('upgrade_ocp_version', '')
kubeadmin_password = environment_variables_dict.get('kubeadmin_password', '')
odf_version = environment_variables_dict.get('odf_version', '')
lso_version = environment_variables_dict.get('lso_version', '')
cnv_version = environment_variables_dict.get('cnv_version', '')
Expand Down Expand Up @@ -163,9 +164,8 @@ def upgrade_ocp_bare_metal(step: str):
This method runs Bare Metal OCP upgrade
:return:
"""
bare_metal_operations = BareMetalOperations(user=provision_user)
bare_metal_operations.connect_to_provisioner()
oc = bare_metal_operations.oc_login()
bare_metal_operations = BareMetalOperations()
oc = bare_metal_operations.oc_login(kubeadmin_password=kubeadmin_password)
if step == 'run_bare_metal_ocp_upgrade':
if not bare_metal_operations.is_ocp_already_upgraded(oc):
bare_metal_operations.run_ocp_upgrade(oc)
Expand All @@ -180,7 +180,7 @@ def upgrade_ocp_bare_metal(step: str):
error_message = f'OCP {upgrade_ocp_version} upgrade failed'
logger.error(error_message)
raise RuntimeError(error_message)
bare_metal_operations.disconnect_from_provisioner()



@logger_time_stamp
Expand Down
27 changes: 3 additions & 24 deletions jenkins/PerfCI_Chaos/04_PerfCI_Chaos_Upgrade_OpenShift_Deployment/Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,20 @@ pipeline {
CNV_VERSION = credentials('perfci_upgrade_target_cnv_version')
LSO_VERSION = credentials('perfci_upgrade_target_lso_version')
ODF_VERSION = credentials('perfci_upgrade_target_odf_version')
PROVISION_USER = credentials('perfci_provision_user')
PROVISION_IP = credentials('perfci_provision_ip')
WAIT_FOR_UPGRADE_VERSION = credentials('perfci_upgrade_target_ocp_version')
UPGRADE_CHANNEL = credentials('perfci_upgrade_channel') // stable (default)/candidate
CONTACT_EMAIL1 = credentials('perfci_contact_email1')
PROVISION_PRIVATE_KEY_FILE = credentials('perfci_provision_private_key_file')
PROVISION_IP = credentials('perfci_provision_ip')
PROVISION_USER = credentials('perfci_provision_user')
CLUSTER_TYPE = credentials('perfci_cluster_type')
EXPECTED_NODES = credentials('perfci_expected_nodes')
OCP_CLIENT_VERSION = credentials('perfci_ocp_client_version')
QUAY_BENCHMARK_RUNNER_REPOSITORY = credentials('perfci_quay_repository')
WORKSPACE = '/home/jenkins'
PROVISION_PORT = 22
KUBEADMIN_PASSWORD_PATH = '/home/jenkins/.kube/kubeadmin-password'
KUBECONFIG_PATH = '/root/.kube/config'
PROVISION_INSTALLER_PATH = '/root/jetlag/./run_jetlag.sh'
PROVISION_INSTALLER_CMD = 'pushd /root/jetlag;/root/jetlag/./run_jetlag.sh 1>/dev/null 2>&1;popd'
PROVISION_INSTALLER_LOG = 'tail -100 /root/jetlag/jetlag.log'
PRIVATE_KEY_PATH = '/home/jenkins/.ssh/provision_private_key'
CONFIG_PATH = '/home/jenkins/.ssh/config'
CONTAINER_PRIVATE_KEY_PATH = '/root/.ssh/provision_private_key'
CONTAINER_CONFIG_PATH = '/root/.ssh/config'
ANSIBLE_TMP_PATH = '/root/.ansible/tmp/'
SCALE_NODES = credentials('perfci_scale_nodes')
Expand Down Expand Up @@ -57,7 +51,6 @@ pipeline {
WINDOWS_SERVER_2022_URL = credentials('perfci_windows_server_2022_url')
LSO_NODE = credentials('perfci_lso_node')
KUBEADMIN_PASSWORD = readFile('/home/jenkins/.kube/kubeadmin-password').trim()
CONTAINER_KUBECONFIG_PATH = '/home/jenkins/.kube/config'
WORKER_DISK_PREFIX = 'wwn-0x'
SAVE_ARTIFACTS_LOCAL = 'False'
ENABLE_PROMETHEUS_SNAPSHOT = 'False'
Expand Down Expand Up @@ -154,9 +147,6 @@ END
echo "Running deployment for ${upgrade_step}"
// deployment code
script {
withCredentials([string(credentialsId: 'perfci_provision_ip', variable: 'PROVISION_IP')]) {
withCredentials([string(credentialsId: 'perfci_provision_user', variable: 'PROVISION_USER')]) {
withCredentials([file(credentialsId: 'perfci_provision_private_key_file', variable: 'PROVISION_PRIVATE_KEY_FILE')]) {
sh """
sudo podman run --rm \
-e UPGRADE_OCP_VERSION="${UPGRADE_OCP_VERSION}" \
Expand All @@ -166,24 +156,13 @@ END
-e ODF_VERSION="${ODF_VERSION}" \
-e CNV_VERSION="${CNV_VERSION}" \
-e EXPECTED_NODES='${EXPECTED_NODES}' \
-e PROVISION_IP="${PROVISION_IP}" \
-e PROVISION_USER="${PROVISION_USER}" \
-e PROVISION_PORT="${PROVISION_PORT}" \
-e KUBEADMIN_PASSWORD="${KUBEADMIN_PASSWORD}" \
-e KUBECONFIG_PATH="${KUBECONFIG_PATH}" \
-e PROVISION_INSTALLER_PATH="${PROVISION_INSTALLER_PATH}" \
-e PROVISION_INSTALLER_CMD="${PROVISION_INSTALLER_CMD}" \
-e PROVISION_INSTALLER_LOG="${PROVISION_INSTALLER_LOG}" \
-e CONTAINER_PRIVATE_KEY_PATH="${CONTAINER_PRIVATE_KEY_PATH}" \
-e PROVISION_TIMEOUT="${PROVISION_TIMEOUT}" \
-e log_level="INFO" -v "${PRIVATE_KEY_PATH}":"${CONTAINER_PRIVATE_KEY_PATH}" \
-e TIMEOUT="${TIMEOUT}" \
-v "${KUBECONFIG_PATH}":"${KUBECONFIG_PATH}" \
--privileged "${QUAY_BENCHMARK_RUNNER_REPOSITORY}"
"""
}
}
}
}
}
}
Expand Down

0 comments on commit 33a2150

Please sign in to comment.