SSLCertVerificationError with skip_tls=True

[asharma1304]

In the Fine-Tune Llama Models with Ray and DeepSpeed on OpenShift AI demo, when I execute the following step, I get the SSLCertVerification error. This is an on-prem cluster running 4.16.x.

From the example code:

# Authenticate the CodeFlare SDK
# On OpenShift, you can retrieve the token by running `oc whoami -t`,
# and the server with `oc cluster-info`.
auth = TokenAuthentication(
    token='sha256~AyvA....',
    server='https://api.ac10-ocp.fpb.local:6443',
    skip_tls=True
)
auth.login()`

I get the following error:

Insecure request warnings have been disabled
---------------------------------------------------------------------------
SSLCertVerificationError                  Traceback (most recent call last)
File /opt/app-root/lib64/python3.9/site-packages/urllib3/connectionpool.py:716, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    715 # Make the request on the httplib connection object.
--> 716 httplib_response = self._make_request(
    717     conn,
    718     method,
    719     url,
    720     timeout=timeout_obj,
    721     body=body,
    722     headers=headers,
    723     chunked=chunked,
    724 )
    726 # If we're going to release the connection in ``finally:``, then
    727 # the response doesn't need to know about the connection. Otherwise
    728 # it will also try to release it and we'll have a double-release
    729 # mess.

File /opt/app-root/lib64/python3.9/site-packages/urllib3/connectionpool.py:404, in HTTPConnectionPool._make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    403 try:
--> 404     self._validate_conn(conn)
    405 except (SocketTimeout, BaseSSLError) as e:
    406     # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.

File /opt/app-root/lib64/python3.9/site-packages/urllib3/connectionpool.py:1061, in HTTPSConnectionPool._validate_conn(self, conn)
   1060 if not getattr(conn, "sock", None):  # AppEngine might not have  `.sock`
-> 1061     conn.connect()
   1063 if not conn.is_verified:

File /opt/app-root/lib64/python3.9/site-packages/urllib3/connection.py:419, in HTTPSConnection.connect(self)
    417     context.load_default_certs()
--> 419 self.sock = ssl_wrap_socket(
    420     sock=conn,
    421     keyfile=self.key_file,
    422     certfile=self.cert_file,
    423     key_password=self.key_password,
    424     ca_certs=self.ca_certs,
    425     ca_cert_dir=self.ca_cert_dir,
    426     ca_cert_data=self.ca_cert_data,
    427     server_hostname=server_hostname,
    428     ssl_context=context,
    429     tls_in_tls=tls_in_tls,
    430 )
    432 # If we're using all defaults and the connection
    433 # is TLSv1 or TLSv1.1 we throw a DeprecationWarning
    434 # for the host.

File /opt/app-root/lib64/python3.9/site-packages/urllib3/util/ssl_.py:458, in ssl_wrap_socket(sock, keyfile, certfile, cert_reqs, ca_certs, server_hostname, ssl_version, ciphers, ssl_context, ca_cert_dir, key_password, ca_cert_data, tls_in_tls)
    457 if send_sni:
--> 458     ssl_sock = _ssl_wrap_socket_impl(
    459         sock, context, tls_in_tls, server_hostname=server_hostname
    460     )
    461 else:

File /opt/app-root/lib64/python3.9/site-packages/urllib3/util/ssl_.py:502, in _ssl_wrap_socket_impl(sock, ssl_context, tls_in_tls, server_hostname)
    501 if server_hostname:
--> 502     return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
    503 else:

File /usr/lib64/python3.9/ssl.py:501, in SSLContext.wrap_socket(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session)
    495 def wrap_socket(self, sock, server_side=False,
    496                 do_handshake_on_connect=True,
    497                 suppress_ragged_eofs=True,
    498                 server_hostname=None, session=None):
    499     # SSLSocket class handles server_hostname encoding before it calls
    500     # ctx._wrap_socket()
--> 501     return self.sslsocket_class._create(
    502         sock=sock,
    503         server_side=server_side,
    504         do_handshake_on_connect=do_handshake_on_connect,
    505         suppress_ragged_eofs=suppress_ragged_eofs,
    506         server_hostname=server_hostname,
    507         context=self,
    508         session=session
    509     )

File /usr/lib64/python3.9/ssl.py:1074, in SSLSocket._create(cls, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, context, session)
   1073             raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
-> 1074         self.do_handshake()
   1075 except (OSError, ValueError):

File /usr/lib64/python3.9/ssl.py:1343, in SSLSocket.do_handshake(self, block)
   1342         self.settimeout(None)
-> 1343     self._sslobj.do_handshake()
   1344 finally:

SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1129)

During handling of the above exception, another exception occurred:

MaxRetryError                             Traceback (most recent call last)
Cell In[3], line 9
      1 # Authenticate the CodeFlare SDK
      2 # On OpenShift, you can retrieve the token by running `oc whoami -t`,
      3 # and the server with `oc cluster-info`.
      4 auth = TokenAuthentication(
      5     token='sha256~Ayv.....',
      6     server='https://api.ac10-ocp.fpb.local:6443/',
      7     skip_tls=True
      8 )
----> 9 auth.login()

File /opt/app-root/lib64/python3.9/site-packages/codeflare_sdk/cluster/auth.py:120, in TokenAuthentication.login(self)
    117     print("Insecure request warnings have been disabled")
    118     configuration.verify_ssl = False
--> 120 client.AuthenticationApi(api_client).get_api_group()
    121 config_path = None
    122 return "Logged into %s" % self.server

File /opt/app-root/lib64/python3.9/site-packages/kubernetes/client/api/authentication_api.py:61, in AuthenticationApi.get_api_group(self, **kwargs)
     40 """get_api_group  # noqa: E501
     41 
     42 get information of a group  # noqa: E501
   (...)
     58          returns the request thread.
     59 """
     60 kwargs['_return_http_data_only'] = True
---> 61 return self.get_api_group_with_http_info(**kwargs)

File /opt/app-root/lib64/python3.9/site-packages/kubernetes/client/api/authentication_api.py:128, in AuthenticationApi.get_api_group_with_http_info(self, **kwargs)
    125 # Authentication setting
    126 auth_settings = ['BearerToken']  # noqa: E501
--> 128 return self.api_client.call_api(
    129     '/apis/authentication.k8s.io/', 'GET',
    130     path_params,
    131     query_params,
    132     header_params,
    133     body=body_params,
    134     post_params=form_params,
    135     files=local_var_files,
    136     response_type='V1APIGroup',  # noqa: E501
    137     auth_settings=auth_settings,
    138     async_req=local_var_params.get('async_req'),
    139     _return_http_data_only=local_var_params.get('_return_http_data_only'),  # noqa: E501
    140     _preload_content=local_var_params.get('_preload_content', True),
    141     _request_timeout=local_var_params.get('_request_timeout'),
    142     collection_formats=collection_formats)

File /opt/app-root/lib64/python3.9/site-packages/kubernetes/client/api_client.py:348, in ApiClient.call_api(self, resource_path, method, path_params, query_params, header_params, body, post_params, files, response_type, auth_settings, async_req, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host)
    311 """Makes the HTTP request (synchronous) and returns deserialized data.
    312 
    313 To make an async_req request, set the async_req parameter.
   (...)
    345     then the method will return the response directly.
    346 """
    347 if not async_req:
--> 348     return self.__call_api(resource_path, method,
    349                            path_params, query_params, header_params,
    350                            body, post_params, files,
    351                            response_type, auth_settings,
    352                            _return_http_data_only, collection_formats,
    353                            _preload_content, _request_timeout, _host)
    355 return self.pool.apply_async(self.__call_api, (resource_path,
    356                                                method, path_params,
    357                                                query_params,
   (...)
    365                                                _request_timeout,
    366                                                _host))

File /opt/app-root/lib64/python3.9/site-packages/kubernetes/client/api_client.py:180, in ApiClient.__call_api(self, resource_path, method, path_params, query_params, header_params, body, post_params, files, response_type, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host)
    177     url = _host + resource_path
    179 # perform request and return response
--> 180 response_data = self.request(
    181     method, url, query_params=query_params, headers=header_params,
    182     post_params=post_params, body=body,
    183     _preload_content=_preload_content,
    184     _request_timeout=_request_timeout)
    186 self.last_response = response_data
    188 return_data = response_data

File /opt/app-root/lib64/python3.9/site-packages/kubernetes/client/api_client.py:373, in ApiClient.request(self, method, url, query_params, headers, post_params, body, _preload_content, _request_timeout)
    371 """Makes the HTTP request using RESTClient."""
    372 if method == "GET":
--> 373     return self.rest_client.GET(url,
    374                                 query_params=query_params,
    375                                 _preload_content=_preload_content,
    376                                 _request_timeout=_request_timeout,
    377                                 headers=headers)
    378 elif method == "HEAD":
    379     return self.rest_client.HEAD(url,
    380                                  query_params=query_params,
    381                                  _preload_content=_preload_content,
    382                                  _request_timeout=_request_timeout,
    383                                  headers=headers)

File /opt/app-root/lib64/python3.9/site-packages/kubernetes/client/rest.py:241, in RESTClientObject.GET(self, url, headers, query_params, _preload_content, _request_timeout)
    239 def GET(self, url, headers=None, query_params=None, _preload_content=True,
    240         _request_timeout=None):
--> 241     return self.request("GET", url,
    242                         headers=headers,
    243                         _preload_content=_preload_content,
    244                         _request_timeout=_request_timeout,
    245                         query_params=query_params)

File /opt/app-root/lib64/python3.9/site-packages/kubernetes/client/rest.py:214, in RESTClientObject.request(self, method, url, query_params, headers, body, post_params, _preload_content, _request_timeout)
    211             raise ApiException(status=0, reason=msg)
    212     # For `GET`, `HEAD`
    213     else:
--> 214         r = self.pool_manager.request(method, url,
    215                                       fields=query_params,
    216                                       preload_content=_preload_content,
    217                                       timeout=timeout,
    218                                       headers=headers)
    219 except urllib3.exceptions.SSLError as e:
    220     msg = "{0}\n{1}".format(type(e).__name__, str(e))

File /opt/app-root/lib64/python3.9/site-packages/urllib3/request.py:77, in RequestMethods.request(self, method, url, fields, headers, **urlopen_kw)
     74 urlopen_kw["request_url"] = url
     76 if method in self._encode_url_methods:
---> 77     return self.request_encode_url(
     78         method, url, fields=fields, headers=headers, **urlopen_kw
     79     )
     80 else:
     81     return self.request_encode_body(
     82         method, url, fields=fields, headers=headers, **urlopen_kw
     83     )

File /opt/app-root/lib64/python3.9/site-packages/urllib3/request.py:99, in RequestMethods.request_encode_url(self, method, url, fields, headers, **urlopen_kw)
     96 if fields:
     97     url += "?" + urlencode(fields)
---> 99 return self.urlopen(method, url, **extra_kw)

File /opt/app-root/lib64/python3.9/site-packages/urllib3/poolmanager.py:376, in PoolManager.urlopen(self, method, url, redirect, **kw)
    374     response = conn.urlopen(method, url, **kw)
    375 else:
--> 376     response = conn.urlopen(method, u.request_uri, **kw)
    378 redirect_location = redirect and response.get_redirect_location()
    379 if not redirect_location:

File /opt/app-root/lib64/python3.9/site-packages/urllib3/connectionpool.py:830, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    825 if not conn:
    826     # Try again
    827     log.warning(
    828         "Retrying (%r) after connection broken by '%r': %s", retries, err, url
    829     )
--> 830     return self.urlopen(
    831         method,
    832         url,
    833         body,
    834         headers,
    835         retries,
    836         redirect,
    837         assert_same_host,
    838         timeout=timeout,
    839         pool_timeout=pool_timeout,
    840         release_conn=release_conn,
    841         chunked=chunked,
    842         body_pos=body_pos,
    843         **response_kw
    844     )
    846 # Handle redirect?
    847 redirect_location = redirect and response.get_redirect_location()

File /opt/app-root/lib64/python3.9/site-packages/urllib3/connectionpool.py:830, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    825 if not conn:
    826     # Try again
    827     log.warning(
    828         "Retrying (%r) after connection broken by '%r': %s", retries, err, url
    829     )
--> 830     return self.urlopen(
    831         method,
    832         url,
    833         body,
    834         headers,
    835         retries,
    836         redirect,
    837         assert_same_host,
    838         timeout=timeout,
    839         pool_timeout=pool_timeout,
    840         release_conn=release_conn,
    841         chunked=chunked,
    842         body_pos=body_pos,
    843         **response_kw
    844     )
    846 # Handle redirect?
    847 redirect_location = redirect and response.get_redirect_location()

File /opt/app-root/lib64/python3.9/site-packages/urllib3/connectionpool.py:830, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    825 if not conn:
    826     # Try again
    827     log.warning(
    828         "Retrying (%r) after connection broken by '%r': %s", retries, err, url
    829     )
--> 830     return self.urlopen(
    831         method,
    832         url,
    833         body,
    834         headers,
    835         retries,
    836         redirect,
    837         assert_same_host,
    838         timeout=timeout,
    839         pool_timeout=pool_timeout,
    840         release_conn=release_conn,
    841         chunked=chunked,
    842         body_pos=body_pos,
    843         **response_kw
    844     )
    846 # Handle redirect?
    847 redirect_location = redirect and response.get_redirect_location()

File /opt/app-root/lib64/python3.9/site-packages/urllib3/connectionpool.py:802, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    799 elif isinstance(e, (SocketError, HTTPException)):
    800     e = ProtocolError("Connection aborted.", e)
--> 802 retries = retries.increment(
    803     method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
    804 )
    805 retries.sleep()
    807 # Keep track of the error for the retry warning.

File /opt/app-root/lib64/python3.9/site-packages/urllib3/util/retry.py:594, in Retry.increment(self, method, url, response, error, _pool, _stacktrace)
    583 new_retry = self.new(
    584     total=total,
    585     connect=connect,
   (...)
    590     history=history,
    591 )
    593 if new_retry.is_exhausted():
--> 594     raise MaxRetryError(_pool, url, error or ResponseError(cause))
    596 log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
    598 return new_retry

MaxRetryError: HTTPSConnectionPool(host='api.ac10-ocp.fpb.local', port=6443): Max retries exceeded with url: /apis/authentication.k8s.io/ (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1129)')))