Keep sessions around between each ConanRequester (#17455)

* Keep sessions around between each ConanRequester

Avoid creating a new session for each ConanRequester instance, this
helps a lot with performance, as it avoids the overhead duplicate handshakes

* Patch new cached value on ConanRequester tests

* Modify adapter as needed

* Fix tests

* Name

* Update conans/client/rest/conan_requester.py

* Refactor requester tests

* Reinit is not part of this PR

* Refactor request usage out of app

* Make max_retries a local variable

conan/api/conan_api.py

@@ -41,9 +41,10 @@ def __init__(self, cache_folder=None):
         migrator = ClientMigrator(self.cache_folder, conan_version)
         migrator.migrate()
 
-        self.command = CommandAPI(self)
+        self.config = ConfigAPI(self)
         self.remotes = RemotesAPI(self)
-        # Search recipes by wildcard and packages filtering by configuracion
+        self.command = CommandAPI(self)
+        # Search recipes by wildcard and packages filtering by configuration
         self.search = SearchAPI(self)
         # Get latest refs and list refs of recipes and packages
         self.list = ListAPI(self)
@@ -52,7 +53,6 @@ def __init__(self, cache_folder=None):
         self.graph = GraphAPI(self)
         self.export = ExportAPI(self)
         self.remove = RemoveAPI(self)
-        self.config = ConfigAPI(self)
         self.new = NewAPI(self)
         self.upload = UploadAPI(self)
         self.download = DownloadAPI(self)

conan/api/subapi/config.py

@@ -36,8 +36,9 @@ def install(self, path_or_url, verify_ssl, config_type=None, args=None,
                 source_folder=None, target_folder=None):
         # TODO: We probably want to split this into git-folder-http cases?
         from conan.internal.api.config.config_installer import configuration_install
-        app = ConanApp(self.conan_api)
-        configuration_install(app, path_or_url, verify_ssl, config_type=config_type, args=args,
+        cache_folder = self.conan_api.cache_folder
+        requester = self.conan_api.remotes.requester
+        configuration_install(cache_folder, requester, path_or_url, verify_ssl, config_type=config_type, args=args,
                               source_folder=source_folder, target_folder=target_folder)
 
     def install_pkg(self, ref, lockfile=None, force=False, remotes=None, profile=None):
@@ -90,7 +91,9 @@ def install_pkg(self, ref, lockfile=None, force=False, remotes=None, profile=Non
                     return pkg.pref  # Already installed, we can skip repeating the install
 
         from conan.internal.api.config.config_installer import configuration_install
-        configuration_install(app, uri=pkg.conanfile.package_folder, verify_ssl=False,
+        cache_folder = self.conan_api.cache_folder
+        requester = self.conan_api.remotes.requester
+        configuration_install(cache_folder, requester, uri=pkg.conanfile.package_folder, verify_ssl=False,
                               config_type="dir", ignore=["conaninfo.txt", "conanmanifest.txt"])
         # We save the current package full reference in the file for future
         # And for ``package_id`` computation

conan/api/subapi/remotes.py

@@ -7,6 +7,7 @@
 from conan.api.output import ConanOutput
 from conan.internal.cache.home_paths import HomePaths
 from conan.internal.conan_app import ConanApp
+from conans.client.rest.conan_requester import ConanRequester
 from conans.client.rest_client_local_recipe_index import add_local_recipes_index_remote, \
     remove_local_recipes_index_remote
 from conan.internal.api.remotes.localdb import LocalDB
@@ -31,6 +32,8 @@ def __init__(self, conan_api):
         self.conan_api = conan_api
         self._home_folder = conan_api.home_folder
         self._remotes_file = HomePaths(self._home_folder).remotes_path
+        # Wraps an http_requester to inject proxies, certs, etc
+        self._requester = ConanRequester(self.conan_api.config.global_conf, self.conan_api.cache_folder)
 
     def list(self, pattern=None, only_enabled=True):
         """
@@ -253,6 +256,10 @@ def user_auth(self, remote: Remote, with_user=False, force=False):
         user, token, _ = localdb.get_login(remote.url)
         return user
 
+    @property
+    def requester(self):
+        return self._requester
+
 
 def _load(remotes_file):
     if not os.path.exists(remotes_file):

conan/api/subapi/upload.py

@@ -113,9 +113,8 @@ def upload_backup_sources(self, files):
             output.info("No backup sources files to upload")
             return files
 
-        app = ConanApp(self.conan_api)
-        # TODO: verify might need a config to force it to False
-        uploader = FileUploader(app.requester, verify=True, config=config, source_credentials=True)
+        requester = self.conan_api.remotes.requester
+        uploader = FileUploader(requester, verify=True, config=config, source_credentials=True)
         # TODO: For Artifactory, we can list all files once and check from there instead
         #  of 1 request per file, but this is more general
         for file in files:

conan/internal/api/config/config_installer.py

@@ -193,10 +193,8 @@ def _is_compressed_file(filename):
     return any(filename.endswith(e) for e in tgz_exts)
 
 
-def configuration_install(app, uri, verify_ssl, config_type=None,
+def configuration_install(cache_folder, requester, uri, verify_ssl, config_type=None,
                           args=None, source_folder=None, target_folder=None, ignore=None):
-    requester = app.requester
-    cache_folder = app.cache_folder
     config = _ConfigOrigin(uri, config_type, verify_ssl, args, source_folder, target_folder)
     try:
         if config.type == "git":

conan/internal/conan_app.py

@@ -46,13 +46,9 @@ def __init__(self, conan_api):
         cache_folder = conan_api.home_folder
         self.cache_folder = cache_folder
         self.cache = PkgCache(self.cache_folder, global_conf)
-
-        # Wraps an http_requester to inject proxies, certs, etc
-        self.requester = ConanRequester(global_conf, cache_folder)
-        # To handle remote connections
         # Wraps RestApiClient to add authentication support (same interface)
         localdb = LocalDB(cache_folder)
-        auth_manager = ConanApiAuthManager(self.requester, cache_folder, localdb, global_conf)
+        auth_manager = ConanApiAuthManager(conan_api.remotes.requester, cache_folder, localdb, global_conf)
         # Handle remote connections
         self.remote_manager = RemoteManager(self.cache, auth_manager, cache_folder)
         global_editables = conan_api.local.editable_packages
@@ -71,6 +67,6 @@ def __init__(self, conan_api):
 
         self.pyreq_loader = PyRequireLoader(self, self.global_conf)
         cmd_wrap = CmdWrapper(HomePaths(self.cache_folder).wrapper_path)
-        conanfile_helpers = ConanFileHelpers(self.requester, cmd_wrap, self.global_conf, self.cache,
-                                             self.cache_folder)
+        conanfile_helpers = ConanFileHelpers(conan_api.remotes.requester, cmd_wrap, self.global_conf,
+                                             self.cache, self.cache_folder)
         self.loader = ConanFileLoader(self.pyreq_loader, conanfile_helpers)

conan/test/utils/tools.py

@@ -277,6 +277,16 @@ def _set_auth_headers(kwargs):
                 kwargs["headers"] = {}
             kwargs["headers"].update(mock_request.headers)
 
+    def mount(self, *args, **kwargs):
+        pass
+
+    def Session(self):
+        return self
+
+    @property
+    def codes(self):
+        return requests.codes
+
 
 class TestServer(object):
     def __init__(self, read_permissions=None,

conans/client/rest/conan_requester.py

@@ -100,18 +100,12 @@ def add_auth(self, url, kwargs):
 class ConanRequester:
 
     def __init__(self, config, cache_folder=None):
-        # TODO: Make all this lazy, to avoid fully configuring Requester, for every api call
-        #  even if it doesn't use it
-        # FIXME: Trick for testing when requests is mocked
-        if hasattr(requests, "Session"):
-            self._http_requester = requests.Session()
-            adapter = HTTPAdapter(max_retries=self._get_retries(config))
-            self._http_requester.mount("http://", adapter)
-            self._http_requester.mount("https://", adapter)
-        else:
-            self._http_requester = requests
-
         self._url_creds = _SourceURLCredentials(cache_folder)
+        _max_retries = config.get("core.net.http:max_retries", default=2, check_type=int)
+        self._http_requester = requests.Session()
+        _adapter = HTTPAdapter(max_retries=self._get_retries(_max_retries))
+        self._http_requester.mount("http://", _adapter)
+        self._http_requester.mount("https://", _adapter)
         self._timeout = config.get("core.net.http:timeout", default=DEFAULT_TIMEOUT)
         self._no_proxy_match = config.get("core.net.http:no_proxy_match", check_type=list)
         self._proxies = config.get("core.net.http:proxies")
@@ -125,8 +119,8 @@ def __init__(self, config, cache_folder=None):
         self._user_agent = "Conan/%s (%s)" % (__version__, platform_info)
 
     @staticmethod
-    def _get_retries(config):
-        retry = config.get("core.net.http:max_retries", default=2, check_type=int)
+    def _get_retries(max_retries):
+        retry = max_retries
         if retry == 0:
             return 0
         retry_status_code_set = {

test/integration/command/remote_verify_ssl_test.py

@@ -1,5 +1,6 @@
 import unittest
 
+import requests
 from requests.models import Response
 
 from conan.test.utils.tools import TestClient
@@ -19,6 +20,15 @@ def get(self, url, *args, **kwargs):
         assert "cacert.pem" in kwargs["verify"], "TEST FAILURE: cacert.pem not in verify kwarg"
         return resp
 
+    def Session(self):
+        return self
+
+    @property
+    def codes(self):
+        return requests.codes
+
+    def mount(self, *args, **kwargs):
+        pass
 
 class RequesterMockFalse(object):
 
@@ -29,6 +39,16 @@ def get(self, url, *args, **kwargs):
         assert kwargs["verify"] is False, "TEST FAILURE: verify arg is not False"
         return resp
 
+    def Session(self):
+        return self
+
+    @property
+    def codes(self):
+        return requests.codes
+
+    def mount(self, *args, **kwargs):
+        pass
+
 
 class VerifySSLTest(unittest.TestCase):
 

test/integration/command/remove_test.py

@@ -356,9 +356,9 @@ def test_package_query_no_package_ref(populated_client):
 
 def _get_all_packages(client, ref, with_remote):
     ref = RecipeReference.loads(ref)
-    api = ConanAPI(client.cache_folder)
-    remote = api.remotes.get("default") if with_remote else None
     with client.mocked_servers():
+        api = ConanAPI(client.cache_folder)
+        remote = api.remotes.get("default") if with_remote else None
         try:
             return set([r.repr_notime() for r in api.list.packages_configurations(ref, remote=remote)])
         except NotFoundException:
@@ -367,9 +367,9 @@ def _get_all_packages(client, ref, with_remote):
 
 def _get_revisions_recipes(client, ref, with_remote):
     ref = RecipeReference.loads(ref)
-    api = ConanAPI(client.cache_folder)
-    remote = api.remotes.get("default") if with_remote else None
     with client.mocked_servers():
+        api = ConanAPI(client.cache_folder)
+        remote = api.remotes.get("default") if with_remote else None
         try:
             return set([r.repr_notime() for r in api.list.recipe_revisions(ref, remote=remote)])
         except NotFoundException:
@@ -378,9 +378,9 @@ def _get_revisions_recipes(client, ref, with_remote):
 
 def _get_revisions_packages(client, pref, with_remote):
     pref = PkgReference.loads(pref)
-    api = ConanAPI(client.cache_folder)
-    remote = api.remotes.get("default") if with_remote else None
     with client.mocked_servers():
+        api = ConanAPI(client.cache_folder)
+        remote = api.remotes.get("default") if with_remote else None
         try:
             return set([r.repr_notime() for r in api.list.package_revisions(pref, remote=remote)])
         except NotFoundException:

test/integration/conan_api/search_test.py

@@ -24,11 +24,11 @@ def test_search_recipes(remote_name):
 
     client.run("upload * -r=default -c")
 
-    # Search all the recipes locally and in the remote
-    api = ConanAPI(client.cache_folder)
-    remote = api.remotes.get(remote_name) if remote_name else None
-
     with client.mocked_servers():
+        # Search all the recipes locally and in the remote
+        api = ConanAPI(client.cache_folder)
+        remote = api.remotes.get(remote_name) if remote_name else None
+
         sot = api.search.recipes(query="f*", remote=remote)
         assert sot == [RecipeReference.loads("felipe/1.0"),
                        RecipeReference.loads("felipe/2.0"),