Merge branch 'release/0.17.0-beta2'

README.md

@@ -1040,8 +1040,8 @@ This is especially useful for CI integration.
 - **MINGW_CONFIGURATIONS**: Specify a list of MinGW builds. See MinGW builds section.
 - **CONAN_BASH_PATH**: Path to a bash executable. Used only in windows to help the tools.run_in_windows_bash() function to locate our Cygwin/MSYS2 bash.
   Set it with the bash executable path if it’s not in the PATH or you want to use a different one.
-- **CONAN_USE_SUDO** Use "sudo" when invoking docker and pip, by default it will use sudo when not using Windows. "False" to deactivate.
-- **CONAN_DOCKER_USE_SUDO** Same as CONAN_USE_SUDO for retro compatibility with previous versions.
+- **CONAN_PIP_USE_SUDO** Use "sudo" when invoking pip, by default it will use sudo when not using Windows. "False" to deactivate.
+- **CONAN_DOCKER_USE_SUDO** Use "sudo" when invoking docker, by default it will use sudo when not using Windows. "False" to deactivate.
 - **CONAN_ALLOW_GCC_MINORS** Declare this variable if you want to allow gcc >=5 versions with the minor (5.1, 6.3 etc).
 - **CONAN_EXCLUDE_VCVARS_PRECOMMAND** For Visual Studio builds, it exclude the vcvars call to set the environment.
 - **CONAN_BUILD_REQUIRES** You can specify additional build requires for the generated profile with an environment variable following the same profile syntax and separated by ","

conan/packager.py

@@ -2,4 +2,4 @@
 # noinspection PyUnresolvedReferences
 from cpt.packager import ConanMultiPackager
 # noinspection PyUnresolvedReferences
-from cpt.builds_generator import split_colon_env
+from cpt.tools import *

cpt/__init__.py

@@ -1,2 +1,2 @@
 
-__version__ = '0.17.0.b1'
+__version__ = '0.17.0.b2'

cpt/builds_generator.py

@@ -4,6 +4,7 @@
 
 from conans.model.ref import ConanFileReference
 from conans.model.version import Version
+from cpt.tools import split_colon_env
 
 default_gcc_versions = ["4.9", "5", "6", "7"]
 default_clang_versions = ["3.8", "3.9", "4.0"]
@@ -32,10 +33,6 @@ def get_mingw_config_from_env():
     return ret
 
 
-def split_colon_env(varname):
-    return [a.strip() for a in list(filter(None, os.getenv(varname, "").split(",")))]
-
-
 class BuildGenerator(object):
 
     def __init__(self, reference, os_name, gcc_versions, apple_clang_versions, clang_versions,

cpt/packager.py

@@ -82,6 +82,7 @@ def __init__(self, args=None, username=None, channel=None, runner=None,
                  build_policy=None,
                  always_update_conan_in_docker=False,
                  conan_api=None,
+                 ci_manager=None,
                  out=None):
 
         self.printer = Printer(out)
@@ -93,7 +94,7 @@ def __init__(self, args=None, username=None, channel=None, runner=None,
         else:
             self.conan_api = conan_api
 
-        self.ci_manager = CIManager(self.printer)
+        self.ci_manager = ci_manager or CIManager(self.printer)
         self.remotes_manager = RemotesManager(self.conan_api, self.printer, remotes, upload)
         self.username = username or os.getenv("CONAN_USERNAME", None)
 
@@ -129,9 +130,8 @@ def __init__(self, args=None, username=None, channel=None, runner=None,
                 name, version = self.partial_reference.split("/")
                 self.reference = ConanFileReference(name, version, self.username, self.channel)
         else:
-            print(os.getcwd())
             if not os.path.exists("conanfile.py"):
-                raise Exception("Conanfile not found")
+                raise Exception("Conanfile not found, specify a 'reference' parameter with name and version")
             conanfile = load_conanfile_class("./conanfile.py")
             name, version = conanfile.name, conanfile.version
             if not name or not version:
@@ -159,15 +159,17 @@ def __init__(self, args=None, username=None, channel=None, runner=None,
 
         self.build_policy = build_policy
 
-        self.sudo_command = ""
+        self.sudo_docker_command = ""
         if "CONAN_DOCKER_USE_SUDO" in os.environ:
-            if get_bool_from_env("CONAN_DOCKER_USE_SUDO"):
-                self.sudo_command = "sudo"
-        elif "CONAN_USE_SUDO" in os.environ:
-            if get_bool_from_env("CONAN_USE_SUDO"):
-                self.sudo_command = "sudo"
+            self.sudo_docker_command = "sudo" if get_bool_from_env("CONAN_DOCKER_USE_SUDO") else ""
         elif platform.system() != "Windows":
-            self.sudo_command = "sudo"
+            self.sudo_docker_command = "sudo"
+
+        self.sudo_pip_command = ""
+        if "CONAN_PIP_USE_SUDO" in os.environ:
+            self.sudo_pip_command = "sudo" if get_bool_from_env("CONAN_PIP_USE_SUDO") else ""
+        elif platform.system() != "Windows":
+            self.sudo_pip_command = "sudo"
 
         self.exclude_vcvars_precommand = exclude_vcvars_precommand or os.getenv("CONAN_EXCLUDE_VCVARS_PRECOMMAND", False)
         self._docker_image_skip_update = docker_image_skip_update or os.getenv("CONAN_DOCKER_IMAGE_SKIP_UPDATE", False)
@@ -309,9 +311,9 @@ def run(self, profile_name=None):
             if not self.skip_check_credentials and self._upload_enabled():
                 self.remotes_manager.add_remotes_to_conan()
                 self.auth_manager.login(self.remotes_manager.upload_remote_name)
-            if self.conan_pip_package:
+            if self.conan_pip_package and not self.use_docker:
                 with self.printer.foldable_output("pip_update"):
-                    self.runner('%s pip install %s' % (self.sudo_command, self.conan_pip_package))
+                    self.runner('%s pip install %s' % (self.sudo_pip_command, self.conan_pip_package))
 
             self.run_builds(profile_name=profile_name)
 
@@ -384,7 +386,8 @@ def run_builds(self, curpage=None, total_pages=None, profile_name=None):
                                                   runner=self.runner,
                                                   docker_image=docker_image,
                                                   docker_image_skip_update=self._docker_image_skip_update,
-                                                  sudo_docker_command=self.sudo_command,
+                                                  sudo_docker_command=self.sudo_docker_command,
+                                                  sudo_pip_command=self.sudo_pip_command,
                                                   always_update_conan_in_docker=self._always_update_conan_in_docker,
                                                   upload=self._upload_enabled())
 
@@ -436,16 +439,23 @@ def _autodetect_docker_base_image(compiler_name, compiler_version):
         return "lasote/conan%s%s" % (compiler_name, compiler_version.replace(".", ""))
 
     def _get_channel(self, specified_channel, stable_channel):
+        if self.stable_branch_pattern:
+            stable_patterns = [self.stable_branch_pattern]
+        else:
+            stable_patterns = ["master$", "release*", "stable*"]
 
-        pattern = self.stable_branch_pattern or "master"
-        prog = re.compile(pattern)
         branch = self.ci_manager.get_branch()
         self.printer.print_message("Branch detected", branch)
 
-        if branch and prog.match(branch):
-            self.printer.print_message("Info", "Redefined channel by CI branch matching with '%s', "
-                                       "setting CONAN_CHANNEL to '%s'" % (pattern, stable_channel))
-            return stable_channel
+        for pattern in stable_patterns:
+            prog = re.compile(pattern)
+
+            if branch and prog.match(branch):
+                self.printer.print_message("Info",
+                                           "Redefined channel by CI branch matching with '%s', "
+                                           "setting CONAN_CHANNEL to '%s'" % (pattern,
+                                                                              stable_channel))
+                return stable_channel
 
         return specified_channel
 

cpt/remotes.py

@@ -105,21 +105,34 @@ def _get_remote_from_str(the_str, var_name, name=None):
         else:
             raise Exception("Invalid %s env var format, check README" % var_name)
 
-    def _get_remote_name(self, remote_url):
+    def _get_remote_by_url(self, remote_url):
         remotes = self._conan_api.remote_list()
         for remote in remotes:
             if remote.url == remote_url:
-                return remote.name
+                return remote.name, remotes
+        return None, remotes
+
+    def _get_remote_by_name(self, remotes, name):
+        for remote in remotes:
+            if remote.name == name:
+                return remote
         return None
 
     def _add_remote(self, url, verify_ssl, name, insert=False):
 
-        remote = self._get_remote_name(url)
+        remote, remote_list = self._get_remote_by_url(url)
         if remote:
             self.printer.print_message("Remote for URL '%s' already exist, "
                                        "keeping the current remote and its name" % url)
             return remote
 
+        remote = self._get_remote_by_name(remote_list, name)
+        # If name is duplicated, but the url is not equal, remove it before adding it
+        # A rename won't be good, because the remote is really different, it happens in local
+        # when "upload_repo" is kept
+        if remote:
+            self._conan_api.remote_remove(name)
+
         self._conan_api.remote_add(name, url, verify_ssl=verify_ssl, insert=insert)
         return name
 

cpt/runner.py

@@ -78,16 +78,20 @@ def run(self):
                 # TODO: Get uploaded packages with Conan 1.3 from the ret json
                 self.printer.print_message("Calling 'conan create'")
                 name, version, user, channel = self._reference
-                self._conan_api.create(self._abs_folder, name=name, version=version, user=user, channel=channel,
-                                       build_modes=self._build_policy,
-                                       profile_name=self._abs_profile_path)
+                # FIXME: chdir Can be removed in 1.3, fixed issue about api changing curdir
+                with tools.chdir(self._abs_folder):
+                    self._conan_api.create(".", name=name, version=version,
+                                           user=user, channel=channel,
+                                           build_modes=self._build_policy,
+                                           profile_name=self._abs_profile_path)
 
                 self._uploader.upload_packages(self._reference, self._upload)
 
 
 class DockerCreateRunner(CreateRunner):
     def __init__(self, profile_text, reference, conan_api, uploader, runner=None,
                  args=None, conan_pip_package=None, docker_image=None, sudo_docker_command=True,
+                 sudo_pip_command=True,
                  docker_image_skip_update=False, build_policy=None,
                  always_update_conan_in_docker=False,
                  upload=False):
@@ -101,20 +105,21 @@ def __init__(self, profile_text, reference, conan_api, uploader, runner=None,
         self._always_update_conan_in_docker = always_update_conan_in_docker
         self._docker_image_skip_update = docker_image_skip_update
         self._sudo_docker_command = sudo_docker_command
+        self._sudo_pip_command = sudo_pip_command
 
     def pip_update_conan_command(self):
         commands = []
         # Hack for testing when retriving cpt from artifactory repo
         if "conan-package-tools" not in self._conan_pip_package:
             commands.append("%s pip install conan_package_tools==%s "
-                            "--upgrade --no-cache" % (self._sudo_docker_command,
+                            "--upgrade --no-cache" % (self._sudo_pip_command,
                                                       package_tools_version))
 
         if self._conan_pip_package:
-            commands.append("%s pip install %s --no-cache" % (self._sudo_docker_command,
+            commands.append("%s pip install %s --no-cache" % (self._sudo_pip_command,
                                                               self._conan_pip_package))
         else:
-            commands.append("%s pip install conan --upgrade --no-cache" % self._sudo_docker_command)
+            commands.append("%s pip install conan --upgrade --no-cache" % self._sudo_pip_command)
 
         command = "&& ".join(commands)
         self.printer.print_command(command)

cpt/test/integration/basic_test.py

@@ -1,9 +1,13 @@
+import os
 import unittest
+import sys
 
-from future.moves import sys
+from conans import tools
+from conans.model.ref import ConanFileReference
 
 from cpt.test.integration.base import BaseTest
 from cpt.packager import ConanMultiPackager
+from cpt.test.unit.utils import MockCIManager
 
 
 class SimpleTest(BaseTest):
@@ -110,3 +114,41 @@ class Pkg(ConanFile):
         self.packager = ConanMultiPackager(username="lasote", reference="lib2/1.0")
         self.packager.add_common_builds(shared_option_name=False)
         self.assertNotIn("lib2:shared", self.packager.items[0].options)
+
+    def test_exported_files(self):
+        conanfile = """from conans import ConanFile
+
+class Pkg(ConanFile):
+    name = "lib"
+    version = "1.0"
+    settings = "os"
+    exports = "*"
+    exports_sources = "source*"
+
+"""
+        ci_manager = MockCIManager()
+        self.save_conanfile(conanfile)
+        tools.save(os.path.join(self.tmp_folder, "other_file"), "Dummy contents")
+        tools.save(os.path.join(self.tmp_folder, "source.cpp"), "Dummy contents")
+        self.packager = ConanMultiPackager(username="lasote", reference="lib/1.0", ci_manager=ci_manager)
+        self.packager.add({}, {}, {}, {})
+        self.packager.run()
+
+        pf = self.client_cache.export(ConanFileReference.loads("lib/1.0@lasote/testing"))
+        found_in_export = False
+        for exported in os.listdir(pf):
+            if "other_file" == exported:
+                found_in_export = True
+                break
+
+        self.assertTrue(found_in_export)
+
+        pf = self.client_cache.export_sources(ConanFileReference.loads("lib/1.0@lasote/testing"))
+        found_in_export_sources = False
+        for exported in os.listdir(pf):
+            if "source.cpp" == exported:
+                found_in_export_sources = True
+                break
+
+        self.assertTrue(found_in_export_sources)
+

cpt/test/integration/docker_test.py

@@ -1,6 +1,7 @@
 import unittest
 import sys
 
+import time
 from conans.client.conan_api import ConanAPIV1
 from conans.model.ref import ConanFileReference
 
@@ -10,14 +11,16 @@
 from cpt.test.integration.base import BaseTest, PYPI_TESTING_REPO, CONAN_UPLOAD_URL, \
     CONAN_UPLOAD_PASSWORD, CONAN_LOGIN_UPLOAD
 from cpt.packager import ConanMultiPackager
+from cpt.test.unit.utils import MockCIManager
 
 
 class DockerTest(BaseTest):
 
     @unittest.skipUnless(sys.platform.startswith("linux"), "Requires Linux")
     def test_docker(self):
         self.deploy_pip()
-
+        ci_manager = MockCIManager()
+        unique_ref = "zlib/%s" % str(time.time())
         conanfile = """from conans import ConanFile
 import os
 
@@ -39,20 +42,23 @@ class Pkg(ConanFile):
                                                gcc_versions=["6"],
                                                archs=["x86", "x86_64"],
                                                build_types=["Release"],
-                                               reference="zlib/1.2.2")
+                                               reference=unique_ref,
+                                               ci_manager=ci_manager)
             self.packager.add_common_builds()
             self.packager.run()
 
+        search_pattern = "%s*" % unique_ref
+        ref = ConanFileReference.loads("%s@lasote/mychannel" % unique_ref)
+
         # Remove from remote
-        self.assertEquals(len(self.api.search_recipes("zlib*", remote="upload_repo")), 1)
-        packages = self.api.search_packages(ConanFileReference.loads("zlib/1.2.2@lasote/mychannel"),
-                                                       remote="upload_repo")[0]
+        self.assertEquals(len(self.api.search_recipes(search_pattern, remote="upload_repo")), 1)
+        packages = self.api.search_packages(ref, remote="upload_repo")[0]
         self.assertEquals(len(packages), 2)
 
         self.api.authenticate(name=CONAN_LOGIN_UPLOAD, password=CONAN_UPLOAD_PASSWORD,
-                         remote="upload_repo")
-        self.api.remove("zlib*", remote="upload_repo", force=True)
-        self.assertEquals(self.api.search_recipes("zlib*"), [])
+                              remote="upload_repo")
+        self.api.remove(search_pattern, remote="upload_repo", force=True)
+        self.assertEquals(self.api.search_recipes(search_pattern), [])
 
         # Try upload only when stable, shouldn't upload anything
         with tools.environment_append({"CONAN_USE_DOCKER": "1",
@@ -67,8 +73,10 @@ class Pkg(ConanFile):
                                                gcc_versions=["6"],
                                                archs=["x86", "x86_64"],
                                                build_types=["Release"],
-                                               reference="zlib/1.2.2")
+                                               reference=unique_ref,
+                                               ci_manager=ci_manager)
             self.packager.add_common_builds()
             self.packager.run()
 
-        self.assertEquals(len(self.api.search_recipes("zlib*", remote="upload_repo")), 0)
+        self.assertEquals(len(self.api.search_recipes(search_pattern, remote="upload_repo")), 0)
+        self.api.remove(search_pattern, remote="upload_repo", force=True)

cpt/test/integration/remotes_test.py

@@ -0,0 +1,39 @@
+from cpt.printer import Printer
+from cpt.remotes import RemotesManager
+from cpt.test.integration.base import BaseTest
+
+
+class RemotesTest(BaseTest):
+
+    def test_duplicated_remotes_with_different_url(self):
+
+        self.api.remote_remove("conan-center")
+        self.api.remote_add("upload_repo", "url_different", True)
+
+        manager = RemotesManager(self.api, Printer(), remotes_input="url1@True@upload_repo",
+                                 upload_input="url1@True@upload_repo")
+
+        remotes = self.api.remote_list()
+        self.assertIsNotNone(manager._get_remote_by_name(remotes, "upload_repo"))
+
+        manager.add_remotes_to_conan()
+
+        self.assertEquals(len(self.api.remote_list()), 1)
+
+        manager.add_remotes_to_conan()
+
+        self.assertEquals(len(self.api.remote_list()), 1)
+
+    def test_duplicated_remotes_with_same_url(self):
+
+        self.api.remote_remove("conan-center")
+        self.api.remote_add("upload_repo", "url1", True)
+
+        manager = RemotesManager(self.api, Printer(), remotes_input="url1@True@upload_repo",
+                                 upload_input="url1@True@upload_repo")
+
+        remotes = self.api.remote_list()
+        self.assertIsNotNone(manager._get_remote_by_name(remotes, "upload_repo"))
+
+        manager.add_remotes_to_conan()
+        self.assertEquals(len(self.api.remote_list()), 1)