From 8f5ec330c99379f186723c70da1fdac07f985fa5 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 7 Dec 2020 22:24:59 +0100 Subject: [PATCH 001/132] Introduce support for reference repository paths ending with /${GIT_URL} to replace by url => funny dir subtree in filesystem --- .../plugins/gitclient/CliGitAPIImpl.java | 33 +++++++++++++++++++ .../plugins/gitclient/JGitAPIImpl.java | 21 ++++++++++++ .../plugins/gitclient/GitClientCloneTest.java | 15 +++++++++ 3 files changed, 69 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index 521958e87d..ab7904b88d 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -762,6 +762,39 @@ public void execute() throws GitException, InterruptedException { if (reference != null && !reference.isEmpty()) { File referencePath = new File(reference); + if (!referencePath.exists()) { + if (reference.endsWith("/${GIT_URL}")) { + // See also JGitAPIImpl.java and keep the two blocks in + // sync (TODO: refactor into a method both would use?) + // For mass-configured jobs, like Organization Folders, + // allow to support parameterized paths to many refrepos. + // End-users can set up webs of symlinks to same repos + // known by different URLs (and/or including their forks + // also cached in same index). Luckily all URL chars are + // valid parts of path name... in Unix... Maybe parse or + // escape chars for URLs=>paths with Windows in mind? + // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions + // Further ideas: beside "GIT_URL" other meta variables + // can be introduced, e.g. to escape non-ascii chars for + // portability? Support SHA or MD5 hashes of URLs as + // pathnames? + reference = reference.replaceAll("\\$\\{GIT_URL\\}$", url).replaceAll("/*$", "").replaceAll(".git$", ""); + referencePath = new File(reference); + if (!referencePath.exists()) { + // Normalize the URLs with or without .git suffix to + // be served by same dir with the refrepo contents + reference += ".git"; + referencePath = new File(reference); + } + // Note: both these logs are needed, they are used in selftest + if (referencePath.exists()) { + listener.getLogger().println("[WARNING] Parameterized reference path replaced with: " + reference); + } else { + listener.getLogger().println("[WARNING] Parameterized reference path replaced with: " + reference + " was not found"); + } + } + } + if (!referencePath.exists()) listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); else if (!referencePath.isDirectory()) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index a1ffa8acb1..4c34b9a386 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1406,6 +1406,27 @@ public void execute() throws GitException, InterruptedException { // the repository builder does not create the alternates file if (reference != null && !reference.isEmpty()) { File referencePath = new File(reference); + if (!referencePath.exists()) { + if (reference.endsWith("/${GIT_URL}")) { + // See comments in CliGitAPIImpl.java for details + // Keep the two blocks in sync (refactor into method?) + reference = reference.replaceAll("\\$\\{GIT_URL\\}$", url).replaceAll("/*$", "").replaceAll(".git$", ""); + referencePath = new File(reference); + if (!referencePath.exists()) { + // Normalize the URLs with or without .git suffix to + // be served by same dir with the refrepo contents + reference += ".git"; + referencePath = new File(reference); + } + // Note: both these logs are needed, they are used in selftest + if (referencePath.exists()) { + listener.getLogger().println("[WARNING] Parameterized reference path replaced with: " + reference); + } else { + listener.getLogger().println("[WARNING] Parameterized reference path replaced with: " + reference + " was not found"); + } + } + } + if (!referencePath.exists()) listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); else if (!referencePath.isDirectory()) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index c44a06689e..d273d46d13 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -238,6 +238,21 @@ public void test_clone_reference() throws Exception, IOException, InterruptedExc assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); } + @Test + public void test_clone_reference_parameterized() throws Exception, IOException, InterruptedException { + testGitClient.clone_().url(workspace.localMirror()).repositoryName("origin").reference(workspace.localMirror() + "/${GIT_URL}").execute(); + testGitClient.checkout().ref("origin/master").branch("master").execute(); + check_remote_url(workspace, testGitClient, "origin"); + // Verify JENKINS-46737 expected log message is written + String messages = StringUtils.join(handler.getMessages(), ";"); + assertThat("Reference repo name-parsing logged in: " + messages, handler.containsMessageSubstring("Parameterized reference path replaced with: "), is(true)); + // Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path structure + //assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); + //assertAlternateFilePointsToLocalMirror(); + //assertBranchesExist(testGitClient.getBranches(), "master"); + //assertNoObjectsInRepository(); + } + private static final String SRC_DIR = (new File(".")).getAbsolutePath(); @Test From 022bbc6d4f1258b7edd6573fb56be3f5601aa9a4 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 8 Dec 2020 16:58:37 +0100 Subject: [PATCH 002/132] Refactor: move findParameterizedReferenceRepository() into LegacyCompatibleGitAPIImpl.java so its logic (expected to grow in complexity) can be shared by both JGitAPIImpl.java and CliGitAPIImpl.java --- .../plugins/gitclient/CliGitAPIImpl.java | 88 ++++++++----------- .../plugins/gitclient/JGitAPIImpl.java | 75 ++++++++-------- .../gitclient/LegacyCompatibleGitAPIImpl.java | 50 +++++++++++ .../plugins/gitclient/GitClientCloneTest.java | 5 +- 4 files changed, 128 insertions(+), 90 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index ab7904b88d..18d8b56c93 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -761,64 +761,50 @@ public void execute() throws GitException, InterruptedException { } if (reference != null && !reference.isEmpty()) { - File referencePath = new File(reference); - if (!referencePath.exists()) { - if (reference.endsWith("/${GIT_URL}")) { - // See also JGitAPIImpl.java and keep the two blocks in - // sync (TODO: refactor into a method both would use?) - // For mass-configured jobs, like Organization Folders, - // allow to support parameterized paths to many refrepos. - // End-users can set up webs of symlinks to same repos - // known by different URLs (and/or including their forks - // also cached in same index). Luckily all URL chars are - // valid parts of path name... in Unix... Maybe parse or - // escape chars for URLs=>paths with Windows in mind? - // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions - // Further ideas: beside "GIT_URL" other meta variables - // can be introduced, e.g. to escape non-ascii chars for - // portability? Support SHA or MD5 hashes of URLs as - // pathnames? - reference = reference.replaceAll("\\$\\{GIT_URL\\}$", url).replaceAll("/*$", "").replaceAll(".git$", ""); - referencePath = new File(reference); - if (!referencePath.exists()) { - // Normalize the URLs with or without .git suffix to - // be served by same dir with the refrepo contents - reference += ".git"; - referencePath = new File(reference); - } + File referencePath = findParameterizedReferenceRepository(reference, url); + if (referencePath == null) { + listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + reference); + } else { + if (!referencePath.getName().equals(reference)) { // Note: both these logs are needed, they are used in selftest + String msg = "Parameterized reference path "; + msg += "'" + reference + "'"; + msg += " replaced with: "; + msg += "'" + referencePath.getName() + "'"; if (referencePath.exists()) { - listener.getLogger().println("[WARNING] Parameterized reference path replaced with: " + reference); + listener.getLogger().println("[WARNING] " + msg); } else { - listener.getLogger().println("[WARNING] Parameterized reference path replaced with: " + reference + " was not found"); + listener.getLogger().println("[WARNING] " + msg + " does not exist"); } + reference = referencePath.getName(); } - } - if (!referencePath.exists()) - listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); - else if (!referencePath.isDirectory()) - listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); - else { - // reference path can either be a normal or a base repository - File objectsPath = new File(referencePath, ".git/objects"); - if (!objectsPath.isDirectory()) { - // reference path is bare repo - objectsPath = new File(referencePath, "objects"); - } - if (!objectsPath.isDirectory()) - listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (not a git repo?): " + objectsPath); + if (!referencePath.exists()) + listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); + else if (!referencePath.isDirectory()) + listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); else { - File alternates = new File(workspace, ".git/objects/info/alternates"); - try (PrintWriter w = new PrintWriter(alternates, Charset.defaultCharset().toString())) { - String absoluteReference = objectsPath.getAbsolutePath().replace('\\', '/'); - listener.getLogger().println("Using reference repository: " + reference); - // git implementations on windows also use - w.print(absoluteReference); - } catch (UnsupportedEncodingException ex) { - listener.error("Default character set is an unsupported encoding"); - } catch (FileNotFoundException e) { - listener.error("Failed to setup reference"); + // reference path can either be a normal or a base repository + File objectsPath = new File(referencePath, ".git/objects"); + if (!objectsPath.isDirectory()) { + // reference path is bare repo + objectsPath = new File(referencePath, "objects"); + } + if (!objectsPath.isDirectory()) + listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (not a git repo?): " + objectsPath); + else { + // Go behind git's back to write a meta file in new workspace + File alternates = new File(workspace, ".git/objects/info/alternates"); + try (PrintWriter w = new PrintWriter(alternates, Charset.defaultCharset().toString())) { + String absoluteReference = objectsPath.getAbsolutePath().replace('\\', '/'); + listener.getLogger().println("Using reference repository: " + reference); + // git implementations on windows also use + w.print(absoluteReference); + } catch (UnsupportedEncodingException ex) { + listener.error("Default character set is an unsupported encoding"); + } catch (FileNotFoundException e) { + listener.error("Failed to setup reference"); + } } } } diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 4c34b9a386..323b386518 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1405,53 +1405,52 @@ public void execute() throws GitException, InterruptedException { // the repository builder does not create the alternates file if (reference != null && !reference.isEmpty()) { - File referencePath = new File(reference); - if (!referencePath.exists()) { - if (reference.endsWith("/${GIT_URL}")) { - // See comments in CliGitAPIImpl.java for details - // Keep the two blocks in sync (refactor into method?) - reference = reference.replaceAll("\\$\\{GIT_URL\\}$", url).replaceAll("/*$", "").replaceAll(".git$", ""); - referencePath = new File(reference); - if (!referencePath.exists()) { - // Normalize the URLs with or without .git suffix to - // be served by same dir with the refrepo contents - reference += ".git"; - referencePath = new File(reference); - } + // Note: keep in sync with similar logic in CliGitAPIImpl.java + File referencePath = findParameterizedReferenceRepository(reference, url); + if (referencePath == null) { + listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + reference); + } else { + if (!referencePath.getName().equals(reference)) { // Note: both these logs are needed, they are used in selftest + String msg = "Parameterized reference path "; + msg += "'" + reference + "'"; + msg += " replaced with: "; + msg += "'" + referencePath.getName() + "'"; if (referencePath.exists()) { - listener.getLogger().println("[WARNING] Parameterized reference path replaced with: " + reference); + listener.getLogger().println("[WARNING] " + msg); } else { - listener.getLogger().println("[WARNING] Parameterized reference path replaced with: " + reference + " was not found"); + listener.getLogger().println("[WARNING] " + msg + " does not exist"); } + reference = referencePath.getName(); } - } - if (!referencePath.exists()) - listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); - else if (!referencePath.isDirectory()) - listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); - else { - // reference path can either be a normal or a base repository - File objectsPath = new File(referencePath, ".git/objects"); - if (!objectsPath.isDirectory()) { - // reference path is bare repo - objectsPath = new File(referencePath, "objects"); - } - if (!objectsPath.isDirectory()) - listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (no git repo?): " + objectsPath); + if (!referencePath.exists()) + listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); + else if (!referencePath.isDirectory()) + listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); else { - try { - File alternates = new File(workspace, ".git/objects/info/alternates"); - String absoluteReference = objectsPath.getAbsolutePath().replace('\\', '/'); - listener.getLogger().println("Using reference repository: " + reference); - // git implementations on windows also use - try (PrintWriter w = new PrintWriter(alternates, "UTF-8")) { + // reference path can either be a normal or a base repository + File objectsPath = new File(referencePath, ".git/objects"); + if (!objectsPath.isDirectory()) { + // reference path is bare repo + objectsPath = new File(referencePath, "objects"); + } + if (!objectsPath.isDirectory()) + listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (no git repo?): " + objectsPath); + else { + // Go behind git's back to write a meta file in new workspace + try { + File alternates = new File(workspace, ".git/objects/info/alternates"); + String absoluteReference = objectsPath.getAbsolutePath().replace('\\', '/'); + listener.getLogger().println("Using reference repository: " + reference); // git implementations on windows also use - w.print(absoluteReference); + try (PrintWriter w = new PrintWriter(alternates, "UTF-8")) { + // git implementations on windows also use + w.print(absoluteReference); + } + } catch (FileNotFoundException e) { + listener.error("Failed to setup reference"); } - } catch (FileNotFoundException e) { - listener.error("Failed to setup reference"); } } } diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 24ec1b7289..8654e4f6a8 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -158,6 +158,56 @@ public void clone(RemoteConfig rc, boolean useShallowClone) throws GitException, clone(source, rc.getName(), useShallowClone, null); } + /** Yield the File object for the reference repository local filesystem + * pathname. Note that the provided string may be suffixed with expandable + * tokens which allow to store a filesystem structure of multiple small + * reference repositories instead of a big combined repository, while + * providing a single inheritable configuration string value. Callers + * can check whether the original path was used or mangled into another + * by comparing their "reference" with returned object's File.getName(). + * + * At some point this plugin might also maintain that filesystem structure. + */ + public File findParameterizedReferenceRepository(String reference, String url) { + if (reference == null || reference.isEmpty()) { + return null; + } + + File referencePath = new File(reference); + if (!referencePath.exists()) { + if (reference.endsWith("/${GIT_URL}")) { + // For mass-configured jobs, like Organization Folders, + // allow to support parameterized paths to many refrepos. + // End-users can set up webs of symlinks to same repos + // known by different URLs (and/or including their forks + // also cached in same index). Luckily all URL chars are + // valid parts of path name... in Unix... Maybe parse or + // escape chars for URLs=>paths with Windows in mind? + // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions + // Further ideas: beside "GIT_URL" other meta variables + // can be introduced, e.g. to escape non-ascii chars for + // portability? Support base64, SHA or MD5 hashes of URLs + // as pathnames? Normalize first (lowercase, .git, ...)? + reference = reference.replaceAll("\\$\\{GIT_URL\\}$", url).replaceAll("/*$", "").replaceAll(".git$", ""); + referencePath = null; // GC + referencePath = new File(reference); + } + } + + if (!referencePath.exists()) { + // Normalize the URLs with or without .git suffix to + // be served by same dir with the refrepo contents + reference += ".git"; + referencePath = null; // GC + referencePath = new File(reference); + } + + // Note that the referenced path may exist or not exist, in the + // latter case it is up to the caller to decide on course of action. + // Maybe create this dir to begin a reference repo (given the options)? + return referencePath; + } + /** {@inheritDoc} */ @Deprecated public List revListBranch(String branchId) throws GitException, InterruptedException { diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index d273d46d13..7fb6d08324 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -245,7 +245,10 @@ public void test_clone_reference_parameterized() throws Exception, IOException, check_remote_url(workspace, testGitClient, "origin"); // Verify JENKINS-46737 expected log message is written String messages = StringUtils.join(handler.getMessages(), ";"); - assertThat("Reference repo name-parsing logged in: " + messages, handler.containsMessageSubstring("Parameterized reference path replaced with: "), is(true)); + assertThat("Reference repo name-parsing logged in: " + messages, + handler.containsMessageSubstring("Parameterized reference path ") && + handler.containsMessageSubstring(" replaced with: "), + is(true)); // Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path structure //assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); //assertAlternateFilePointsToLocalMirror(); From 6210adbb38e47fa69666113e8daec65fcca170d5 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 8 Dec 2020 17:52:03 +0100 Subject: [PATCH 003/132] Add isParameterizedReferenceRepository() and support for parameterized ref-repos in submodule checkouts (only CliGitAPIImpl.java has it) --- .../plugins/gitclient/CliGitAPIImpl.java | 37 +++++++++++++++---- .../gitclient/LegacyCompatibleGitAPIImpl.java | 14 +++++++ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index 18d8b56c93..1743f98257 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -1396,13 +1396,15 @@ public void execute() throws GitException, InterruptedException { } } if ((ref != null) && !ref.isEmpty()) { - File referencePath = new File(ref); - if (!referencePath.exists()) - listener.getLogger().println("[WARNING] Reference path does not exist: " + ref); - else if (!referencePath.isDirectory()) - listener.getLogger().println("[WARNING] Reference path is not a directory: " + ref); - else - args.add("--reference", ref); + if (!isParameterizedReferenceRepository(ref)) { + File referencePath = new File(ref); + if (!referencePath.exists()) + listener.getLogger().println("[WARNING] Reference path does not exist: " + ref); + else if (!referencePath.isDirectory()) + listener.getLogger().println("[WARNING] Reference path is not a directory: " + ref); + else + args.add("--reference", ref); + } // else handled below in per-module loop } if (shallow) { if (depth == null) { @@ -1453,6 +1455,27 @@ else if (!referencePath.isDirectory()) throw new GitException("Invalid repository for " + sModuleName); } + if (isParameterizedReferenceRepository(ref)) { + File referencePath = findParameterizedReferenceRepository(ref, urIish); + if (referencePath == null) { + listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + ref); + } else { + String expRef = null; + if (referencePath.getName().equals(ref)) { + expRef = ref; + } else { + expRef = referencePath.getName(); + expRef += " (expanded from " + ref + ")"; + } + if (!referencePath.exists()) + listener.getLogger().println("[WARNING] Reference path does not exist: " + expRef); + else if (!referencePath.isDirectory()) + listener.getLogger().println("[WARNING] Reference path is not a directory: " + expRef); + else + args.add("--reference", referencePath.getName()); + } + } + // Find credentials for this URL StandardCredentials cred = credentials.get(urIish.toPrivateString()); if (parentCredentials) { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 8654e4f6a8..e956eeaa6b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -158,6 +158,20 @@ public void clone(RemoteConfig rc, boolean useShallowClone) throws GitException, clone(source, rc.getName(), useShallowClone, null); } + /** Handle magic strings in the reference pathname to sort out patterns + * classified as evaluated by parametrization, as handled below */ + public Boolean isParameterizedReferenceRepository(String reference) { + if (reference == null || reference.isEmpty()) { + return false; + } + + if (reference.endsWith("/${GIT_URL}")) { + return true; + } + + return false; + } + /** Yield the File object for the reference repository local filesystem * pathname. Note that the provided string may be suffixed with expandable * tokens which allow to store a filesystem structure of multiple small From 06d4859c5244c14e199e9f12d8ba350da14664c5 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 8 Dec 2020 18:01:32 +0100 Subject: [PATCH 004/132] Handle the possibility that the URL for parameterized checkout is null String --- .../org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java | 8 ++++++-- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index 1743f98257..deb3c9b990 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -1454,9 +1454,13 @@ else if (!referencePath.isDirectory()) listener.error("Invalid repository for " + sModuleName); throw new GitException("Invalid repository for " + sModuleName); } + String strURIish = null; + if (urIish != null) { + strURIish = urIish.toPrivateString(); + } if (isParameterizedReferenceRepository(ref)) { - File referencePath = findParameterizedReferenceRepository(ref, urIish); + File referencePath = findParameterizedReferenceRepository(ref, strURIish); if (referencePath == null) { listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + ref); } else { @@ -1477,7 +1481,7 @@ else if (!referencePath.isDirectory()) } // Find credentials for this URL - StandardCredentials cred = credentials.get(urIish.toPrivateString()); + StandardCredentials cred = credentials.get(strURIish); if (parentCredentials) { String parentUrl = getRemoteUrl(getDefaultRemote()); URIish parentUri = null; diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index e956eeaa6b..5e143bb8b2 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -188,7 +188,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { } File referencePath = new File(reference); - if (!referencePath.exists()) { + if (!referencePath.exists() && url != null && !url.isEmpty()) { if (reference.endsWith("/${GIT_URL}")) { // For mass-configured jobs, like Organization Folders, // allow to support parameterized paths to many refrepos. From b5a6a7b1bae0f9f5c5dfdc7efd61689f38f96a87 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 8 Dec 2020 18:30:01 +0100 Subject: [PATCH 005/132] LegacyCompatibleGitAPIImpl.java: document params for new routines --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 5e143bb8b2..b0dde55f6f 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -159,7 +159,11 @@ public void clone(RemoteConfig rc, boolean useShallowClone) throws GitException, } /** Handle magic strings in the reference pathname to sort out patterns - * classified as evaluated by parametrization, as handled below */ + * classified as evaluated by parametrization, as handled below + * + * @params + * reference Pathname (maybe with magic suffix) to reference repo + */ public Boolean isParameterizedReferenceRepository(String reference) { if (reference == null || reference.isEmpty()) { return false; @@ -181,6 +185,11 @@ public Boolean isParameterizedReferenceRepository(String reference) { * by comparing their "reference" with returned object's File.getName(). * * At some point this plugin might also maintain that filesystem structure. + * + * @params + * reference Pathname (maybe with magic suffix) to reference repo + * url URL of the repository being cloned, to help choose a + * suitable parameterized reference repo subdirectory. */ public File findParameterizedReferenceRepository(String reference, String url) { if (reference == null || reference.isEmpty()) { From f90bdca702d70e18f0f3643d57c488bfede6689c Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 8 Dec 2020 18:31:10 +0100 Subject: [PATCH 006/132] LegacyCompatibleGitAPIImpl.java: refactor urlNormalized generation in findParameterizedReferenceRepository() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index b0dde55f6f..f602259685 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -198,6 +198,26 @@ public File findParameterizedReferenceRepository(String reference, String url) { File referencePath = new File(reference); if (!referencePath.exists() && url != null && !url.isEmpty()) { + // Note: this normalization might crush several URLs into one, + // and as far as is known this would be the goal - people tend + // to use or omit .git suffix, or spell with varied case, while + // it means the same thing to known Git platforms. + // The actual reference repository directory may choose to have + // original URL strings added as remotes (in case some are case + // sensitive and different). + String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + if (!url.contains("://") && + !url.startsWith("/") && + !url.startsWith(".") + ) { + // Not an URL with schema, not an absolute or relative pathname + File urlPath = new File(url); + if (!urlPath.exists()) { + // Also not a subdirectory without "./" prefix... + urlNormalized = "ssh://" + urlNormalized; + } + } + if (reference.endsWith("/${GIT_URL}")) { // For mass-configured jobs, like Organization Folders, // allow to support parameterized paths to many refrepos. @@ -211,7 +231,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { // can be introduced, e.g. to escape non-ascii chars for // portability? Support base64, SHA or MD5 hashes of URLs // as pathnames? Normalize first (lowercase, .git, ...)? - reference = reference.replaceAll("\\$\\{GIT_URL\\}$", url).replaceAll("/*$", "").replaceAll(".git$", ""); + reference = reference.replaceAll("\\$\\{GIT_URL\\}$", urlNormalized); referencePath = null; // GC referencePath = new File(reference); } From e09eff1eeeea78f96dd8bf7ce0d98d39468f688f Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 8 Dec 2020 18:34:14 +0100 Subject: [PATCH 007/132] LegacyCompatibleGitAPIImpl.java: introduce support for .../${GIT_URL_SHA256} reference repos --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f602259685..c9721a734c 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -1,6 +1,7 @@ package org.jenkinsci.plugins.gitclient; import static java.util.Arrays.copyOfRange; +import org.apache.commons.codec.digest.DigestUtils; import static org.apache.commons.lang.StringUtils.join; import hudson.model.TaskListener; import hudson.plugins.git.GitException; @@ -173,6 +174,10 @@ public Boolean isParameterizedReferenceRepository(String reference) { return true; } + if (reference.endsWith("/${GIT_URL_SHA256}")) { + return true; + } + return false; } @@ -234,6 +239,12 @@ public File findParameterizedReferenceRepository(String reference, String url) { reference = reference.replaceAll("\\$\\{GIT_URL\\}$", urlNormalized); referencePath = null; // GC referencePath = new File(reference); + } else if (reference.endsWith("/${GIT_URL_SHA256}")) { + // This may be the more portable solution with regard to filesystems + reference = reference.replaceAll("\\$\\{GIT_URL_SHA256\\}$", + org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + referencePath = null; // GC + referencePath = new File(reference); } } From 17010ca0849ec4dfcd1703309122bc848593ea32 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 8 Dec 2020 22:36:03 +0100 Subject: [PATCH 008/132] WorkspaceWithRepo.java (test): extend localMirror() with customizable name for the mirrored repo --- .../org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java index 4602ad9acd..26192748d7 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java @@ -76,11 +76,14 @@ private void renameAndDeleteDir(Path srcDir, String destDirName) { * @throws InterruptedException when execption is interrupted */ String localMirror() throws IOException, InterruptedException { + return localMirror("clone.git"); + } + + String localMirror(String cloneDirName) throws IOException, InterruptedException { File base = new File(".").getAbsoluteFile(); for (File f = base; f != null; f = f.getParentFile()) { File targetDir = new File(f, "target"); if (targetDir.exists()) { - String cloneDirName = "clone.git"; File clone = new File(targetDir, cloneDirName); if (!clone.exists()) { /* Clone to a temporary directory then move the From 72a4a05dc5c873896f0b6d2496fb67a202f9f960 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 8 Dec 2020 22:36:33 +0100 Subject: [PATCH 009/132] GitClientCloneTest.java: add test_clone_reference_parameterized_sha256() with real sanity checks --- .../plugins/gitclient/GitClientCloneTest.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 7fb6d08324..79718d159d 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -6,6 +6,7 @@ import hudson.plugins.git.GitException; import hudson.remoting.VirtualChannel; import hudson.util.StreamTaskListener; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.eclipse.jgit.lib.ConfigConstants; @@ -256,6 +257,53 @@ public void test_clone_reference_parameterized() throws Exception, IOException, //assertNoObjectsInRepository(); } + @Test + public void test_clone_reference_parameterized_sha256() throws Exception, IOException, InterruptedException { + String wsMirror = workspace.localMirror(); + /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java */ + String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + String wsMirrorHash = org.apache.commons.codec.digest.DigestUtils.sha256Hex(wsMirrorNormalized); + + /* Make a new repo replica to use as refrepo, in specified location */ + // Start of the path to pass into `git clone` call + File fRefrepoBase = new File("refrepo.git").getAbsoluteFile(); + String wsRefrepoBase = fRefrepoBase.getName(); // String with full pathname + String wsRefrepo = null; + try { + if (fRefrepoBase.mkdirs()) { + /* Note: per parser of magic suffix, use slash - not OS separator char + * And be sure to use relative paths here (see + * WorkspaceWithRepo.java::localMirror()): + */ + wsRefrepo = workspace.localMirror("refrepo.git/" + wsMirrorHash); + } + } catch (RuntimeException e) { + wsRefrepo = null; + Util.deleteRecursive(fRefrepoBase); + // At worst, the test would log that it mangled and parsed + // the provided string, as we check in log below + } + + testGitClient.clone_().url(wsMirror).repositoryName("origin").reference(wsRefrepoBase + "/${GIT_URL_SHA256}").execute(); + + testGitClient.checkout().ref("origin/master").branch("master").execute(); + check_remote_url(workspace, testGitClient, "origin"); + // Verify JENKINS-46737 expected log message is written + String messages = StringUtils.join(handler.getMessages(), ";"); + assertThat("Reference repo name-parsing logged in: " + messages, + handler.containsMessageSubstring("Parameterized reference path ") && + handler.containsMessageSubstring(" replaced with: ") && + (wsRefrepo == null || handler.containsMessageSubstring(wsRefrepo)) , + is(true)); + + if (wsRefrepo != null) { + assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); + assertAlternateFilePointsToLocalMirror(); + assertBranchesExist(testGitClient.getBranches(), "master"); + assertNoObjectsInRepository(); + } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path structure + } + private static final String SRC_DIR = (new File(".")).getAbsolutePath(); @Test From c6c010fe3472159b20aad000e8a77add122e8e33 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 9 Dec 2020 01:13:03 +0100 Subject: [PATCH 010/132] LegacyCompatibleGitAPIImpl.java: findParameterizedReferenceRepository(): do not bother normalizing the URL if the string is not with supported suffix --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index c9721a734c..b9398b4c6b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -202,7 +202,10 @@ public File findParameterizedReferenceRepository(String reference, String url) { } File referencePath = new File(reference); - if (!referencePath.exists() && url != null && !url.isEmpty()) { + if (!referencePath.exists() && + isParameterizedReferenceRepository(reference) && + url != null && !url.isEmpty() + ) { // Note: this normalization might crush several URLs into one, // and as far as is known this would be the goal - people tend // to use or omit .git suffix, or spell with varied case, while From bbedefd508038fe9e2a9f9c296887b9870479e6b Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 9 Dec 2020 01:29:07 +0100 Subject: [PATCH 011/132] Fix referencePath getName() => getPath() --- .../jenkinsci/plugins/gitclient/CliGitAPIImpl.java | 12 ++++++------ .../org/jenkinsci/plugins/gitclient/JGitAPIImpl.java | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index deb3c9b990..40fca347a8 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -765,18 +765,18 @@ public void execute() throws GitException, InterruptedException { if (referencePath == null) { listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + reference); } else { - if (!referencePath.getName().equals(reference)) { + if (!referencePath.getPath().equals(reference)) { // Note: both these logs are needed, they are used in selftest String msg = "Parameterized reference path "; msg += "'" + reference + "'"; msg += " replaced with: "; - msg += "'" + referencePath.getName() + "'"; + msg += "'" + referencePath.getPath() + "'"; if (referencePath.exists()) { listener.getLogger().println("[WARNING] " + msg); } else { listener.getLogger().println("[WARNING] " + msg + " does not exist"); } - reference = referencePath.getName(); + reference = referencePath.getPath(); } if (!referencePath.exists()) @@ -1465,10 +1465,10 @@ else if (!referencePath.isDirectory()) listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + ref); } else { String expRef = null; - if (referencePath.getName().equals(ref)) { + if (referencePath.getPath().equals(ref)) { expRef = ref; } else { - expRef = referencePath.getName(); + expRef = referencePath.getPath(); expRef += " (expanded from " + ref + ")"; } if (!referencePath.exists()) @@ -1476,7 +1476,7 @@ else if (!referencePath.isDirectory()) else if (!referencePath.isDirectory()) listener.getLogger().println("[WARNING] Reference path is not a directory: " + expRef); else - args.add("--reference", referencePath.getName()); + args.add("--reference", referencePath.getPath()); } } diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 323b386518..5a03460604 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1410,18 +1410,18 @@ public void execute() throws GitException, InterruptedException { if (referencePath == null) { listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + reference); } else { - if (!referencePath.getName().equals(reference)) { + if (!referencePath.getPath().equals(reference)) { // Note: both these logs are needed, they are used in selftest String msg = "Parameterized reference path "; msg += "'" + reference + "'"; msg += " replaced with: "; - msg += "'" + referencePath.getName() + "'"; + msg += "'" + referencePath.getPath() + "'"; if (referencePath.exists()) { listener.getLogger().println("[WARNING] " + msg); } else { listener.getLogger().println("[WARNING] " + msg + " does not exist"); } - reference = referencePath.getName(); + reference = referencePath.getPath(); } if (!referencePath.exists()) From 5306163d6c69233e5de53ac1b5044ef45351da1e Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 9 Dec 2020 03:00:58 +0100 Subject: [PATCH 012/132] Update tests for fRefrepoBase being under target/ dir from CWD of the test runner --- .../org/jenkinsci/plugins/gitclient/GitClientCloneTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 79718d159d..ba8f25b1a6 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -265,8 +265,10 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce String wsMirrorHash = org.apache.commons.codec.digest.DigestUtils.sha256Hex(wsMirrorNormalized); /* Make a new repo replica to use as refrepo, in specified location */ - // Start of the path to pass into `git clone` call - File fRefrepoBase = new File("refrepo.git").getAbsoluteFile(); + // Start of the path to pass into `git clone` call; note that per + // WorkspaceWithRepo.java the test workspaces are under target/ + // where the executed test binaries live + File fRefrepoBase = new File("target/refrepo.git").getAbsoluteFile(); String wsRefrepoBase = fRefrepoBase.getName(); // String with full pathname String wsRefrepo = null; try { From bc7e19819657f1fbfc5c902bb274fc4916a1bbe0 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 9 Dec 2020 03:25:24 +0100 Subject: [PATCH 013/132] GitClientCloneTest: check if refrepo dir exists already --- .../org/jenkinsci/plugins/gitclient/GitClientCloneTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index ba8f25b1a6..b9a89f629d 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -268,16 +268,16 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce // Start of the path to pass into `git clone` call; note that per // WorkspaceWithRepo.java the test workspaces are under target/ // where the executed test binaries live - File fRefrepoBase = new File("target/refrepo.git").getAbsoluteFile(); + File fRefrepoBase = new File("target/refrepo256.git").getAbsoluteFile(); String wsRefrepoBase = fRefrepoBase.getName(); // String with full pathname String wsRefrepo = null; try { - if (fRefrepoBase.mkdirs()) { + if (fRefrepoBase.exists() || fRefrepoBase.mkdirs()) { /* Note: per parser of magic suffix, use slash - not OS separator char * And be sure to use relative paths here (see * WorkspaceWithRepo.java::localMirror()): */ - wsRefrepo = workspace.localMirror("refrepo.git/" + wsMirrorHash); + wsRefrepo = workspace.localMirror("refrepo256.git/" + wsMirrorHash); } } catch (RuntimeException e) { wsRefrepo = null; From 3ff19df7fb0d2c4ff468a30c829782b6880eb993 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 04:06:43 +0100 Subject: [PATCH 014/132] WorkspaceWithRepo.java: localMirror(): normalize() the returned pathname string --- .../org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java index 26192748d7..effea085aa 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java @@ -16,6 +16,7 @@ import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; @@ -124,7 +125,8 @@ String localMirror(String cloneDirName) throws IOException, InterruptedException Util.deleteRecursive(tempClonePath.toFile()); } } - return clone.getPath(); + // Strip away the "/./" in "...git-client-plugin/./target/..." + return Paths.get(clone.getPath()).normalize().toString(); } } throw new IllegalStateException(); From a509386b15596f91faa9d1f1c9a6803d162c1b9c Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 04:12:59 +0100 Subject: [PATCH 015/132] GitClientCloneTest.java: extend API choices for assertAlternateFilePointsToLocal*Mirror() with custom paths and bare vs workspace repos --- .../plugins/gitclient/GitClientCloneTest.java | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index b9a89f629d..d4a4545588 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -414,12 +414,51 @@ private void assertNoObjectsInRepository() { } + // Most tests use this method, expecting a non-bare repo private void assertAlternateFilePointsToLocalMirror() throws IOException, InterruptedException { + assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir); + } + + private void assertAlternateFilePointsToLocalWorkspaceMirror() throws IOException, InterruptedException { + assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir); + } + + private void assertAlternateFilePointsToLocalWorkspaceMirror(File _testGitDir) throws IOException, InterruptedException { + assertAlternateFilePointsToLocalWorkspaceMirror(_testGitDir.getPath()); + } + + private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir) throws IOException, InterruptedException { + assertAlternateFilePointsToLocalWorkspaceMirror(_testGitDir, workspace.localMirror()); + } + + private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir, String _testAltDir) throws IOException, InterruptedException { final String alternates = ".git" + File.separator + "objects" + File.separator + "info" + File.separator + "alternates"; + assertAlternateFilePointsToLocalMirror(_testGitDir, _testAltDir, alternates); + } - assertThat(new File(testGitDir, alternates), is(anExistingFile())); - final String expectedContent = workspace.localMirror().replace("\\", "/") + "/objects"; - final String actualContent = FileUtils.readFileToString(new File(testGitDir, alternates), "UTF-8"); + // Similar for bare repos, without ".git/" dir + private void assertAlternateFilePointsToLocalBareMirror() throws IOException, InterruptedException { + assertAlternateFilePointsToLocalBareMirror(testGitDir); + } + + private void assertAlternateFilePointsToLocalBareMirror(File _testGitDir) throws IOException, InterruptedException { + assertAlternateFilePointsToLocalBareMirror(_testGitDir.getPath()); + } + + private void assertAlternateFilePointsToLocalBareMirror(String _testGitDir) throws IOException, InterruptedException { + assertAlternateFilePointsToLocalBareMirror(_testGitDir, workspace.localMirror()); + } + + private void assertAlternateFilePointsToLocalBareMirror(String _testGitDir, String _testAltDir) throws IOException, InterruptedException { + final String alternates = "objects" + File.separator + "info" + File.separator + "alternates"; + assertAlternateFilePointsToLocalMirror(_testGitDir, _testAltDir, alternates); + } + + private void assertAlternateFilePointsToLocalMirror(String _testGitDir, String _testAltDir, String alternates) throws IOException, InterruptedException { + assertThat("Did not find '" + alternates + "' under '" + _testGitDir + "'", + new File(_testGitDir, alternates), is(anExistingFile())); + final String expectedContent = _testAltDir.replace("\\", "/") + "/objects"; + final String actualContent = FileUtils.readFileToString(new File(_testGitDir, alternates), "UTF-8"); assertThat("Alternates file content", actualContent, is(expectedContent)); final File alternatesDir = new File(actualContent); assertThat(alternatesDir, is(anExistingDirectory())); From 603e3900e96a8875853a3cca926025b4f1d0ebc0 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 04:51:16 +0100 Subject: [PATCH 016/132] GitClientCloneTest.java: fix test_clone_reference_parameterized_sha256() to find expected paths --- .../org/jenkinsci/plugins/gitclient/GitClientCloneTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index d4a4545588..b0a986ebfb 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -269,7 +269,7 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce // WorkspaceWithRepo.java the test workspaces are under target/ // where the executed test binaries live File fRefrepoBase = new File("target/refrepo256.git").getAbsoluteFile(); - String wsRefrepoBase = fRefrepoBase.getName(); // String with full pathname + String wsRefrepoBase = fRefrepoBase.getPath(); // String with full pathname String wsRefrepo = null; try { if (fRefrepoBase.exists() || fRefrepoBase.mkdirs()) { @@ -300,7 +300,7 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce if (wsRefrepo != null) { assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); - assertAlternateFilePointsToLocalMirror(); + assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir.getPath(), wsRefrepo); assertBranchesExist(testGitClient.getBranches(), "master"); assertNoObjectsInRepository(); } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path structure From 66e23a1322ac7d241908d071f5d5416ca56fa70c Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 20:45:47 +0100 Subject: [PATCH 017/132] LegacyCompatibleGitAPIImpl.java CliGitAPIImpl.java JGitAPIImpl.java: refactor getObjectPath(referencePath) to check on git dirs elsewhere later --- .../plugins/gitclient/CliGitAPIImpl.java | 9 +--- .../plugins/gitclient/JGitAPIImpl.java | 9 +--- .../gitclient/LegacyCompatibleGitAPIImpl.java | 47 +++++++++++++++++++ 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index 40fca347a8..b8c011a08c 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -784,13 +784,8 @@ public void execute() throws GitException, InterruptedException { else if (!referencePath.isDirectory()) listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); else { - // reference path can either be a normal or a base repository - File objectsPath = new File(referencePath, ".git/objects"); - if (!objectsPath.isDirectory()) { - // reference path is bare repo - objectsPath = new File(referencePath, "objects"); - } - if (!objectsPath.isDirectory()) + File objectsPath = getObjectPath(referencePath); + if (objectsPath == null || !objectsPath.isDirectory()) listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (not a git repo?): " + objectsPath); else { // Go behind git's back to write a meta file in new workspace diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 5a03460604..a83981b144 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1429,13 +1429,8 @@ public void execute() throws GitException, InterruptedException { else if (!referencePath.isDirectory()) listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); else { - // reference path can either be a normal or a base repository - File objectsPath = new File(referencePath, ".git/objects"); - if (!objectsPath.isDirectory()) { - // reference path is bare repo - objectsPath = new File(referencePath, "objects"); - } - if (!objectsPath.isDirectory()) + File objectsPath = getObjectPath(referencePath); + if (objectsPath == null || !objectsPath.isDirectory()) listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (no git repo?): " + objectsPath); else { // Go behind git's back to write a meta file in new workspace diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index b9398b4c6b..2c5ab04961 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -159,6 +159,53 @@ public void clone(RemoteConfig rc, boolean useShallowClone) throws GitException, clone(source, rc.getName(), useShallowClone, null); } + /** For referenced directory check if it is a full or bare git repo + * and return the File object for its "objects" sub-directory. + * If there is nothing to find, or inputs are bad, returns null. + * The idea is that checking for null allows to rule out non-git + * paths, while a not-null return value is instantly usable by + * some code which plays with git under its hood. + */ + public File getObjectPath(String reference) { + if (reference == null || reference.isEmpty()) { + return null; + } + return getObjectPath(new File(reference)); + } + + public File getObjectPath(File referencePath) { + if (referencePath == null) { + return referencePath; + } + + if (!referencePath.exists()) + return null; + + if (!referencePath.isDirectory()) + return null; + + // reference path can either be a normal or a base repository + File objectsPath = new File(referencePath, ".git/objects"); + if (objectsPath == null) { + return objectsPath; // Some Java error, could not make object from the paths involved + } + + if (!objectsPath.isDirectory()) { + // reference path is bare repo + objectsPath = new File(referencePath, "objects"); + if (objectsPath == null) { + return objectsPath; // Some Java error, could not make object from the paths involved + } + } + + if (!objectsPath.isDirectory()) + return null; + + // If we get here, we have a non-null File referencing a + // "(.git/)objects" subdir under original referencePath + return objectsPath; + } + /** Handle magic strings in the reference pathname to sort out patterns * classified as evaluated by parametrization, as handled below * From 5af44004edcdaa52e84a823676ac5c36ad6f816a Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 20:46:42 +0100 Subject: [PATCH 018/132] LegacyCompatibleGitAPIImpl.java: add variants of findParameterizedReferenceRepository() and isParameterizedReferenceRepository() taking a File reference (not only a String) object --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 2c5ab04961..ca98eeedc2 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -212,6 +212,13 @@ public File getObjectPath(File referencePath) { * @params * reference Pathname (maybe with magic suffix) to reference repo */ + public Boolean isParameterizedReferenceRepository(File reference) { + if (reference == null) { + return false; + } + return isParameterizedReferenceRepository(reference.getPath()); + } + public Boolean isParameterizedReferenceRepository(String reference) { if (reference == null || reference.isEmpty()) { return false; @@ -243,6 +250,13 @@ public Boolean isParameterizedReferenceRepository(String reference) { * url URL of the repository being cloned, to help choose a * suitable parameterized reference repo subdirectory. */ + public File findParameterizedReferenceRepository(File reference, String url) { + if (reference == null) { + return reference; + } + return findParameterizedReferenceRepository(reference.getPath(), url); + } + public File findParameterizedReferenceRepository(String reference, String url) { if (reference == null || reference.isEmpty()) { return null; From 509c5ecd44fdc21f7119eda70d81197b53c20b7e Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 20:48:13 +0100 Subject: [PATCH 019/132] [TMP] LegacyCompatibleGitAPIImpl.java: trace findParameterizedReferenceRepository() processing to stderr --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index ca98eeedc2..f9f9bba49d 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -287,6 +287,10 @@ public File findParameterizedReferenceRepository(String reference, String url) { } } + System.err.println("reference='" + reference + "'\n" + + "url='" + url + "'\n" + + "urlNormalized='" + urlNormalized + "'\n"); + if (reference.endsWith("/${GIT_URL}")) { // For mass-configured jobs, like Organization Folders, // allow to support parameterized paths to many refrepos. @@ -310,6 +314,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { referencePath = null; // GC referencePath = new File(reference); } + System.err.println("reference after='" + reference + "'\n"); } if (!referencePath.exists()) { From 0b0fcbfceeb0fdd9de36d9e2abde0f8bd48201e0 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 20:48:51 +0100 Subject: [PATCH 020/132] [TMP] WorkspaceWithRepo.java (test): trace localMirror() processing to stderr --- .../jenkinsci/plugins/gitclient/GitClientCloneTest.java | 7 +++++++ .../jenkinsci/plugins/gitclient/WorkspaceWithRepo.java | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index b0a986ebfb..873dda9bb3 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -286,12 +286,19 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce // the provided string, as we check in log below } + System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + + "wsRefrepo='" + wsRefrepo); + testGitClient.clone_().url(wsMirror).repositoryName("origin").reference(wsRefrepoBase + "/${GIT_URL_SHA256}").execute(); testGitClient.checkout().ref("origin/master").branch("master").execute(); check_remote_url(workspace, testGitClient, "origin"); + // Verify JENKINS-46737 expected log message is written String messages = StringUtils.join(handler.getMessages(), ";"); + + System.err.println("clone output:\n======\n" + messages + "\n======\n"); + assertThat("Reference repo name-parsing logged in: " + messages, handler.containsMessageSubstring("Parameterized reference path ") && handler.containsMessageSubstring(" replaced with: ") && diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java index effea085aa..22bdb883a7 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java @@ -82,10 +82,13 @@ String localMirror() throws IOException, InterruptedException { String localMirror(String cloneDirName) throws IOException, InterruptedException { File base = new File(".").getAbsoluteFile(); + System.err.println("=== Beginning to search for cloneDirName='" + cloneDirName + "' from " + base.getPath()); for (File f = base; f != null; f = f.getParentFile()) { + System.err.println("Looking for 'target' in " + f.getPath()); File targetDir = new File(f, "target"); if (targetDir.exists()) { File clone = new File(targetDir, cloneDirName); + System.err.println("Looking for cloneDirName " + cloneDirName + " in " + targetDir.getPath()); if (!clone.exists()) { /* Clone to a temporary directory then move the * temporary directory to the final destination @@ -96,8 +99,10 @@ String localMirror(String cloneDirName) throws IOException, InterruptedException * the final destination directory. */ Path tempClonePath = Files.createTempDirectory(targetDir.toPath(), "clone-"); + System.err.println("tempClonePath=" + tempClonePath + " => (FQPN)" + tempClonePath.toFile().getAbsolutePath()); cliGitCommand.run("clone", "--reference", f.getCanonicalPath(), "--mirror", repoURL, tempClonePath.toFile().getAbsolutePath()); if (!clone.exists()) { // Still a race condition, but a narrow race handled by Files.move() + System.err.println("moving tempClonePath to cloneDirName=" + cloneDirName); renameAndDeleteDir(tempClonePath, cloneDirName); } else { /* @@ -122,8 +127,11 @@ String localMirror(String cloneDirName) throws IOException, InterruptedException * deleteRecursive() will discard a clone that * 'lost the race'. */ + System.err.println("removing extra tempClonePath, we already (race?) have cloneDirName=" + cloneDirName); Util.deleteRecursive(tempClonePath.toFile()); } + } else { + System.err.println("FOUND cloneDirName " + cloneDirName + " in " + targetDir.getPath()); } // Strip away the "/./" in "...git-client-plugin/./target/..." return Paths.get(clone.getPath()).normalize().toString(); From 6a9c79d8dc63dfbfb63df0182e489966b50d9696 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 21:07:37 +0100 Subject: [PATCH 021/132] LegacyCompatibleGitAPIImpl.java: refactor to use referenceExpanded to keep original reference intact, and as indicator to recreate referencePath object once for many cases --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f9f9bba49d..d84e2402da 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -291,6 +291,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { "url='" + url + "'\n" + "urlNormalized='" + urlNormalized + "'\n"); + String referenceExpanded = null; if (reference.endsWith("/${GIT_URL}")) { // For mass-configured jobs, like Organization Folders, // allow to support parameterized paths to many refrepos. @@ -304,13 +305,16 @@ public File findParameterizedReferenceRepository(String reference, String url) { // can be introduced, e.g. to escape non-ascii chars for // portability? Support base64, SHA or MD5 hashes of URLs // as pathnames? Normalize first (lowercase, .git, ...)? - reference = reference.replaceAll("\\$\\{GIT_URL\\}$", urlNormalized); - referencePath = null; // GC - referencePath = new File(reference); + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL\\}$", urlNormalized); } else if (reference.endsWith("/${GIT_URL_SHA256}")) { // This may be the more portable solution with regard to filesystems - reference = reference.replaceAll("\\$\\{GIT_URL_SHA256\\}$", + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_SHA256\\}$", org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + } + } + + if (referenceExpanded != null) { + reference = referenceExpanded; referencePath = null; // GC referencePath = new File(reference); } From 264197188a27f41651bfb9458f753423903e5d61 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 21:09:08 +0100 Subject: [PATCH 022/132] LegacyCompatibleGitAPIImpl.java: add GIT_URL_FALLBACK and GIT_URL_SHA256_FALLBACK suffixes for using unsuffixed directory if expanded path points nowhere useful --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index d84e2402da..e67276ea9c 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -232,6 +232,14 @@ public Boolean isParameterizedReferenceRepository(String reference) { return true; } + if (reference.endsWith("/${GIT_URL_FALLBACK}")) { + return true; + } + + if (reference.endsWith("/${GIT_URL_SHA256_FALLBACK}")) { + return true; + } + return false; } @@ -305,11 +313,50 @@ public File findParameterizedReferenceRepository(String reference, String url) { // can be introduced, e.g. to escape non-ascii chars for // portability? Support base64, SHA or MD5 hashes of URLs // as pathnames? Normalize first (lowercase, .git, ...)? + + // TODO: employ git submodules - there we can reliably match + // remote URLs (easily multiple) to particular modules, and + // yet keep separate git index directories per module with + // smaller scopes - much quicker to check out from than one + // huge combined repo. It would also be much more native to + // tools and custom scriptware that can be involved. + + // TODO: Config option whether to populate absent reference + // repos (If the expanded path does not have git repo data + // right now, populate it in the location expanded below) + // or update existing ones before pulling commits, and how + // to achieve that. Note that this is done by caller with + // their implementation of git, or in case of a build farm it + // is more likely to be a shared path only readable to Jenkins + // and its agents, so write-operations would be done by helper + // scripts that log into the shared storage server to populate + // or update reference repositories. Note that users may also + // want to run their own scripts to "populate" reference repos + // as symlinks to existing other repos, to support combined + // repo setup for different URLs pointing to same upstream, + // or storing multiple closely related forks together. + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL\\}$", urlNormalized); } else if (reference.endsWith("/${GIT_URL_SHA256}")) { // This may be the more portable solution with regard to filesystems referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_SHA256\\}$", org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + } else if (reference.endsWith("/${GIT_URL_SHA256_FALLBACK}")) { + // The safest option - fall back to parent directory if + // the expanded one does not have git repo data right now: + // it allows to gradually convert a big combined reference + // repository into smaller chunks without breaking builds. + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_SHA256_FALLBACK\\}$", + org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + if (getObjectPath(referenceExpanded) == null) { + // chop it off, use main directory + referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_SHA256_FALLBACK\\}$", ""); + } + } else if (reference.endsWith("/${GIT_URL_FALLBACK}")) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_FALLBACK\\}$", urlNormalized); + if (getObjectPath(referenceExpanded) == null) { + // chop it off, use main directory + referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_FALLBACK\\}$", ""); } } From c10231ca971f9856b43200d3c0cb2baab0364124 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 21:50:32 +0100 Subject: [PATCH 023/132] GitClientCloneTest.java: add cases for parameterized reference repo with fallback modes of operation --- .../plugins/gitclient/GitClientCloneTest.java | 93 ++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 873dda9bb3..26f7953690 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -250,13 +250,46 @@ public void test_clone_reference_parameterized() throws Exception, IOException, handler.containsMessageSubstring("Parameterized reference path ") && handler.containsMessageSubstring(" replaced with: "), is(true)); - // Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path structure + // TODO: Actually construct the local filesystem path to match + // the literally substituted URL. Note this may not work for + // filesystems and operating systems that constrain character + // ranges usable for filenames (Windows at least) so failure + // to create the paths is an option to handle in test. + // Be sure to clean away this path at end of test, so that the + // test_clone_reference_parameterized_fallback() below is not + // confused - it expects this location to not exist. + // Skip: Missing if clone failed - currently would, with bogus + // path above and not yet pre-created path structure. //assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); //assertAlternateFilePointsToLocalMirror(); //assertBranchesExist(testGitClient.getBranches(), "master"); //assertNoObjectsInRepository(); } + @Test + public void test_clone_reference_parameterized_fallback() throws Exception, IOException, InterruptedException { + // TODO: Currently we do not make paths which would invalidate + // this test, but note the test above might do just that later. + testGitClient.clone_().url(workspace.localMirror()).repositoryName("origin").reference(workspace.localMirror() + "/${GIT_URL_FALLBACK}").execute(); + testGitClient.checkout().ref("origin/master").branch("master").execute(); + check_remote_url(workspace, testGitClient, "origin"); + // Verify JENKINS-46737 expected log message is written + String messages = StringUtils.join(handler.getMessages(), ";"); + assertThat("Reference repo name-parsing logged in: " + messages, + handler.containsMessageSubstring("Parameterized reference path ") && + handler.containsMessageSubstring(" replaced with: '" + workspace.localMirror() + "'"), + is(true)); + // With fallback mode, and nonexistent parameterized reference + // repository, and a usable repository in the common path (what + // remains if the parameterizing suffix is just discarded), + // this common path should be used. So it should overall behave + // same as the non-parameterized test_clone_reference() above. + assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); + assertAlternateFilePointsToLocalMirror(); + assertBranchesExist(testGitClient.getBranches(), "master"); + assertNoObjectsInRepository(); + } + @Test public void test_clone_reference_parameterized_sha256() throws Exception, IOException, InterruptedException { String wsMirror = workspace.localMirror(); @@ -313,6 +346,64 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path structure } + @Test + public void test_clone_reference_parameterized_sha256_fallback() throws Exception, IOException, InterruptedException { + String wsMirror = workspace.localMirror(); + /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java */ + String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + String wsMirrorHash = org.apache.commons.codec.digest.DigestUtils.sha256Hex(wsMirrorNormalized); + + /* Make a new repo replica to use as refrepo, in specified location */ + // Start of the path to pass into `git clone` call; note that per + // WorkspaceWithRepo.java the test workspaces are under target/ + // where the executed test binaries live + File fRefrepoBase = new File("target/refrepo256.git").getAbsoluteFile(); + String wsRefrepoBase = fRefrepoBase.getPath(); // String with full pathname + String wsRefrepo = null; + try { + if (fRefrepoBase.exists() || fRefrepoBase.mkdirs()) { + /* Note: per parser of magic suffix, use slash - not OS separator char + * And be sure to use relative paths here (see + * WorkspaceWithRepo.java::localMirror()): + */ + wsRefrepo = workspace.localMirror("refrepo256.git/" + wsMirrorHash); + } + } catch (RuntimeException e) { + wsRefrepo = null; + Util.deleteRecursive(fRefrepoBase); + // At worst, the test would log that it mangled and parsed + // the provided string, as we check in log below + } + + System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + + "wsRefrepo='" + wsRefrepo); + + testGitClient.clone_().url(wsMirror).repositoryName("origin").reference(wsRefrepoBase + "/${GIT_URL_SHA256_FALLBACK}").execute(); + + testGitClient.checkout().ref("origin/master").branch("master").execute(); + check_remote_url(workspace, testGitClient, "origin"); + + // Verify JENKINS-46737 expected log message is written + String messages = StringUtils.join(handler.getMessages(), ";"); + + System.err.println("clone output:\n======\n" + messages + "\n======\n"); + + assertThat("Reference repo name-parsing logged in: " + messages, + handler.containsMessageSubstring("Parameterized reference path ") && + handler.containsMessageSubstring(" replaced with: ") && + (wsRefrepo == null || handler.containsMessageSubstring(wsRefrepo)) , + is(true)); + + // Barring filesystem errors, if we have the "custom" refrepo + // we expect it to be used (fallback mode is not triggered) + if (wsRefrepo != null) { + assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); + assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir.getPath(), wsRefrepo); + assertBranchesExist(testGitClient.getBranches(), "master"); + assertNoObjectsInRepository(); + } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path structure + } + private static final String SRC_DIR = (new File(".")).getAbsolutePath(); @Test From 201671907eb9bf49964dd022325da15fffe7ba18 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 22:14:56 +0100 Subject: [PATCH 024/132] LegacyCompatibleGitAPIImpl.java: refactor normalizeGitUrl() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index e67276ea9c..b4eea9cdc7 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -243,6 +243,32 @@ public Boolean isParameterizedReferenceRepository(String reference) { return false; } + /** There are many ways to spell an URL to the same repository even if + * using the same access protocol. This routine converts the "url" string + * in a way that helps us confirm whether two spellings mean same thing. + */ + public String normalizeGitUrl(String url, Boolean checkLocalPaths) { + String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + if (!url.contains("://") && + !url.startsWith("/") && + !url.startsWith(".") + ) { + // Not an URL with schema, not an absolute or relative pathname + if (checkLocalPaths) { + File urlPath = new File(url); + if (!urlPath.exists()) { + // Also not a subdirectory of current dir without "./" prefix... + urlNormalized = "ssh://" + urlNormalized; + } + } else { + // Assume it is not a path + urlNormalized = "ssh://" + urlNormalized; + } + } + + return urlNormalized; + } + /** Yield the File object for the reference repository local filesystem * pathname. Note that the provided string may be suffixed with expandable * tokens which allow to store a filesystem structure of multiple small @@ -282,18 +308,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { // The actual reference repository directory may choose to have // original URL strings added as remotes (in case some are case // sensitive and different). - String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); - if (!url.contains("://") && - !url.startsWith("/") && - !url.startsWith(".") - ) { - // Not an URL with schema, not an absolute or relative pathname - File urlPath = new File(url); - if (!urlPath.exists()) { - // Also not a subdirectory without "./" prefix... - urlNormalized = "ssh://" + urlNormalized; - } - } + String urlNormalized = normalizeGitUrl(url, true); System.err.println("reference='" + reference + "'\n" + "url='" + url + "'\n" + From 71c0850d440690e672b8d1cb5af65745e97c2a1c Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 10 Dec 2020 23:07:01 +0100 Subject: [PATCH 025/132] [WIP] LegacyCompatibleGitAPIImpl.java: skeleton for GIT_SUBMODULES and GIT_SUBMODULES_FALLBACK support --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index b4eea9cdc7..d1441d9e00 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -240,6 +240,14 @@ public Boolean isParameterizedReferenceRepository(String reference) { return true; } + if (reference.endsWith("/${GIT_SUBMODULES}")) { + return true; + } + + if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { + return true; + } + return false; } @@ -373,6 +381,14 @@ public File findParameterizedReferenceRepository(String reference, String url) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_FALLBACK\\}$", ""); } + } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") ) { + // Assuming the provided "reference" directory already hosts + // submodules, we use git tools to find the one subdir which + // has a registered remote URL equivalent (per normalization) + // to the provided "url". If there is no hit, the non-fallback + // mode suggests a new directory name to host the submodule + // (via same rules as for SHA256), and the fallback mode would + // return the main directory. } if (referenceExpanded != null) { From ab04ae7ae43da4aa8f1484c962d2264ce1dd1768 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Fri, 11 Dec 2020 01:22:35 +0100 Subject: [PATCH 026/132] Introduce getRemoteUrls() and getRemotePushUrls() --- .../plugins/gitclient/CliGitAPIImpl.java | 42 ++++++++++++++++++ .../plugins/gitclient/GitClient.java | 22 ++++++++++ .../plugins/gitclient/JGitAPIImpl.java | 44 +++++++++++++++++++ .../plugins/gitclient/RemoteGitImpl.java | 10 +++++ 4 files changed, 118 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index b8c011a08c..c2b7873e0f 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -1584,6 +1584,48 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr return StringUtils.trim(firstLine(result)); } + /** {@inheritDoc} + * + * Note: unlike the version in JGitAPIImpl, this one does not filter + * ASCII or Private URLs, and just conveys what git config contains. + */ + @Override + public @CheckForNull Map getRemoteUrls() throws GitException, InterruptedException { + String result = launchCommand( "config", "--local", "--list" ); + Map uriNames = new HashMap<>(); + for (String line : result.split("\\R+")) { + line = StringUtils.trim(line); + if (!line.startsWith("remote.") || !line.contains(".url=")) { + continue; + } + + String remoteName = StringUtils.substringBetween(line, "remote.", ".url="); + String remoteUri = StringUtils.substringAfter(line, ".url="); + + // If uri String values end up identical, Map only stores one entry + uriNames.put(remoteUri, remoteName); + } + return uriNames; + } + + /** {@inheritDoc} */ + @Override + public @CheckForNull Map getRemotePushUrls() throws GitException, InterruptedException { + String result = launchCommand( "config", "--local", "--list" ); + Map uriNames = new HashMap<>(); + for (String line : result.split("\\R+")) { + line = StringUtils.trim(line); + if (!line.startsWith("remote.") || !line.contains(".pushurl=")) { + continue; + } + + String remoteName = StringUtils.substringBetween(line, "remote.", ".pushurl="); + String remoteUri = StringUtils.substringAfter(line, ".pushurl="); + uriNames.put(remoteUri, remoteName); + } + return uriNames; + } + /** {@inheritDoc} */ @Override public void setRemoteUrl(String name, String url) throws GitException, InterruptedException { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java index 644a4ff236..498d73bdc1 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java @@ -211,6 +211,28 @@ public interface GitClient { */ String getRemoteUrl(String name) throws GitException, InterruptedException; + /** + * getRemoteUrls. + * + * @return a Map where String keys represent URIs (with and + * without passwords, if any; ASCII or not, if + * applicable) for all remotes configured in this + * repository/workspace, and values represent names. + * There may be several URIs corresponding to same name. + */ + public Map getRemoteUrls() throws GitException, InterruptedException; + + /** + * getRemotePushUrls. + * + * @return a Map where String keys represent push-only URIs + * (with and without passwords, if any; ASCII or not, + * if applicable) for all remotes configured in this + * repository/workspace, and values represent names. + * There may be several URIs corresponding to same name. + */ + public Map getRemotePushUrls() throws GitException, InterruptedException; + /** * For a given repository, set a remote's URL * diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index a83981b144..cf609ebcb9 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -911,6 +911,50 @@ public String getRemoteUrl(String name) throws GitException { } } + /** {@inheritDoc} */ + @Override + public Map getRemoteUrls() throws GitException, InterruptedException { + Map uriNames = new HashMap<>(); + try (Repository repo = getRepository()) { + Config c = repo.getConfig(); + for (RemoteConfig rc : RemoteConfig.getAllRemoteConfigs(c)) { + String remoteName = rc.getName(); + for (URIish u : rc.getURIs()) { + // If uri String values end up identical, Map only stores one entry + uriNames.put(u.toString(), remoteName); + uriNames.put(u.toPrivateString(), remoteName); + uriNames.put(u.toASCIIString(), remoteName); + uriNames.put(u.toPrivateASCIIString(), remoteName); + } + } + } catch (URISyntaxException ue) { + throw new GitException(ue.toString()); + } + return uriNames; + } + + /** {@inheritDoc} */ + @Override + public Map getRemotePushUrls() throws GitException, InterruptedException { + Map uriNames = new HashMap<>(); + try (Repository repo = getRepository()) { + Config c = repo.getConfig(); + for (RemoteConfig rc : RemoteConfig.getAllRemoteConfigs(c)) { + String remoteName = rc.getName(); + for (URIish u : rc.getPushURIs()) { + // If uri String values end up identical, Map only stores one entry + uriNames.put(u.toString(), remoteName); + uriNames.put(u.toPrivateString(), remoteName); + uriNames.put(u.toASCIIString(), remoteName); + uriNames.put(u.toPrivateASCIIString(), remoteName); + } + } + } catch (URISyntaxException ue) { + throw new GitException(ue.toString()); + } + return uriNames; + } + /** * getRepository. * diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java index 63b504168d..68a116010c 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java @@ -294,6 +294,16 @@ public String getRemoteUrl(String name) throws GitException, InterruptedExceptio return proxy.getRemoteUrl(name); } + /** {@inheritDoc} */ + public Map getRemoteUrls() throws GitException, InterruptedException { + return proxy.getRemoteUrls(); + } + + /** {@inheritDoc} */ + public Map getRemotePushUrls() throws GitException, InterruptedException { + return proxy.getRemotePushUrls(); + } + /** {@inheritDoc} */ public void setRemoteUrl(String name, String url) throws GitException, InterruptedException { proxy.setRemoteUrl(name, url); From 179f5c192919b1b1f15fc48c8ab49744617e39ea Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Fri, 11 Dec 2020 02:08:24 +0100 Subject: [PATCH 027/132] LegacyCompatibleGitAPIImpl.java: normalizeGitUrl(): revise for local path names --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index d1441d9e00..a7639ecae4 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -21,6 +21,8 @@ import java.io.File; import java.io.IOException; import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -257,20 +259,30 @@ public Boolean isParameterizedReferenceRepository(String reference) { */ public String normalizeGitUrl(String url, Boolean checkLocalPaths) { String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); - if (!url.contains("://") && - !url.startsWith("/") && - !url.startsWith(".") - ) { - // Not an URL with schema, not an absolute or relative pathname - if (checkLocalPaths) { - File urlPath = new File(url); - if (!urlPath.exists()) { - // Also not a subdirectory of current dir without "./" prefix... + if (!url.contains("://")) { + if (!url.startsWith("/") && + !url.startsWith(".") + ) { + // Not an URL with schema, not an absolute or relative pathname + if (checkLocalPaths) { + File urlPath = new File(url); + if (urlPath.exists()) { + urlNormalized = "file://" + Paths.get( Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized ).normalize().toString();; + } else { + // Also not a subdirectory of current dir without "./" prefix... + urlNormalized = "ssh://" + urlNormalized; + } + } else { + // Assume it is not a path urlNormalized = "ssh://" + urlNormalized; } } else { - // Assume it is not a path - urlNormalized = "ssh://" + urlNormalized; + // Looks like a local path + if (url.startsWith("/")) { + urlNormalized = "file://" + urlNormalized; + } else { + urlNormalized = "file://" + Paths.get( Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized ).normalize().toString();; + } } } From 7c0ad4f4de7edc8e7028692f1c0a1e1ba276837b Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Fri, 11 Dec 2020 02:17:06 +0100 Subject: [PATCH 028/132] [WIP] LegacyCompatibleGitAPIImpl.java: skeleton for GIT_SUBMODULES and GIT_SUBMODULES_FALLBACK support --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index a7639ecae4..cf1a0b1d1a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -401,6 +401,9 @@ public File findParameterizedReferenceRepository(String reference, String url) { // mode suggests a new directory name to host the submodule // (via same rules as for SHA256), and the fallback mode would // return the main directory. + // For each current workspace (recurse): + // try { getSubmodules("HEAD") ... } => List filtered for "commit" items + // getRemoteUrls() => Map } if (referenceExpanded != null) { From ead03a60380837b161733b2f2690880362781d91 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Fri, 11 Dec 2020 09:50:11 +0100 Subject: [PATCH 029/132] CliGitAPIImpl.java: extend getRemote{,Push}Urls() with filters for ACSII and for "safe" URLs without username/password, to match jGit version --- .../plugins/gitclient/CliGitAPIImpl.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index c2b7873e0f..b1875b757b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -1584,11 +1584,7 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr return StringUtils.trim(firstLine(result)); } - /** {@inheritDoc} - * - * Note: unlike the version in JGitAPIImpl, this one does not filter - * ASCII or Private URLs, and just conveys what git config contains. - */ + /** {@inheritDoc} */ @Override public @CheckForNull Map getRemoteUrls() throws GitException, InterruptedException { String result = launchCommand( "config", "--local", "--list" ); @@ -1604,6 +1600,14 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr // If uri String values end up identical, Map only stores one entry uriNames.put(remoteUri, remoteName); + + try { + URI u = new URI(remoteUri); + uriNames.put(u.toASCIIString(), remoteName); + URI uSafe = new URI(u.getScheme(), u.getHost(), u.getPath(), u.getFragment()); + uriNames.put(uSafe.toString(), remoteName); + uriNames.put(uSafe.toASCIIString(), remoteName); + } catch (URISyntaxException ue) {} // ignore, move along } return uriNames; } @@ -1622,6 +1626,14 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr String remoteName = StringUtils.substringBetween(line, "remote.", ".pushurl="); String remoteUri = StringUtils.substringAfter(line, ".pushurl="); uriNames.put(remoteUri, remoteName); + + try { + URI u = new URI(remoteUri); + uriNames.put(u.toASCIIString(), remoteName); + URI uSafe = new URI(u.getScheme(), u.getHost(), u.getPath(), u.getFragment()); + uriNames.put(uSafe.toString(), remoteName); + uriNames.put(uSafe.toASCIIString(), remoteName); + } catch (URISyntaxException ue) {} // ignore, move along } return uriNames; } From 5fde3c916b70c018c7094776d93fde581654944b Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Fri, 11 Dec 2020 10:01:14 +0100 Subject: [PATCH 030/132] GitClientCloneTest.java: if we expect particular wsRefrepo spelling, note it in assertion stacktrace --- .../org/jenkinsci/plugins/gitclient/GitClientCloneTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 26f7953690..0bfcd72c75 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -332,7 +332,8 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce System.err.println("clone output:\n======\n" + messages + "\n======\n"); - assertThat("Reference repo name-parsing logged in: " + messages, + assertThat("Reference repo name-parsing logged in: " + messages + + (wsRefrepo == null ? "" : ("\n...and replaced with: '" + wsRefrepo + "'")) , handler.containsMessageSubstring("Parameterized reference path ") && handler.containsMessageSubstring(" replaced with: ") && (wsRefrepo == null || handler.containsMessageSubstring(wsRefrepo)) , @@ -388,7 +389,8 @@ public void test_clone_reference_parameterized_sha256_fallback() throws Exceptio System.err.println("clone output:\n======\n" + messages + "\n======\n"); - assertThat("Reference repo name-parsing logged in: " + messages, + assertThat("Reference repo name-parsing logged in: " + messages + + (wsRefrepo == null ? "" : ("\n...and replaced with: '" + wsRefrepo + "'")) , handler.containsMessageSubstring("Parameterized reference path ") && handler.containsMessageSubstring(" replaced with: ") && (wsRefrepo == null || handler.containsMessageSubstring(wsRefrepo)) , From 6c7374ccb4f039c94eefe92ffb6e68048646c6a2 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Fri, 11 Dec 2020 10:04:50 +0100 Subject: [PATCH 031/132] GitClientCloneTest.java: for fallback case, expect wsRefrepoBase itself --- .../org/jenkinsci/plugins/gitclient/GitClientCloneTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 0bfcd72c75..426719ce78 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -390,10 +390,9 @@ public void test_clone_reference_parameterized_sha256_fallback() throws Exceptio System.err.println("clone output:\n======\n" + messages + "\n======\n"); assertThat("Reference repo name-parsing logged in: " + messages + - (wsRefrepo == null ? "" : ("\n...and replaced with: '" + wsRefrepo + "'")) , + "\n...and replaced with: '" + wsRefrepoBase + "'", handler.containsMessageSubstring("Parameterized reference path ") && - handler.containsMessageSubstring(" replaced with: ") && - (wsRefrepo == null || handler.containsMessageSubstring(wsRefrepo)) , + handler.containsMessageSubstring(" replaced with: '" + wsRefrepoBase + "'"), is(true)); // Barring filesystem errors, if we have the "custom" refrepo From a2cf84f23d9cdcf9b2c65223ef2fd6534b6e08f8 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Fri, 11 Dec 2020 10:11:10 +0100 Subject: [PATCH 032/132] GitClientCloneTest.java: use the LegacyCompatibleGitAPIImpl.normalizeGitUrl() directly --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 2 +- .../plugins/gitclient/GitClientCloneTest.java | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index cf1a0b1d1a..8c998c3b8b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -257,7 +257,7 @@ public Boolean isParameterizedReferenceRepository(String reference) { * using the same access protocol. This routine converts the "url" string * in a way that helps us confirm whether two spellings mean same thing. */ - public String normalizeGitUrl(String url, Boolean checkLocalPaths) { + public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); if (!url.contains("://")) { if (!url.startsWith("/") && diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 426719ce78..fcfd9149ff 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -293,8 +293,10 @@ public void test_clone_reference_parameterized_fallback() throws Exception, IOEx @Test public void test_clone_reference_parameterized_sha256() throws Exception, IOException, InterruptedException { String wsMirror = workspace.localMirror(); - /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java */ - String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java + * should be okay for network URLs but are too complex for local pathnames */ + //String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + String wsMirrorNormalized = LegacyCompatibleGitAPIImpl.normalizeGitUrl(wsMirror, true); String wsMirrorHash = org.apache.commons.codec.digest.DigestUtils.sha256Hex(wsMirrorNormalized); /* Make a new repo replica to use as refrepo, in specified location */ @@ -350,8 +352,10 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce @Test public void test_clone_reference_parameterized_sha256_fallback() throws Exception, IOException, InterruptedException { String wsMirror = workspace.localMirror(); - /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java */ - String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java + * should be okay for network URLs but are too complex for local pathnames */ + //String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + String wsMirrorNormalized = LegacyCompatibleGitAPIImpl.normalizeGitUrl(wsMirror, true); String wsMirrorHash = org.apache.commons.codec.digest.DigestUtils.sha256Hex(wsMirrorNormalized); /* Make a new repo replica to use as refrepo, in specified location */ From 823bd0f0697d4bffc6a1a82ef9c5a59833c75ac3 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Fri, 11 Dec 2020 10:49:25 +0100 Subject: [PATCH 033/132] GitClientCloneTest.java: for fallback case, expect wsRefrepoBase itself (might have or not the SHA dir after it) --- .../org/jenkinsci/plugins/gitclient/GitClientCloneTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index fcfd9149ff..5bcb91bf80 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -393,10 +393,12 @@ public void test_clone_reference_parameterized_sha256_fallback() throws Exceptio System.err.println("clone output:\n======\n" + messages + "\n======\n"); + // Note: we do not expect the closing single quote after wsRefrepoBase + // because other tests might pollute our test area, and SHA dir is there assertThat("Reference repo name-parsing logged in: " + messages + - "\n...and replaced with: '" + wsRefrepoBase + "'", + "\n...and replaced with: '" + wsRefrepoBase, handler.containsMessageSubstring("Parameterized reference path ") && - handler.containsMessageSubstring(" replaced with: '" + wsRefrepoBase + "'"), + handler.containsMessageSubstring(" replaced with: '" + wsRefrepoBase), is(true)); // Barring filesystem errors, if we have the "custom" refrepo From 0e5e0cd9673ded2fee8cf3b4a49544d156cf4d86 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sat, 12 Dec 2020 01:31:31 +0100 Subject: [PATCH 034/132] LegacyCompatibleGitAPIImpl.java: skeleton for getSubmodulesUrls() as needed for GIT_SUBMODULES --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 54 ++++++++++++++++--- 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 8c998c3b8b..4186c00a4f 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -24,8 +24,10 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -289,6 +291,45 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { return urlNormalized; } + /** Find referenced URLs in this repo and its submodules, recursively. + * Current primary use is for parameterized refrepo/${GIT_SUBMODULES} + * + * @return a Set of (unique) String arrays, representing: + * [0] directory of nested submodule (relative to current workspace root) + * The current workspace would be listed as directory "" and consumers + * should check these entries last if they care for most-specific hits + * with smaller-scope reference repositories. + * [1] url as returned by getRemoteUrls() - fetch URLs, maybe several entries per remote + * [2] urlNormalized from normalizeGitUrl(url, true) (local pathnames fully qualified) + * [3] remoteName as defined in that nested submodule + * + * @param needle - a normalized URL (coming from normalizeGitUrl(url, true)) + * which we want to find if it is not null - so stop and + * return just hits for it as soon as we have something. + */ + public Set getSubmodulesUrls(String needle) { + Set result = new HashSet<>(); + // For each current workspace (recurse?): + // subGit(subdir) + // try { getSubmodules("HEAD") ... } => List filtered for "commit" items + // getRemoteUrls() => Map + + // If needle is not null, look in SHA256 named dir that can match it; + // note that this use-case might pan out also if "this" repo is bare + // and can not have "proper" git submodules. If it is bare, look at + // remote URLs in current dir after the guessed subdir and return then. + + // Otherwise first dig into submodules, when there is no deeper to drill, + // report remote URLs and step back from recursion. + return result; + } + + /** See above. With null, returns all we can find (slower) for the caller + * to parse */ + public Set getSubmodulesUrls() { + return getSubmodulesUrls(null); + } + /** Yield the File object for the reference repository local filesystem * pathname. Note that the provided string may be suffixed with expandable * tokens which allow to store a filesystem structure of multiple small @@ -397,13 +438,12 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Assuming the provided "reference" directory already hosts // submodules, we use git tools to find the one subdir which // has a registered remote URL equivalent (per normalization) - // to the provided "url". If there is no hit, the non-fallback - // mode suggests a new directory name to host the submodule - // (via same rules as for SHA256), and the fallback mode would - // return the main directory. - // For each current workspace (recurse): - // try { getSubmodules("HEAD") ... } => List filtered for "commit" items - // getRemoteUrls() => Map + // to the provided "url". + // If there is no hit, the non-fallback mode suggests a new + // directory name to host the submodule (same rules as SHA), + // and the fallback mode would return the main directory. + + // getSubmodulesUrls(urlNormalized) => dirname if any } if (referenceExpanded != null) { From 172e40eeb2bff56b1415d29b6738601ce07d0d84 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 14 Dec 2020 10:19:40 +0100 Subject: [PATCH 035/132] LegacyCompatibleGitAPIImpl.java: update comments for ideas about submodule searches --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 4186c00a4f..50d638887a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -299,8 +299,10 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * The current workspace would be listed as directory "" and consumers * should check these entries last if they care for most-specific hits * with smaller-scope reference repositories. - * [1] url as returned by getRemoteUrls() - fetch URLs, maybe several entries per remote - * [2] urlNormalized from normalizeGitUrl(url, true) (local pathnames fully qualified) + * [1] url as returned by getRemoteUrls() - fetch URLs, maybe several + * entries per remote + * [2] urlNormalized from normalizeGitUrl(url, true) (local pathnames + * fully qualified) * [3] remoteName as defined in that nested submodule * * @param needle - a normalized URL (coming from normalizeGitUrl(url, true)) @@ -308,19 +310,34 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * return just hits for it as soon as we have something. */ public Set getSubmodulesUrls(String needle) { + // Keep track of where we've already looked in the "result" Set, to + // avoid looking in same places (different strategies below) twice. + // And eventually return this Set or part of it as the answer. Set result = new HashSet<>(); - // For each current workspace (recurse?): - // subGit(subdir) + + // For each current workspace (recurse or big loop in same context?): + // public GitClient subGit(String subdir) => would this.subGit(...) + // give us a copy of this applied class instance (CLI Git vs jGit)? // try { getSubmodules("HEAD") ... } => List filtered for "commit" items // getRemoteUrls() => Map - // If needle is not null, look in SHA256 named dir that can match it; - // note that this use-case might pan out also if "this" repo is bare - // and can not have "proper" git submodules. If it is bare, look at - // remote URLs in current dir after the guessed subdir and return then. - - // Otherwise first dig into submodules, when there is no deeper to drill, - // report remote URLs and step back from recursion. + // If needle is not null, first look perhaps in the subdir(s) named + // with base-name of the URL with and without a ".git" suffix, then + // in SHA256 named dir that can match it; note that this use-case + // might pan out also if "this" repo is bare and can not have "proper" + // git submodules - but was prepared for our other options. + + // If current repo *is* bare (can't have proper submodules) and the + // needle is not null, follow up with: + // * Maybe also direct child dirs that have a ".git" FS object inside?.. + // * Look at remote URLs in current dir after the guessed subdir and + // return then. + + // If current dir does have submodules, first dig into submodules, + // when there is no deeper to drill, report remote URLs and step + // back from recursion. This way we have least specific repo last, + // if several have the replica (assuming the first hits are smaller + // scopes). return result; } From d6cebe891fdf4a204b90ffdb6aa3b7ea8ed3db77 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 14 Dec 2020 14:03:44 +0100 Subject: [PATCH 036/132] LegacyCompatibleGitAPIImpl.java: getSubmodulesUrls(): implement the search for "needle" in bare repo case --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 128 +++++++++++++++++- 1 file changed, 122 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 50d638887a..0f6b0a3ce6 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -25,6 +25,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -313,7 +314,15 @@ public Set getSubmodulesUrls(String needle) { // Keep track of where we've already looked in the "result" Set, to // avoid looking in same places (different strategies below) twice. // And eventually return this Set or part of it as the answer. - Set result = new HashSet<>(); + Set result = new LinkedHashSet<>(); // Retain order of insertion + File f = null; + + Boolean isBare = false; + try { + isBare = this.isBareRepository(); + } catch (InterruptedException e) { + isBare = false; // At least try to look into submodules... + } // For each current workspace (recurse or big loop in same context?): // public GitClient subGit(String subdir) => would this.subGit(...) @@ -326,12 +335,119 @@ public Set getSubmodulesUrls(String needle) { // in SHA256 named dir that can match it; note that this use-case // might pan out also if "this" repo is bare and can not have "proper" // git submodules - but was prepared for our other options. + if (needle != null && !needle.isEmpty()) { + int sep = needle.lastIndexOf("/"); + String needleNorm = normalizeGitUrl(needle, true); + String needleBasename; + if (sep < 0) { + needleBasename = needle; + } else { + needleBasename = needle.substring(sep + 1); + } + needleBasename = needleBasename.replaceAll(".git$", ""); + + // Try with the basename without .git extension, and then with one. + ArrayList arrDirnames = new ArrayList(); + arrDirnames.add(needleBasename); + arrDirnames.add(needleBasename + ".git"); + String needleBasenameLC = needleBasename.toLowerCase(); + if (!needleBasenameLC.equals(needleBasename)) { + // Retry with lowercased dirname + arrDirnames.add(needleBasenameLC); + arrDirnames.add(needleBasenameLC + ".git"); + } + String needleSha = org.apache.commons.codec.digest.DigestUtils.sha256Hex(needleNorm); + arrDirnames.add(needleSha); + arrDirnames.add(needleSha + ".git"); + + for (String dirname : arrDirnames) { + f = new File(".", dirname); + if (f.exists() && f.isDirectory()) { + try { + //LegacyCompatibleGitAPIImpl? + GitClient g = this.subGit(needleBasename); + Map uriNames = g.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new HashSet<>(); + result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); + return result; + } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. + result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); + } + } catch (Exception e) { + // ignore, go to next slide + } + } + } - // If current repo *is* bare (can't have proper submodules) and the - // needle is not null, follow up with: - // * Maybe also direct child dirs that have a ".git" FS object inside?.. - // * Look at remote URLs in current dir after the guessed subdir and - // return then. + // If current repo *is* bare (can't have proper submodules) and the + // needle is not null, follow up with: + // * Maybe also direct child dirs that have a ".git" FS object inside?.. + // * Look at remote URLs in current dir after the guessed subdir and + // return then. + if (isBare) { + arrDirnames.clear(); + + // Check subdirs that are git workspaces + Set checkedDirs = new HashSet<>(); + for (String[] resultEntry : result) { + checkedDirs.add(resultEntry[0]); + } + File[] directories = new File(".").listFiles(File::isDirectory); + for (File dir : directories) { + f = new File(dir, ".git"); + if (f.exists()) { // May be a file or directory... or symlink to those... + String dirname = dir.getPath().replaceAll("/*$", ""); + if (!checkedDirs.contains(dirname)) { + arrDirnames.add(dirname); + } + } + } + + // Finally check parent dir + arrDirnames.add("."); + + for (String dirname : arrDirnames) { + f = new File(".", dirname); + if (f.exists() && f.isDirectory()) { + try { + //LegacyCompatibleGitAPIImpl? + GitClient g = this.subGit(needleBasename); + Map uriNames = g.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new HashSet<>(); + result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); + return result; + } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. + // Commented away: currently we exit after this loop. + //result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); + } + } catch (Exception e) { + // ignore, go to next slide + } + } + } + + // Nothing found, and for bare top-level repo not much to search for + return new HashSet<>(); + } + } else { + // ...and if there is no needle? + /* if (isBare) { ... } */ + // fall through currently, let git decide if it has submodules here and now + } // If current dir does have submodules, first dig into submodules, // when there is no deeper to drill, report remote URLs and step From 864e34febd4f936d4dfeca1a51084cdf49179759 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 10 Jan 2021 01:04:13 +0100 Subject: [PATCH 037/132] LegacyCompatibleGitAPIImpl.java: move arrDirNames declaration higher in scope --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 0f6b0a3ce6..cb219ea427 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -316,6 +316,8 @@ public Set getSubmodulesUrls(String needle) { // And eventually return this Set or part of it as the answer. Set result = new LinkedHashSet<>(); // Retain order of insertion File f = null; + // Helper list storage in loops below + ArrayList arrDirnames = new ArrayList(); Boolean isBare = false; try { @@ -347,7 +349,6 @@ public Set getSubmodulesUrls(String needle) { needleBasename = needleBasename.replaceAll(".git$", ""); // Try with the basename without .git extension, and then with one. - ArrayList arrDirnames = new ArrayList(); arrDirnames.add(needleBasename); arrDirnames.add(needleBasename + ".git"); String needleBasenameLC = needleBasename.toLowerCase(); From 55fda14f7673b4fb82b169b180577d12a1ab8b72 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 10 Jan 2021 01:04:56 +0100 Subject: [PATCH 038/132] LegacyCompatibleGitAPIImpl.java: update commented thoughts --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index cb219ea427..de42fbeb00 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -390,12 +390,14 @@ public Set getSubmodulesUrls(String needle) { // If current repo *is* bare (can't have proper submodules) and the // needle is not null, follow up with: // * Maybe also direct child dirs that have a ".git" FS object inside?.. - // * Look at remote URLs in current dir after the guessed subdir and - // return then. + // * Look at remote URLs in current dir after the guessed subdirs failed, + // and return then. if (isBare) { arrDirnames.clear(); // Check subdirs that are git workspaces + // TODO: Refactor to avoid lookups of dirs that may prove not + // needed in the end (aim for less I/Os to find the goal) Set checkedDirs = new HashSet<>(); for (String[] resultEntry : result) { checkedDirs.add(resultEntry[0]); @@ -442,6 +444,7 @@ public Set getSubmodulesUrls(String needle) { } // Nothing found, and for bare top-level repo not much to search for + // Or maybe also fall through to let git decide? return new HashSet<>(); } } else { From 3e81723af9dc9def9b904cbf481c732db8ce822c Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 10 Jan 2021 01:05:44 +0100 Subject: [PATCH 039/132] LegacyCompatibleGitAPIImpl.java: prepare to parse submodule workspaces --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index de42fbeb00..b2556bf144 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -326,12 +326,6 @@ public Set getSubmodulesUrls(String needle) { isBare = false; // At least try to look into submodules... } - // For each current workspace (recurse or big loop in same context?): - // public GitClient subGit(String subdir) => would this.subGit(...) - // give us a copy of this applied class instance (CLI Git vs jGit)? - // try { getSubmodules("HEAD") ... } => List filtered for "commit" items - // getRemoteUrls() => Map - // If needle is not null, first look perhaps in the subdir(s) named // with base-name of the URL with and without a ".git" suffix, then // in SHA256 named dir that can match it; note that this use-case @@ -453,6 +447,22 @@ public Set getSubmodulesUrls(String needle) { // fall through currently, let git decide if it has submodules here and now } + // For each current workspace (recurse or big loop in same context?): + // public GitClient subGit(String subdir) => would this.subGit(...) + // give us a copy of this applied class instance (CLI Git vs jGit)? + // get submodule name-vs-one-url from .gitmodules if present, for a + // faster possible answer (only bother if needle is not null?) + // try { getSubmodules("HEAD") ... } => List filtered for + // "commit" items + // getRemoteUrls() => Map + arrDirnames.clear(); + + // Check subdirs that are git workspaces + Set checkedDirs = new HashSet<>(); + for (String[] resultEntry : result) { + checkedDirs.add(resultEntry[0]); + } + // If current dir does have submodules, first dig into submodules, // when there is no deeper to drill, report remote URLs and step // back from recursion. This way we have least specific repo last, From ff65341e1c15809c906c61a81dae05cb5e21e348 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 10 Jan 2021 01:06:29 +0100 Subject: [PATCH 040/132] LegacyCompatibleGitAPIImpl.java: uncomment needle tracking in a loop that we might not-abort after all --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index b2556bf144..77562022a0 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -428,8 +428,7 @@ public Set getSubmodulesUrls(String needle) { // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop // which by construct looks at different dirs so far. - // Commented away: currently we exit after this loop. - //result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); + result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); } } catch (Exception e) { // ignore, go to next slide From 122746fe54e74a3c6f2565cff189ed6657a86d70 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 10 Jan 2021 02:28:37 +0100 Subject: [PATCH 041/132] LegacyCompatibleGitAPIImpl.java: fix @param usage --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 77562022a0..2af842e0fd 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -214,8 +214,7 @@ public File getObjectPath(File referencePath) { /** Handle magic strings in the reference pathname to sort out patterns * classified as evaluated by parametrization, as handled below * - * @params - * reference Pathname (maybe with magic suffix) to reference repo + * @param reference Pathname (maybe with magic suffix) to reference repo */ public Boolean isParameterizedReferenceRepository(File reference) { if (reference == null) { @@ -486,10 +485,9 @@ public Set getSubmodulesUrls() { * * At some point this plugin might also maintain that filesystem structure. * - * @params - * reference Pathname (maybe with magic suffix) to reference repo - * url URL of the repository being cloned, to help choose a - * suitable parameterized reference repo subdirectory. + * @param reference Pathname (maybe with magic suffix) to reference repo + * @param url URL of the repository being cloned, to help choose a + * suitable parameterized reference repo subdirectory. */ public File findParameterizedReferenceRepository(File reference, String url) { if (reference == null) { From bf978eec0544589e7781085f6664a0f0a29dfd0f Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 10 Jan 2021 02:48:36 +0100 Subject: [PATCH 042/132] LegacyCompatibleGitAPIImpl.java: implement handling output of getSubmodulesUrls() in findParameterizedReferenceRepository() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 2af842e0fd..716462daab 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -406,7 +406,7 @@ public Set getSubmodulesUrls(String needle) { } } - // Finally check parent dir + // Finally check pattern's parent dir arrDirnames.add("."); for (String dirname : arrDirnames) { @@ -583,11 +583,32 @@ public File findParameterizedReferenceRepository(String reference, String url) { // submodules, we use git tools to find the one subdir which // has a registered remote URL equivalent (per normalization) // to the provided "url". - // If there is no hit, the non-fallback mode suggests a new - // directory name to host the submodule (same rules as SHA), - // and the fallback mode would return the main directory. - // getSubmodulesUrls(urlNormalized) => dirname if any + Set subEntries = getSubmodulesUrls(urlNormalized); + if (!subEntries.isEmpty()) { + // Normally we should only have one entry here, as sorted by the routine + referenceExpanded = subEntries.iterator().next()[0]; + if (getObjectPath(referenceExpanded) == null) { + // chop it off, use main directory + referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", "").replaceAll("/\\$\\{GIT_SUBMODULES\\}$", ""); + } + } else { + // If there is no hit, the non-fallback mode suggests a new + // directory name to host the submodule (same rules as SHA), + // and the fallback mode would return the main directory. + if (reference.endsWith("/${GIT_SUBMODULES}")) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", + org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + if (getObjectPath(referenceExpanded) == null) { + // chop it off, use main directory + referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES\\}$", ""); + } + } + else if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { + // chop it off, use main directory + referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", ""); + } + } } if (referenceExpanded != null) { From aacc0cdd13f8a767a3561063d576a860d9601b83 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 10 Jan 2021 03:14:36 +0100 Subject: [PATCH 043/132] LegacyCompatibleGitAPIImpl.java: getSubmodulesUrls() / findParameterizedReferenceRepository(): deal with LinkedHashSet instead of plain Set to maintain insertion order as the priority --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 716462daab..ef2796ba68 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -309,11 +309,11 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * which we want to find if it is not null - so stop and * return just hits for it as soon as we have something. */ - public Set getSubmodulesUrls(String needle) { + public LinkedHashSet getSubmodulesUrls(String needle) { // Keep track of where we've already looked in the "result" Set, to // avoid looking in same places (different strategies below) twice. // And eventually return this Set or part of it as the answer. - Set result = new LinkedHashSet<>(); // Retain order of insertion + LinkedHashSet result = new LinkedHashSet<>(); // Retain order of insertion File f = null; // Helper list storage in loops below ArrayList arrDirnames = new ArrayList(); @@ -365,7 +365,7 @@ public Set getSubmodulesUrls(String needle) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new HashSet<>(); + result = new LinkedHashSet<>(); result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); return result; } @@ -391,7 +391,7 @@ public Set getSubmodulesUrls(String needle) { // Check subdirs that are git workspaces // TODO: Refactor to avoid lookups of dirs that may prove not // needed in the end (aim for less I/Os to find the goal) - Set checkedDirs = new HashSet<>(); + LinkedHashSet checkedDirs = new LinkedHashSet<>(); for (String[] resultEntry : result) { checkedDirs.add(resultEntry[0]); } @@ -420,7 +420,7 @@ public Set getSubmodulesUrls(String needle) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new HashSet<>(); + result = new LinkedHashSet<>(); result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); return result; } @@ -437,7 +437,7 @@ public Set getSubmodulesUrls(String needle) { // Nothing found, and for bare top-level repo not much to search for // Or maybe also fall through to let git decide? - return new HashSet<>(); + return new LinkedHashSet<>(); } } else { // ...and if there is no needle? @@ -456,7 +456,7 @@ public Set getSubmodulesUrls(String needle) { arrDirnames.clear(); // Check subdirs that are git workspaces - Set checkedDirs = new HashSet<>(); + LinkedHashSet checkedDirs = new LinkedHashSet<>(); for (String[] resultEntry : result) { checkedDirs.add(resultEntry[0]); } @@ -471,7 +471,7 @@ public Set getSubmodulesUrls(String needle) { /** See above. With null, returns all we can find (slower) for the caller * to parse */ - public Set getSubmodulesUrls() { + public LinkedHashSet getSubmodulesUrls() { return getSubmodulesUrls(null); } @@ -584,7 +584,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { // has a registered remote URL equivalent (per normalization) // to the provided "url". - Set subEntries = getSubmodulesUrls(urlNormalized); + LinkedHashSet subEntries = getSubmodulesUrls(urlNormalized); if (!subEntries.isEmpty()) { // Normally we should only have one entry here, as sorted by the routine referenceExpanded = subEntries.iterator().next()[0]; From 1a8306aec5326e4723a09ad196d6291e7c241f34 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 10 Jan 2021 03:16:38 +0100 Subject: [PATCH 044/132] LegacyCompatibleGitAPIImpl.java: findParameterizedReferenceRepository(): comment a TODO in case there are several hits in the getSubmodulesUrls() return --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index ef2796ba68..f92f058394 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -587,6 +587,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { LinkedHashSet subEntries = getSubmodulesUrls(urlNormalized); if (!subEntries.isEmpty()) { // Normally we should only have one entry here, as sorted by the routine + // TODO: If several entries are present after all, iterate until first existing hit referenceExpanded = subEntries.iterator().next()[0]; if (getObjectPath(referenceExpanded) == null) { // chop it off, use main directory From 769a2d4b6c28cb00934e4148a78557e3f2a9785d Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 04:22:26 +0100 Subject: [PATCH 045/132] LegacyCompatibleGitAPIImpl.java: log-tracing for getSubmodulesUrls() --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f92f058394..fc9c04df99 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -359,11 +359,13 @@ public LinkedHashSet getSubmodulesUrls(String needle) { if (f.exists() && f.isDirectory()) { try { //LegacyCompatibleGitAPIImpl? + System.err.println("getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'\n"); GitClient g = this.subGit(needleBasename); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); + System.err.println("getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')\n"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); @@ -414,11 +416,13 @@ public LinkedHashSet getSubmodulesUrls(String needle) { if (f.exists() && f.isDirectory()) { try { //LegacyCompatibleGitAPIImpl? + System.err.println("getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'\n"); GitClient g = this.subGit(needleBasename); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); + System.err.println("getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')\n"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); @@ -589,11 +593,13 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Normally we should only have one entry here, as sorted by the routine // TODO: If several entries are present after all, iterate until first existing hit referenceExpanded = subEntries.iterator().next()[0]; + System.err.println("findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + "' from subEntries\n"); if (getObjectPath(referenceExpanded) == null) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", "").replaceAll("/\\$\\{GIT_SUBMODULES\\}$", ""); } } else { + System.err.println("findParameterizedReferenceRepository(): got no subEntries\n"); // If there is no hit, the non-fallback mode suggests a new // directory name to host the submodule (same rules as SHA), // and the fallback mode would return the main directory. From 654362a0218fb1114be68c3cc163f4139c9323cf Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 04:22:51 +0100 Subject: [PATCH 046/132] LegacyCompatibleGitAPIImpl.java: for now look in subdirs regardless of isbare --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index fc9c04df99..89148cc84e 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -387,7 +387,8 @@ public LinkedHashSet getSubmodulesUrls(String needle) { // * Maybe also direct child dirs that have a ".git" FS object inside?.. // * Look at remote URLs in current dir after the guessed subdirs failed, // and return then. - if (isBare) { + // TODO: Remove "|| true" when other logic fo look for real submodules is complete + if (isBare || true) { arrDirnames.clear(); // Check subdirs that are git workspaces @@ -441,7 +442,9 @@ public LinkedHashSet getSubmodulesUrls(String needle) { // Nothing found, and for bare top-level repo not much to search for // Or maybe also fall through to let git decide? - return new LinkedHashSet<>(); + if (isBare) { + return new LinkedHashSet<>(); + } } } else { // ...and if there is no needle? @@ -459,7 +462,7 @@ public LinkedHashSet getSubmodulesUrls(String needle) { // getRemoteUrls() => Map arrDirnames.clear(); - // Check subdirs that are git workspaces + // TODO: Check subdirs that are git workspaces, and remove "|| true" above LinkedHashSet checkedDirs = new LinkedHashSet<>(); for (String[] resultEntry : result) { checkedDirs.add(resultEntry[0]); From 5b6ff6d699ef8e042a8283e47409be76226325e7 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 04:23:01 +0100 Subject: [PATCH 047/132] LegacyCompatibleGitAPIImpl.java: comment updated --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 89148cc84e..82c8f1f824 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -447,7 +447,8 @@ public LinkedHashSet getSubmodulesUrls(String needle) { } } } else { - // ...and if there is no needle? + // ...and if there is no needle? Return info on all git remotes + // found under this directory... /* if (isBare) { ... } */ // fall through currently, let git decide if it has submodules here and now } From 8d8514c93ab67738bc09e2baa09b8fa92e14a63c Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 14:01:25 +0100 Subject: [PATCH 048/132] LegacyCompatibleGitAPIImpl.java: rename getObjectPath() to less misleading getObjectsFile(), and varnames inside it too --- .../plugins/gitclient/CliGitAPIImpl.java | 2 +- .../plugins/gitclient/JGitAPIImpl.java | 2 +- .../gitclient/LegacyCompatibleGitAPIImpl.java | 40 +++++++++---------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index b1875b757b..d6dda9cdaa 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -784,7 +784,7 @@ public void execute() throws GitException, InterruptedException { else if (!referencePath.isDirectory()) listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); else { - File objectsPath = getObjectPath(referencePath); + File objectsPath = getObjectsFile(referencePath); if (objectsPath == null || !objectsPath.isDirectory()) listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (not a git repo?): " + objectsPath); else { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index cf609ebcb9..a46330a4ca 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1473,7 +1473,7 @@ public void execute() throws GitException, InterruptedException { else if (!referencePath.isDirectory()) listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); else { - File objectsPath = getObjectPath(referencePath); + File objectsPath = getObjectsFile(referencePath); if (objectsPath == null || !objectsPath.isDirectory()) listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (no git repo?): " + objectsPath); else { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 82c8f1f824..69fb29ffe2 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -171,44 +171,44 @@ public void clone(RemoteConfig rc, boolean useShallowClone) throws GitException, * paths, while a not-null return value is instantly usable by * some code which plays with git under its hood. */ - public File getObjectPath(String reference) { + public File getObjectsFile(String reference) { if (reference == null || reference.isEmpty()) { return null; } - return getObjectPath(new File(reference)); + return getObjectsFile(new File(reference)); } - public File getObjectPath(File referencePath) { - if (referencePath == null) { - return referencePath; + public File getObjectsFile(File reference) { + if (reference == null) { + return reference; } - if (!referencePath.exists()) + if (!reference.exists()) return null; - if (!referencePath.isDirectory()) + if (!reference.isDirectory()) return null; - // reference path can either be a normal or a base repository - File objectsPath = new File(referencePath, ".git/objects"); - if (objectsPath == null) { - return objectsPath; // Some Java error, could not make object from the paths involved + // reference pathname can either point to a normal or a base repository + File objects = new File(reference, ".git/objects"); + if (objects == null) { + return objects; // Some Java error, could not make object from the paths involved } - if (!objectsPath.isDirectory()) { + if (!objects.isDirectory()) { // reference path is bare repo - objectsPath = new File(referencePath, "objects"); - if (objectsPath == null) { - return objectsPath; // Some Java error, could not make object from the paths involved + objects = new File(reference, "objects"); + if (objects == null) { + return objects; // Some Java error, could not make object from the paths involved } } - if (!objectsPath.isDirectory()) + if (!objects.isDirectory()) return null; // If we get here, we have a non-null File referencing a // "(.git/)objects" subdir under original referencePath - return objectsPath; + return objects; } /** Handle magic strings in the reference pathname to sort out patterns @@ -576,7 +576,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { // repository into smaller chunks without breaking builds. referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_SHA256_FALLBACK\\}$", org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); - if (getObjectPath(referenceExpanded) == null) { + if (getObjectsFile(referenceExpanded) == null) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_SHA256_FALLBACK\\}$", ""); } @@ -598,7 +598,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { // TODO: If several entries are present after all, iterate until first existing hit referenceExpanded = subEntries.iterator().next()[0]; System.err.println("findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + "' from subEntries\n"); - if (getObjectPath(referenceExpanded) == null) { + if (getObjectsFile(referenceExpanded) == null) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", "").replaceAll("/\\$\\{GIT_SUBMODULES\\}$", ""); } @@ -610,7 +610,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { if (reference.endsWith("/${GIT_SUBMODULES}")) { referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); - if (getObjectPath(referenceExpanded) == null) { + if (getObjectsFile(referenceExpanded) == null) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES\\}$", ""); } From 925eabae288c8607865436bb7ba9627a42dfb74c Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 14:04:05 +0100 Subject: [PATCH 049/132] LegacyCompatibleGitAPIImpl.java: in findParameterizedReferenceRepository() fallbacks, quick-check for ".git" dirname existence and usability too --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 69fb29ffe2..2a8a8d6034 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -576,13 +576,13 @@ public File findParameterizedReferenceRepository(String reference, String url) { // repository into smaller chunks without breaking builds. referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_SHA256_FALLBACK\\}$", org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); - if (getObjectsFile(referenceExpanded) == null) { + if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_SHA256_FALLBACK\\}$", ""); } } else if (reference.endsWith("/${GIT_URL_FALLBACK}")) { referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_FALLBACK\\}$", urlNormalized); - if (getObjectPath(referenceExpanded) == null) { + if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_FALLBACK\\}$", ""); } @@ -598,7 +598,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { // TODO: If several entries are present after all, iterate until first existing hit referenceExpanded = subEntries.iterator().next()[0]; System.err.println("findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + "' from subEntries\n"); - if (getObjectsFile(referenceExpanded) == null) { + if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", "").replaceAll("/\\$\\{GIT_SUBMODULES\\}$", ""); } @@ -610,7 +610,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { if (reference.endsWith("/${GIT_SUBMODULES}")) { referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); - if (getObjectsFile(referenceExpanded) == null) { + if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES\\}$", ""); } From a103bdda97f7e0a394d5c8d05b3bbb1410d7b9a2 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 14:05:08 +0100 Subject: [PATCH 050/132] LegacyCompatibleGitAPIImpl.java: in findParameterizedReferenceRepository() move GIT_URL_FALLBACK closer to GIT_URL --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 2a8a8d6034..5056f51dc6 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -232,11 +232,11 @@ public Boolean isParameterizedReferenceRepository(String reference) { return true; } - if (reference.endsWith("/${GIT_URL_SHA256}")) { + if (reference.endsWith("/${GIT_URL_FALLBACK}")) { return true; } - if (reference.endsWith("/${GIT_URL_FALLBACK}")) { + if (reference.endsWith("/${GIT_URL_SHA256}")) { return true; } @@ -565,6 +565,12 @@ public File findParameterizedReferenceRepository(String reference, String url) { // or storing multiple closely related forks together. referenceExpanded = reference.replaceAll("\\$\\{GIT_URL\\}$", urlNormalized); + } else if (reference.endsWith("/${GIT_URL_FALLBACK}")) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_FALLBACK\\}$", urlNormalized); + if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + // chop it off, use main directory + referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_FALLBACK\\}$", ""); + } } else if (reference.endsWith("/${GIT_URL_SHA256}")) { // This may be the more portable solution with regard to filesystems referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_SHA256\\}$", @@ -580,12 +586,6 @@ public File findParameterizedReferenceRepository(String reference, String url) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_SHA256_FALLBACK\\}$", ""); } - } else if (reference.endsWith("/${GIT_URL_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_FALLBACK\\}$", urlNormalized); - if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { - // chop it off, use main directory - referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_FALLBACK\\}$", ""); - } } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") ) { // Assuming the provided "reference" directory already hosts // submodules, we use git tools to find the one subdir which From aaaef9c0130a33907c15febbcdcb19e407046d21 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 14:54:15 +0100 Subject: [PATCH 051/132] LegacyCompatibleGitAPIImpl.java: in findParameterizedReferenceRepository() add a GIT_URL_BASENAME/GIT_URL_BASENAME_FALLBACK option --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 5056f51dc6..063e67a218 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -244,6 +244,14 @@ public Boolean isParameterizedReferenceRepository(String reference) { return true; } + if (reference.endsWith("/${GIT_URL_BASENAME}")) { + return true; + } + + if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { + return true; + } + if (reference.endsWith("/${GIT_SUBMODULES}")) { return true; } @@ -586,6 +594,28 @@ public File findParameterizedReferenceRepository(String reference, String url) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_SHA256_FALLBACK\\}$", ""); } + } else if (reference.endsWith("/${GIT_URL_BASENAME}") || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { + // This may be the more portable solution with regard to filesystems + int sep = urlNormalized.lastIndexOf("/"); + String needleBasename; + if (sep < 0) { + needleBasename = urlNormalized; + } else { + needleBasename = urlNormalized.substring(sep + 1); + } + needleBasename = needleBasename.replaceAll(".git$", ""); + + if (reference.endsWith("/${GIT_URL_BASENAME}")) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", + needleBasename); + } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", + needleBasename); + if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + // chop it off, use main directory + referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", ""); + } + } } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") ) { // Assuming the provided "reference" directory already hosts // submodules, we use git tools to find the one subdir which From a77dab9df9abf6efe292e08a0bf4e801de41cd16 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 14:54:43 +0100 Subject: [PATCH 052/132] LegacyCompatibleGitAPIImpl.java: in findParameterizedReferenceRepository() revise the GIT_SUBMODULES/GIT_SUBMODULES_FALLBACK handling --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 063e67a218..e8f19a237b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -624,13 +624,24 @@ public File findParameterizedReferenceRepository(String reference, String url) { LinkedHashSet subEntries = getSubmodulesUrls(urlNormalized); if (!subEntries.isEmpty()) { - // Normally we should only have one entry here, as sorted by the routine - // TODO: If several entries are present after all, iterate until first existing hit - referenceExpanded = subEntries.iterator().next()[0]; + // Normally we should only have one entry here, as sorted + // by the routine, and prefer that first option if a new + // reference repo would have to be made (and none exists). + // If several entries are present after all, iterate until + // first existing hit and return the first entry otherwise. + for (String[] subEntry : subEntries) { + if (getObjectsFile(subEntry[0]) != null || getObjectsFile(subEntry[0] + ".git") != null) { + referenceExpanded = subEntry[0]; + break; + } + } + if (referenceExpanded == null) { + referenceExpanded = subEntries.iterator().next()[0]; + } System.err.println("findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + "' from subEntries\n"); - if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory - referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", "").replaceAll("/\\$\\{GIT_SUBMODULES\\}$", ""); + referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", ""); } } else { System.err.println("findParameterizedReferenceRepository(): got no subEntries\n"); @@ -640,12 +651,8 @@ public File findParameterizedReferenceRepository(String reference, String url) { if (reference.endsWith("/${GIT_SUBMODULES}")) { referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); - if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { - // chop it off, use main directory - referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES\\}$", ""); - } } - else if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { + else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", ""); } From d664eb348d209a26f6e5bc8afe061c6598aece9e Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 19:29:05 +0100 Subject: [PATCH 053/132] LegacyCompatibleGitAPIImpl.java: pass not-normalized URL from findParameterizedReferenceRepository() to getSubmodulesUrls(), in case users manually cloned refrepos with whatever casing they use throughout --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index e8f19a237b..7256ab735e 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -313,9 +313,10 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * fully qualified) * [3] remoteName as defined in that nested submodule * - * @param needle - a normalized URL (coming from normalizeGitUrl(url, true)) - * which we want to find if it is not null - so stop and - * return just hits for it as soon as we have something. + * @param needle - an URL which (or its normalized variant coming from + * normalizeGitUrl(url, true)) we want to find: + * if it is not null - then stop and return just hits + * for it as soon as we have something. */ public LinkedHashSet getSubmodulesUrls(String needle) { // Keep track of where we've already looked in the "result" Set, to @@ -340,16 +341,26 @@ public LinkedHashSet getSubmodulesUrls(String needle) { // git submodules - but was prepared for our other options. if (needle != null && !needle.isEmpty()) { int sep = needle.lastIndexOf("/"); - String needleNorm = normalizeGitUrl(needle, true); String needleBasename; if (sep < 0) { needleBasename = needle; } else { needleBasename = needle.substring(sep + 1); } - needleBasename = needleBasename.replaceAll(".git$", ""); + needleBasename = needleBasename.replaceAll(".[Gg][Ii][Tt]$", ""); + + String needleNorm = normalizeGitUrl(needle, true); + sep = needleNorm.lastIndexOf("/"); + String needleNormBasename; + if (sep < 0) { + needleNormBasename = needleNorm; + } else { + needleNormBasename = needleNorm.substring(sep + 1); + } + needleNormBasename = needleNormBasename.replaceAll(".git$", ""); // Try with the basename without .git extension, and then with one. + // First we try the caller-provided string casing, then normalized. arrDirnames.add(needleBasename); arrDirnames.add(needleBasename + ".git"); String needleBasenameLC = needleBasename.toLowerCase(); @@ -358,6 +369,11 @@ public LinkedHashSet getSubmodulesUrls(String needle) { arrDirnames.add(needleBasenameLC); arrDirnames.add(needleBasenameLC + ".git"); } + if (!needleNormBasename.equals(needleBasenameLC)) { + arrDirnames.add(needleNormBasename); + arrDirnames.add(needleNormBasename + ".git"); + } + String needleSha = org.apache.commons.codec.digest.DigestUtils.sha256Hex(needleNorm); arrDirnames.add(needleSha); arrDirnames.add(needleSha + ".git"); @@ -622,7 +638,9 @@ public File findParameterizedReferenceRepository(String reference, String url) { // has a registered remote URL equivalent (per normalization) // to the provided "url". - LinkedHashSet subEntries = getSubmodulesUrls(urlNormalized); + // Note: we pass "url" here, the routine differentiates original + // naming vs. normalization. + LinkedHashSet subEntries = getSubmodulesUrls(url); if (!subEntries.isEmpty()) { // Normally we should only have one entry here, as sorted // by the routine, and prefer that first option if a new From e29d9478c7cfce95329b0e58f24507f2c8262259 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 19:29:45 +0100 Subject: [PATCH 054/132] LegacyCompatibleGitAPIImpl.java: check also not-normalized URL in findParameterizedReferenceRepository() GIT_URL_BASENAME, in case users manually cloned refrepos with whatever casing they use throughout --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 7256ab735e..3af1194aa2 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -612,12 +612,14 @@ public File findParameterizedReferenceRepository(String reference, String url) { } } else if (reference.endsWith("/${GIT_URL_BASENAME}") || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { // This may be the more portable solution with regard to filesystems - int sep = urlNormalized.lastIndexOf("/"); + // First try with original user-provided casing of the URL (if local + // dirs were cloned manually) + int sep = url.lastIndexOf("/"); String needleBasename; if (sep < 0) { - needleBasename = urlNormalized; + needleBasename = url; } else { - needleBasename = urlNormalized.substring(sep + 1); + needleBasename = url.substring(sep + 1); } needleBasename = needleBasename.replaceAll(".git$", ""); @@ -627,11 +629,34 @@ public File findParameterizedReferenceRepository(String reference, String url) { } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", needleBasename); - if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { - // chop it off, use main directory + if (url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + // chop it off, use main directory (only if we do not check urlNormalized separately below) referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", ""); } } + + if (!url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + // Retry with automation-ready normalized URL + sep = urlNormalized.lastIndexOf("/"); + if (sep < 0) { + needleBasename = urlNormalized; + } else { + needleBasename = urlNormalized.substring(sep + 1); + } + needleBasename = needleBasename.replaceAll(".git$", ""); + + if (reference.endsWith("/${GIT_URL_BASENAME}")) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", + needleBasename); + } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", + needleBasename); + if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + // chop it off, use main directory + referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", ""); + } + } + } } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") ) { // Assuming the provided "reference" directory already hosts // submodules, we use git tools to find the one subdir which From 18166325deddaa3e7ec7f2efb328acecaccf583f Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 19:50:36 +0100 Subject: [PATCH 055/132] LegacyCompatibleGitAPIImpl.java: only add ".git" to reference if it is not there yet (might be from *_FALLBACK cases) --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 3af1194aa2..35ee5925d9 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -710,7 +710,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { System.err.println("reference after='" + reference + "'\n"); } - if (!referencePath.exists()) { + if (!referencePath.exists() && !reference.endsWith(".git")) { // Normalize the URLs with or without .git suffix to // be served by same dir with the refrepo contents reference += ".git"; From 8b693fd3e2be9ff141dd2b1e97565f6b84cac669 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 19:51:12 +0100 Subject: [PATCH 056/132] LegacyCompatibleGitAPIImpl.java: fix tracing into private LOGGER instead of stderr (except cases currently anticipated by unit-tests) --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 35ee5925d9..67262b5cbb 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -29,6 +29,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -43,6 +45,8 @@ */ abstract class LegacyCompatibleGitAPIImpl extends AbstractGitAPIImpl implements IGitAPI { + private static final Logger LOGGER = Logger.getLogger(LegacyCompatibleGitAPIImpl.class.getName()); + /** * isBareRepository. * @@ -296,6 +300,7 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { } } + LOGGER.log(Level.FINEST, "normalizeGitUrl('" + url + "', " + checkLocalPaths.toString() + ") => " + urlNormalized); return urlNormalized; } @@ -383,13 +388,13 @@ public LinkedHashSet getSubmodulesUrls(String needle) { if (f.exists() && f.isDirectory()) { try { //LegacyCompatibleGitAPIImpl? - System.err.println("getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'\n"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'"); GitClient g = this.subGit(needleBasename); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); - System.err.println("getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')\n"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); @@ -441,13 +446,13 @@ public LinkedHashSet getSubmodulesUrls(String needle) { if (f.exists() && f.isDirectory()) { try { //LegacyCompatibleGitAPIImpl? - System.err.println("getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'\n"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'"); GitClient g = this.subGit(needleBasename); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); - System.err.println("getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')\n"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); @@ -547,6 +552,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { // sensitive and different). String urlNormalized = normalizeGitUrl(url, true); + // Note: currently unit-tests expect this markup on stderr: System.err.println("reference='" + reference + "'\n" + "url='" + url + "'\n" + "urlNormalized='" + urlNormalized + "'\n"); @@ -681,13 +687,13 @@ public File findParameterizedReferenceRepository(String reference, String url) { if (referenceExpanded == null) { referenceExpanded = subEntries.iterator().next()[0]; } - System.err.println("findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + "' from subEntries\n"); + LOGGER.log(Level.FINE, "findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + "' from subEntries"); if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", ""); } } else { - System.err.println("findParameterizedReferenceRepository(): got no subEntries\n"); + LOGGER.log(Level.FINE, "findParameterizedReferenceRepository(): got no subEntries"); // If there is no hit, the non-fallback mode suggests a new // directory name to host the submodule (same rules as SHA), // and the fallback mode would return the main directory. @@ -707,6 +713,8 @@ public File findParameterizedReferenceRepository(String reference, String url) { referencePath = null; // GC referencePath = new File(reference); } + + // Note: currently unit-tests expect this markup on stderr: System.err.println("reference after='" + reference + "'\n"); } From e49b985cdb388a27f9932e9e060d06e99c28f227 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 23:07:00 +0100 Subject: [PATCH 057/132] LegacyCompatibleGitAPIImpl.java: trace looking for the needle in arrDirnames --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 67262b5cbb..58b5b59ffc 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -383,13 +383,16 @@ public LinkedHashSet getSubmodulesUrls(String needle) { arrDirnames.add(needleSha); arrDirnames.add(needleSha + ".git"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like arrDirnames: " + arrDirnames.toString() + " under " + Paths.get("").toAbsolutePath().toString()); + for (String dirname : arrDirnames) { f = new File(".", dirname); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => file '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { try { //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'"); - GitClient g = this.subGit(needleBasename); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in dir '" + dirname + "'"); + GitClient g = this.subGit(needleBasename); //FIXME? needle or f? Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); @@ -441,13 +444,15 @@ public LinkedHashSet getSubmodulesUrls(String needle) { // Finally check pattern's parent dir arrDirnames.add("."); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) per arrDirnames: " + arrDirnames.toString() + " under " + Paths.get("").toAbsolutePath().toString()); + for (String dirname : arrDirnames) { f = new File(".", dirname); if (f.exists() && f.isDirectory()) { try { //LegacyCompatibleGitAPIImpl? LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'"); - GitClient g = this.subGit(needleBasename); + GitClient g = this.subGit(needleBasename); // needle or f? Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); From ab29c684d1dc70c71664e36cbdfc29c4183fd7f5 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 23:21:17 +0100 Subject: [PATCH 058/132] LegacyCompatibleGitAPIImpl.java: comment reshuffle --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 58b5b59ffc..e7790d1abd 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -551,7 +551,8 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Note: this normalization might crush several URLs into one, // and as far as is known this would be the goal - people tend // to use or omit .git suffix, or spell with varied case, while - // it means the same thing to known Git platforms. + // it means the same thing to known Git platforms, except local + // dirs on case-sensitive filesystems. // The actual reference repository directory may choose to have // original URL strings added as remotes (in case some are case // sensitive and different). @@ -577,13 +578,6 @@ public File findParameterizedReferenceRepository(String reference, String url) { // portability? Support base64, SHA or MD5 hashes of URLs // as pathnames? Normalize first (lowercase, .git, ...)? - // TODO: employ git submodules - there we can reliably match - // remote URLs (easily multiple) to particular modules, and - // yet keep separate git index directories per module with - // smaller scopes - much quicker to check out from than one - // huge combined repo. It would also be much more native to - // tools and custom scriptware that can be involved. - // TODO: Config option whether to populate absent reference // repos (If the expanded path does not have git repo data // right now, populate it in the location expanded below) @@ -669,6 +663,13 @@ public File findParameterizedReferenceRepository(String reference, String url) { } } } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") ) { + // Here we employ git submodules - so we can reliably match + // remote URLs (easily multiple) to particular modules, and + // yet keep separate git index directories per module with + // smaller scopes - much quicker to check out from than one + // huge combined repo. It would also be much more native to + // tools and custom scriptware that can be involved. + // Assuming the provided "reference" directory already hosts // submodules, we use git tools to find the one subdir which // has a registered remote URL equivalent (per normalization) From 66798fd7dcf409a26c720d7cd21b621fab1419b8 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 23:27:26 +0100 Subject: [PATCH 059/132] LegacyCompatibleGitAPIImpl.java: refactor findParameterizedReferenceRepository() with referenceBaseDir --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index e7790d1abd..f3a85dde76 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -558,6 +558,9 @@ public File findParameterizedReferenceRepository(String reference, String url) { // sensitive and different). String urlNormalized = normalizeGitUrl(url, true); + // Drop the trailing keyword to know the root refrepo dirname + String referenceBaseDir = reference.replaceAll("/\\$\\{GIT_[^\\}]*\\}$", ""); + // Note: currently unit-tests expect this markup on stderr: System.err.println("reference='" + reference + "'\n" + "url='" + url + "'\n" + @@ -598,7 +601,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_FALLBACK\\}$", urlNormalized); if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory - referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_FALLBACK\\}$", ""); + referenceExpanded = referenceBaseDir; } } else if (reference.endsWith("/${GIT_URL_SHA256}")) { // This may be the more portable solution with regard to filesystems @@ -613,7 +616,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory - referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_SHA256_FALLBACK\\}$", ""); + referenceExpanded = referenceBaseDir; } } else if (reference.endsWith("/${GIT_URL_BASENAME}") || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { // This may be the more portable solution with regard to filesystems @@ -636,7 +639,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { needleBasename); if (url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory (only if we do not check urlNormalized separately below) - referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", ""); + referenceExpanded = referenceBaseDir; } } @@ -658,7 +661,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { needleBasename); if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory - referenceExpanded = reference.replaceAll("/\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", ""); + referenceExpanded = referenceBaseDir; } } } @@ -696,7 +699,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { LOGGER.log(Level.FINE, "findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + "' from subEntries"); if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory - referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", ""); + referenceExpanded = referenceBaseDir; } } else { LOGGER.log(Level.FINE, "findParameterizedReferenceRepository(): got no subEntries"); @@ -709,7 +712,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { } else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { // chop it off, use main directory - referenceExpanded = reference.replaceAll("/\\$\\{GIT_SUBMODULES_FALLBACK\\}$", ""); + referenceExpanded = referenceBaseDir; } } } From 776a249db1bf8398645e5422cc384e05d8c5797e Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 23:45:51 +0100 Subject: [PATCH 060/132] LegacyCompatibleGitAPIImpl.java: trace which sub-git workspaces we do actually look at in getSubmodulesUrls() --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f3a85dde76..de427cd71b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -3,6 +3,7 @@ import static java.util.Arrays.copyOfRange; import org.apache.commons.codec.digest.DigestUtils; import static org.apache.commons.lang.StringUtils.join; +import hudson.FilePath; import hudson.model.TaskListener; import hudson.plugins.git.GitException; import hudson.plugins.git.IGitAPI; @@ -393,6 +394,7 @@ public LinkedHashSet getSubmodulesUrls(String needle) { //LegacyCompatibleGitAPIImpl? LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in dir '" + dirname + "'"); GitClient g = this.subGit(needleBasename); //FIXME? needle or f? + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); @@ -453,6 +455,7 @@ public LinkedHashSet getSubmodulesUrls(String needle) { //LegacyCompatibleGitAPIImpl? LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'"); GitClient g = this.subGit(needleBasename); // needle or f? + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); From 094de4df709dd0d2c91a7f7713506f5759eca7a6 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 11 Jan 2021 23:35:31 +0100 Subject: [PATCH 061/132] LegacyCompatibleGitAPIImpl.java: fix getSubmodulesUrls() to explicitly know referenceBaseDir --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index de427cd71b..8a219be0ee 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -319,12 +319,13 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * fully qualified) * [3] remoteName as defined in that nested submodule * + * @param referenceBaseDir - the reference repository, or container thereof * @param needle - an URL which (or its normalized variant coming from * normalizeGitUrl(url, true)) we want to find: * if it is not null - then stop and return just hits * for it as soon as we have something. */ - public LinkedHashSet getSubmodulesUrls(String needle) { + public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String needle) { // Keep track of where we've already looked in the "result" Set, to // avoid looking in same places (different strategies below) twice. // And eventually return this Set or part of it as the answer. @@ -384,15 +385,15 @@ public LinkedHashSet getSubmodulesUrls(String needle) { arrDirnames.add(needleSha); arrDirnames.add(needleSha + ".git"); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like arrDirnames: " + arrDirnames.toString() + " under " + Paths.get("").toAbsolutePath().toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like subdirs under '" + referenceBaseDir + "' per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { - f = new File(".", dirname); + f = new File(referenceBaseDir, dirname); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => file '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { try { //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in dir '" + dirname + "'"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + f.getAbsolutePath().toString() + "'"); GitClient g = this.subGit(needleBasename); //FIXME? needle or f? LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); @@ -446,14 +447,14 @@ public LinkedHashSet getSubmodulesUrls(String needle) { // Finally check pattern's parent dir arrDirnames.add("."); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) per arrDirnames: " + arrDirnames.toString() + " under " + Paths.get("").toAbsolutePath().toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) under '" + referenceBaseDir + "' per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { - f = new File(".", dirname); + f = new File(referenceBaseDir, dirname); if (f.exists() && f.isDirectory()) { try { //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for needle='" + needle + "' in dir '" + dirname + "'"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + f.getAbsolutePath().toString() + "'"); GitClient g = this.subGit(needleBasename); // needle or f? LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); @@ -516,9 +517,11 @@ public LinkedHashSet getSubmodulesUrls(String needle) { /** See above. With null, returns all we can find (slower) for the caller * to parse */ - public LinkedHashSet getSubmodulesUrls() { - return getSubmodulesUrls(null); + public LinkedHashSet getSubmodulesUrls(String referenceBaseDir) { + return getSubmodulesUrls(referenceBaseDir, null); } + /* Do we need a parameter-less variant to look under current work dir + * aka Paths.get("").toAbsolutePath().toString() ?.. */ /** Yield the File object for the reference repository local filesystem * pathname. Note that the provided string may be suffixed with expandable @@ -682,8 +685,9 @@ public File findParameterizedReferenceRepository(String reference, String url) { // to the provided "url". // Note: we pass "url" here, the routine differentiates original - // naming vs. normalization. - LinkedHashSet subEntries = getSubmodulesUrls(url); + // naming vs. normalization while looking for its needle in the + // haystack. + LinkedHashSet subEntries = getSubmodulesUrls(referenceBaseDir, url); if (!subEntries.isEmpty()) { // Normally we should only have one entry here, as sorted // by the routine, and prefer that first option if a new From d30de211983004dc49593137e50013c4e2ef5d2e Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 00:04:13 +0100 Subject: [PATCH 062/132] LegacyCompatibleGitAPIImpl.java: fix getSubmodulesUrls() to look in "f" dirname as the git workspace, not needle --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 8a219be0ee..e732b6ec0a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -392,9 +392,10 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => file '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { try { + String fAbs = f.getAbsolutePath().toString(); //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + f.getAbsolutePath().toString() + "'"); - GitClient g = this.subGit(needleBasename); //FIXME? needle or f? + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + fAbs + "'"); + GitClient g = this.subGit(fAbs); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { @@ -453,9 +454,11 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String f = new File(referenceBaseDir, dirname); if (f.exists() && f.isDirectory()) { try { + + String fAbs = f.getAbsolutePath().toString(); //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + f.getAbsolutePath().toString() + "'"); - GitClient g = this.subGit(needleBasename); // needle or f? + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + fAbs + "'"); + GitClient g = this.subGit(fAbs); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { From 7ed0f0f6b2aaf4ca4c04badac5c6555cb3c8f452 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 01:08:09 +0100 Subject: [PATCH 063/132] GitClient and its implementations: extend API with a newGit(somedir) to inspect other repos with same class as currently used --- src/main/java/hudson/plugins/git/GitAPI.java | 5 +++++ .../jenkinsci/plugins/gitclient/CliGitAPIImpl.java | 6 ++++++ .../org/jenkinsci/plugins/gitclient/GitClient.java | 11 +++++++++++ .../org/jenkinsci/plugins/gitclient/JGitAPIImpl.java | 6 ++++++ .../jenkinsci/plugins/gitclient/RemoteGitImpl.java | 5 +++++ 5 files changed, 33 insertions(+) diff --git a/src/main/java/hudson/plugins/git/GitAPI.java b/src/main/java/hudson/plugins/git/GitAPI.java index ce275fdacc..0d02281fdb 100644 --- a/src/main/java/hudson/plugins/git/GitAPI.java +++ b/src/main/java/hudson/plugins/git/GitAPI.java @@ -156,6 +156,11 @@ public GitClient subGit(String subdir) { return Git.USE_CLI ? super.subGit(subdir) : jgit.subGit(subdir); } + /** {@inheritDoc} */ + public GitClient newGit(String somedir) { + return Git.USE_CLI ? super.newGit(somedir) : jgit.newGit(somedir); + } + /** {@inheritDoc} */ public void setRemoteUrl(String name, String url) throws GitException, InterruptedException { if (Git.USE_CLI) super.setRemoteUrl(name, url); else jgit.setRemoteUrl(name, url); diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index d6dda9cdaa..7c3c0cc92c 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -321,6 +321,12 @@ public GitClient subGit(String subdir) { return new CliGitAPIImpl(gitExe, new File(workspace, subdir), listener, environment); } + /** {@inheritDoc} */ + @Override + public GitClient newGit(String somedir) { + return new CliGitAPIImpl(gitExe, new File(somedir), listener, environment); + } + /** * Initialize an empty repository for further git operations. * diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java index 498d73bdc1..afe038b65a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java @@ -713,6 +713,17 @@ public interface GitClient { List revList(String ref) throws GitException, InterruptedException; + // --- new instance of same applied class + + /** + * newGit. + * + * @return a IGitAPI implementation to manage another git repository + * with same general settings and implementation as the current one. + * @param somedir a {@link java.lang.String} object. + */ + GitClient newGit(String somedir); + // --- submodules /** diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index a46330a4ca..5ff02d0eb6 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -214,6 +214,12 @@ public GitClient subGit(String subdir) { return new JGitAPIImpl(new File(workspace, subdir), listener); } + /** {@inheritDoc} */ + @Override + public GitClient newGit(String somedir) { + return new JGitAPIImpl(new File(somedir), listener); + } + /** {@inheritDoc} */ @Override public void setAuthor(String name, String email) throws GitException { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java index 68a116010c..5f15406f01 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java @@ -617,6 +617,11 @@ public GitClient subGit(String subdir) { return proxy.subGit(subdir); } + /** {@inheritDoc} */ + public GitClient newGit(String somedir) { + return proxy.newGit(somedir); + } + /** * hasGitModules. * From 14cf3451c3aa60dc634c188e41ce41817363ea9c Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 01:15:49 +0100 Subject: [PATCH 064/132] LegacyCompatibleGitAPIImpl.java: fix getSubmodulesUrls() to look in referenceGit --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index e732b6ec0a..7837155757 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -334,9 +334,14 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Helper list storage in loops below ArrayList arrDirnames = new ArrayList(); + // "this" during a checkout typically represents the job workspace, + // but we want to inspect the reference repository located elsewhere + // with the same implementation as the end-user set up (CliGit/jGit) + GitClient referenceGit = this.newGit(referenceBaseDir); + Boolean isBare = false; try { - isBare = this.isBareRepository(); + isBare = ((hudson.plugins.git.IGitAPI)referenceGit).isBareRepository(); } catch (InterruptedException e) { isBare = false; // At least try to look into submodules... } @@ -395,7 +400,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String String fAbs = f.getAbsolutePath().toString(); //LegacyCompatibleGitAPIImpl? LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + fAbs + "'"); - GitClient g = this.subGit(fAbs); + GitClient g = referenceGit.subGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { @@ -458,7 +463,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String String fAbs = f.getAbsolutePath().toString(); //LegacyCompatibleGitAPIImpl? LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + fAbs + "'"); - GitClient g = this.subGit(fAbs); + GitClient g = referenceGit.subGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); for (Map.Entry pair : uriNames.entrySet()) { From 2b61767eb81a805e7a94863b61883c0ee84cc9dc Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 01:16:19 +0100 Subject: [PATCH 065/132] LegacyCompatibleGitAPIImpl.java: fix getSubmodulesUrls() to return fAbs string in case of a hit --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 7837155757..f0aebb3602 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -409,7 +409,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); - result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); return result; } // Cache the finding to avoid the dirname later, if we @@ -472,7 +472,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); - result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); return result; } // Cache the finding to avoid the dirname later, if we From 1c9a8f570ae41c0cbc9e2c9edc795d7517fb31e0 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 01:16:36 +0100 Subject: [PATCH 066/132] LegacyCompatibleGitAPIImpl.java: fix getSubmodulesUrls() to track relative dirname string in case of no hit --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f0aebb3602..021b6f0186 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -415,7 +415,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop // which by construct looks at different dirs so far. - result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); + result.add(new String[]{dirname, uri, uriNorm, pair.getValue()}); } } catch (Exception e) { // ignore, go to next slide @@ -478,7 +478,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop // which by construct looks at different dirs so far. - result.add(new String[]{needleBasename, uri, uriNorm, pair.getValue()}); + result.add(new String[]{dirname, uri, uriNorm, pair.getValue()}); } } catch (Exception e) { // ignore, go to next slide From a29b00ce1d35c8c3ba8f30ce273c94b6105ab6e3 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 02:41:27 +0100 Subject: [PATCH 067/132] LegacyCompatibleGitAPIImpl.java: clearer trace messages in getSubmodulesUrls() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 021b6f0186..b0a942198d 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -390,16 +390,16 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String arrDirnames.add(needleSha); arrDirnames.add(needleSha + ".git"); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like subdirs under '" + referenceBaseDir + "' per arrDirnames: " + arrDirnames.toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like subdirs under refrepo '" + referenceBaseDir + "' per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { f = new File(referenceBaseDir, dirname); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => file '" + f.getAbsolutePath().toString() + "'"); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { try { String fAbs = f.getAbsolutePath().toString(); //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + fAbs + "'"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); GitClient g = referenceGit.subGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); @@ -453,16 +453,16 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Finally check pattern's parent dir arrDirnames.add("."); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) under '" + referenceBaseDir + "' per arrDirnames: " + arrDirnames.toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) under refrepo '" + referenceBaseDir + "' per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { f = new File(referenceBaseDir, dirname); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { try { - String fAbs = f.getAbsolutePath().toString(); //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing dir '" + dirname + "' => '" + fAbs + "'"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); GitClient g = referenceGit.subGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); From 25bef67503b3892fa5e875548e8f96582a323403 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 02:58:39 +0100 Subject: [PATCH 068/132] LegacyCompatibleGitAPIImpl.java: track referenceBaseDirAbs to compare pathnames (cached hits) more reliably in getSubmodulesUrls() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index b0a942198d..65407e5db0 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -333,11 +333,12 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String File f = null; // Helper list storage in loops below ArrayList arrDirnames = new ArrayList(); + String referenceBaseDirAbs = Paths.get(referenceBaseDir).toAbsolutePath().toString(); // "this" during a checkout typically represents the job workspace, // but we want to inspect the reference repository located elsewhere // with the same implementation as the end-user set up (CliGit/jGit) - GitClient referenceGit = this.newGit(referenceBaseDir); + GitClient referenceGit = this.newGit(referenceBaseDirAbs); Boolean isBare = false; try { @@ -390,10 +391,10 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String arrDirnames.add(needleSha); arrDirnames.add(needleSha + ".git"); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like subdirs under refrepo '" + referenceBaseDir + "' per arrDirnames: " + arrDirnames.toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like subdirs under refrepo '" + referenceBaseDirAbs + "' per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { - f = new File(referenceBaseDir, dirname); + f = new File(referenceBaseDirAbs, dirname); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { try { @@ -439,7 +440,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String for (String[] resultEntry : result) { checkedDirs.add(resultEntry[0]); } - File[] directories = new File(".").listFiles(File::isDirectory); + File[] directories = new File(referenceBaseDirAbs).listFiles(File::isDirectory); for (File dir : directories) { f = new File(dir, ".git"); if (f.exists()) { // May be a file or directory... or symlink to those... @@ -453,10 +454,10 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Finally check pattern's parent dir arrDirnames.add("."); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) under refrepo '" + referenceBaseDir + "' per arrDirnames: " + arrDirnames.toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) under refrepo '" + referenceBaseDirAbs + "' per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { - f = new File(referenceBaseDir, dirname); + f = new File(referenceBaseDirAbs, dirname); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { try { From 39dd9b4c9a7709d2b0b5bf4ccb9241f866d77037 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 03:00:37 +0100 Subject: [PATCH 069/132] LegacyCompatibleGitAPIImpl.java: fix getSubmodulesUrls() to return fAbs string in case of a hit --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 65407e5db0..acceb57c78 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -416,7 +416,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop // which by construct looks at different dirs so far. - result.add(new String[]{dirname, uri, uriNorm, pair.getValue()}); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); } } catch (Exception e) { // ignore, go to next slide @@ -479,7 +479,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop // which by construct looks at different dirs so far. - result.add(new String[]{dirname, uri, uriNorm, pair.getValue()}); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); } } catch (Exception e) { // ignore, go to next slide From c349b2525e9ccedbe59fe41d331efcdcade342ae Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 03:00:57 +0100 Subject: [PATCH 070/132] LegacyCompatibleGitAPIImpl.java: comment the ends of long logical blocks --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index acceb57c78..90aa5fdfdb 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -492,7 +492,9 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String if (isBare) { return new LinkedHashSet<>(); } + // else... ||true above :) } + // else: look for needle in submodules defined in current non-bare refrepo } else { // ...and if there is no needle? Return info on all git remotes // found under this directory... From 6a7bb18121a501a2514ea55cb9bf33e60c5327a9 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 03:24:45 +0100 Subject: [PATCH 071/132] LegacyCompatibleGitAPIImpl.java: use absolute pathnames "natively" in second phase of needle-searching, and only look for submodule definitions in dirs that ARE git workspaces (avoid looking into parent refrepoBaseDir all the time for simple subdirs) --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 75 ++++++++++--------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 90aa5fdfdb..6fb36e2d2c 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -399,24 +399,26 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String if (f.exists() && f.isDirectory()) { try { String fAbs = f.getAbsolutePath().toString(); - //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); - GitClient g = referenceGit.subGit(dirname); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); - Map uriNames = g.getRemoteUrls(); - for (Map.Entry pair : uriNames.entrySet()) { - String uri = pair.getKey(); - String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); - if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new LinkedHashSet<>(); + if (getObjectsFile(fAbs) != null) { + //LegacyCompatibleGitAPIImpl? + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); + GitClient g = referenceGit.subGit(dirname); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); + Map uriNames = g.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new LinkedHashSet<>(); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + return result; + } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); - return result; } - // Cache the finding to avoid the dirname later, if we - // get to that; but no checks are needed in this loop - // which by construct looks at different dirs so far. - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); } } catch (Exception e) { // ignore, go to next slide @@ -433,7 +435,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String if (isBare || true) { arrDirnames.clear(); - // Check subdirs that are git workspaces + // Check subdirs that are git workspaces; note these are absolute pathnames // TODO: Refactor to avoid lookups of dirs that may prove not // needed in the end (aim for less I/Os to find the goal) LinkedHashSet checkedDirs = new LinkedHashSet<>(); @@ -452,34 +454,35 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String } // Finally check pattern's parent dir - arrDirnames.add("."); + arrDirnames.add(referenceBaseDirAbs); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) under refrepo '" + referenceBaseDirAbs + "' per arrDirnames: " + arrDirnames.toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) under refrepo '" + referenceBaseDirAbs + "' per absolute arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { - f = new File(referenceBaseDirAbs, dirname); + f = new File(dirname); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { try { String fAbs = f.getAbsolutePath().toString(); - //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); - GitClient g = referenceGit.subGit(dirname); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); - Map uriNames = g.getRemoteUrls(); - for (Map.Entry pair : uriNames.entrySet()) { - String uri = pair.getKey(); - String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); - if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new LinkedHashSet<>(); + if (getObjectsFile(fAbs) != null) { + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); + GitClient g = referenceGit.subGit(dirname); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); + Map uriNames = g.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new LinkedHashSet<>(); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + return result; + } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); - return result; } - // Cache the finding to avoid the dirname later, if we - // get to that; but no checks are needed in this loop - // which by construct looks at different dirs so far. - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); } } catch (Exception e) { // ignore, go to next slide From f4756062cb64bb9c7d8b4cf567431de54a22f7e3 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 03:55:27 +0100 Subject: [PATCH 072/132] LegacyCompatibleGitAPIImpl.java: do not need to check bare-mode dirs for being git repos, twice --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 6fb36e2d2c..0205a1dbce 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -456,7 +456,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Finally check pattern's parent dir arrDirnames.add(referenceBaseDirAbs); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) under refrepo '" + referenceBaseDirAbs + "' per absolute arrDirnames: " + arrDirnames.toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) that have a .git, under refrepo '" + referenceBaseDirAbs + "' per absolute arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { f = new File(dirname); @@ -464,25 +464,23 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String if (f.exists() && f.isDirectory()) { try { String fAbs = f.getAbsolutePath().toString(); - if (getObjectsFile(fAbs) != null) { - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); - GitClient g = referenceGit.subGit(dirname); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); - Map uriNames = g.getRemoteUrls(); - for (Map.Entry pair : uriNames.entrySet()) { - String uri = pair.getKey(); - String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); - if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new LinkedHashSet<>(); - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); - return result; - } - // Cache the finding to avoid the dirname later, if we - // get to that; but no checks are needed in this loop - // which by construct looks at different dirs so far. + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); + GitClient g = referenceGit.subGit(dirname); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); + Map uriNames = g.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new LinkedHashSet<>(); result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + return result; } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); } } catch (Exception e) { // ignore, go to next slide From 2ddd7df74e33613ae3d5b073bff19ed49fe2f4f4 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 03:57:56 +0100 Subject: [PATCH 073/132] LegacyCompatibleGitAPIImpl.java: check basename-like dirs for being git repos, only (submodules do not host their index and have no "objects" directly inside) --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 0205a1dbce..e5039075e2 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -399,7 +399,8 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String if (f.exists() && f.isDirectory()) { try { String fAbs = f.getAbsolutePath().toString(); - if (getObjectsFile(fAbs) != null) { + File fGit = new File(f, ".git"); + if (fGit.exists()) { // file, dir or symlink to those //LegacyCompatibleGitAPIImpl? LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); GitClient g = referenceGit.subGit(dirname); From 721d791b4a5cfd1bc969b7c5278e96890ac60346 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 04:01:49 +0100 Subject: [PATCH 074/132] LegacyCompatibleGitAPIImpl.java: check the parent refrepo dir for hosting a .git object too --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index e5039075e2..7550500e39 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -455,7 +455,9 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String } // Finally check pattern's parent dir - arrDirnames.add(referenceBaseDirAbs); + if (new File(referenceBaseDirAbs, ".git").exists()) { + arrDirnames.add(referenceBaseDirAbs); + } LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) that have a .git, under refrepo '" + referenceBaseDirAbs + "' per absolute arrDirnames: " + arrDirnames.toString()); From 69f5b59eb638ed8026f7380a651d77963a3b0ccc Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 04:36:46 +0100 Subject: [PATCH 075/132] LegacyCompatibleGitAPIImpl.java: trace what getRemoteUrls() gave us --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 7550500e39..f168631275 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -406,6 +406,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String GitClient g = referenceGit.subGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); @@ -471,6 +472,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String GitClient g = referenceGit.subGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); From 9c48cb6685f7dd7e08703be6ff70c05f93189508 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 04:37:26 +0100 Subject: [PATCH 076/132] LegacyCompatibleGitAPIImpl.java: leave a TODO block for checking submodule definitions in current dir (might recurse that eventually) --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f168631275..d467c96cc6 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -428,6 +428,39 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String } } +/* +// TBD: Needs a way to list submodules in given workspace and convert +// that into (relative) subdirs, possibly buried some levels deep, for +// cases where the submodule is defined in parent with the needle URL. +// Maybe merge with current "if isBare" below, to optionally seed +// same arrDirnames with different values and check remotes listed +// in those repos. + // If current repo *is NOT* bare - check its submodules + if (!isBare || true) { + try { + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in submodules of refrepo, if any"); + Map uriNames = referenceGit.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new LinkedHashSet<>(); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + return result; + } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + } + } catch (Exception e) { + // ignore, go to next slide + } + } +*/ + // If current repo *is* bare (can't have proper submodules) and the // needle is not null, follow up with: // * Maybe also direct child dirs that have a ".git" FS object inside?.. From f465258fb50c7c73fdfa4c274160bed53c769af0 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 09:56:24 +0100 Subject: [PATCH 077/132] LegacyCompatibleGitAPIImpl.java: trace mis-processing an assumed git workspace with an exception --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index d467c96cc6..356e08e916 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -424,6 +424,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String } } catch (Exception e) { // ignore, go to next slide + LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception (will go to next item):\n" + e.toString()); } } } @@ -522,6 +523,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String } } catch (Exception e) { // ignore, go to next slide + LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception (will go to next item):\n" + e.toString()); } } } From f6e756acebeae90111acca4eea690cf751f0774a Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 09:56:41 +0100 Subject: [PATCH 078/132] LegacyCompatibleGitAPIImpl.java: comment about abs dirnames in second phase --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 356e08e916..c49763ad30 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -497,6 +497,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) that have a .git, under refrepo '" + referenceBaseDirAbs + "' per absolute arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { + // Note that here dirnames deal in absolutes f = new File(dirname); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { From ce1827e7759ffa96b830525adea471702f599e20 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 10:09:24 +0100 Subject: [PATCH 079/132] LegacyCompatibleGitAPIImpl.java: fix GitClient for abs dirnames in second phase --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index c49763ad30..f67891f94a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -504,7 +504,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String try { String fAbs = f.getAbsolutePath().toString(); LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); - GitClient g = referenceGit.subGit(dirname); + GitClient g = this.newGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); From c5e863fed53b6ff7371c3464ccd8bca397f3656d Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 10:34:49 +0100 Subject: [PATCH 080/132] LegacyCompatibleGitAPIImpl.java: Log into build console the start and outcome of resolving the parameterized Git reference repository, to explain the clutter from "git" calls --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f67891f94a..a355bd477c 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -603,6 +603,9 @@ public File findParameterizedReferenceRepository(String reference, String url) { } File referencePath = new File(reference); + // Note: Initially we expect the reference to be a realistic dirname + // with a special suffix to substitute after the logic below, so the + // referencePath for that verbatim funny string should not exist now: if (!referencePath.exists() && isParameterizedReferenceRepository(reference) && url != null && !url.isEmpty() @@ -625,6 +628,10 @@ public File findParameterizedReferenceRepository(String reference, String url) { "url='" + url + "'\n" + "urlNormalized='" + urlNormalized + "'\n"); + // Let users know why there are many "git config --list" lines in their build log: + LOGGER.log(Level.INFO, "Trying to resolve parameterized Git reference repository '" + + reference + "' into a specific (sub-)directory to use for URL '" + url + "' ..."); + String referenceExpanded = null; if (reference.endsWith("/${GIT_URL}")) { // For mass-configured jobs, like Organization Folders, @@ -785,7 +792,10 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Note: currently unit-tests expect this markup on stderr: System.err.println("reference after='" + reference + "'\n"); - } + + LOGGER.log(Level.INFO, "After resolving the parameterized Git reference repository, " + + "decided to use '" + reference + "' directory for URL '" + url + "'"); + } // if referencePath is the replaceable token and not existing directory if (!referencePath.exists() && !reference.endsWith(".git")) { // Normalize the URLs with or without .git suffix to From d0bf23c5aeb55ef1931ae4a3d9effa0412194f37 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 11:25:34 +0100 Subject: [PATCH 081/132] LegacyCompatibleGitAPIImpl.java: Refactor a useless needle to be null, and move declarations of values derived from nun-null needle --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index a355bd477c..338dc57c53 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -347,14 +347,25 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String isBare = false; // At least try to look into submodules... } + // Simplify checks below by stating a useless needle is null + if (needle != null && needle.isEmpty()) { + needle = null; + } + // This is only used and populated if needle is not null + String needleNorm = null; + // If needle is not null, first look perhaps in the subdir(s) named // with base-name of the URL with and without a ".git" suffix, then // in SHA256 named dir that can match it; note that this use-case // might pan out also if "this" repo is bare and can not have "proper" // git submodules - but was prepared for our other options. - if (needle != null && !needle.isEmpty()) { - int sep = needle.lastIndexOf("/"); + if (needle != null) { String needleBasename; + String needleBasenameLC; + String needleNormBasename; + String needleSha; + + int sep = needle.lastIndexOf("/"); if (sep < 0) { needleBasename = needle; } else { @@ -362,9 +373,8 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String } needleBasename = needleBasename.replaceAll(".[Gg][Ii][Tt]$", ""); - String needleNorm = normalizeGitUrl(needle, true); + needleNorm = normalizeGitUrl(needle, true); sep = needleNorm.lastIndexOf("/"); - String needleNormBasename; if (sep < 0) { needleNormBasename = needleNorm; } else { @@ -376,7 +386,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // First we try the caller-provided string casing, then normalized. arrDirnames.add(needleBasename); arrDirnames.add(needleBasename + ".git"); - String needleBasenameLC = needleBasename.toLowerCase(); + needleBasenameLC = needleBasename.toLowerCase(); if (!needleBasenameLC.equals(needleBasename)) { // Retry with lowercased dirname arrDirnames.add(needleBasenameLC); @@ -387,7 +397,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String arrDirnames.add(needleNormBasename + ".git"); } - String needleSha = org.apache.commons.codec.digest.DigestUtils.sha256Hex(needleNorm); + needleSha = org.apache.commons.codec.digest.DigestUtils.sha256Hex(needleNorm); arrDirnames.add(needleSha); arrDirnames.add(needleSha + ".git"); From 0b761da482573e775839ca92ca61d1d27496fea2 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 11:27:11 +0100 Subject: [PATCH 082/132] LegacyCompatibleGitAPIImpl.java: Refactor getSubmodulesUrls() to unify logic needed for both search for a needle or full listing --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 231 +++++++++--------- 1 file changed, 120 insertions(+), 111 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 338dc57c53..72c551d8a0 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -438,6 +438,33 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String } } } + } // if needle, look in basename-like and SHA dirs first + + // Needle or not, the rest of directory walk to collect data is the + // same, so follow a list of whom we want to visit in likely-quickest + // hit order. Note that if needle is null, we walk over everything + // that makes sense to visit, to return info on all git remotes found + // in or under this directory; however if it is not-null, we still + // try to have minimal overhead to complete as soon as we match it. + // TODO: Refactor to avoid lookups of dirs that may prove not needed + // in the end (aim for less I/Os to find the goal)... or is one dir + // listing a small price to pay for maintaining one unified logic? + + // If current dir does have submodules, first dig into submodules, + // when there is no deeper to drill, report remote URLs and step + // back from recursion. This way we have least specific repo last, + // if several have the replica (assuming the first hits are smaller + // scopes). + + // Will build a new shopping list to browse + arrDirnames.clear(); + + // Track where we have looked already; note that values in result[] + // (if any from needle-search above) are absolute pathnames + LinkedHashSet checkedDirs = new LinkedHashSet<>(); + for (String[] resultEntry : result) { + checkedDirs.add(resultEntry[0]); + } /* // TBD: Needs a way to list submodules in given workspace and convert @@ -446,135 +473,117 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Maybe merge with current "if isBare" below, to optionally seed // same arrDirnames with different values and check remotes listed // in those repos. - // If current repo *is NOT* bare - check its submodules - if (!isBare || true) { + // If current repo *is NOT* bare - check its submodules + if (!isBare || true) { + try { + // For each current workspace (recurse or big loop in same context?): + // public GitClient subGit(String subdir) => would this.subGit(...) + // give us a copy of this applied class instance (CLI Git vs jGit)? + // get submodule name-vs-one-url from .gitmodules if present, for a + // faster possible answer (only bother if needle is not null?) + // try { getSubmodules("HEAD") ... } => List filtered for + // "commit" items + // getRemoteUrls() => Map +// arrDirnames.clear(); + + // TODO: Check subdirs that are git workspaces, and remove "|| true" above +// LinkedHashSet checkedDirs = new LinkedHashSet<>(); +// for (String[] resultEntry : result) { +// checkedDirs.add(resultEntry[0]); +// } + + + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in submodules of refrepo, if any"); + Map uriNames = referenceGit.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new LinkedHashSet<>(); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + return result; + } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + } + } catch (Exception e) { + // ignore, go to next slide + } + } +*/ + + // If current repo *is* bare (can't have proper submodules), or if the + // end-users just cloned or linked some more repos into this container, + // follow up with direct child dirs that have a ".git" FS object inside: + + // Check subdirs that are git workspaces; note that values in checkedDirs + // are absolute pathnames + File[] directories = new File(referenceBaseDirAbs).listFiles(File::isDirectory); + for (File dir : directories) { + f = new File(dir, ".git"); + if (f.exists()) { // May be a file or directory... or symlink to those... + String dirname = dir.getPath().replaceAll("/*$", ""); + if (!checkedDirs.contains(dirname)) { + arrDirnames.add(dirname); + } + } + } + + // Finally check pattern's parent dir + // * Look at remote URLs in current dir after the guessed subdirs failed, + // and return then. + if (new File(referenceBaseDirAbs, ".git").exists()) { + arrDirnames.add(referenceBaseDirAbs); + } + + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at " + + ((isBare ? "" : "submodules first, then ")) + + "all subdirs that have a .git, under refrepo '" + referenceBaseDirAbs + + "' per absolute arrDirnames: " + arrDirnames.toString()); + + for (String dirname : arrDirnames) { + // Note that here dirnames deal in absolutes + f = new File(dirname); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); + if (f.exists() && f.isDirectory()) { try { - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in submodules of refrepo, if any"); - Map uriNames = referenceGit.getRemoteUrls(); + String fAbs = f.getAbsolutePath().toString(); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo dir '" + dirname + "' => '" + fAbs + "'"); + GitClient g = this.newGit(dirname); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); + Map uriNames = g.getRemoteUrls(); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + uriNames.toString()); for (Map.Entry pair : uriNames.entrySet()) { String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); - if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + if (needle != null && needleNorm != null && (needleNorm.equals(uriNorm) || needle.equals(uri)) ) { result = new LinkedHashSet<>(); result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); return result; } - // Cache the finding to avoid the dirname later, if we - // get to that; but no checks are needed in this loop - // which by construct looks at different dirs so far. + // Cache the finding to return eventually. result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); } + // TODO: Here is a good spot to recurse this routine into + // a subdir that is a known git workspace, to add its data + // and/or return a found needle. } catch (Exception e) { // ignore, go to next slide + LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception (will go to next item):\n" + e.toString()); } } -*/ - - // If current repo *is* bare (can't have proper submodules) and the - // needle is not null, follow up with: - // * Maybe also direct child dirs that have a ".git" FS object inside?.. - // * Look at remote URLs in current dir after the guessed subdirs failed, - // and return then. - // TODO: Remove "|| true" when other logic fo look for real submodules is complete - if (isBare || true) { - arrDirnames.clear(); - - // Check subdirs that are git workspaces; note these are absolute pathnames - // TODO: Refactor to avoid lookups of dirs that may prove not - // needed in the end (aim for less I/Os to find the goal) - LinkedHashSet checkedDirs = new LinkedHashSet<>(); - for (String[] resultEntry : result) { - checkedDirs.add(resultEntry[0]); - } - File[] directories = new File(referenceBaseDirAbs).listFiles(File::isDirectory); - for (File dir : directories) { - f = new File(dir, ".git"); - if (f.exists()) { // May be a file or directory... or symlink to those... - String dirname = dir.getPath().replaceAll("/*$", ""); - if (!checkedDirs.contains(dirname)) { - arrDirnames.add(dirname); - } - } - } - - // Finally check pattern's parent dir - if (new File(referenceBaseDirAbs, ".git").exists()) { - arrDirnames.add(referenceBaseDirAbs); - } - - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at all subdirs (bare repo) that have a .git, under refrepo '" + referenceBaseDirAbs + "' per absolute arrDirnames: " + arrDirnames.toString()); - - for (String dirname : arrDirnames) { - // Note that here dirnames deal in absolutes - f = new File(dirname); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); - if (f.exists() && f.isDirectory()) { - try { - String fAbs = f.getAbsolutePath().toString(); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); - GitClient g = this.newGit(dirname); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); - Map uriNames = g.getRemoteUrls(); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); - for (Map.Entry pair : uriNames.entrySet()) { - String uri = pair.getKey(); - String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); - if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new LinkedHashSet<>(); - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); - return result; - } - // Cache the finding to avoid the dirname later, if we - // get to that; but no checks are needed in this loop - // which by construct looks at different dirs so far. - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); - } - } catch (Exception e) { - // ignore, go to next slide - LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception (will go to next item):\n" + e.toString()); - } - } - } - - // Nothing found, and for bare top-level repo not much to search for - // Or maybe also fall through to let git decide? - if (isBare) { - return new LinkedHashSet<>(); - } - // else... ||true above :) - } - // else: look for needle in submodules defined in current non-bare refrepo - } else { - // ...and if there is no needle? Return info on all git remotes - // found under this directory... - /* if (isBare) { ... } */ - // fall through currently, let git decide if it has submodules here and now - } - - // For each current workspace (recurse or big loop in same context?): - // public GitClient subGit(String subdir) => would this.subGit(...) - // give us a copy of this applied class instance (CLI Git vs jGit)? - // get submodule name-vs-one-url from .gitmodules if present, for a - // faster possible answer (only bother if needle is not null?) - // try { getSubmodules("HEAD") ... } => List filtered for - // "commit" items - // getRemoteUrls() => Map - arrDirnames.clear(); + } - // TODO: Check subdirs that are git workspaces, and remove "|| true" above - LinkedHashSet checkedDirs = new LinkedHashSet<>(); - for (String[] resultEntry : result) { - checkedDirs.add(resultEntry[0]); + // Nothing found, so if we had a needle - report there are no hits + if (needle != null) { + return new LinkedHashSet<>(); } - // If current dir does have submodules, first dig into submodules, - // when there is no deeper to drill, report remote URLs and step - // back from recursion. This way we have least specific repo last, - // if several have the replica (assuming the first hits are smaller - // scopes). return result; } From 6046f4b1d2cfa1064a83d4f7f4789f3ef3d1e5f7 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 11:44:10 +0100 Subject: [PATCH 083/132] LegacyCompatibleGitAPIImpl.java: Refactor getSubmodulesUrls() to name uriNames pair value a "remoteName" --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 72c551d8a0..e7e047a6d1 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -418,18 +418,19 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String Map uriNames = g.getRemoteUrls(); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); for (Map.Entry pair : uriNames.entrySet()) { + String remoteName = pair.getValue(); // whatever the git workspace config calls it String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + result.add(new String[]{fAbs, uri, uriNorm, remoteName}); return result; } // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop // which by construct looks at different dirs so far. - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + result.add(new String[]{fAbs, uri, uriNorm, remoteName}); } } } catch (Exception e) { @@ -558,16 +559,17 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String Map uriNames = g.getRemoteUrls(); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + uriNames.toString()); for (Map.Entry pair : uriNames.entrySet()) { + String remoteName = pair.getValue(); // whatever the git workspace config calls it String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); if (needle != null && needleNorm != null && (needleNorm.equals(uriNorm) || needle.equals(uri)) ) { result = new LinkedHashSet<>(); - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + result.add(new String[]{fAbs, uri, uriNorm, remoteName}); return result; } // Cache the finding to return eventually. - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + result.add(new String[]{fAbs, uri, uriNorm, remoteName}); } // TODO: Here is a good spot to recurse this routine into // a subdir that is a known git workspace, to add its data From 3f7b4e4e5ad025cfa33830a6cabc48cfea8a9805 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 14:54:59 +0100 Subject: [PATCH 084/132] LegacyCompatibleGitAPIImpl.java: re-use arrDirnames in getSubmodulesUrls() if it has "prioritized" pathnames, to recurse into them first --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 61 +++++++++++-------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index e7e047a6d1..b634402307 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -457,9 +457,6 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // if several have the replica (assuming the first hits are smaller // scopes). - // Will build a new shopping list to browse - arrDirnames.clear(); - // Track where we have looked already; note that values in result[] // (if any from needle-search above) are absolute pathnames LinkedHashSet checkedDirs = new LinkedHashSet<>(); @@ -522,7 +519,9 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // follow up with direct child dirs that have a ".git" FS object inside: // Check subdirs that are git workspaces; note that values in checkedDirs - // are absolute pathnames + // are absolute pathnames. If we did look for the needle, array already + // starts with some "prioritized" pathnames which we should not directly + // inspect again... but should recurse into first anyhow. File[] directories = new File(referenceBaseDirAbs).listFiles(File::isDirectory); for (File dir : directories) { f = new File(dir, ".git"); @@ -551,33 +550,41 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String f = new File(dirname); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); if (f.exists() && f.isDirectory()) { - try { - String fAbs = f.getAbsolutePath().toString(); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo dir '" + dirname + "' => '" + fAbs + "'"); - GitClient g = this.newGit(dirname); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); - Map uriNames = g.getRemoteUrls(); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + uriNames.toString()); - for (Map.Entry pair : uriNames.entrySet()) { - String remoteName = pair.getValue(); // whatever the git workspace config calls it - String uri = pair.getKey(); - String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); - if (needle != null && needleNorm != null && (needleNorm.equals(uriNorm) || needle.equals(uri)) ) { - result = new LinkedHashSet<>(); + if (!checkedDirs.contains(dirname)) { + try { + String fAbs = f.getAbsolutePath().toString(); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking " + ((needle == null) ? "" : "for submodule URL needle='" + needle + "' ") + "in existing refrepo dir '" + dirname + "' => '" + fAbs + "'"); + GitClient g = this.newGit(dirname); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); + Map uriNames = g.getRemoteUrls(); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + uriNames.toString()); + for (Map.Entry pair : uriNames.entrySet()) { + String remoteName = pair.getValue(); // whatever the git workspace config calls it + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + if (needle != null && needleNorm != null && (needleNorm.equals(uriNorm) || needle.equals(uri)) ) { + result = new LinkedHashSet<>(); + result.add(new String[]{fAbs, uri, uriNorm, remoteName}); + return result; + } + // Cache the finding to return eventually, for each remote: + // * absolute dirname of a Git workspace + // * original remote URI from that workspace's config + // * normalized remote URI + // * name of the remote from that workspace's config ("origin" etc) result.add(new String[]{fAbs, uri, uriNorm, remoteName}); - return result; } - // Cache the finding to return eventually. - result.add(new String[]{fAbs, uri, uriNorm, remoteName}); + } catch (Exception e) { + // ignore, go to next slide + LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception (will go to next item):\n" + e.toString()); } - // TODO: Here is a good spot to recurse this routine into - // a subdir that is a known git workspace, to add its data - // and/or return a found needle. - } catch (Exception e) { - // ignore, go to next slide - LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception (will go to next item):\n" + e.toString()); } + + // TODO: Here is a good spot to recurse this routine into + // a subdir that is a known git workspace, to add its data + // and/or return a found needle. + } } From 61a7d0bed352546b79231abfbd788b602a05eba9 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Tue, 12 Jan 2021 14:55:22 +0100 Subject: [PATCH 085/132] LegacyCompatibleGitAPIImpl.java: comment the plan about submodule search from .gitmodules data --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index b634402307..351a4eacb1 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -472,7 +472,9 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // same arrDirnames with different values and check remotes listed // in those repos. // If current repo *is NOT* bare - check its submodules - if (!isBare || true) { + // (the .gitmodules => submodule.MODNAME.{url,path} mapping) + // but this essentially does not look into any subdirectory + if (!isBare) { try { // For each current workspace (recurse or big loop in same context?): // public GitClient subGit(String subdir) => would this.subGit(...) @@ -481,6 +483,11 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // faster possible answer (only bother if needle is not null?) // try { getSubmodules("HEAD") ... } => List filtered for // "commit" items + // * if we are recursed into a "leaf" project and inspect ITS + // submodules, look at all git tips or even commits, to find + // and inspect all unique (by hash) .gitmodule objects, since + // over time or in different branches a "leaf" project could + // reference different subs? // getRemoteUrls() => Map // arrDirnames.clear(); From b91e00b939e299fe4295bf796e9a83946b13e88f Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 13 Jan 2021 13:50:55 +0100 Subject: [PATCH 086/132] LegacyCompatibleGitAPIImpl.java: update comments around shorter getSubmodulesUrls() signature --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 351a4eacb1..924dac704e 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -603,13 +603,14 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String return result; } - /** See above. With null, returns all we can find (slower) for the caller - * to parse */ + /** See above. With null needle, returns all data we can find under the + * referenceBaseDir tree (can take a while) for the caller to parse */ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir) { - return getSubmodulesUrls(referenceBaseDir, null); + return getSubmodulesUrls(referenceBaseDir, null, true); } - /* Do we need a parameter-less variant to look under current work dir - * aka Paths.get("").toAbsolutePath().toString() ?.. */ + /* Do we need a completely parameter-less variant to look under current + * work dir aka Paths.get("").toAbsolutePath().toString(), or under "this" + * GitClient workspace ?.. */ /** Yield the File object for the reference repository local filesystem * pathname. Note that the provided string may be suffixed with expandable From 65684f31b22d57b076a32fe84a26401655c3307f Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 13 Jan 2021 17:57:53 +0100 Subject: [PATCH 087/132] LegacyCompatibleGitAPIImpl.java: only track absolute dirname values in getSubmodulesUrls() right away --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 924dac704e..5f2b2cd392 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -384,53 +384,55 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Try with the basename without .git extension, and then with one. // First we try the caller-provided string casing, then normalized. - arrDirnames.add(needleBasename); - arrDirnames.add(needleBasename + ".git"); + // Note that only after this first smaller pass which we hope to + // succeed quickly, we engage in heavier (by I/O and computation) + // investigation of submodules, and then similar loop against any + // remaining direct subdirs that contain a ".git" FS object. + arrDirnames.add(referenceBaseDirAbs + "/" + needleBasename); + arrDirnames.add(referenceBaseDirAbs + "/" + needleBasename + ".git"); needleBasenameLC = needleBasename.toLowerCase(); if (!needleBasenameLC.equals(needleBasename)) { // Retry with lowercased dirname - arrDirnames.add(needleBasenameLC); - arrDirnames.add(needleBasenameLC + ".git"); + arrDirnames.add(referenceBaseDirAbs + "/" + needleBasenameLC); + arrDirnames.add(referenceBaseDirAbs + "/" + needleBasenameLC + ".git"); } if (!needleNormBasename.equals(needleBasenameLC)) { - arrDirnames.add(needleNormBasename); - arrDirnames.add(needleNormBasename + ".git"); + arrDirnames.add(referenceBaseDirAbs + "/" + needleNormBasename); + arrDirnames.add(referenceBaseDirAbs + "/" + needleNormBasename + ".git"); } needleSha = org.apache.commons.codec.digest.DigestUtils.sha256Hex(needleNorm); - arrDirnames.add(needleSha); - arrDirnames.add(needleSha + ".git"); + arrDirnames.add(referenceBaseDirAbs + "/" + needleSha); + arrDirnames.add(referenceBaseDirAbs + "/" + needleSha + ".git"); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like subdirs under refrepo '" + referenceBaseDirAbs + "' per arrDirnames: " + arrDirnames.toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like subdirs under base refrepo '" + referenceBaseDirAbs + "', per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { - f = new File(referenceBaseDirAbs, dirname); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); + f = new File(dirname); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir at abs pathname '" + dirname + "' if it exists"); if (f.exists() && f.isDirectory()) { try { - String fAbs = f.getAbsolutePath().toString(); File fGit = new File(f, ".git"); if (fGit.exists()) { // file, dir or symlink to those - //LegacyCompatibleGitAPIImpl? - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "' => '" + fAbs + "'"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "'"); GitClient g = referenceGit.subGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + uriNames.toString()); for (Map.Entry pair : uriNames.entrySet()) { String remoteName = pair.getValue(); // whatever the git workspace config calls it String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "') vs needle"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); - result.add(new String[]{fAbs, uri, uriNorm, remoteName}); + result.add(new String[]{dirname, uri, uriNorm, remoteName}); return result; } // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop // which by construct looks at different dirs so far. - result.add(new String[]{fAbs, uri, uriNorm, remoteName}); + result.add(new String[]{dirname, uri, uriNorm, remoteName}); } } } catch (Exception e) { @@ -555,12 +557,11 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String for (String dirname : arrDirnames) { // Note that here dirnames deal in absolutes f = new File(dirname); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' => abs pathname '" + f.getAbsolutePath().toString() + "'"); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' if it exists"); if (f.exists() && f.isDirectory()) { if (!checkedDirs.contains(dirname)) { try { - String fAbs = f.getAbsolutePath().toString(); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking " + ((needle == null) ? "" : "for submodule URL needle='" + needle + "' ") + "in existing refrepo dir '" + dirname + "' => '" + fAbs + "'"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking " + ((needle == null) ? "" : "for submodule URL needle='" + needle + "' ") + "in existing refrepo dir '" + dirname + "'"); GitClient g = this.newGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); Map uriNames = g.getRemoteUrls(); @@ -569,10 +570,10 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String String remoteName = pair.getValue(); // whatever the git workspace config calls it String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "') vs needle"); if (needle != null && needleNorm != null && (needleNorm.equals(uriNorm) || needle.equals(uri)) ) { result = new LinkedHashSet<>(); - result.add(new String[]{fAbs, uri, uriNorm, remoteName}); + result.add(new String[]{dirname, uri, uriNorm, remoteName}); return result; } // Cache the finding to return eventually, for each remote: @@ -580,7 +581,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // * original remote URI from that workspace's config // * normalized remote URI // * name of the remote from that workspace's config ("origin" etc) - result.add(new String[]{fAbs, uri, uriNorm, remoteName}); + result.add(new String[]{dirname, uri, uriNorm, remoteName}); } } catch (Exception e) { // ignore, go to next slide From 0c99c95dc11796433b97a679eb6f7c9a4e972560 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 13 Jan 2021 17:58:09 +0100 Subject: [PATCH 088/132] LegacyCompatibleGitAPIImpl.java: comment the plan for submodule handling --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 5f2b2cd392..282d6ca52b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -475,7 +475,10 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // in those repos. // If current repo *is NOT* bare - check its submodules // (the .gitmodules => submodule.MODNAME.{url,path} mapping) - // but this essentially does not look into any subdirectory + // but this essentially does not look into any subdirectory. + // But we can add at higher priority submodule path(s) whose + // basename of the URL matches the needleBasename. And then + // other submodule paths to inspect before arbitrary subdirs. if (!isBare) { try { // For each current workspace (recurse or big loop in same context?): From 327cb48e56eff2043eca2126b893f368db8f8bef Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 13 Jan 2021 13:50:30 +0100 Subject: [PATCH 089/132] LegacyCompatibleGitAPIImpl.java: finally, do recursion in getSubmodulesUrls() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 282d6ca52b..c621d54bf4 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -324,8 +324,14 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * normalizeGitUrl(url, true)) we want to find: * if it is not null - then stop and return just hits * for it as soon as we have something. + * @param checkRemotesInReferenceBaseDir - if true (reasonable default for + * external callers), the referenceBaseDir would be added + * to the list of dirs for listing known remotes in search + * for a needle match or for the big listing. Set to false + * when recursing, since this directory was checked already + * as part of parent directory inspection. */ - public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String needle) { + public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String needle, Boolean checkRemotesInReferenceBaseDir) { // Keep track of where we've already looked in the "result" Set, to // avoid looking in same places (different strategies below) twice. // And eventually return this Set or part of it as the answer. @@ -548,7 +554,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Finally check pattern's parent dir // * Look at remote URLs in current dir after the guessed subdirs failed, // and return then. - if (new File(referenceBaseDirAbs, ".git").exists()) { + if (checkRemotesInReferenceBaseDir && new File(referenceBaseDirAbs, ".git").exists()) { arrDirnames.add(referenceBaseDirAbs); } @@ -592,10 +598,20 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String } } - // TODO: Here is a good spot to recurse this routine into - // a subdir that is a known git workspace, to add its data - // and/or return a found needle. - + // Here is a good spot to recurse this routine into a + // subdir that is already a a known git workspace, to + // add its data to list and/or return a found needle. + LOGGER.log(Level.FINE, "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); + LinkedHashSet subEntries = getSubmodulesUrls(dirname, needle, false); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): returned from recursing into dir '" + dirname + "' with " + subEntries.size() + " found mappings"); + if (!subEntries.isEmpty()) { + if (needle != null) { + // We found nothing... until now! Bubble it up! + LOGGER.log(Level.FINE, "getSubmodulesUrls(): got a needle match from recursing into dir '" + dirname + "': " + subEntries.iterator().next()[0]); + return subEntries; + } + result.addAll(subEntries); + } } } @@ -787,7 +803,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Note: we pass "url" here, the routine differentiates original // naming vs. normalization while looking for its needle in the // haystack. - LinkedHashSet subEntries = getSubmodulesUrls(referenceBaseDir, url); + LinkedHashSet subEntries = getSubmodulesUrls(referenceBaseDir, url, true); if (!subEntries.isEmpty()) { // Normally we should only have one entry here, as sorted // by the routine, and prefer that first option if a new From 45d268cc7b3424f5acacb05605d2ff41cd049024 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sat, 16 Jan 2021 18:04:01 +0100 Subject: [PATCH 090/132] LegacyCompatibleGitAPIImpl.java: check not only for ".git" but also "objects" in getSubmodulesUrls() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index c621d54bf4..f06d42aee7 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -393,7 +393,8 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Note that only after this first smaller pass which we hope to // succeed quickly, we engage in heavier (by I/O and computation) // investigation of submodules, and then similar loop against any - // remaining direct subdirs that contain a ".git" FS object. + // remaining direct subdirs that contain a ".git" (or "objects") + // FS object. arrDirnames.add(referenceBaseDirAbs + "/" + needleBasename); arrDirnames.add(referenceBaseDirAbs + "/" + needleBasename + ".git"); needleBasenameLC = needleBasename.toLowerCase(); @@ -418,8 +419,9 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir at abs pathname '" + dirname + "' if it exists"); if (f.exists() && f.isDirectory()) { try { - File fGit = new File(f, ".git"); - if (fGit.exists()) { // file, dir or symlink to those + File fGit = new File(f, ".git"); // workspace - file, dir or symlink to those + File fObj = new File(f, "objects"); // bare + if (fGit.exists() || fObj.exists()) { LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "'"); GitClient g = referenceGit.subGit(dirname); LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); @@ -534,7 +536,8 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // If current repo *is* bare (can't have proper submodules), or if the // end-users just cloned or linked some more repos into this container, - // follow up with direct child dirs that have a ".git" FS object inside: + // follow up with direct child dirs that have a ".git" (or "objects") + // FS object inside: // Check subdirs that are git workspaces; note that values in checkedDirs // are absolute pathnames. If we did look for the needle, array already @@ -542,8 +545,9 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // inspect again... but should recurse into first anyhow. File[] directories = new File(referenceBaseDirAbs).listFiles(File::isDirectory); for (File dir : directories) { - f = new File(dir, ".git"); - if (f.exists()) { // May be a file or directory... or symlink to those... + File fGit = new File(f, ".git"); // workspace - file, dir or symlink to those + File fObj = new File(f, "objects"); // bare + if (fGit.exists() || fObj.exists()) { String dirname = dir.getPath().replaceAll("/*$", ""); if (!checkedDirs.contains(dirname)) { arrDirnames.add(dirname); @@ -554,8 +558,12 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // Finally check pattern's parent dir // * Look at remote URLs in current dir after the guessed subdirs failed, // and return then. - if (checkRemotesInReferenceBaseDir && new File(referenceBaseDirAbs, ".git").exists()) { - arrDirnames.add(referenceBaseDirAbs); + if (checkRemotesInReferenceBaseDir) { + File fGit = new File(referenceBaseDirAbs, ".git"); // workspace - file, dir or symlink to those + File fObj = new File(referenceBaseDirAbs, "objects"); // bare + if (fGit.exists() || fObj.exists()) { + arrDirnames.add(referenceBaseDirAbs); + } } LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at " + @@ -568,6 +576,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String f = new File(dirname); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' if it exists"); if (f.exists() && f.isDirectory()) { + // No checks for ".git" or "objects" this time, already checked above if (!checkedDirs.contains(dirname)) { try { LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking " + ((needle == null) ? "" : "for submodule URL needle='" + needle + "' ") + "in existing refrepo dir '" + dirname + "'"); From 0413d95fd4f88aaf221f292232621294f3eb5332 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sat, 16 Jan 2021 18:21:47 +0100 Subject: [PATCH 091/132] LegacyCompatibleGitAPIImpl.java: for GIT_SUBMODULES mode search, if there were no hits, suggest the basename(URL) subdir (friendlier for co-hosting forks) --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f06d42aee7..72e7af1565 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -803,15 +803,20 @@ public File findParameterizedReferenceRepository(String reference, String url) { // smaller scopes - much quicker to check out from than one // huge combined repo. It would also be much more native to // tools and custom scriptware that can be involved. + // Beside git-submodule parsing (that only points to one URL + // at a time) his also covers a search for subdirectories + // that host a git repository whose remotes match the URL, + // to handle co-hosting of several remotes (different URLs + // to same repository, e.g. SSH and HTTPS; mirrors; forks). // Assuming the provided "reference" directory already hosts // submodules, we use git tools to find the one subdir which // has a registered remote URL equivalent (per normalization) // to the provided "url". - // Note: we pass "url" here, the routine differentiates original - // naming vs. normalization while looking for its needle in the - // haystack. + // Note: we pass the unmodified "url" value here, the routine + // differentiates original spelling vs. normalization while + // looking for its needle in the haystack. LinkedHashSet subEntries = getSubmodulesUrls(referenceBaseDir, url, true); if (!subEntries.isEmpty()) { // Normally we should only have one entry here, as sorted @@ -836,15 +841,29 @@ public File findParameterizedReferenceRepository(String reference, String url) { } else { LOGGER.log(Level.FINE, "findParameterizedReferenceRepository(): got no subEntries"); // If there is no hit, the non-fallback mode suggests a new - // directory name to host the submodule (same rules as SHA), + // directory name to host the submodule (same rules as for + // the refrepo forks' co-hosting friendly basename search), // and the fallback mode would return the main directory. + int sep = url.lastIndexOf("/"); + String needleBasename; + if (sep < 0) { + needleBasename = url; + } else { + needleBasename = url.substring(sep + 1); + } + needleBasename = needleBasename.replaceAll(".git$", ""); + if (reference.endsWith("/${GIT_SUBMODULES}")) { referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", - org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + needleBasename); } else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { - // chop it off, use main directory - referenceExpanded = referenceBaseDir; + referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", + needleBasename); + if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + // chop it off, use main directory + referenceExpanded = referenceBaseDir; + } } } } From fad3727ab577e8b4a5bd97fffb79d9c0cebcf6c9 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sat, 16 Jan 2021 18:58:42 +0100 Subject: [PATCH 092/132] LegacyCompatibleGitAPIImpl.java: implement a use-case of ".git" file with "gitdir" entry, for getObjectsFile() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 66 +++++++++++++++++-- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 72e7af1565..0a1b7f439d 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -19,7 +19,9 @@ import org.eclipse.jgit.transport.RemoteConfig; import org.eclipse.jgit.transport.URIish; +import java.io.BufferedReader; import java.io.File; +import java.io.FileReader; import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Path; @@ -184,6 +186,9 @@ public File getObjectsFile(String reference) { } public File getObjectsFile(File reference) { + // reference pathname can either point to a "normal" workspace + // checkout or a bare repository + if (reference == null) { return reference; } @@ -194,14 +199,63 @@ public File getObjectsFile(File reference) { if (!reference.isDirectory()) return null; - // reference pathname can either point to a normal or a base repository - File objects = new File(reference, ".git/objects"); - if (objects == null) { - return objects; // Some Java error, could not make object from the paths involved + File fGit = new File(reference, ".git"); // workspace - file, dir or symlink to those + //File fObj = new File(f, "objects"); // bare + File objects = null; + + if (fGit.exists()) { + if (fGit.isDirectory()) { + objects = new File(fGit, "objects"); + if (objects == null) { + return objects; // Some Java error, could not make object from the paths involved + } + } else { + // If ".git" FS object exists and is a not-empty file (and + // is not a dir), then its content may point to some other + // filesystem location for the Git-internal data. + // For example, a checked-out submodule workspace can point + // to the index and other metadata stored in its "parent" + // repository's directory: + // "gitdir: ../.git/modules/childRepoName" + BufferedReader reader = null; + try { + String line; + reader = new BufferedReader(new FileReader(fGit)); + while ((line = reader.readLine()) != null) + { + String[] parts = line.split(":", 2); + if (parts.length >= 2) + { + String key = parts[0].trim(); + String value = parts[1].trim(); + if (key.equals("gitdir")) { + objects = new File(reference, value); + LOGGER.log(Level.FINE, "getObjectsFile(): while looking for 'gitdir:' in '" + + fGit.getAbsolutePath().toString() + "', found reference to objects " + + "which should be at: '" + objects.getAbsolutePath().toString() + "'"); + break; + } + LOGGER.log(Level.FINEST, "getObjectsFile(): while looking for 'gitdir:' in '" + + fGit.getAbsolutePath().toString() + "', ignoring line: " + line); + } + } + if (objects == null) { + LOGGER.log(Level.WARNING, "getObjectsFile(): failed to parse '" + fGit.getAbsolutePath().toString() + "': did not contain a 'gitdir:' entry"); + } + } catch (IOException e) { + LOGGER.log(Level.SEVERE, "getObjectsFile(): failed to parse '" + fGit.getAbsolutePath().toString() + "': " + e.toString()); + objects = null; + } + try { + if (reader != null) { + reader.close(); + } + } catch (IOException e) {} + } } - if (!objects.isDirectory()) { - // reference path is bare repo + if (objects == null || !objects.isDirectory()) { + // reference path is bare repo (no .git inside) or failed interpreting ".git" contents objects = new File(reference, "objects"); if (objects == null) { return objects; // Some Java error, could not make object from the paths involved From d37fe49ca7d9d20be918e46d4fa140005cd61d67 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sat, 16 Jan 2021 19:54:22 +0100 Subject: [PATCH 093/132] LegacyCompatibleGitAPIImpl.java: refactor getSubmodulesUrls() to use common getObjectsFile() instead of direct FS object queries --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 52 ++++++++----------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 0a1b7f439d..37c26fc193 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -471,31 +471,27 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String for (String dirname : arrDirnames) { f = new File(dirname); LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir at abs pathname '" + dirname + "' if it exists"); - if (f.exists() && f.isDirectory()) { + if (getObjectsFile(f) != null) { try { - File fGit = new File(f, ".git"); // workspace - file, dir or symlink to those - File fObj = new File(f, "objects"); // bare - if (fGit.exists() || fObj.exists()) { - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "'"); - GitClient g = referenceGit.subGit(dirname); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); - Map uriNames = g.getRemoteUrls(); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + uriNames.toString()); - for (Map.Entry pair : uriNames.entrySet()) { - String remoteName = pair.getValue(); // whatever the git workspace config calls it - String uri = pair.getKey(); - String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "') vs needle"); - if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new LinkedHashSet<>(); - result.add(new String[]{dirname, uri, uriNorm, remoteName}); - return result; - } - // Cache the finding to avoid the dirname later, if we - // get to that; but no checks are needed in this loop - // which by construct looks at different dirs so far. + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "'"); + GitClient g = referenceGit.subGit(dirname); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); + Map uriNames = g.getRemoteUrls(); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + uriNames.toString()); + for (Map.Entry pair : uriNames.entrySet()) { + String remoteName = pair.getValue(); // whatever the git workspace config calls it + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "') vs needle"); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new LinkedHashSet<>(); result.add(new String[]{dirname, uri, uriNorm, remoteName}); + return result; } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. + result.add(new String[]{dirname, uri, uriNorm, remoteName}); } } catch (Exception e) { // ignore, go to next slide @@ -599,9 +595,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // inspect again... but should recurse into first anyhow. File[] directories = new File(referenceBaseDirAbs).listFiles(File::isDirectory); for (File dir : directories) { - File fGit = new File(f, ".git"); // workspace - file, dir or symlink to those - File fObj = new File(f, "objects"); // bare - if (fGit.exists() || fObj.exists()) { + if (getObjectsFile(dir) != null) { String dirname = dir.getPath().replaceAll("/*$", ""); if (!checkedDirs.contains(dirname)) { arrDirnames.add(dirname); @@ -613,9 +607,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // * Look at remote URLs in current dir after the guessed subdirs failed, // and return then. if (checkRemotesInReferenceBaseDir) { - File fGit = new File(referenceBaseDirAbs, ".git"); // workspace - file, dir or symlink to those - File fObj = new File(referenceBaseDirAbs, "objects"); // bare - if (fGit.exists() || fObj.exists()) { + if (getObjectsFile(referenceBaseDirAbs) != null) { arrDirnames.add(referenceBaseDirAbs); } } @@ -631,6 +623,8 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' if it exists"); if (f.exists() && f.isDirectory()) { // No checks for ".git" or "objects" this time, already checked above + // by getObjectsFile(). Probably should not check exists/dir either, + // but better be on the safe side :) if (!checkedDirs.contains(dirname)) { try { LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking " + ((needle == null) ? "" : "for submodule URL needle='" + needle + "' ") + "in existing refrepo dir '" + dirname + "'"); @@ -662,7 +656,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String } // Here is a good spot to recurse this routine into a - // subdir that is already a a known git workspace, to + // subdir that is already a known git workspace, to // add its data to list and/or return a found needle. LOGGER.log(Level.FINE, "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); LinkedHashSet subEntries = getSubmodulesUrls(dirname, needle, false); From 6afacdab311161e83b62ff88038f6fc9f4138736 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sat, 16 Jan 2021 19:46:25 +0100 Subject: [PATCH 094/132] LegacyCompatibleGitAPIImpl.java: refactor getSubmodulesUrls() to return both a Boolean whether it had an exact match, and the result Set --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 67 +++++++++++++------ 1 file changed, 48 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 37c26fc193..67c19b60d6 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -26,6 +26,7 @@ import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; import java.util.HashSet; import java.util.LinkedHashSet; @@ -359,10 +360,13 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { return urlNormalized; } - /** Find referenced URLs in this repo and its submodules, recursively. - * Current primary use is for parameterized refrepo/${GIT_SUBMODULES} + /** Find referenced URLs in this repo and its submodules (or other + * subdirs with git repos), recursively. Current primary use is for + * parameterized refrepo/${GIT_SUBMODULES} handling. * - * @return a Set of (unique) String arrays, representing: + * @return an AbstractMap.SimpleEntry, containing a Boolean to denote + * an exact match (or lack thereof) for the needle (if searched for), + * and a Set of (unique) String arrays, representing: * [0] directory of nested submodule (relative to current workspace root) * The current workspace would be listed as directory "" and consumers * should check these entries last if they care for most-specific hits @@ -373,6 +377,17 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * fully qualified) * [3] remoteName as defined in that nested submodule * + * If the returned SimpleEntry has the Boolean flag as False but also + * a Set which is not empty, and a search for "needle" was requested, + * then that Set lists some not-exact matches for existing sub-dirs + * with repositories that seem likely to be close hits (e.g. remotes + * there *probably* point to other URLs of same repo as the needle, + * or its forks - so these directories are more likely than others to + * contain the reference commits needed for the faster git checkouts). + * + * For a search with needle==null, the Boolean flag would be False too, + * and the Set would just detail all found sub-repositories. + * * @param referenceBaseDir - the reference repository, or container thereof * @param needle - an URL which (or its normalized variant coming from * normalizeGitUrl(url, true)) we want to find: @@ -385,7 +400,7 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * when recursing, since this directory was checked already * as part of parent directory inspection. */ - public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String needle, Boolean checkRemotesInReferenceBaseDir) { + public SimpleEntry> getSubmodulesUrls(String referenceBaseDir, String needle, Boolean checkRemotesInReferenceBaseDir) { // Keep track of where we've already looked in the "result" Set, to // avoid looking in same places (different strategies below) twice. // And eventually return this Set or part of it as the answer. @@ -486,7 +501,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); result.add(new String[]{dirname, uri, uriNorm, remoteName}); - return result; + return new SimpleEntry<>(true, result); } // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop @@ -640,7 +655,7 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String if (needle != null && needleNorm != null && (needleNorm.equals(uriNorm) || needle.equals(uri)) ) { result = new LinkedHashSet<>(); result.add(new String[]{dirname, uri, uriNorm, remoteName}); - return result; + return new SimpleEntry<>(true, result); } // Cache the finding to return eventually, for each remote: // * absolute dirname of a Git workspace @@ -659,30 +674,40 @@ public LinkedHashSet getSubmodulesUrls(String referenceBaseDir, String // subdir that is already a known git workspace, to // add its data to list and/or return a found needle. LOGGER.log(Level.FINE, "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); - LinkedHashSet subEntries = getSubmodulesUrls(dirname, needle, false); + SimpleEntry > subEntriesRet = getSubmodulesUrls(dirname, needle, false); + Boolean subEntriesExactMatched = subEntriesRet.getKey(); + LinkedHashSet subEntries = subEntriesRet.getValue(); LOGGER.log(Level.FINE, "getSubmodulesUrls(): returned from recursing into dir '" + dirname + "' with " + subEntries.size() + " found mappings"); if (!subEntries.isEmpty()) { - if (needle != null) { + if (needle != null && subEntriesExactMatched) { // We found nothing... until now! Bubble it up! - LOGGER.log(Level.FINE, "getSubmodulesUrls(): got a needle match from recursing into dir '" + dirname + "': " + subEntries.iterator().next()[0]); - return subEntries; + LOGGER.log(Level.FINE, "getSubmodulesUrls(): got an exact needle match from recursing into dir '" + dirname + "': " + subEntries.iterator().next()[0]); + return subEntriesRet; } result.addAll(subEntries); } } } - // Nothing found, so if we had a needle - report there are no hits + // Nothing found, if we had a needle - so report there are no hits if (needle != null) { - return new LinkedHashSet<>(); + // TODO: Handle suggestions (not-exact matches) if something from + // results looks like it is related to the needle. +/* + if (checkRemotesInReferenceBaseDir) { + // Overload the flag's meaning to only parse results once? + } +*/ + return new SimpleEntry<>(false, new LinkedHashSet<>()); } - return result; + // Did not look for anything in particular + return new SimpleEntry<>(false, result); } /** See above. With null needle, returns all data we can find under the * referenceBaseDir tree (can take a while) for the caller to parse */ - public LinkedHashSet getSubmodulesUrls(String referenceBaseDir) { + public SimpleEntry> getSubmodulesUrls(String referenceBaseDir) { return getSubmodulesUrls(referenceBaseDir, null, true); } /* Do we need a completely parameter-less variant to look under current @@ -865,17 +890,21 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Note: we pass the unmodified "url" value here, the routine // differentiates original spelling vs. normalization while // looking for its needle in the haystack. - LinkedHashSet subEntries = getSubmodulesUrls(referenceBaseDir, url, true); + SimpleEntry > subEntriesRet = getSubmodulesUrls(referenceBaseDir, url, true); + Boolean subEntriesExactMatched = subEntriesRet.getKey(); + LinkedHashSet subEntries = subEntriesRet.getValue(); if (!subEntries.isEmpty()) { // Normally we should only have one entry here, as sorted // by the routine, and prefer that first option if a new // reference repo would have to be made (and none exists). // If several entries are present after all, iterate until // first existing hit and return the first entry otherwise. - for (String[] subEntry : subEntries) { - if (getObjectsFile(subEntry[0]) != null || getObjectsFile(subEntry[0] + ".git") != null) { - referenceExpanded = subEntry[0]; - break; + if (!subEntriesExactMatched) { // else look at first entry below + for (String[] subEntry : subEntries) { + if (getObjectsFile(subEntry[0]) != null || getObjectsFile(subEntry[0] + ".git") != null) { + referenceExpanded = subEntry[0]; + break; + } } } if (referenceExpanded == null) { From b278523e4127d916eb5ccd7441af4276444e9f6f Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 17 Jan 2021 13:30:46 +0100 Subject: [PATCH 095/132] LegacyCompatibleGitAPIImpl.java: add logging for getObjectsFile() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 67c19b60d6..b0737c1790 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -210,6 +210,8 @@ public File getObjectsFile(File reference) { if (objects == null) { return objects; // Some Java error, could not make object from the paths involved } + LOGGER.log(Level.FINEST, "getObjectsFile(): found an fGit '" + + fGit.getAbsolutePath().toString() + "' which is a directory"); } else { // If ".git" FS object exists and is a not-empty file (and // is not a dir), then its content may point to some other @@ -218,6 +220,8 @@ public File getObjectsFile(File reference) { // to the index and other metadata stored in its "parent" // repository's directory: // "gitdir: ../.git/modules/childRepoName" + LOGGER.log(Level.FINEST, "getObjectsFile(): found an fGit '" + + fGit.getAbsolutePath().toString() + "' which is NOT a directory"); BufferedReader reader = null; try { String line; @@ -253,6 +257,9 @@ public File getObjectsFile(File reference) { } } catch (IOException e) {} } + } else { + LOGGER.log(Level.FINEST, "getObjectsFile(): did not find any checked-out '" + + fGit.getAbsolutePath().toString() + "'"); } if (objects == null || !objects.isDirectory()) { @@ -261,6 +268,19 @@ public File getObjectsFile(File reference) { if (objects == null) { return objects; // Some Java error, could not make object from the paths involved } + // This clause below is redundant for production, but useful for troubleshooting + if (objects.exists()) { + if (objects.isDirectory()) { + LOGGER.log(Level.FINEST, "getObjectsFile(): found a bare '" + + objects.getAbsolutePath().toString() + "' which is a directory"); + } else { + LOGGER.log(Level.FINEST, "getObjectsFile(): found a bare '" + + objects.getAbsolutePath().toString() + "' which is NOT a directory"); + } + } else { + LOGGER.log(Level.FINEST, "getObjectsFile(): did not find any bare '" + + objects.getAbsolutePath().toString() + "'"); + } } if (!objects.isDirectory()) From 446844355ee5072bc19303038e53db13b2a3b476 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 17 Jan 2021 14:33:39 +0100 Subject: [PATCH 096/132] LegacyCompatibleGitAPIImpl.java: in getObjectsFile(), comment why it does not return "object" dir File named with getCanonicalPath() --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index b0737c1790..7ef63e1a00 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -238,6 +238,14 @@ public File getObjectsFile(File reference) { LOGGER.log(Level.FINE, "getObjectsFile(): while looking for 'gitdir:' in '" + fGit.getAbsolutePath().toString() + "', found reference to objects " + "which should be at: '" + objects.getAbsolutePath().toString() + "'"); + // Note: we don't use getCanonicalPath() here to avoid further filesystem + // access and possible exceptions (the getAbsolutePath() is about string + // processing), but callers might benefit from canonicising and ensuring + // unique pathnames (for equality checks etc.) with relative components + // and symlinks resolved. + // On another hand, keeping the absolute paths, possibly, relative to a + // parent directory as prefix, allows callers to match/compare such parent + // prefixes for the contexts the callers would define for themselves. break; } LOGGER.log(Level.FINEST, "getObjectsFile(): while looking for 'gitdir:' in '" + From 0edcc0c2c2a30977eca15aec2506caa2c73c3f38 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 17 Jan 2021 14:46:40 +0100 Subject: [PATCH 097/132] JGitAPIImpl.java / CliGitAPIImpl.java: inform the build console log reader why they see lots of "git rev-parse --is-bare-repository" + "git config --local --list" messages --- .../java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java | 5 +++++ .../java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index 7c3c0cc92c..34e97fb5f1 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -767,6 +767,11 @@ public void execute() throws GitException, InterruptedException { } if (reference != null && !reference.isEmpty()) { + if (isParameterizedReferenceRepository(reference)) { + // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via listener + listener.getLogger().println("[INFO] The git reference repository path is parameterized, " + + "it may take a few git queries logged below to resolve it into a particular directory name"); + } File referencePath = findParameterizedReferenceRepository(reference, url); if (referencePath == null) { listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + reference); diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 5ff02d0eb6..c407f7b1ec 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1456,6 +1456,11 @@ public void execute() throws GitException, InterruptedException { // the repository builder does not create the alternates file if (reference != null && !reference.isEmpty()) { // Note: keep in sync with similar logic in CliGitAPIImpl.java + if (isParameterizedReferenceRepository(reference)) { + // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via listener + listener.getLogger().println("[INFO] The git reference repository path is parameterized, " + + "it may take a few git queries logged below to resolve it into a particular directory name"); + } File referencePath = findParameterizedReferenceRepository(reference, url); if (referencePath == null) { listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + reference); From 0767c2a3ecb67b2c27ef213abe548abfa483f2cc Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 17 Jan 2021 15:33:26 +0100 Subject: [PATCH 098/132] LegacyCompatibleGitAPIImpl.java: in getObjectsFile() method comment, clarify when "objects" may be not under "reference" --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 7ef63e1a00..1cd4a225cc 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -174,6 +174,8 @@ public void clone(RemoteConfig rc, boolean useShallowClone) throws GitException, /** For referenced directory check if it is a full or bare git repo * and return the File object for its "objects" sub-directory. + * (Note that for submodules and other cases with externalized Git + * metadata, the "objects" directory may be NOT under "reference"). * If there is nothing to find, or inputs are bad, returns null. * The idea is that checking for null allows to rule out non-git * paths, while a not-null return value is instantly usable by From 03b30564402acc42465b750a4d3b2ed5e9fd22c6 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 17 Jan 2021 15:36:26 +0100 Subject: [PATCH 099/132] LegacyCompatibleGitAPIImpl.java: little clean-up in getObjectsFile() --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 1cd4a225cc..2f74e8c305 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -203,7 +203,6 @@ public File getObjectsFile(File reference) { return null; File fGit = new File(reference, ".git"); // workspace - file, dir or symlink to those - //File fObj = new File(f, "objects"); // bare File objects = null; if (fGit.exists()) { @@ -273,8 +272,9 @@ public File getObjectsFile(File reference) { } if (objects == null || !objects.isDirectory()) { - // reference path is bare repo (no .git inside) or failed interpreting ".git" contents - objects = new File(reference, "objects"); + // either reference path is bare repo (no ".git" inside), + // or we have failed interpreting ".git" contents above + objects = new File(reference, "objects"); // bare if (objects == null) { return objects; // Some Java error, could not make object from the paths involved } From fd88900a6946537bb004a358a7305e5f7abea181 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 17 Jan 2021 16:57:30 +0100 Subject: [PATCH 100/132] LegacyCompatibleGitAPIImpl.java: referenceBaseDirAbs in getSubmodulesUrls() should be canonical (thus unique), if possible --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 2f74e8c305..283aa3ffcf 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -438,7 +438,19 @@ public SimpleEntry> getSubmodulesUrls(String re File f = null; // Helper list storage in loops below ArrayList arrDirnames = new ArrayList(); - String referenceBaseDirAbs = Paths.get(referenceBaseDir).toAbsolutePath().toString(); + + // Note: an absolute path is not necessarily the canonical one + // We want to hit same dirs only once, so canonicize paths below + String referenceBaseDirAbs; + try { + referenceBaseDirAbs = new File(referenceBaseDir).getAbsoluteFile().getCanonicalPath().toString(); + } catch (IOException e) { + // Access error while dereferencing some parent?.. + referenceBaseDirAbs = new File(referenceBaseDir).getAbsoluteFile().toString(); + LOGGER.log(Level.SEVERE, "getSubmodulesUrls(): failed to canonicize '" + + referenceBaseDir + "' => '" + referenceBaseDirAbs + "': " + e.toString()); + //return new SimpleEntry<>(false, result); + } // "this" during a checkout typically represents the job workspace, // but we want to inspect the reference repository located elsewhere From d72999be39aac7a14f01ba4306c8b43d84ad87c9 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 17 Jan 2021 16:58:09 +0100 Subject: [PATCH 101/132] LegacyCompatibleGitAPIImpl.java: suggest a selection of results[] *similar* to needle in getSubmodulesUrls() if no exact match happened --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 96 +++++++++++++++++-- 1 file changed, 87 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 283aa3ffcf..583875937a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -471,17 +471,19 @@ public SimpleEntry> getSubmodulesUrls(String re // This is only used and populated if needle is not null String needleNorm = null; + // This is only used and populated if needle is not null, and + // can be used in the end to filter not-exact match suggestions + String needleBasename = null; + String needleBasenameLC = null; + String needleNormBasename = null; + String needleSha = null; + // If needle is not null, first look perhaps in the subdir(s) named // with base-name of the URL with and without a ".git" suffix, then // in SHA256 named dir that can match it; note that this use-case // might pan out also if "this" repo is bare and can not have "proper" // git submodules - but was prepared for our other options. if (needle != null) { - String needleBasename; - String needleBasenameLC; - String needleNormBasename; - String needleSha; - int sep = needle.lastIndexOf("/"); if (sep < 0) { needleBasename = needle; @@ -726,21 +728,97 @@ public SimpleEntry> getSubmodulesUrls(String re LOGGER.log(Level.FINE, "getSubmodulesUrls(): got an exact needle match from recursing into dir '" + dirname + "': " + subEntries.iterator().next()[0]); return subEntriesRet; } + // ...else collect results to inspect and/or propagate later result.addAll(subEntries); } } } // Nothing found, if we had a needle - so report there are no hits + // If we did not have a needle, we did not search for it - return + // below whatever result we have, then. if (needle != null) { - // TODO: Handle suggestions (not-exact matches) if something from + if (result.size() == 0) { + // Completely nothing git-like found here, return quickly + return new SimpleEntry<>(false, result); + } + + // Handle suggestions (not-exact matches) if something from // results looks like it is related to the needle. + LinkedHashSet resultFiltered = new LinkedHashSet<>(); + /* - if (checkRemotesInReferenceBaseDir) { - // Overload the flag's meaning to only parse results once? + if (!checkRemotesInReferenceBaseDir) { + // Overload the flag's meaning to only parse results once, + // in the parent dir? + return new SimpleEntry<>(false, resultFiltered); } */ - return new SimpleEntry<>(false, new LinkedHashSet<>()); + + // Separate lists by suggestion priority: + // 1) URI basename similarity + // 2) Directory basename similarity + LinkedHashSet resultFiltered1 = new LinkedHashSet<>(); + LinkedHashSet resultFiltered2 = new LinkedHashSet<>(); + + LinkedHashSet suggestedDirs = new LinkedHashSet<>(); + for (String[] resultEntry : result) { + checkedDirs.add(resultEntry[0]); + } + + for (String[] subEntry : result) { + // Iterating to filter suggestions in order of original + // directory-walk prioritization under current reference + String dirName = subEntry[0]; + String uri = subEntry[1]; + String uriNorm = subEntry[2]; + String remoteName = subEntry[3]; + Integer sep; + String uriNormBasename; + String dirBasename; + + // Match basename of needle vs. a remote tracked by an + // existing git directory (automation-ready normalized URL) + sep = uriNorm.lastIndexOf("/"); + if (sep < 0) { + uriNormBasename = uriNorm; + } else { + uriNormBasename = uriNorm.substring(sep + 1); + } + uriNormBasename = uriNormBasename.replaceAll(".git$", ""); + + if (uriNormBasename.equals(needleNormBasename)) { + resultFiltered1.add(subEntry); + } + + if (!suggestedDirs.contains(dirName)) { + // Here just match basename of needle vs. an existing + // sub-git directory base name + suggestedDirs.add(dirName); + + sep = dirName.lastIndexOf("/"); + if (sep < 0) { + dirBasename = dirName; + } else { + dirBasename = dirName.substring(sep + 1); + } + dirBasename = dirBasename.replaceAll(".git$", ""); + + if (dirBasename.equals(needleNormBasename)) { + resultFiltered2.add(subEntry); + } + } + } + + // Concatenate suggestions in order of priority. + // Hopefully the Set should deduplicate entries + // if something matched twice :) + resultFiltered.addAll(resultFiltered1); // URLs + resultFiltered.addAll(resultFiltered2); // Dirnames + + // Note: flag is false since matches (if any) are + // not exactly for the Git URL requested by caller + return new SimpleEntry<>(false, resultFiltered); } // Did not look for anything in particular From f1f478a0009bd18d2c1bfe5eef9ff62ee772c4ed Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 18 Jan 2021 02:05:46 +0100 Subject: [PATCH 102/132] LegacyCompatibleGitAPIImpl.java: mark helpers as public static --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 583875937a..10af938135 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -181,14 +181,14 @@ public void clone(RemoteConfig rc, boolean useShallowClone) throws GitException, * paths, while a not-null return value is instantly usable by * some code which plays with git under its hood. */ - public File getObjectsFile(String reference) { + public static File getObjectsFile(String reference) { if (reference == null || reference.isEmpty()) { return null; } return getObjectsFile(new File(reference)); } - public File getObjectsFile(File reference) { + public static File getObjectsFile(File reference) { // reference pathname can either point to a "normal" workspace // checkout or a bare repository @@ -306,14 +306,14 @@ public File getObjectsFile(File reference) { * * @param reference Pathname (maybe with magic suffix) to reference repo */ - public Boolean isParameterizedReferenceRepository(File reference) { + public static Boolean isParameterizedReferenceRepository(File reference) { if (reference == null) { return false; } return isParameterizedReferenceRepository(reference.getPath()); } - public Boolean isParameterizedReferenceRepository(String reference) { + public static Boolean isParameterizedReferenceRepository(String reference) { if (reference == null || reference.isEmpty()) { return false; } From 91ed630ff9ad751c4ba98b8691a0ae353f1976d4 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 18 Jan 2021 15:19:38 +0100 Subject: [PATCH 103/132] LegacyCompatibleGitAPIImpl.java: check if referenceBaseDir is usable (exists and is a dir) before trying a useless and failure-prone search in findParameterizedReferenceRepository() --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 10af938135..267342b10a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -868,6 +868,19 @@ public File findParameterizedReferenceRepository(String reference, String url) { isParameterizedReferenceRepository(reference) && url != null && !url.isEmpty() ) { + // Drop the trailing keyword to know the root refrepo dirname + String referenceBaseDir = reference.replaceAll("/\\$\\{GIT_[^\\}]*\\}$", ""); + + File referenceBaseDirFile = new File(referenceBaseDir); + if (!referenceBaseDirFile.exists()) { + LOGGER.log(Level.WARNING, "Base Git reference directory " + referenceBaseDir + " does not exist"); + return null; + } + if (!referenceBaseDirFile.isDirectory()) { + LOGGER.log(Level.WARNING, "Base Git reference directory " + referenceBaseDir + " is not a directory"); + return null; + } + // Note: this normalization might crush several URLs into one, // and as far as is known this would be the goal - people tend // to use or omit .git suffix, or spell with varied case, while @@ -878,9 +891,6 @@ public File findParameterizedReferenceRepository(String reference, String url) { // sensitive and different). String urlNormalized = normalizeGitUrl(url, true); - // Drop the trailing keyword to know the root refrepo dirname - String referenceBaseDir = reference.replaceAll("/\\$\\{GIT_[^\\}]*\\}$", ""); - // Note: currently unit-tests expect this markup on stderr: System.err.println("reference='" + reference + "'\n" + "url='" + url + "'\n" + From de98cd13daf9aaa983d645d9545e57d4d206cb57 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 31 Jan 2021 18:04:06 +0100 Subject: [PATCH 104/132] GitClientCloneTest.java: reword original test for GIT_URL into checking GIT_URL_BASENAME --- .../plugins/gitclient/GitClientCloneTest.java | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 5bcb91bf80..e9ae5bcab5 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -240,8 +240,8 @@ public void test_clone_reference() throws Exception, IOException, InterruptedExc } @Test - public void test_clone_reference_parameterized() throws Exception, IOException, InterruptedException { - testGitClient.clone_().url(workspace.localMirror()).repositoryName("origin").reference(workspace.localMirror() + "/${GIT_URL}").execute(); + public void test_clone_reference_parameterized_basename() throws Exception, IOException, InterruptedException { + testGitClient.clone_().url(workspace.localMirror()).repositoryName("origin").reference(workspace.localMirror() + "/${GIT_URL_BASENAME}").execute(); testGitClient.checkout().ref("origin/master").branch("master").execute(); check_remote_url(workspace, testGitClient, "origin"); // Verify JENKINS-46737 expected log message is written @@ -251,13 +251,10 @@ public void test_clone_reference_parameterized() throws Exception, IOException, handler.containsMessageSubstring(" replaced with: "), is(true)); // TODO: Actually construct the local filesystem path to match - // the literally substituted URL. Note this may not work for - // filesystems and operating systems that constrain character - // ranges usable for filenames (Windows at least) so failure - // to create the paths is an option to handle in test. - // Be sure to clean away this path at end of test, so that the - // test_clone_reference_parameterized_fallback() below is not - // confused - it expects this location to not exist. + // the last pathname component from the URL (plus/minus ".git" + // extension). Be sure to clean away this path at end of test, + // so that the test_clone_reference_parameterized_basename_fallback() + // below is not confused - it expects this location to not exist. // Skip: Missing if clone failed - currently would, with bogus // path above and not yet pre-created path structure. //assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); @@ -267,10 +264,10 @@ public void test_clone_reference_parameterized() throws Exception, IOException, } @Test - public void test_clone_reference_parameterized_fallback() throws Exception, IOException, InterruptedException { + public void test_clone_reference_parameterized_basename_fallback() throws Exception, IOException, InterruptedException { // TODO: Currently we do not make paths which would invalidate // this test, but note the test above might do just that later. - testGitClient.clone_().url(workspace.localMirror()).repositoryName("origin").reference(workspace.localMirror() + "/${GIT_URL_FALLBACK}").execute(); + testGitClient.clone_().url(workspace.localMirror()).repositoryName("origin").reference(workspace.localMirror() + "/${GIT_URL_BASENAME_FALLBACK}").execute(); testGitClient.checkout().ref("origin/master").branch("master").execute(); check_remote_url(workspace, testGitClient, "origin"); // Verify JENKINS-46737 expected log message is written @@ -281,9 +278,9 @@ public void test_clone_reference_parameterized_fallback() throws Exception, IOEx is(true)); // With fallback mode, and nonexistent parameterized reference // repository, and a usable repository in the common path (what - // remains if the parameterizing suffix is just discarded), - // this common path should be used. So it should overall behave - // same as the non-parameterized test_clone_reference() above. + // remains if the parameterizing suffix is just discarded), this + // common path should be used. So it should overall behave same + // as the non-parameterized test_clone_reference_basename() above. assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); assertAlternateFilePointsToLocalMirror(); assertBranchesExist(testGitClient.getBranches(), "master"); From aa9eaadadc85c6d4f7f88f5bf97cc486fb3a3d28 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 31 Jan 2021 18:18:26 +0100 Subject: [PATCH 105/132] LegacyCompatibleGitAPIImpl.java: findParameterizedReferenceRepository(): move generic comments about the design from GIT_URL handling to head of function --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 47 ++++++++++++------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 267342b10a..34d918f41b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -861,6 +861,36 @@ public File findParameterizedReferenceRepository(String reference, String url) { } File referencePath = new File(reference); + // For mass-configured jobs, like Organization Folders, which inherit + // a refrepo setting (String) into generated MultiBranch Pipelines and + // leaf jobs made for each repo branch and PR, the code below allows + // us to support parameterized paths, with one string leading to many + // reference repositories fanned out under a common location. + // This also works for legacy jobs using a Git SCM. + + // TODO: Consider a config option whether to populate absent reference + // repos (If the expanded path does not have git repo data right now, + // should we populate it into the location expanded by logic below), + // or update existing ones before pulling commits, and how to achieve + // that. Currently this is something that comments elsewhere in the + // git-client-plugin and/or articles on reference repository setup + // considered to be explicitly out of scope of the plugin. + // Note that this pre-population (or update) is done by caller with + // their implementation of git and site-specific connectivity and + // storage desigh, e.g. in case of a build farm it is more likely + // to be a shared path from a common storage server only readable to + // Jenkins and its agents, so write-operations would be done by helper + // scripts that log into the shared storage server to populate or + // update the reference repositories. Note that users may also + // want to run their own scripts to "populate" reference repos + // as symlinks to existing other repos, to support combined + // repo setup for different URLs pointing to same upstream, + // or storing multiple closely related forks together. + // This feature was developed along with a shell script to manage + // reference repositories, both in original combined-monolith layout, + // and in the subdirectory fanout compatible with plugin code below: + // https://github.com/jimklimov/git-scripts/blob/master/register-git-cache.sh + // Note: Initially we expect the reference to be a realistic dirname // with a special suffix to substitute after the logic below, so the // referencePath for that verbatim funny string should not exist now: @@ -902,8 +932,6 @@ public File findParameterizedReferenceRepository(String reference, String url) { String referenceExpanded = null; if (reference.endsWith("/${GIT_URL}")) { - // For mass-configured jobs, like Organization Folders, - // allow to support parameterized paths to many refrepos. // End-users can set up webs of symlinks to same repos // known by different URLs (and/or including their forks // also cached in same index). Luckily all URL chars are @@ -915,21 +943,6 @@ public File findParameterizedReferenceRepository(String reference, String url) { // portability? Support base64, SHA or MD5 hashes of URLs // as pathnames? Normalize first (lowercase, .git, ...)? - // TODO: Config option whether to populate absent reference - // repos (If the expanded path does not have git repo data - // right now, populate it in the location expanded below) - // or update existing ones before pulling commits, and how - // to achieve that. Note that this is done by caller with - // their implementation of git, or in case of a build farm it - // is more likely to be a shared path only readable to Jenkins - // and its agents, so write-operations would be done by helper - // scripts that log into the shared storage server to populate - // or update reference repositories. Note that users may also - // want to run their own scripts to "populate" reference repos - // as symlinks to existing other repos, to support combined - // repo setup for different URLs pointing to same upstream, - // or storing multiple closely related forks together. - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL\\}$", urlNormalized); } else if (reference.endsWith("/${GIT_URL_FALLBACK}")) { referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_FALLBACK\\}$", urlNormalized); From 82a43eb918d7f07c0edc1101bed2cc679909c6b9 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 31 Jan 2021 18:19:54 +0100 Subject: [PATCH 106/132] LegacyCompatibleGitAPIImpl.java: findParameterizedReferenceRepository(): drop original-experimental support for verbatim GIT_URL as the sub-pathname, due to its little portability --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 29 +------------------ 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 34d918f41b..f4452aa1b9 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -318,14 +318,6 @@ public static Boolean isParameterizedReferenceRepository(String reference) { return false; } - if (reference.endsWith("/${GIT_URL}")) { - return true; - } - - if (reference.endsWith("/${GIT_URL_FALLBACK}")) { - return true; - } - if (reference.endsWith("/${GIT_URL_SHA256}")) { return true; } @@ -931,26 +923,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { reference + "' into a specific (sub-)directory to use for URL '" + url + "' ..."); String referenceExpanded = null; - if (reference.endsWith("/${GIT_URL}")) { - // End-users can set up webs of symlinks to same repos - // known by different URLs (and/or including their forks - // also cached in same index). Luckily all URL chars are - // valid parts of path name... in Unix... Maybe parse or - // escape chars for URLs=>paths with Windows in mind? - // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions - // Further ideas: beside "GIT_URL" other meta variables - // can be introduced, e.g. to escape non-ascii chars for - // portability? Support base64, SHA or MD5 hashes of URLs - // as pathnames? Normalize first (lowercase, .git, ...)? - - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL\\}$", urlNormalized); - } else if (reference.endsWith("/${GIT_URL_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_FALLBACK\\}$", urlNormalized); - if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { - // chop it off, use main directory - referenceExpanded = referenceBaseDir; - } - } else if (reference.endsWith("/${GIT_URL_SHA256}")) { + if (reference.endsWith("/${GIT_URL_SHA256}")) { // This may be the more portable solution with regard to filesystems referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_SHA256\\}$", org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); From 23745a8719fbb95cfccab30c61b462e91b2e763f Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 31 Jan 2021 20:24:55 +0100 Subject: [PATCH 107/132] LegacyCompatibleGitAPIImpl.java: fix spotbugs complaints: listFiles() can return null --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f4452aa1b9..423ebc0223 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -645,11 +645,15 @@ public SimpleEntry> getSubmodulesUrls(String re // starts with some "prioritized" pathnames which we should not directly // inspect again... but should recurse into first anyhow. File[] directories = new File(referenceBaseDirAbs).listFiles(File::isDirectory); - for (File dir : directories) { - if (getObjectsFile(dir) != null) { - String dirname = dir.getPath().replaceAll("/*$", ""); - if (!checkedDirs.contains(dirname)) { - arrDirnames.add(dirname); + if (directories != null) { + // listFiles() "...returns null if this abstract pathname + // does not denote a directory, or if an I/O error occurs" + for (File dir : directories) { + if (getObjectsFile(dir) != null) { + String dirname = dir.getPath().replaceAll("/*$", ""); + if (!checkedDirs.contains(dirname)) { + arrDirnames.add(dirname); + } } } } From f51e60f4554236031270357020d1b5fd7c03e9f0 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 31 Jan 2021 20:32:12 +0100 Subject: [PATCH 108/132] LegacyCompatibleGitAPIImpl.java: fix spotbugs complaints: getSubmodulesUrls(): catch generic Throwable, not just Exception, while digging into submodules --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 423ebc0223..98fa23319b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -544,9 +544,9 @@ public SimpleEntry> getSubmodulesUrls(String re // which by construct looks at different dirs so far. result.add(new String[]{dirname, uri, uriNorm, remoteName}); } - } catch (Exception e) { + } catch (Throwable t) { // ignore, go to next slide - LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception (will go to next item):\n" + e.toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception or error (will go to next item):\n" + t.toString()); } } } @@ -704,9 +704,9 @@ public SimpleEntry> getSubmodulesUrls(String re // * name of the remote from that workspace's config ("origin" etc) result.add(new String[]{dirname, uri, uriNorm, remoteName}); } - } catch (Exception e) { + } catch (Throwable t) { // ignore, go to next slide - LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception (will go to next item):\n" + e.toString()); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception or error (will go to next item):\n" + t.toString()); } } From a5fdbd3783e19cbf87c2e18cae051fc975e3f429 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 31 Jan 2021 20:34:36 +0100 Subject: [PATCH 109/132] LegacyCompatibleGitAPIImpl.java: fix spotbugs complaints: have non-zero handling for caught IOExpection --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 98fa23319b..ed38cd9752 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -264,7 +264,9 @@ public static File getObjectsFile(File reference) { if (reader != null) { reader.close(); } - } catch (IOException e) {} + } catch (IOException e) { + LOGGER.log(Level.SEVERE, "getObjectsFile(): failed to close file after parsing '" + fGit.getAbsolutePath().toString() + "': " + e.toString()); + } } } else { LOGGER.log(Level.FINEST, "getObjectsFile(): did not find any checked-out '" + From 4341cd2831dfa1173c47953e86e5bed23a232206 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 31 Jan 2021 21:03:58 +0100 Subject: [PATCH 110/132] LegacyCompatibleGitAPIImpl.java: fix spotbugs complaints: "new File()" should never return null --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index ed38cd9752..0c20978b25 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -207,10 +207,12 @@ public static File getObjectsFile(File reference) { if (fGit.exists()) { if (fGit.isDirectory()) { - objects = new File(fGit, "objects"); - if (objects == null) { + objects = new File(fGit, "objects"); // this might not exist or not be a dir - checked below +/* + if (objects == null) { // spotbugs dislikes this, since "new File()" should not return null return objects; // Some Java error, could not make object from the paths involved } +*/ LOGGER.log(Level.FINEST, "getObjectsFile(): found an fGit '" + fGit.getAbsolutePath().toString() + "' which is a directory"); } else { @@ -277,9 +279,11 @@ public static File getObjectsFile(File reference) { // either reference path is bare repo (no ".git" inside), // or we have failed interpreting ".git" contents above objects = new File(reference, "objects"); // bare +/* if (objects == null) { return objects; // Some Java error, could not make object from the paths involved } +*/ // This clause below is redundant for production, but useful for troubleshooting if (objects.exists()) { if (objects.isDirectory()) { @@ -295,6 +299,9 @@ public static File getObjectsFile(File reference) { } } + if (!objects.exists()) + return null; + if (!objects.isDirectory()) return null; From 4cd45c82d688642443838ee2ac78732d0dcf245e Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Sun, 31 Jan 2021 21:10:21 +0100 Subject: [PATCH 111/132] LegacyCompatibleGitAPIImpl.java: fix spotbugs complaints: use a file reader with specified encoding --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 0c20978b25..eac141bd18 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -21,8 +21,9 @@ import java.io.BufferedReader; import java.io.File; -import java.io.FileReader; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStreamReader; import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; @@ -228,7 +229,7 @@ public static File getObjectsFile(File reference) { BufferedReader reader = null; try { String line; - reader = new BufferedReader(new FileReader(fGit)); + reader = new BufferedReader(new InputStreamReader(new FileInputStream(fGit), "UTF-8")); while ((line = reader.readLine()) != null) { String[] parts = line.split(":", 2); From fd293d78c71c67f4e377404c60287214010cf2bc Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 22 Feb 2021 16:17:49 +0100 Subject: [PATCH 112/132] LegacyCompatibleGitAPIImpl.java: normalizeUrl() failed with full paths (tried as relative) on Windows --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index eac141bd18..d4eb8895a1 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -369,7 +369,21 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { if (checkLocalPaths) { File urlPath = new File(url); if (urlPath.exists()) { - urlNormalized = "file://" + Paths.get( Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized ).normalize().toString();; + try { + // Check if the string in urlNormalized is a valid + // relative path (subdir) in current working directory + urlNormalized = "file://" + Paths.get( Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized ).normalize().toString(); + } catch (java.nio.file.InvalidPathException ipe1) { + // e.g. Illegal char <:> at index 30: C:\jenkins\git-client-plugin/c:\jenkins\git-client-plugin\target\clone + try { + // Re-check in another manner + urlNormalized = "file://" + Paths.get( Paths.get("", urlNormalized).toAbsolutePath().toString() ).normalize().toString(); + } catch (java.nio.file.InvalidPathException ipe2) { + // Finally, fall back to checking the originally + // fully-qualified path + urlNormalized = "file://" + Paths.get( Paths.get("/", urlNormalized).toAbsolutePath().toString() ).normalize().toString(); + } + } } else { // Also not a subdirectory of current dir without "./" prefix... urlNormalized = "ssh://" + urlNormalized; From c45cedd75c07a5e563be8d14816c51ff32d2ebc1 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 22 Feb 2021 16:56:41 +0100 Subject: [PATCH 113/132] LegacyCompatibleGitAPIImpl.java: normalizeUrl(): @SuppressFBWarnings for absolute "/" path in some lines --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index d4eb8895a1..d28fec590a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -359,6 +359,9 @@ public static Boolean isParameterizedReferenceRepository(String reference) { * using the same access protocol. This routine converts the "url" string * in a way that helps us confirm whether two spellings mean same thing. */ + @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", + justification = "Path operations below intentionally use absolute '/' in some cases" + ) public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); if (!url.contains("://")) { From be57864f0b937810ee055bc94868dd37f5c17fde Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 26 Jan 2022 21:58:18 +0100 Subject: [PATCH 114/132] CliGitAPIImpl.java: launchCommandIn(): report workDir when throwing exception --- .../org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index 34e97fb5f1..dd1ded53d7 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -2553,7 +2553,12 @@ private String launchCommandIn(ArgumentListBuilder args, File workDir, EnvVars e } if (status != 0) { - throw new GitException("Command \"" + command + "\" returned status code " + status + ":\nstdout: " + stdout + "\nstderr: "+ stderr); + if (workDir == null) + workDir = java.nio.file.Paths.get(".").toAbsolutePath().normalize().toFile(); + throw new GitException("Command \"" + command + + "\" executed in workdir \"" + workDir.toString() + + "\" returned status code " + status + + ":\nstdout: " + stdout + "\nstderr: " + stderr); } return stdout; From 8f843d452bbcfb87350a1b26f23e6e676c48a5ba Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 26 Jan 2022 23:28:53 +0100 Subject: [PATCH 115/132] LegacyCompatibleGitAPIImpl.java: getSubmodulesUrls(): do not fail checkout due to not-initialized referenceBaseDir (no git repo there) --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index d28fec590a..453346d12d 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -479,8 +479,25 @@ public SimpleEntry> getSubmodulesUrls(String re Boolean isBare = false; try { isBare = ((hudson.plugins.git.IGitAPI)referenceGit).isBareRepository(); - } catch (InterruptedException e) { - isBare = false; // At least try to look into submodules... + } catch (InterruptedException | GitException e) { + // Proposed base directory whose subdirs contain refrepos might + // itself be not a repo. Shouldn't be per reference scripts, but... + if (e.toString().contains("GIT_DISCOVERY_ACROSS_FILESYSTEM")) { + // Note the message may be localized, envvar name should not be: + // stderr: fatal: not a git repository (or any parent up to mount point /some/path) + // Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set). + // As far as the logic below is currently concerned, we do not + // look for submodules directly in a bare repo. + isBare = true; + } else { + // Some other error + isBare = false; // At least try to look into submodules... + isBare = false; + } + + LOGGER.log(Level.SEVERE, "getSubmodulesUrls(): failed to determine " + + "isBareRepository() in '" + referenceBaseDirAbs + "'; " + + "assuming '" + isBare + "': " + e.toString()); } // Simplify checks below by stating a useless needle is null From 4ee6935f48f6d1ad98fc5c4063e009238de104d6 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 27 Jan 2022 00:06:34 +0100 Subject: [PATCH 116/132] JGitAPIImpl.java + CliGitAPIImpl.java: report "reference" pathname that may take a few git queries to resolve --- .../java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java | 5 +++-- .../java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index dd1ded53d7..3f59bffa6d 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -769,8 +769,9 @@ public void execute() throws GitException, InterruptedException { if (reference != null && !reference.isEmpty()) { if (isParameterizedReferenceRepository(reference)) { // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via listener - listener.getLogger().println("[INFO] The git reference repository path is parameterized, " + - "it may take a few git queries logged below to resolve it into a particular directory name"); + listener.getLogger().println("[INFO] The git reference repository path '" + + reference + "' is parameterized, it may take a few git queries logged " + + "below to resolve it into a particular directory name"); } File referencePath = findParameterizedReferenceRepository(reference, url); if (referencePath == null) { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index c407f7b1ec..63707260b2 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1458,8 +1458,9 @@ public void execute() throws GitException, InterruptedException { // Note: keep in sync with similar logic in CliGitAPIImpl.java if (isParameterizedReferenceRepository(reference)) { // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via listener - listener.getLogger().println("[INFO] The git reference repository path is parameterized, " + - "it may take a few git queries logged below to resolve it into a particular directory name"); + listener.getLogger().println("[INFO] The git reference repository path '" + + reference + "'is parameterized, it may take a few git queries logged " + + "below to resolve it into a particular directory name"); } File referencePath = findParameterizedReferenceRepository(reference, url); if (referencePath == null) { From 558d07600423882c450277a28501c010396be416 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 1 Aug 2022 11:52:28 +0200 Subject: [PATCH 117/132] [JENKINS-69193] GitAPITestCase: test_submodule_update_with_error(): rely less on strict wording of messages from git CLI tool --- .../jenkinsci/plugins/gitclient/GitAPITestCase.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitAPITestCase.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitAPITestCase.java index 1d0716d789..bee3f8fc80 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitAPITestCase.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitAPITestCase.java @@ -1225,7 +1225,16 @@ public void test_submodule_update_with_error() throws Exception { w.git.submoduleUpdate().execute(); fail("Did not throw expected submodule update exception"); } catch (GitException e) { - assertThat(e.getMessage(), containsString("Command \"git submodule update modules/ntp\" returned status code 1")); + /* Depending on git program implementation/version, the string can be either short: + * Command "git submodule update modules/ntp" returned status code 1" + * or detailed: + * Command "git submodule update modules/ntp" executed in workdir "C:\Users\..." returned status code 1 + * so we catch below the two common parts separately. + * NOTE: git codebase itself goes to great extents to forbid their + * own unit-testing code from relying on emitted text messages. + */ + assertThat(e.getMessage(), containsString("Command \"git submodule update modules/ntp\" ")); + assertThat(e.getMessage(), containsString(" returned status code 1")); } } From 4402820b8029493267006e3c34176f66d0e58b68 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 10 Aug 2022 10:40:11 +0200 Subject: [PATCH 118/132] [JENKINS-69193] GitAPITestUpdateCliGit: rely less on strict wording of messages from git CLI tool (after PR #824 mass renaming) --- .../plugins/gitclient/GitAPITestUpdateCliGit.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitAPITestUpdateCliGit.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitAPITestUpdateCliGit.java index 502257c8b2..335341b202 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitAPITestUpdateCliGit.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitAPITestUpdateCliGit.java @@ -48,7 +48,16 @@ public void testSubmoduleUpdateWithError() throws Exception { w.git.submoduleUpdate().execute(); fail("Did not throw expected submodule update exception"); } catch (GitException e) { - assertThat(e.getMessage(), containsString("Command \"git submodule update modules/ntp\" returned status code 1")); + /* Depending on git program implementation/version, the string can be either short: + * Command "git submodule update modules/ntp" returned status code 1" + * or detailed: + * Command "git submodule update modules/ntp" executed in workdir "C:\Users\..." returned status code 1 + * so we catch below the two common parts separately. + * NOTE: git codebase itself goes to great extents to forbid their + * own unit-testing code from relying on emitted text messages. + */ + assertThat(e.getMessage(), containsString("Command \"git submodule update modules/ntp\" ")); + assertThat(e.getMessage(), containsString(" returned status code 1")); } } From c8820502a797ad28934708bbb5db66a6e99e6f34 Mon Sep 17 00:00:00 2001 From: Mark Waite Date: Tue, 11 Apr 2023 12:18:54 -0600 Subject: [PATCH 119/132] Format source changes --- src/main/java/hudson/plugins/git/GitAPI.java | 2 +- .../plugins/gitclient/CliGitAPIImpl.java | 56 +- .../plugins/gitclient/GitClient.java | 1 - .../plugins/gitclient/JGitAPIImpl.java | 26 +- .../gitclient/LegacyCompatibleGitAPIImpl.java | 512 +++++++++++------- .../plugins/gitclient/GitClientCloneTest.java | 173 +++--- .../plugins/gitclient/WorkspaceWithRepo.java | 3 +- 7 files changed, 462 insertions(+), 311 deletions(-) diff --git a/src/main/java/hudson/plugins/git/GitAPI.java b/src/main/java/hudson/plugins/git/GitAPI.java index adb36d73e0..506f0a6dbe 100644 --- a/src/main/java/hudson/plugins/git/GitAPI.java +++ b/src/main/java/hudson/plugins/git/GitAPI.java @@ -175,7 +175,7 @@ public GitClient subGit(String subdir) { /** {@inheritDoc} */ public GitClient newGit(String somedir) { - return Git.USE_CLI ? super.newGit(somedir) : jgit.newGit(somedir); + return Git.USE_CLI ? super.newGit(somedir) : jgit.newGit(somedir); } /** {@inheritDoc} */ diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index 9929eb21e5..12c518334d 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -836,13 +836,16 @@ public void execute() throws GitException, InterruptedException { if (reference != null && !reference.isEmpty()) { if (isParameterizedReferenceRepository(reference)) { // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via listener - listener.getLogger().println("[INFO] The git reference repository path '" + - reference + "' is parameterized, it may take a few git queries logged " + - "below to resolve it into a particular directory name"); + listener.getLogger() + .println("[INFO] The git reference repository path '" + reference + + "' is parameterized, it may take a few git queries logged " + + "below to resolve it into a particular directory name"); } File referencePath = findParameterizedReferenceRepository(reference, url); if (referencePath == null) { - listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + reference); + listener.getLogger() + .println("[ERROR] Could not make File object from reference path, skipping its use: " + + reference); } else { if (!referencePath.getPath().equals(reference)) { // Note: both these logs are needed, they are used in selftest @@ -865,12 +868,17 @@ else if (!referencePath.isDirectory()) else { File objectsPath = getObjectsFile(referencePath); if (objectsPath == null || !objectsPath.isDirectory()) - listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (not a git repo?): " + objectsPath); + listener.getLogger() + .println( + "[WARNING] Reference path does not contain an objects directory (not a git repo?): " + + objectsPath); else { // Go behind git's back to write a meta file in new workspace File alternates = new File(workspace, ".git/objects/info/alternates"); - try (PrintWriter w = new PrintWriter(alternates, Charset.defaultCharset().toString())) { - String absoluteReference = objectsPath.getAbsolutePath().replace('\\', '/'); + try (PrintWriter w = new PrintWriter( + alternates, Charset.defaultCharset().toString())) { + String absoluteReference = + objectsPath.getAbsolutePath().replace('\\', '/'); listener.getLogger().println("Using reference repository: " + reference); // git implementations on windows also use w.print(absoluteReference); @@ -1514,8 +1522,7 @@ public void execute() throws GitException, InterruptedException { listener.getLogger().println("[WARNING] Reference path does not exist: " + ref); else if (!referencePath.isDirectory()) listener.getLogger().println("[WARNING] Reference path is not a directory: " + ref); - else - args.add("--reference", ref); + else args.add("--reference", ref); } // else handled below in per-module loop } if (shallow) { @@ -1577,7 +1584,10 @@ else if (!referencePath.isDirectory()) if (isParameterizedReferenceRepository(ref)) { File referencePath = findParameterizedReferenceRepository(ref, strURIish); if (referencePath == null) { - listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + ref); + listener.getLogger() + .println( + "[ERROR] Could not make File object from reference path, skipping its use: " + + ref); } else { String expRef = null; if (referencePath.getPath().equals(ref)) { @@ -1590,8 +1600,7 @@ else if (!referencePath.isDirectory()) listener.getLogger().println("[WARNING] Reference path does not exist: " + expRef); else if (!referencePath.isDirectory()) listener.getLogger().println("[WARNING] Reference path is not a directory: " + expRef); - else - args.add("--reference", referencePath.getPath()); + else args.add("--reference", referencePath.getPath()); } } @@ -1709,7 +1718,7 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr /** {@inheritDoc} */ @Override public @CheckForNull Map getRemoteUrls() throws GitException, InterruptedException { - String result = launchCommand( "config", "--local", "--list" ); + String result = launchCommand("config", "--local", "--list"); Map uriNames = new HashMap<>(); for (String line : result.split("\\R+")) { line = StringUtils.trim(line); @@ -1729,7 +1738,8 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr URI uSafe = new URI(u.getScheme(), u.getHost(), u.getPath(), u.getFragment()); uriNames.put(uSafe.toString(), remoteName); uriNames.put(uSafe.toASCIIString(), remoteName); - } catch (URISyntaxException ue) {} // ignore, move along + } catch (URISyntaxException ue) { + } // ignore, move along } return uriNames; } @@ -1737,7 +1747,7 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr /** {@inheritDoc} */ @Override public @CheckForNull Map getRemotePushUrls() throws GitException, InterruptedException { - String result = launchCommand( "config", "--local", "--list" ); + String result = launchCommand("config", "--local", "--list"); Map uriNames = new HashMap<>(); for (String line : result.split("\\R+")) { line = StringUtils.trim(line); @@ -1755,7 +1765,8 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr URI uSafe = new URI(u.getScheme(), u.getHost(), u.getPath(), u.getFragment()); uriNames.put(uSafe.toString(), remoteName); uriNames.put(uSafe.toASCIIString(), remoteName); - } catch (URISyntaxException ue) {} // ignore, move along + } catch (URISyntaxException ue) { + } // ignore, move along } return uriNames; } @@ -2944,11 +2955,14 @@ private String launchCommandIn(ArgumentListBuilder args, File workDir, EnvVars e if (status != 0) { if (workDir == null) - workDir = java.nio.file.Paths.get(".").toAbsolutePath().normalize().toFile(); - throw new GitException("Command \"" + command + - "\" executed in workdir \"" + workDir.toString() + - "\" returned status code " + status + - ":\nstdout: " + stdout + "\nstderr: " + stderr); + workDir = java.nio.file.Paths.get(".") + .toAbsolutePath() + .normalize() + .toFile(); + throw new GitException("Command \"" + command + "\" executed in workdir \"" + + workDir.toString() + "\" returned status code " + + status + ":\nstdout: " + + stdout + "\nstderr: " + stderr); } return stdout; diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java index 49342863cb..5111804ff9 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java @@ -719,7 +719,6 @@ Map getRemoteSymbolicReferences(String remoteRepoUrl, String pat */ List revList(String ref) throws GitException, InterruptedException; - // --- new instance of same applied class /** diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 14750bab2c..978f7473b2 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1555,14 +1555,19 @@ public void execute() throws GitException { if (reference != null && !reference.isEmpty()) { // Note: keep in sync with similar logic in CliGitAPIImpl.java if (isParameterizedReferenceRepository(reference)) { - // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via listener - listener.getLogger().println("[INFO] The git reference repository path '" + - reference + "'is parameterized, it may take a few git queries logged " + - "below to resolve it into a particular directory name"); + // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via + // listener + listener.getLogger() + .println("[INFO] The git reference repository path '" + reference + + "'is parameterized, it may take a few git queries logged " + + "below to resolve it into a particular directory name"); } File referencePath = findParameterizedReferenceRepository(reference, url); if (referencePath == null) { - listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + reference); + listener.getLogger() + .println( + "[ERROR] Could not make File object from reference path, skipping its use: " + + reference); } else { if (!referencePath.getPath().equals(reference)) { // Note: both these logs are needed, they are used in selftest @@ -1581,16 +1586,21 @@ public void execute() throws GitException { if (!referencePath.exists()) listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); else if (!referencePath.isDirectory()) - listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); + listener.getLogger() + .println("[WARNING] Reference path is not a directory: " + reference); else { File objectsPath = getObjectsFile(referencePath); if (objectsPath == null || !objectsPath.isDirectory()) - listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (no git repo?): " + objectsPath); + listener.getLogger() + .println( + "[WARNING] Reference path does not contain an objects directory (no git repo?): " + + objectsPath); else { // Go behind git's back to write a meta file in new workspace try { File alternates = new File(workspace, ".git/objects/info/alternates"); - String absoluteReference = objectsPath.getAbsolutePath().replace('\\', '/'); + String absoluteReference = + objectsPath.getAbsolutePath().replace('\\', '/'); listener.getLogger().println("Using reference repository: " + reference); // git implementations on windows also use try (PrintWriter w = new PrintWriter(alternates, "UTF-8")) { diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 2fb4a100e6..02eeffb3de 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -1,9 +1,8 @@ package org.jenkinsci.plugins.gitclient; import static java.util.Arrays.copyOfRange; -import org.apache.commons.codec.digest.DigestUtils; import static org.apache.commons.lang.StringUtils.join; -import hudson.FilePath; + import hudson.model.TaskListener; import hudson.plugins.git.GitException; import hudson.plugins.git.IGitAPI; @@ -11,32 +10,26 @@ import hudson.plugins.git.Revision; import hudson.plugins.git.Tag; import hudson.remoting.Channel; - -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.transport.RefSpec; -import org.eclipse.jgit.transport.RemoteConfig; -import org.eclipse.jgit.transport.URIish; - import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.net.URISyntaxException; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; -import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; - +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.transport.RefSpec; +import org.eclipse.jgit.transport.RemoteConfig; +import org.eclipse.jgit.transport.URIish; import org.jenkinsci.plugins.gitclient.verifier.HostKeyVerifierFactory; /** @@ -221,11 +214,9 @@ public static File getObjectsFile(File reference) { return reference; } - if (!reference.exists()) - return null; + if (!reference.exists()) return null; - if (!reference.isDirectory()) - return null; + if (!reference.isDirectory()) return null; File fGit = new File(reference, ".git"); // workspace - file, dir or symlink to those File objects = null; @@ -233,13 +224,15 @@ public static File getObjectsFile(File reference) { if (fGit.exists()) { if (fGit.isDirectory()) { objects = new File(fGit, "objects"); // this might not exist or not be a dir - checked below -/* - if (objects == null) { // spotbugs dislikes this, since "new File()" should not return null - return objects; // Some Java error, could not make object from the paths involved - } -*/ - LOGGER.log(Level.FINEST, "getObjectsFile(): found an fGit '" + - fGit.getAbsolutePath().toString() + "' which is a directory"); + /* + if (objects == null) { // spotbugs dislikes this, since "new File()" should not return null + return objects; // Some Java error, could not make object from the paths involved + } + */ + LOGGER.log( + Level.FINEST, + "getObjectsFile(): found an fGit '" + + fGit.getAbsolutePath().toString() + "' which is a directory"); } else { // If ".git" FS object exists and is a not-empty file (and // is not a dir), then its content may point to some other @@ -248,24 +241,27 @@ public static File getObjectsFile(File reference) { // to the index and other metadata stored in its "parent" // repository's directory: // "gitdir: ../.git/modules/childRepoName" - LOGGER.log(Level.FINEST, "getObjectsFile(): found an fGit '" + - fGit.getAbsolutePath().toString() + "' which is NOT a directory"); + LOGGER.log( + Level.FINEST, + "getObjectsFile(): found an fGit '" + + fGit.getAbsolutePath().toString() + "' which is NOT a directory"); BufferedReader reader = null; try { String line; reader = new BufferedReader(new InputStreamReader(new FileInputStream(fGit), "UTF-8")); - while ((line = reader.readLine()) != null) - { + while ((line = reader.readLine()) != null) { String[] parts = line.split(":", 2); - if (parts.length >= 2) - { + if (parts.length >= 2) { String key = parts[0].trim(); String value = parts[1].trim(); if (key.equals("gitdir")) { objects = new File(reference, value); - LOGGER.log(Level.FINE, "getObjectsFile(): while looking for 'gitdir:' in '" + - fGit.getAbsolutePath().toString() + "', found reference to objects " + - "which should be at: '" + objects.getAbsolutePath().toString() + "'"); + LOGGER.log( + Level.FINE, + "getObjectsFile(): while looking for 'gitdir:' in '" + + fGit.getAbsolutePath().toString() + + "', found reference to objects " + "which should be at: '" + + objects.getAbsolutePath().toString() + "'"); // Note: we don't use getCanonicalPath() here to avoid further filesystem // access and possible exceptions (the getAbsolutePath() is about string // processing), but callers might benefit from canonicising and ensuring @@ -276,15 +272,23 @@ public static File getObjectsFile(File reference) { // prefixes for the contexts the callers would define for themselves. break; } - LOGGER.log(Level.FINEST, "getObjectsFile(): while looking for 'gitdir:' in '" + - fGit.getAbsolutePath().toString() + "', ignoring line: " + line); + LOGGER.log( + Level.FINEST, + "getObjectsFile(): while looking for 'gitdir:' in '" + + fGit.getAbsolutePath().toString() + "', ignoring line: " + line); } } if (objects == null) { - LOGGER.log(Level.WARNING, "getObjectsFile(): failed to parse '" + fGit.getAbsolutePath().toString() + "': did not contain a 'gitdir:' entry"); + LOGGER.log( + Level.WARNING, + "getObjectsFile(): failed to parse '" + + fGit.getAbsolutePath().toString() + "': did not contain a 'gitdir:' entry"); } } catch (IOException e) { - LOGGER.log(Level.SEVERE, "getObjectsFile(): failed to parse '" + fGit.getAbsolutePath().toString() + "': " + e.toString()); + LOGGER.log( + Level.SEVERE, + "getObjectsFile(): failed to parse '" + + fGit.getAbsolutePath().toString() + "': " + e.toString()); objects = null; } try { @@ -292,43 +296,52 @@ public static File getObjectsFile(File reference) { reader.close(); } } catch (IOException e) { - LOGGER.log(Level.SEVERE, "getObjectsFile(): failed to close file after parsing '" + fGit.getAbsolutePath().toString() + "': " + e.toString()); + LOGGER.log( + Level.SEVERE, + "getObjectsFile(): failed to close file after parsing '" + + fGit.getAbsolutePath().toString() + "': " + e.toString()); } } } else { - LOGGER.log(Level.FINEST, "getObjectsFile(): did not find any checked-out '" + - fGit.getAbsolutePath().toString() + "'"); + LOGGER.log( + Level.FINEST, + "getObjectsFile(): did not find any checked-out '" + + fGit.getAbsolutePath().toString() + "'"); } if (objects == null || !objects.isDirectory()) { // either reference path is bare repo (no ".git" inside), // or we have failed interpreting ".git" contents above objects = new File(reference, "objects"); // bare -/* - if (objects == null) { - return objects; // Some Java error, could not make object from the paths involved - } -*/ + /* + if (objects == null) { + return objects; // Some Java error, could not make object from the paths involved + } + */ // This clause below is redundant for production, but useful for troubleshooting if (objects.exists()) { if (objects.isDirectory()) { - LOGGER.log(Level.FINEST, "getObjectsFile(): found a bare '" + - objects.getAbsolutePath().toString() + "' which is a directory"); + LOGGER.log( + Level.FINEST, + "getObjectsFile(): found a bare '" + + objects.getAbsolutePath().toString() + "' which is a directory"); } else { - LOGGER.log(Level.FINEST, "getObjectsFile(): found a bare '" + - objects.getAbsolutePath().toString() + "' which is NOT a directory"); + LOGGER.log( + Level.FINEST, + "getObjectsFile(): found a bare '" + + objects.getAbsolutePath().toString() + "' which is NOT a directory"); } } else { - LOGGER.log(Level.FINEST, "getObjectsFile(): did not find any bare '" + - objects.getAbsolutePath().toString() + "'"); + LOGGER.log( + Level.FINEST, + "getObjectsFile(): did not find any bare '" + + objects.getAbsolutePath().toString() + "'"); } } - if (!objects.exists()) - return null; + if (!objects.exists()) return null; - if (!objects.isDirectory()) - return null; + if (!objects.isDirectory()) return null; // If we get here, we have a non-null File referencing a // "(.git/)objects" subdir under original referencePath @@ -383,15 +396,13 @@ public static Boolean isParameterizedReferenceRepository(String reference) { * using the same access protocol. This routine converts the "url" string * in a way that helps us confirm whether two spellings mean same thing. */ - @SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME", - justification = "Path operations below intentionally use absolute '/' in some cases" - ) + @SuppressFBWarnings( + value = "DMI_HARDCODED_ABSOLUTE_FILENAME", + justification = "Path operations below intentionally use absolute '/' in some cases") public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); if (!url.contains("://")) { - if (!url.startsWith("/") && - !url.startsWith(".") - ) { + if (!url.startsWith("/") && !url.startsWith(".")) { // Not an URL with schema, not an absolute or relative pathname if (checkLocalPaths) { File urlPath = new File(url); @@ -399,16 +410,30 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { try { // Check if the string in urlNormalized is a valid // relative path (subdir) in current working directory - urlNormalized = "file://" + Paths.get( Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized ).normalize().toString(); + urlNormalized = "file://" + + Paths.get(Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized) + .normalize() + .toString(); } catch (java.nio.file.InvalidPathException ipe1) { - // e.g. Illegal char <:> at index 30: C:\jenkins\git-client-plugin/c:\jenkins\git-client-plugin\target\clone + // e.g. Illegal char <:> at index 30: + // C:\jenkins\git-client-plugin/c:\jenkins\git-client-plugin\target\clone try { // Re-check in another manner - urlNormalized = "file://" + Paths.get( Paths.get("", urlNormalized).toAbsolutePath().toString() ).normalize().toString(); + urlNormalized = "file://" + + Paths.get(Paths.get("", urlNormalized) + .toAbsolutePath() + .toString()) + .normalize() + .toString(); } catch (java.nio.file.InvalidPathException ipe2) { // Finally, fall back to checking the originally // fully-qualified path - urlNormalized = "file://" + Paths.get( Paths.get("/", urlNormalized).toAbsolutePath().toString() ).normalize().toString(); + urlNormalized = "file://" + + Paths.get(Paths.get("/", urlNormalized) + .toAbsolutePath() + .toString()) + .normalize() + .toString(); } } } else { @@ -424,12 +449,17 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { if (url.startsWith("/")) { urlNormalized = "file://" + urlNormalized; } else { - urlNormalized = "file://" + Paths.get( Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized ).normalize().toString();; + urlNormalized = "file://" + + Paths.get(Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized) + .normalize() + .toString(); + ; } } } - LOGGER.log(Level.FINEST, "normalizeGitUrl('" + url + "', " + checkLocalPaths.toString() + ") => " + urlNormalized); + LOGGER.log( + Level.FINEST, "normalizeGitUrl('" + url + "', " + checkLocalPaths.toString() + ") => " + urlNormalized); return urlNormalized; } @@ -473,7 +503,8 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { * when recursing, since this directory was checked already * as part of parent directory inspection. */ - public SimpleEntry> getSubmodulesUrls(String referenceBaseDir, String needle, Boolean checkRemotesInReferenceBaseDir) { + public SimpleEntry> getSubmodulesUrls( + String referenceBaseDir, String needle, Boolean checkRemotesInReferenceBaseDir) { // Keep track of where we've already looked in the "result" Set, to // avoid looking in same places (different strategies below) twice. // And eventually return this Set or part of it as the answer. @@ -486,13 +517,18 @@ public SimpleEntry> getSubmodulesUrls(String re // We want to hit same dirs only once, so canonicize paths below String referenceBaseDirAbs; try { - referenceBaseDirAbs = new File(referenceBaseDir).getAbsoluteFile().getCanonicalPath().toString(); + referenceBaseDirAbs = new File(referenceBaseDir) + .getAbsoluteFile() + .getCanonicalPath() + .toString(); } catch (IOException e) { // Access error while dereferencing some parent?.. referenceBaseDirAbs = new File(referenceBaseDir).getAbsoluteFile().toString(); - LOGGER.log(Level.SEVERE, "getSubmodulesUrls(): failed to canonicize '" + - referenceBaseDir + "' => '" + referenceBaseDirAbs + "': " + e.toString()); - //return new SimpleEntry<>(false, result); + LOGGER.log( + Level.SEVERE, + "getSubmodulesUrls(): failed to canonicize '" + referenceBaseDir + "' => '" + referenceBaseDirAbs + + "': " + e.toString()); + // return new SimpleEntry<>(false, result); } // "this" during a checkout typically represents the job workspace, @@ -502,7 +538,7 @@ public SimpleEntry> getSubmodulesUrls(String re Boolean isBare = false; try { - isBare = ((hudson.plugins.git.IGitAPI)referenceGit).isBareRepository(); + isBare = ((hudson.plugins.git.IGitAPI) referenceGit).isBareRepository(); } catch (InterruptedException | GitException e) { // Proposed base directory whose subdirs contain refrepos might // itself be not a repo. Shouldn't be per reference scripts, but... @@ -519,9 +555,11 @@ public SimpleEntry> getSubmodulesUrls(String re isBare = false; } - LOGGER.log(Level.SEVERE, "getSubmodulesUrls(): failed to determine " + - "isBareRepository() in '" + referenceBaseDirAbs + "'; " + - "assuming '" + isBare + "': " + e.toString()); + LOGGER.log( + Level.SEVERE, + "getSubmodulesUrls(): failed to determine " + "isBareRepository() in '" + + referenceBaseDirAbs + "'; " + "assuming '" + + isBare + "': " + e.toString()); } // Simplify checks below by stating a useless needle is null @@ -585,36 +623,57 @@ public SimpleEntry> getSubmodulesUrls(String re arrDirnames.add(referenceBaseDirAbs + "/" + needleSha); arrDirnames.add(referenceBaseDirAbs + "/" + needleSha + ".git"); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at basename-like subdirs under base refrepo '" + referenceBaseDirAbs + "', per arrDirnames: " + arrDirnames.toString()); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): looking at basename-like subdirs under base refrepo '" + referenceBaseDirAbs + + "', per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { f = new File(dirname); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir at abs pathname '" + dirname + "' if it exists"); + LOGGER.log( + Level.FINEST, + "getSubmodulesUrls(): probing dir at abs pathname '" + dirname + "' if it exists"); if (getObjectsFile(f) != null) { try { - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in existing refrepo subdir '" + dirname + "'"); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): looking for submodule URL needle='" + needle + + "' in existing refrepo subdir '" + dirname + "'"); GitClient g = referenceGit.subGit(dirname); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); - Map uriNames = g.getRemoteUrls(); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + uriNames.toString()); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): checking git workspace in dir '" + + g.getWorkTree().absolutize().toString() + "'"); + Map uriNames = g.getRemoteUrls(); + LOGGER.log( + Level.FINEST, + "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + + uriNames.toString()); for (Map.Entry pair : uriNames.entrySet()) { String remoteName = pair.getValue(); // whatever the git workspace config calls it String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "') vs needle"); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + + "') vs needle"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); - result.add(new String[]{dirname, uri, uriNorm, remoteName}); + result.add(new String[] {dirname, uri, uriNorm, remoteName}); return new SimpleEntry<>(true, result); } // Cache the finding to avoid the dirname later, if we // get to that; but no checks are needed in this loop // which by construct looks at different dirs so far. - result.add(new String[]{dirname, uri, uriNorm, remoteName}); + result.add(new String[] {dirname, uri, uriNorm, remoteName}); } } catch (Throwable t) { // ignore, go to next slide - LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception or error (will go to next item):\n" + t.toString()); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): probing dir '" + dirname + + "' resulted in an exception or error (will go to next item):\n" + + t.toString()); } } } @@ -643,65 +702,65 @@ public SimpleEntry> getSubmodulesUrls(String re checkedDirs.add(resultEntry[0]); } -/* -// TBD: Needs a way to list submodules in given workspace and convert -// that into (relative) subdirs, possibly buried some levels deep, for -// cases where the submodule is defined in parent with the needle URL. -// Maybe merge with current "if isBare" below, to optionally seed -// same arrDirnames with different values and check remotes listed -// in those repos. - // If current repo *is NOT* bare - check its submodules - // (the .gitmodules => submodule.MODNAME.{url,path} mapping) - // but this essentially does not look into any subdirectory. - // But we can add at higher priority submodule path(s) whose - // basename of the URL matches the needleBasename. And then - // other submodule paths to inspect before arbitrary subdirs. - if (!isBare) { - try { - // For each current workspace (recurse or big loop in same context?): - // public GitClient subGit(String subdir) => would this.subGit(...) - // give us a copy of this applied class instance (CLI Git vs jGit)? - // get submodule name-vs-one-url from .gitmodules if present, for a - // faster possible answer (only bother if needle is not null?) - // try { getSubmodules("HEAD") ... } => List filtered for - // "commit" items - // * if we are recursed into a "leaf" project and inspect ITS - // submodules, look at all git tips or even commits, to find - // and inspect all unique (by hash) .gitmodule objects, since - // over time or in different branches a "leaf" project could - // reference different subs? - // getRemoteUrls() => Map -// arrDirnames.clear(); - - // TODO: Check subdirs that are git workspaces, and remove "|| true" above -// LinkedHashSet checkedDirs = new LinkedHashSet<>(); -// for (String[] resultEntry : result) { -// checkedDirs.add(resultEntry[0]); -// } - - - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in submodules of refrepo, if any"); - Map uriNames = referenceGit.getRemoteUrls(); - for (Map.Entry pair : uriNames.entrySet()) { - String uri = pair.getKey(); - String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); - if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new LinkedHashSet<>(); - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); - return result; + /* + // TBD: Needs a way to list submodules in given workspace and convert + // that into (relative) subdirs, possibly buried some levels deep, for + // cases where the submodule is defined in parent with the needle URL. + // Maybe merge with current "if isBare" below, to optionally seed + // same arrDirnames with different values and check remotes listed + // in those repos. + // If current repo *is NOT* bare - check its submodules + // (the .gitmodules => submodule.MODNAME.{url,path} mapping) + // but this essentially does not look into any subdirectory. + // But we can add at higher priority submodule path(s) whose + // basename of the URL matches the needleBasename. And then + // other submodule paths to inspect before arbitrary subdirs. + if (!isBare) { + try { + // For each current workspace (recurse or big loop in same context?): + // public GitClient subGit(String subdir) => would this.subGit(...) + // give us a copy of this applied class instance (CLI Git vs jGit)? + // get submodule name-vs-one-url from .gitmodules if present, for a + // faster possible answer (only bother if needle is not null?) + // try { getSubmodules("HEAD") ... } => List filtered for + // "commit" items + // * if we are recursed into a "leaf" project and inspect ITS + // submodules, look at all git tips or even commits, to find + // and inspect all unique (by hash) .gitmodule objects, since + // over time or in different branches a "leaf" project could + // reference different subs? + // getRemoteUrls() => Map + // arrDirnames.clear(); + + // TODO: Check subdirs that are git workspaces, and remove "|| true" above + // LinkedHashSet checkedDirs = new LinkedHashSet<>(); + // for (String[] resultEntry : result) { + // checkedDirs.add(resultEntry[0]); + // } + + + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in submodules of refrepo, if any"); + Map uriNames = referenceGit.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new LinkedHashSet<>(); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + return result; + } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + } + } catch (Exception e) { + // ignore, go to next slide } - // Cache the finding to avoid the dirname later, if we - // get to that; but no checks are needed in this loop - // which by construct looks at different dirs so far. - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); } - } catch (Exception e) { - // ignore, go to next slide - } - } -*/ + */ // If current repo *is* bare (can't have proper submodules), or if the // end-users just cloned or linked some more repos into this container, @@ -735,10 +794,12 @@ public SimpleEntry> getSubmodulesUrls(String re } } - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking at " + - ((isBare ? "" : "submodules first, then ")) + - "all subdirs that have a .git, under refrepo '" + referenceBaseDirAbs + - "' per absolute arrDirnames: " + arrDirnames.toString()); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): looking at " + ((isBare ? "" : "submodules first, then ")) + + "all subdirs that have a .git, under refrepo '" + + referenceBaseDirAbs + "' per absolute arrDirnames: " + + arrDirnames.toString()); for (String dirname : arrDirnames) { // Note that here dirnames deal in absolutes @@ -750,19 +811,34 @@ public SimpleEntry> getSubmodulesUrls(String re // but better be on the safe side :) if (!checkedDirs.contains(dirname)) { try { - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking " + ((needle == null) ? "" : "for submodule URL needle='" + needle + "' ") + "in existing refrepo dir '" + dirname + "'"); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): looking " + + ((needle == null) ? "" : "for submodule URL needle='" + needle + "' ") + + "in existing refrepo dir '" + dirname + "'"); GitClient g = this.newGit(dirname); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking git workspace in dir '" + g.getWorkTree().absolutize().toString() + "'"); - Map uriNames = g.getRemoteUrls(); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + uriNames.toString()); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): checking git workspace in dir '" + + g.getWorkTree().absolutize().toString() + "'"); + Map uriNames = g.getRemoteUrls(); + LOGGER.log( + Level.FINEST, + "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map uriNames: " + + uriNames.toString()); for (Map.Entry pair : uriNames.entrySet()) { String remoteName = pair.getValue(); // whatever the git workspace config calls it String uri = pair.getKey(); String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "') vs needle"); - if (needle != null && needleNorm != null && (needleNorm.equals(uriNorm) || needle.equals(uri)) ) { + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + + "') vs needle"); + if (needle != null + && needleNorm != null + && (needleNorm.equals(uriNorm) || needle.equals(uri))) { result = new LinkedHashSet<>(); - result.add(new String[]{dirname, uri, uriNorm, remoteName}); + result.add(new String[] {dirname, uri, uriNorm, remoteName}); return new SimpleEntry<>(true, result); } // Cache the finding to return eventually, for each remote: @@ -770,11 +846,15 @@ public SimpleEntry> getSubmodulesUrls(String re // * original remote URI from that workspace's config // * normalized remote URI // * name of the remote from that workspace's config ("origin" etc) - result.add(new String[]{dirname, uri, uriNorm, remoteName}); + result.add(new String[] {dirname, uri, uriNorm, remoteName}); } } catch (Throwable t) { // ignore, go to next slide - LOGGER.log(Level.FINE, "getSubmodulesUrls(): probing dir '" + dirname + "' resulted in an exception or error (will go to next item):\n" + t.toString()); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): probing dir '" + dirname + + "' resulted in an exception or error (will go to next item):\n" + + t.toString()); } } @@ -782,14 +862,20 @@ public SimpleEntry> getSubmodulesUrls(String re // subdir that is already a known git workspace, to // add its data to list and/or return a found needle. LOGGER.log(Level.FINE, "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); - SimpleEntry > subEntriesRet = getSubmodulesUrls(dirname, needle, false); + SimpleEntry> subEntriesRet = getSubmodulesUrls(dirname, needle, false); Boolean subEntriesExactMatched = subEntriesRet.getKey(); LinkedHashSet subEntries = subEntriesRet.getValue(); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): returned from recursing into dir '" + dirname + "' with " + subEntries.size() + " found mappings"); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): returned from recursing into dir '" + dirname + "' with " + + subEntries.size() + " found mappings"); if (!subEntries.isEmpty()) { if (needle != null && subEntriesExactMatched) { // We found nothing... until now! Bubble it up! - LOGGER.log(Level.FINE, "getSubmodulesUrls(): got an exact needle match from recursing into dir '" + dirname + "': " + subEntries.iterator().next()[0]); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): got an exact needle match from recursing into dir '" + dirname + + "': " + subEntries.iterator().next()[0]); return subEntriesRet; } // ...else collect results to inspect and/or propagate later @@ -811,13 +897,13 @@ public SimpleEntry> getSubmodulesUrls(String re // results looks like it is related to the needle. LinkedHashSet resultFiltered = new LinkedHashSet<>(); -/* - if (!checkRemotesInReferenceBaseDir) { - // Overload the flag's meaning to only parse results once, - // in the parent dir? - return new SimpleEntry<>(false, resultFiltered); - } -*/ + /* + if (!checkRemotesInReferenceBaseDir) { + // Overload the flag's meaning to only parse results once, + // in the parent dir? + return new SimpleEntry<>(false, resultFiltered); + } + */ // Separate lists by suggestion priority: // 1) URI basename similarity @@ -833,9 +919,9 @@ public SimpleEntry> getSubmodulesUrls(String re for (String[] subEntry : result) { // Iterating to filter suggestions in order of original // directory-walk prioritization under current reference - String dirName = subEntry[0]; - String uri = subEntry[1]; - String uriNorm = subEntry[2]; + String dirName = subEntry[0]; + String uri = subEntry[1]; + String uriNorm = subEntry[2]; String remoteName = subEntry[3]; Integer sep; String uriNormBasename; @@ -958,10 +1044,7 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Note: Initially we expect the reference to be a realistic dirname // with a special suffix to substitute after the logic below, so the // referencePath for that verbatim funny string should not exist now: - if (!referencePath.exists() && - isParameterizedReferenceRepository(reference) && - url != null && !url.isEmpty() - ) { + if (!referencePath.exists() && isParameterizedReferenceRepository(reference) && url != null && !url.isEmpty()) { // Drop the trailing keyword to know the root refrepo dirname String referenceBaseDir = reference.replaceAll("/\\$\\{GIT_[^\\}]*\\}$", ""); @@ -986,31 +1069,36 @@ public File findParameterizedReferenceRepository(String reference, String url) { String urlNormalized = normalizeGitUrl(url, true); // Note: currently unit-tests expect this markup on stderr: - System.err.println("reference='" + reference + "'\n" + - "url='" + url + "'\n" + - "urlNormalized='" + urlNormalized + "'\n"); + System.err.println("reference='" + reference + "'\n" + "url='" + + url + "'\n" + "urlNormalized='" + + urlNormalized + "'\n"); // Let users know why there are many "git config --list" lines in their build log: - LOGGER.log(Level.INFO, "Trying to resolve parameterized Git reference repository '" + - reference + "' into a specific (sub-)directory to use for URL '" + url + "' ..."); + LOGGER.log( + Level.INFO, + "Trying to resolve parameterized Git reference repository '" + reference + + "' into a specific (sub-)directory to use for URL '" + url + "' ..."); String referenceExpanded = null; if (reference.endsWith("/${GIT_URL_SHA256}")) { // This may be the more portable solution with regard to filesystems - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_SHA256\\}$", - org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_SHA256\\}$", + org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); } else if (reference.endsWith("/${GIT_URL_SHA256_FALLBACK}")) { // The safest option - fall back to parent directory if // the expanded one does not have git repo data right now: // it allows to gradually convert a big combined reference // repository into smaller chunks without breaking builds. - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_SHA256_FALLBACK\\}$", - org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_SHA256_FALLBACK\\}$", + org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } - } else if (reference.endsWith("/${GIT_URL_BASENAME}") || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { + } else if (reference.endsWith("/${GIT_URL_BASENAME}") + || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { // This may be the more portable solution with regard to filesystems // First try with original user-provided casing of the URL (if local // dirs were cloned manually) @@ -1024,18 +1112,20 @@ public File findParameterizedReferenceRepository(String reference, String url) { needleBasename = needleBasename.replaceAll(".git$", ""); if (reference.endsWith("/${GIT_URL_BASENAME}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", - needleBasename); + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", needleBasename); } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", - needleBasename); - if (url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", needleBasename); + if (url.equals(urlNormalized) + && getObjectsFile(referenceExpanded) == null + && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory (only if we do not check urlNormalized separately below) referenceExpanded = referenceBaseDir; } } - if (!url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + if (!url.equals(urlNormalized) + && getObjectsFile(referenceExpanded) == null + && getObjectsFile(referenceExpanded + ".git") == null) { // Retry with automation-ready normalized URL sep = urlNormalized.lastIndexOf("/"); if (sep < 0) { @@ -1046,18 +1136,17 @@ public File findParameterizedReferenceRepository(String reference, String url) { needleBasename = needleBasename.replaceAll(".git$", ""); if (reference.endsWith("/${GIT_URL_BASENAME}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", - needleBasename); + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", needleBasename); } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", - needleBasename); - if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", needleBasename); + if (getObjectsFile(referenceExpanded) == null + && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } } } - } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") ) { + } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { // Here we employ git submodules - so we can reliably match // remote URLs (easily multiple) to particular modules, and // yet keep separate git index directories per module with @@ -1078,7 +1167,8 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Note: we pass the unmodified "url" value here, the routine // differentiates original spelling vs. normalization while // looking for its needle in the haystack. - SimpleEntry > subEntriesRet = getSubmodulesUrls(referenceBaseDir, url, true); + SimpleEntry> subEntriesRet = + getSubmodulesUrls(referenceBaseDir, url, true); Boolean subEntriesExactMatched = subEntriesRet.getKey(); LinkedHashSet subEntries = subEntriesRet.getValue(); if (!subEntries.isEmpty()) { @@ -1098,8 +1188,13 @@ public File findParameterizedReferenceRepository(String reference, String url) { if (referenceExpanded == null) { referenceExpanded = subEntries.iterator().next()[0]; } - LOGGER.log(Level.FINE, "findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + "' from subEntries"); - if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + LOGGER.log( + Level.FINE, + "findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + + "' from subEntries"); + if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") + && getObjectsFile(referenceExpanded) == null + && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } @@ -1119,13 +1214,12 @@ public File findParameterizedReferenceRepository(String reference, String url) { needleBasename = needleBasename.replaceAll(".git$", ""); if (reference.endsWith("/${GIT_SUBMODULES}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", - needleBasename); - } - else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", - needleBasename); - if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", needleBasename); + } else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", needleBasename); + if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") + && getObjectsFile(referenceExpanded) == null + && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } @@ -1142,8 +1236,10 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Note: currently unit-tests expect this markup on stderr: System.err.println("reference after='" + reference + "'\n"); - LOGGER.log(Level.INFO, "After resolving the parameterized Git reference repository, " + - "decided to use '" + reference + "' directory for URL '" + url + "'"); + LOGGER.log( + Level.INFO, + "After resolving the parameterized Git reference repository, " + "decided to use '" + reference + + "' directory for URL '" + url + "'"); } // if referencePath is the replaceable token and not existing directory if (!referencePath.exists() && !reference.endsWith(".git")) { diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 97fa5c6faa..f6814fade4 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -12,20 +12,6 @@ import hudson.plugins.git.GitException; import hudson.remoting.VirtualChannel; import hudson.util.StreamTaskListener; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; -import org.eclipse.jgit.lib.ConfigConstants; -import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.transport.RefSpec; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -38,6 +24,7 @@ import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; @@ -280,15 +267,21 @@ public void test_clone_reference() throws Exception { @Test public void test_clone_reference_parameterized_basename() throws Exception, IOException, InterruptedException { - testGitClient.clone_().url(workspace.localMirror()).repositoryName("origin").reference(workspace.localMirror() + "/${GIT_URL_BASENAME}").execute(); + testGitClient + .clone_() + .url(workspace.localMirror()) + .repositoryName("origin") + .reference(workspace.localMirror() + "/${GIT_URL_BASENAME}") + .execute(); testGitClient.checkout().ref("origin/master").branch("master").execute(); check_remote_url(workspace, testGitClient, "origin"); // Verify JENKINS-46737 expected log message is written String messages = StringUtils.join(handler.getMessages(), ";"); - assertThat("Reference repo name-parsing logged in: " + messages, - handler.containsMessageSubstring("Parameterized reference path ") && - handler.containsMessageSubstring(" replaced with: "), - is(true)); + assertThat( + "Reference repo name-parsing logged in: " + messages, + handler.containsMessageSubstring("Parameterized reference path ") + && handler.containsMessageSubstring(" replaced with: "), + is(true)); // TODO: Actually construct the local filesystem path to match // the last pathname component from the URL (plus/minus ".git" // extension). Be sure to clean away this path at end of test, @@ -296,31 +289,42 @@ public void test_clone_reference_parameterized_basename() throws Exception, IOEx // below is not confused - it expects this location to not exist. // Skip: Missing if clone failed - currently would, with bogus // path above and not yet pre-created path structure. - //assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); - //assertAlternateFilePointsToLocalMirror(); - //assertBranchesExist(testGitClient.getBranches(), "master"); - //assertNoObjectsInRepository(); + // assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference + // repository: "), is(true)); + // assertAlternateFilePointsToLocalMirror(); + // assertBranchesExist(testGitClient.getBranches(), "master"); + // assertNoObjectsInRepository(); } @Test - public void test_clone_reference_parameterized_basename_fallback() throws Exception, IOException, InterruptedException { + public void test_clone_reference_parameterized_basename_fallback() + throws Exception, IOException, InterruptedException { // TODO: Currently we do not make paths which would invalidate // this test, but note the test above might do just that later. - testGitClient.clone_().url(workspace.localMirror()).repositoryName("origin").reference(workspace.localMirror() + "/${GIT_URL_BASENAME_FALLBACK}").execute(); + testGitClient + .clone_() + .url(workspace.localMirror()) + .repositoryName("origin") + .reference(workspace.localMirror() + "/${GIT_URL_BASENAME_FALLBACK}") + .execute(); testGitClient.checkout().ref("origin/master").branch("master").execute(); check_remote_url(workspace, testGitClient, "origin"); // Verify JENKINS-46737 expected log message is written String messages = StringUtils.join(handler.getMessages(), ";"); - assertThat("Reference repo name-parsing logged in: " + messages, - handler.containsMessageSubstring("Parameterized reference path ") && - handler.containsMessageSubstring(" replaced with: '" + workspace.localMirror() + "'"), - is(true)); + assertThat( + "Reference repo name-parsing logged in: " + messages, + handler.containsMessageSubstring("Parameterized reference path ") + && handler.containsMessageSubstring(" replaced with: '" + workspace.localMirror() + "'"), + is(true)); // With fallback mode, and nonexistent parameterized reference // repository, and a usable repository in the common path (what // remains if the parameterizing suffix is just discarded), this // common path should be used. So it should overall behave same // as the non-parameterized test_clone_reference_basename() above. - assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); + assertThat( + "Reference repo logged in: " + messages, + handler.containsMessageSubstring("Using reference repository: "), + is(true)); assertAlternateFilePointsToLocalMirror(); assertBranchesExist(testGitClient.getBranches(), "master"); assertNoObjectsInRepository(); @@ -331,7 +335,7 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce String wsMirror = workspace.localMirror(); /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java * should be okay for network URLs but are too complex for local pathnames */ - //String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + // String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); String wsMirrorNormalized = LegacyCompatibleGitAPIImpl.normalizeGitUrl(wsMirror, true); String wsMirrorHash = org.apache.commons.codec.digest.DigestUtils.sha256Hex(wsMirrorNormalized); @@ -344,10 +348,10 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce String wsRefrepo = null; try { if (fRefrepoBase.exists() || fRefrepoBase.mkdirs()) { - /* Note: per parser of magic suffix, use slash - not OS separator char - * And be sure to use relative paths here (see - * WorkspaceWithRepo.java::localMirror()): - */ + /* Note: per parser of magic suffix, use slash - not OS separator char + * And be sure to use relative paths here (see + * WorkspaceWithRepo.java::localMirror()): + */ wsRefrepo = workspace.localMirror("refrepo256.git/" + wsMirrorHash); } } catch (RuntimeException e) { @@ -357,10 +361,14 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce // the provided string, as we check in log below } - System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + - "wsRefrepo='" + wsRefrepo); + System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + "wsRefrepo='" + wsRefrepo); - testGitClient.clone_().url(wsMirror).repositoryName("origin").reference(wsRefrepoBase + "/${GIT_URL_SHA256}").execute(); + testGitClient + .clone_() + .url(wsMirror) + .repositoryName("origin") + .reference(wsRefrepoBase + "/${GIT_URL_SHA256}") + .execute(); testGitClient.checkout().ref("origin/master").branch("master").execute(); check_remote_url(workspace, testGitClient, "origin"); @@ -370,27 +378,33 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce System.err.println("clone output:\n======\n" + messages + "\n======\n"); - assertThat("Reference repo name-parsing logged in: " + messages + - (wsRefrepo == null ? "" : ("\n...and replaced with: '" + wsRefrepo + "'")) , - handler.containsMessageSubstring("Parameterized reference path ") && - handler.containsMessageSubstring(" replaced with: ") && - (wsRefrepo == null || handler.containsMessageSubstring(wsRefrepo)) , - is(true)); + assertThat( + "Reference repo name-parsing logged in: " + messages + + (wsRefrepo == null ? "" : ("\n...and replaced with: '" + wsRefrepo + "'")), + handler.containsMessageSubstring("Parameterized reference path ") + && handler.containsMessageSubstring(" replaced with: ") + && (wsRefrepo == null || handler.containsMessageSubstring(wsRefrepo)), + is(true)); if (wsRefrepo != null) { - assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); + assertThat( + "Reference repo logged in: " + messages, + handler.containsMessageSubstring("Using reference repository: "), + is(true)); assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir.getPath(), wsRefrepo); assertBranchesExist(testGitClient.getBranches(), "master"); assertNoObjectsInRepository(); - } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path structure + } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path + // structure } @Test - public void test_clone_reference_parameterized_sha256_fallback() throws Exception, IOException, InterruptedException { + public void test_clone_reference_parameterized_sha256_fallback() + throws Exception, IOException, InterruptedException { String wsMirror = workspace.localMirror(); /* Same rules of URL normalization as in LegacyCompatibleGitAPIImpl.java * should be okay for network URLs but are too complex for local pathnames */ - //String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + // String wsMirrorNormalized = wsMirror.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); String wsMirrorNormalized = LegacyCompatibleGitAPIImpl.normalizeGitUrl(wsMirror, true); String wsMirrorHash = org.apache.commons.codec.digest.DigestUtils.sha256Hex(wsMirrorNormalized); @@ -403,10 +417,10 @@ public void test_clone_reference_parameterized_sha256_fallback() throws Exceptio String wsRefrepo = null; try { if (fRefrepoBase.exists() || fRefrepoBase.mkdirs()) { - /* Note: per parser of magic suffix, use slash - not OS separator char - * And be sure to use relative paths here (see - * WorkspaceWithRepo.java::localMirror()): - */ + /* Note: per parser of magic suffix, use slash - not OS separator char + * And be sure to use relative paths here (see + * WorkspaceWithRepo.java::localMirror()): + */ wsRefrepo = workspace.localMirror("refrepo256.git/" + wsMirrorHash); } } catch (RuntimeException e) { @@ -416,10 +430,14 @@ public void test_clone_reference_parameterized_sha256_fallback() throws Exceptio // the provided string, as we check in log below } - System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + - "wsRefrepo='" + wsRefrepo); + System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + "wsRefrepo='" + wsRefrepo); - testGitClient.clone_().url(wsMirror).repositoryName("origin").reference(wsRefrepoBase + "/${GIT_URL_SHA256_FALLBACK}").execute(); + testGitClient + .clone_() + .url(wsMirror) + .repositoryName("origin") + .reference(wsRefrepoBase + "/${GIT_URL_SHA256_FALLBACK}") + .execute(); testGitClient.checkout().ref("origin/master").branch("master").execute(); check_remote_url(workspace, testGitClient, "origin"); @@ -431,20 +449,24 @@ public void test_clone_reference_parameterized_sha256_fallback() throws Exceptio // Note: we do not expect the closing single quote after wsRefrepoBase // because other tests might pollute our test area, and SHA dir is there - assertThat("Reference repo name-parsing logged in: " + messages + - "\n...and replaced with: '" + wsRefrepoBase, - handler.containsMessageSubstring("Parameterized reference path ") && - handler.containsMessageSubstring(" replaced with: '" + wsRefrepoBase), - is(true)); + assertThat( + "Reference repo name-parsing logged in: " + messages + "\n...and replaced with: '" + wsRefrepoBase, + handler.containsMessageSubstring("Parameterized reference path ") + && handler.containsMessageSubstring(" replaced with: '" + wsRefrepoBase), + is(true)); // Barring filesystem errors, if we have the "custom" refrepo // we expect it to be used (fallback mode is not triggered) if (wsRefrepo != null) { - assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference repository: "), is(true)); + assertThat( + "Reference repo logged in: " + messages, + handler.containsMessageSubstring("Using reference repository: "), + is(true)); assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir.getPath(), wsRefrepo); assertBranchesExist(testGitClient.getBranches(), "master"); assertNoObjectsInRepository(); - } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path structure + } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path + // structure } private static final String SRC_DIR = (new File(".")).getAbsolutePath(); @@ -640,16 +662,20 @@ private void assertAlternateFilePointsToLocalWorkspaceMirror() throws IOExceptio assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir); } - private void assertAlternateFilePointsToLocalWorkspaceMirror(File _testGitDir) throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalWorkspaceMirror(File _testGitDir) + throws IOException, InterruptedException { assertAlternateFilePointsToLocalWorkspaceMirror(_testGitDir.getPath()); } - private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir) throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir) + throws IOException, InterruptedException { assertAlternateFilePointsToLocalWorkspaceMirror(_testGitDir, workspace.localMirror()); } - private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir, String _testAltDir) throws IOException, InterruptedException { - final String alternates = ".git" + File.separator + "objects" + File.separator + "info" + File.separator + "alternates"; + private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir, String _testAltDir) + throws IOException, InterruptedException { + final String alternates = + ".git" + File.separator + "objects" + File.separator + "info" + File.separator + "alternates"; assertAlternateFilePointsToLocalMirror(_testGitDir, _testAltDir, alternates); } @@ -662,18 +688,23 @@ private void assertAlternateFilePointsToLocalBareMirror(File _testGitDir) throws assertAlternateFilePointsToLocalBareMirror(_testGitDir.getPath()); } - private void assertAlternateFilePointsToLocalBareMirror(String _testGitDir) throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalBareMirror(String _testGitDir) + throws IOException, InterruptedException { assertAlternateFilePointsToLocalBareMirror(_testGitDir, workspace.localMirror()); } - private void assertAlternateFilePointsToLocalBareMirror(String _testGitDir, String _testAltDir) throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalBareMirror(String _testGitDir, String _testAltDir) + throws IOException, InterruptedException { final String alternates = "objects" + File.separator + "info" + File.separator + "alternates"; assertAlternateFilePointsToLocalMirror(_testGitDir, _testAltDir, alternates); } - private void assertAlternateFilePointsToLocalMirror(String _testGitDir, String _testAltDir, String alternates) throws IOException, InterruptedException { - assertThat("Did not find '" + alternates + "' under '" + _testGitDir + "'", - new File(_testGitDir, alternates), is(anExistingFile())); + private void assertAlternateFilePointsToLocalMirror(String _testGitDir, String _testAltDir, String alternates) + throws IOException, InterruptedException { + assertThat( + "Did not find '" + alternates + "' under '" + _testGitDir + "'", + new File(_testGitDir, alternates), + is(anExistingFile())); final String expectedContent = _testAltDir.replace("\\", "/") + "/objects"; final String actualContent = FileUtils.readFileToString(new File(_testGitDir, alternates), "UTF-8"); assertThat("Alternates file content", actualContent, is(expectedContent)); diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java index 6d316aa267..1f225036e1 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/WorkspaceWithRepo.java @@ -142,7 +142,8 @@ String localMirror(String cloneDirName) throws IOException, InterruptedException * deleteRecursive() will discard a clone that * 'lost the race'. */ - System.err.println("removing extra tempClonePath, we already (race?) have cloneDirName=" + cloneDirName); + System.err.println( + "removing extra tempClonePath, we already (race?) have cloneDirName=" + cloneDirName); Util.deleteRecursive(tempClonePath.toFile()); } } else { From 9863085827f04abe753045b0dd8acd16fa0f1703 Mon Sep 17 00:00:00 2001 From: Mark Waite Date: Tue, 11 Apr 2023 15:11:50 -0600 Subject: [PATCH 120/132] Fix merge mistake --- .../jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 02eeffb3de..8dd94c6385 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -3,6 +3,7 @@ import static java.util.Arrays.copyOfRange; import static org.apache.commons.lang.StringUtils.join; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import hudson.model.TaskListener; import hudson.plugins.git.GitException; import hudson.plugins.git.IGitAPI; From eeb47b42383bca343bb52f16901e020617c36b9b Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Wed, 14 Jun 2023 23:50:44 +0200 Subject: [PATCH 121/132] GitClient.java: minor fix to javadoc for subGit() --- src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java index 5111804ff9..54c0ac5644 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java @@ -735,7 +735,7 @@ Map getRemoteSymbolicReferences(String remoteRepoUrl, String pat /** * subGit. * - * @return a IGitAPI implementation to manage git submodule repository + * @return an {@link IGitAPI} implementation to manage git submodule repository * @param subdir a {@link java.lang.String} object. */ GitClient subGit(String subdir); From 9f162aaa352610ac2837836c6681c31c2a67f8bd Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 15 Jun 2023 00:24:43 +0200 Subject: [PATCH 122/132] Reconcile code style with changes in the PR #644 (added by @markewaite) and fix some Keeping an eye out for comprehensibility of the resulting source code text --- .../plugins/gitclient/CliGitAPIImpl.java | 59 ++-- .../plugins/gitclient/JGitAPIImpl.java | 32 +- .../gitclient/LegacyCompatibleGitAPIImpl.java | 274 ++++++++++-------- .../plugins/gitclient/GitClientCloneTest.java | 39 ++- 4 files changed, 232 insertions(+), 172 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index d50e1568c1..a65efcffc0 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -837,15 +837,15 @@ public void execute() throws GitException, InterruptedException { if (isParameterizedReferenceRepository(reference)) { // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via listener listener.getLogger() - .println("[INFO] The git reference repository path '" + reference - + "' is parameterized, it may take a few git queries logged " + .println("[INFO] The git reference repository path '" + reference + "' " + + "is parameterized, it may take a few git queries logged " + "below to resolve it into a particular directory name"); } File referencePath = findParameterizedReferenceRepository(reference, url); if (referencePath == null) { listener.getLogger() - .println("[ERROR] Could not make File object from reference path, skipping its use: " - + reference); + .println("[ERROR] Could not make File object from reference path, " + + "skipping its use: " + reference); } else { if (!referencePath.getPath().equals(reference)) { // Note: both these logs are needed, they are used in selftest @@ -861,18 +861,17 @@ public void execute() throws GitException, InterruptedException { reference = referencePath.getPath(); } - if (!referencePath.exists()) + if (!referencePath.exists()) { listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); - else if (!referencePath.isDirectory()) + } else if (!referencePath.isDirectory()) { listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference); - else { + } else { File objectsPath = getObjectsFile(referencePath); - if (objectsPath == null || !objectsPath.isDirectory()) + if (objectsPath == null || !objectsPath.isDirectory()) { listener.getLogger() - .println( - "[WARNING] Reference path does not contain an objects directory (not a git repo?): " - + objectsPath); - else { + .println("[WARNING] Reference path does not contain an objects directory " + + "(not a git repo?): " + objectsPath); + } else { // Go behind git's back to write a meta file in new workspace File alternates = new File(workspace, ".git/objects/info/alternates"); try (PrintWriter w = new PrintWriter( @@ -1518,11 +1517,13 @@ public void execute() throws GitException, InterruptedException { if ((ref != null) && !ref.isEmpty()) { if (!isParameterizedReferenceRepository(ref)) { File referencePath = new File(ref); - if (!referencePath.exists()) + if (!referencePath.exists()) { listener.getLogger().println("[WARNING] Reference path does not exist: " + ref); - else if (!referencePath.isDirectory()) + } else if (!referencePath.isDirectory()) { listener.getLogger().println("[WARNING] Reference path is not a directory: " + ref); - else args.add("--reference", ref); + } else { + args.add("--reference", ref); + } } // else handled below in per-module loop } if (shallow) { @@ -1585,9 +1586,8 @@ else if (!referencePath.isDirectory()) File referencePath = findParameterizedReferenceRepository(ref, strURIish); if (referencePath == null) { listener.getLogger() - .println( - "[ERROR] Could not make File object from reference path, skipping its use: " - + ref); + .println("[ERROR] Could not make File object from reference path, " + + "skipping its use: " + ref); } else { String expRef = null; if (referencePath.getPath().equals(ref)) { @@ -1596,11 +1596,15 @@ else if (!referencePath.isDirectory()) expRef = referencePath.getPath(); expRef += " (expanded from " + ref + ")"; } - if (!referencePath.exists()) - listener.getLogger().println("[WARNING] Reference path does not exist: " + expRef); - else if (!referencePath.isDirectory()) - listener.getLogger().println("[WARNING] Reference path is not a directory: " + expRef); - else args.add("--reference", referencePath.getPath()); + if (!referencePath.exists()) { + listener.getLogger() + .println("[WARNING] Reference path does not exist: " + expRef); + } else if (!referencePath.isDirectory()) { + listener.getLogger() + .println("[WARNING] Reference path is not a directory: " + expRef); + } else { + args.add("--reference", referencePath.getPath()); + } } } @@ -2959,10 +2963,11 @@ private String launchCommandIn(ArgumentListBuilder args, File workDir, EnvVars e .toAbsolutePath() .normalize() .toFile(); - throw new GitException("Command \"" + command + "\" executed in workdir \"" - + workDir.toString() + "\" returned status code " - + status + ":\nstdout: " - + stdout + "\nstderr: " + stderr); + throw new GitException("Command \"" + command + + "\" executed in workdir \"" + workDir.toString() + + "\" returned status code " + status + + ":\nstdout: " + stdout + + "\nstderr: " + stderr); } return stdout; diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 978f7473b2..44c95dac05 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1555,19 +1555,18 @@ public void execute() throws GitException { if (reference != null && !reference.isEmpty()) { // Note: keep in sync with similar logic in CliGitAPIImpl.java if (isParameterizedReferenceRepository(reference)) { - // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via - // listener + // LegacyCompatibleGitAPIImpl.java has a logging trace, + // but not into build console via listener listener.getLogger() - .println("[INFO] The git reference repository path '" + reference - + "'is parameterized, it may take a few git queries logged " + .println("[INFO] The git reference repository path '" + reference + "' " + + "is parameterized, it may take a few git queries logged " + "below to resolve it into a particular directory name"); } File referencePath = findParameterizedReferenceRepository(reference, url); if (referencePath == null) { listener.getLogger() - .println( - "[ERROR] Could not make File object from reference path, skipping its use: " - + reference); + .println("[ERROR] Could not make File object from reference path, " + + "skipping its use: " + reference); } else { if (!referencePath.getPath().equals(reference)) { // Note: both these logs are needed, they are used in selftest @@ -1583,26 +1582,25 @@ public void execute() throws GitException { reference = referencePath.getPath(); } - if (!referencePath.exists()) - listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); - else if (!referencePath.isDirectory()) + if (!referencePath.exists()) { + listener.getLogger() + .println("[WARNING] Reference path does not exist: " + reference); + } else if (!referencePath.isDirectory()) { listener.getLogger() .println("[WARNING] Reference path is not a directory: " + reference); - else { + } else { File objectsPath = getObjectsFile(referencePath); - if (objectsPath == null || !objectsPath.isDirectory()) + if (objectsPath == null || !objectsPath.isDirectory()) { listener.getLogger() - .println( - "[WARNING] Reference path does not contain an objects directory (no git repo?): " - + objectsPath); - else { + .println("[WARNING] Reference path does not contain an objects directory " + + "(no git repo?): " + objectsPath); + } else { // Go behind git's back to write a meta file in new workspace try { File alternates = new File(workspace, ".git/objects/info/alternates"); String absoluteReference = objectsPath.getAbsolutePath().replace('\\', '/'); listener.getLogger().println("Using reference repository: " + reference); - // git implementations on windows also use try (PrintWriter w = new PrintWriter(alternates, "UTF-8")) { // git implementations on windows also use w.print(absoluteReference); diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 8dd94c6385..9e60a3dc6b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -215,9 +215,13 @@ public static File getObjectsFile(File reference) { return reference; } - if (!reference.exists()) return null; + if (!reference.exists()) { + return null; + } - if (!reference.isDirectory()) return null; + if (!reference.isDirectory()) { + return null; + } File fGit = new File(reference, ".git"); // workspace - file, dir or symlink to those File objects = null; @@ -226,9 +230,9 @@ public static File getObjectsFile(File reference) { if (fGit.isDirectory()) { objects = new File(fGit, "objects"); // this might not exist or not be a dir - checked below /* - if (objects == null) { // spotbugs dislikes this, since "new File()" should not return null - return objects; // Some Java error, could not make object from the paths involved - } + if (objects == null) { // spotbugs dislikes this, since "new File()" should not return null + return objects; // Some Java error, could not make object from the paths involved + } */ LOGGER.log( Level.FINEST, @@ -261,7 +265,7 @@ public static File getObjectsFile(File reference) { Level.FINE, "getObjectsFile(): while looking for 'gitdir:' in '" + fGit.getAbsolutePath().toString() - + "', found reference to objects " + "which should be at: '" + + "', found reference to objects which should be at: '" + objects.getAbsolutePath().toString() + "'"); // Note: we don't use getCanonicalPath() here to avoid further filesystem // access and possible exceptions (the getAbsolutePath() is about string @@ -315,9 +319,9 @@ public static File getObjectsFile(File reference) { // or we have failed interpreting ".git" contents above objects = new File(reference, "objects"); // bare /* - if (objects == null) { - return objects; // Some Java error, could not make object from the paths involved - } + if (objects == null) { + return objects; // Some Java error, could not make object from the paths involved + } */ // This clause below is redundant for production, but useful for troubleshooting if (objects.exists()) { @@ -340,9 +344,13 @@ public static File getObjectsFile(File reference) { } } - if (!objects.exists()) return null; + if (!objects.exists()) { + return null; + } - if (!objects.isDirectory()) return null; + if (!objects.isDirectory()) { + return null; + } // If we get here, we have a non-null File referencing a // "(.git/)objects" subdir under original referencePath @@ -454,13 +462,13 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { + Paths.get(Paths.get("").toAbsolutePath().toString() + "/" + urlNormalized) .normalize() .toString(); - ; } } } LOGGER.log( - Level.FINEST, "normalizeGitUrl('" + url + "', " + checkLocalPaths.toString() + ") => " + urlNormalized); + Level.FINEST, + "normalizeGitUrl('" + url + "', " + checkLocalPaths.toString() + ") => " + urlNormalized); return urlNormalized; } @@ -527,8 +535,9 @@ public SimpleEntry> getSubmodulesUrls( referenceBaseDirAbs = new File(referenceBaseDir).getAbsoluteFile().toString(); LOGGER.log( Level.SEVERE, - "getSubmodulesUrls(): failed to canonicize '" + referenceBaseDir + "' => '" + referenceBaseDirAbs - + "': " + e.toString()); + "getSubmodulesUrls(): failed to canonicize '" + + referenceBaseDir + "' => '" + referenceBaseDirAbs + "': " + + e.toString()); // return new SimpleEntry<>(false, result); } @@ -558,9 +567,9 @@ public SimpleEntry> getSubmodulesUrls( LOGGER.log( Level.SEVERE, - "getSubmodulesUrls(): failed to determine " + "isBareRepository() in '" - + referenceBaseDirAbs + "'; " + "assuming '" - + isBare + "': " + e.toString()); + "getSubmodulesUrls(): failed to determine isBareRepository() in '" + + referenceBaseDirAbs + "'; " + "assuming '" + isBare + "': " + + e.toString()); } // Simplify checks below by stating a useless needle is null @@ -626,8 +635,8 @@ public SimpleEntry> getSubmodulesUrls( LOGGER.log( Level.FINE, - "getSubmodulesUrls(): looking at basename-like subdirs under base refrepo '" + referenceBaseDirAbs - + "', per arrDirnames: " + arrDirnames.toString()); + "getSubmodulesUrls(): looking at basename-like subdirs under base refrepo '" + + referenceBaseDirAbs + "', per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { f = new File(dirname); @@ -656,8 +665,8 @@ public SimpleEntry> getSubmodulesUrls( String uriNorm = normalizeGitUrl(uri, true); LOGGER.log( Level.FINE, - "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm - + "') vs needle"); + "getSubmodulesUrls(): checking uri='" + uri + + "' (uriNorm='" + uriNorm + "') vs needle"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); result.add(new String[] {dirname, uri, uriNorm, remoteName}); @@ -710,57 +719,56 @@ public SimpleEntry> getSubmodulesUrls( // Maybe merge with current "if isBare" below, to optionally seed // same arrDirnames with different values and check remotes listed // in those repos. - // If current repo *is NOT* bare - check its submodules - // (the .gitmodules => submodule.MODNAME.{url,path} mapping) - // but this essentially does not look into any subdirectory. - // But we can add at higher priority submodule path(s) whose - // basename of the URL matches the needleBasename. And then - // other submodule paths to inspect before arbitrary subdirs. - if (!isBare) { - try { - // For each current workspace (recurse or big loop in same context?): - // public GitClient subGit(String subdir) => would this.subGit(...) - // give us a copy of this applied class instance (CLI Git vs jGit)? - // get submodule name-vs-one-url from .gitmodules if present, for a - // faster possible answer (only bother if needle is not null?) - // try { getSubmodules("HEAD") ... } => List filtered for - // "commit" items - // * if we are recursed into a "leaf" project and inspect ITS - // submodules, look at all git tips or even commits, to find - // and inspect all unique (by hash) .gitmodule objects, since - // over time or in different branches a "leaf" project could - // reference different subs? - // getRemoteUrls() => Map - // arrDirnames.clear(); - - // TODO: Check subdirs that are git workspaces, and remove "|| true" above - // LinkedHashSet checkedDirs = new LinkedHashSet<>(); - // for (String[] resultEntry : result) { - // checkedDirs.add(resultEntry[0]); - // } - - - LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in submodules of refrepo, if any"); - Map uriNames = referenceGit.getRemoteUrls(); - for (Map.Entry pair : uriNames.entrySet()) { - String uri = pair.getKey(); - String uriNorm = normalizeGitUrl(uri, true); - LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); - if (needleNorm.equals(uriNorm) || needle.equals(uri)) { - result = new LinkedHashSet<>(); - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); - return result; - } - // Cache the finding to avoid the dirname later, if we - // get to that; but no checks are needed in this loop - // which by construct looks at different dirs so far. - result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); - } - } catch (Exception e) { - // ignore, go to next slide + // If current repo *is NOT* bare - check its submodules + // (the .gitmodules => submodule.MODNAME.{url,path} mapping) + // but this essentially does not look into any subdirectory. + // But we can add at higher priority submodule path(s) whose + // basename of the URL matches the needleBasename. And then + // other submodule paths to inspect before arbitrary subdirs. + if (!isBare) { + try { + // For each current workspace (recurse or big loop in same context?): + // public GitClient subGit(String subdir) => would this.subGit(...) + // give us a copy of this applied class instance (CLI Git vs jGit)? + // get submodule name-vs-one-url from .gitmodules if present, for a + // faster possible answer (only bother if needle is not null?) + // try { getSubmodules("HEAD") ... } => List filtered for + // "commit" items + // * if we are recursed into a "leaf" project and inspect ITS + // submodules, look at all git tips or even commits, to find + // and inspect all unique (by hash) .gitmodule objects, since + // over time or in different branches a "leaf" project could + // reference different subs? + // getRemoteUrls() => Map + //// arrDirnames.clear(); + + // TODO: Check subdirs that are git workspaces, and remove "|| true" above + //// LinkedHashSet checkedDirs = new LinkedHashSet<>(); + //// for (String[] resultEntry : result) { + //// checkedDirs.add(resultEntry[0]); + //// } + + LOGGER.log(Level.FINE, "getSubmodulesUrls(): looking for submodule URL needle='" + needle + "' in submodules of refrepo, if any"); + Map uriNames = referenceGit.getRemoteUrls(); + for (Map.Entry pair : uriNames.entrySet()) { + String uri = pair.getKey(); + String uriNorm = normalizeGitUrl(uri, true); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + "')"); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): sub-git getRemoteUrls() returned this Map: " + uriNames.toString()); + if (needleNorm.equals(uriNorm) || needle.equals(uri)) { + result = new LinkedHashSet<>(); + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); + return result; } + // Cache the finding to avoid the dirname later, if we + // get to that; but no checks are needed in this loop + // which by construct looks at different dirs so far. + result.add(new String[]{fAbs, uri, uriNorm, pair.getValue()}); } + } catch (Exception e) { + // ignore, go to next slide + } + } */ // If current repo *is* bare (can't have proper submodules), or if the @@ -797,7 +805,8 @@ public SimpleEntry> getSubmodulesUrls( LOGGER.log( Level.FINE, - "getSubmodulesUrls(): looking at " + ((isBare ? "" : "submodules first, then ")) + "getSubmodulesUrls(): looking at " + + ((isBare ? "" : "submodules first, then ")) + "all subdirs that have a .git, under refrepo '" + referenceBaseDirAbs + "' per absolute arrDirnames: " + arrDirnames.toString()); @@ -805,7 +814,9 @@ public SimpleEntry> getSubmodulesUrls( for (String dirname : arrDirnames) { // Note that here dirnames deal in absolutes f = new File(dirname); - LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' if it exists"); + LOGGER.log( + Level.FINEST, + "getSubmodulesUrls(): probing dir '" + dirname + "' if it exists"); if (f.exists() && f.isDirectory()) { // No checks for ".git" or "objects" this time, already checked above // by getObjectsFile(). Probably should not check exists/dir either, @@ -833,11 +844,12 @@ public SimpleEntry> getSubmodulesUrls( String uriNorm = normalizeGitUrl(uri, true); LOGGER.log( Level.FINE, - "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm - + "') vs needle"); + "getSubmodulesUrls(): checking uri='" + uri + + "' (uriNorm='" + uriNorm + "') vs needle"); if (needle != null && needleNorm != null - && (needleNorm.equals(uriNorm) || needle.equals(uri))) { + && (needleNorm.equals(uriNorm) || needle.equals(uri)) + ) { result = new LinkedHashSet<>(); result.add(new String[] {dirname, uri, uriNorm, remoteName}); return new SimpleEntry<>(true, result); @@ -862,21 +874,24 @@ public SimpleEntry> getSubmodulesUrls( // Here is a good spot to recurse this routine into a // subdir that is already a known git workspace, to // add its data to list and/or return a found needle. - LOGGER.log(Level.FINE, "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); - SimpleEntry> subEntriesRet = getSubmodulesUrls(dirname, needle, false); + LOGGER.log( + Level.FINE, + "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); + SimpleEntry> subEntriesRet = + getSubmodulesUrls(dirname, needle, false); Boolean subEntriesExactMatched = subEntriesRet.getKey(); LinkedHashSet subEntries = subEntriesRet.getValue(); LOGGER.log( Level.FINE, - "getSubmodulesUrls(): returned from recursing into dir '" + dirname + "' with " - + subEntries.size() + " found mappings"); + "getSubmodulesUrls(): returned from recursing into dir '" + dirname + + "' with " + subEntries.size() + " found mappings"); if (!subEntries.isEmpty()) { if (needle != null && subEntriesExactMatched) { // We found nothing... until now! Bubble it up! LOGGER.log( Level.FINE, - "getSubmodulesUrls(): got an exact needle match from recursing into dir '" + dirname - + "': " + subEntries.iterator().next()[0]); + "getSubmodulesUrls(): got an exact needle match from recursing into dir '" + + dirname + "': " + subEntries.iterator().next()[0]); return subEntriesRet; } // ...else collect results to inspect and/or propagate later @@ -899,11 +914,11 @@ public SimpleEntry> getSubmodulesUrls( LinkedHashSet resultFiltered = new LinkedHashSet<>(); /* - if (!checkRemotesInReferenceBaseDir) { - // Overload the flag's meaning to only parse results once, - // in the parent dir? - return new SimpleEntry<>(false, resultFiltered); - } + if (!checkRemotesInReferenceBaseDir) { + // Overload the flag's meaning to only parse results once, + // in the parent dir? + return new SimpleEntry<>(false, resultFiltered); + } */ // Separate lists by suggestion priority: @@ -1045,17 +1060,25 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Note: Initially we expect the reference to be a realistic dirname // with a special suffix to substitute after the logic below, so the // referencePath for that verbatim funny string should not exist now: - if (!referencePath.exists() && isParameterizedReferenceRepository(reference) && url != null && !url.isEmpty()) { + if (!referencePath.exists() + && isParameterizedReferenceRepository(reference) + && url != null + && !url.isEmpty() + ) { // Drop the trailing keyword to know the root refrepo dirname String referenceBaseDir = reference.replaceAll("/\\$\\{GIT_[^\\}]*\\}$", ""); File referenceBaseDirFile = new File(referenceBaseDir); if (!referenceBaseDirFile.exists()) { - LOGGER.log(Level.WARNING, "Base Git reference directory " + referenceBaseDir + " does not exist"); + LOGGER.log( + Level.WARNING, + "Base Git reference directory " + referenceBaseDir + " does not exist"); return null; } if (!referenceBaseDirFile.isDirectory()) { - LOGGER.log(Level.WARNING, "Base Git reference directory " + referenceBaseDir + " is not a directory"); + LOGGER.log( + Level.WARNING, + "Base Git reference directory " + referenceBaseDir + " is not a directory"); return null; } @@ -1070,9 +1093,9 @@ public File findParameterizedReferenceRepository(String reference, String url) { String urlNormalized = normalizeGitUrl(url, true); // Note: currently unit-tests expect this markup on stderr: - System.err.println("reference='" + reference + "'\n" + "url='" - + url + "'\n" + "urlNormalized='" - + urlNormalized + "'\n"); + System.err.println("reference='" + reference + "'\n" + + "url='" + url + "'\n" + + "urlNormalized='" + urlNormalized + "'\n"); // Let users know why there are many "git config --list" lines in their build log: LOGGER.log( @@ -1094,12 +1117,15 @@ public File findParameterizedReferenceRepository(String reference, String url) { referenceExpanded = reference.replaceAll( "\\$\\{GIT_URL_SHA256_FALLBACK\\}$", org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); - if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { + if (getObjectsFile(referenceExpanded) == null + && getObjectsFile(referenceExpanded + ".git") == null + ) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } } else if (reference.endsWith("/${GIT_URL_BASENAME}") - || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { + || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}") + ) { // This may be the more portable solution with regard to filesystems // First try with original user-provided casing of the URL (if local // dirs were cloned manually) @@ -1113,12 +1139,17 @@ public File findParameterizedReferenceRepository(String reference, String url) { needleBasename = needleBasename.replaceAll(".git$", ""); if (reference.endsWith("/${GIT_URL_BASENAME}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", needleBasename); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_BASENAME\\}$", + needleBasename); } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", needleBasename); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", + needleBasename); if (url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null) { + && getObjectsFile(referenceExpanded + ".git") == null + ) { // chop it off, use main directory (only if we do not check urlNormalized separately below) referenceExpanded = referenceBaseDir; } @@ -1126,7 +1157,8 @@ && getObjectsFile(referenceExpanded + ".git") == null) { if (!url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null) { + && getObjectsFile(referenceExpanded + ".git") == null + ) { // Retry with automation-ready normalized URL sep = urlNormalized.lastIndexOf("/"); if (sep < 0) { @@ -1137,17 +1169,24 @@ && getObjectsFile(referenceExpanded + ".git") == null) { needleBasename = needleBasename.replaceAll(".git$", ""); if (reference.endsWith("/${GIT_URL_BASENAME}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", needleBasename); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_BASENAME\\}$", + needleBasename); } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", needleBasename); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", + needleBasename); if (getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null) { + && getObjectsFile(referenceExpanded + ".git") == null + ) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } } } - } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { + } else if (reference.endsWith("/${GIT_SUBMODULES}") + || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") + ) { // Here we employ git submodules - so we can reliably match // remote URLs (easily multiple) to particular modules, and // yet keep separate git index directories per module with @@ -1191,16 +1230,19 @@ && getObjectsFile(referenceExpanded + ".git") == null) { } LOGGER.log( Level.FINE, - "findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded - + "' from subEntries"); + "findParameterizedReferenceRepository(): got referenceExpanded='" + + referenceExpanded + "' from subEntries"); if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null) { + && getObjectsFile(referenceExpanded + ".git") == null + ) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } } else { - LOGGER.log(Level.FINE, "findParameterizedReferenceRepository(): got no subEntries"); + LOGGER.log( + Level.FINE, + "findParameterizedReferenceRepository(): got no subEntries"); // If there is no hit, the non-fallback mode suggests a new // directory name to host the submodule (same rules as for // the refrepo forks' co-hosting friendly basename search), @@ -1215,12 +1257,18 @@ && getObjectsFile(referenceExpanded + ".git") == null) { needleBasename = needleBasename.replaceAll(".git$", ""); if (reference.endsWith("/${GIT_SUBMODULES}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", needleBasename); - } else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { - referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", needleBasename); + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_SUBMODULES\\}$", + needleBasename); + } + else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { + referenceExpanded = reference.replaceAll( + "\\$\\{GIT_SUBMODULES\\}$", + needleBasename); if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null) { + && getObjectsFile(referenceExpanded + ".git") == null + ) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } @@ -1239,8 +1287,8 @@ && getObjectsFile(referenceExpanded + ".git") == null) { LOGGER.log( Level.INFO, - "After resolving the parameterized Git reference repository, " + "decided to use '" + reference - + "' directory for URL '" + url + "'"); + "After resolving the parameterized Git reference repository, " + + "decided to use '" + reference + "' directory for URL '" + url + "'"); } // if referencePath is the replaceable token and not existing directory if (!referencePath.exists() && !reference.endsWith(".git")) { diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index f6814fade4..fa82bf2c99 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -289,11 +289,14 @@ public void test_clone_reference_parameterized_basename() throws Exception, IOEx // below is not confused - it expects this location to not exist. // Skip: Missing if clone failed - currently would, with bogus // path above and not yet pre-created path structure. - // assertThat("Reference repo logged in: " + messages, handler.containsMessageSubstring("Using reference - // repository: "), is(true)); - // assertAlternateFilePointsToLocalMirror(); - // assertBranchesExist(testGitClient.getBranches(), "master"); - // assertNoObjectsInRepository(); + /* + assertThat("Reference repo logged in: " + messages, + handler.containsMessageSubstring( + "Using reference repository: "), is(true)); + assertAlternateFilePointsToLocalMirror(); + assertBranchesExist(testGitClient.getBranches(), "master"); + assertNoObjectsInRepository(); + */ } @Test @@ -361,7 +364,8 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce // the provided string, as we check in log below } - System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + "wsRefrepo='" + wsRefrepo); + System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + + "wsRefrepo='" + wsRefrepo); testGitClient .clone_() @@ -394,8 +398,8 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir.getPath(), wsRefrepo); assertBranchesExist(testGitClient.getBranches(), "master"); assertNoObjectsInRepository(); - } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path - // structure + } // else Skip: Missing if clone failed - currently would, + // with bogus path above and not pre-created path structure } @Test @@ -430,7 +434,8 @@ public void test_clone_reference_parameterized_sha256_fallback() // the provided string, as we check in log below } - System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + "wsRefrepo='" + wsRefrepo); + System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + + "wsRefrepo='" + wsRefrepo); testGitClient .clone_() @@ -465,8 +470,8 @@ public void test_clone_reference_parameterized_sha256_fallback() assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir.getPath(), wsRefrepo); assertBranchesExist(testGitClient.getBranches(), "master"); assertNoObjectsInRepository(); - } // else Skip: Missing if clone failed - currently would, with bogus path above and not pre-created path - // structure + } // else Skip: Missing if clone failed - currently would, + // with bogus path above and not pre-created path structure } private static final String SRC_DIR = (new File(".")).getAbsolutePath(); @@ -654,11 +659,13 @@ private void assertNoObjectsInRepository() { } // Most tests use this method, expecting a non-bare repo - private void assertAlternateFilePointsToLocalMirror() throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalMirror() + throws IOException, InterruptedException { assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir); } - private void assertAlternateFilePointsToLocalWorkspaceMirror() throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalWorkspaceMirror() + throws IOException, InterruptedException { assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir); } @@ -680,11 +687,13 @@ private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir, } // Similar for bare repos, without ".git/" dir - private void assertAlternateFilePointsToLocalBareMirror() throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalBareMirror() + throws IOException, InterruptedException { assertAlternateFilePointsToLocalBareMirror(testGitDir); } - private void assertAlternateFilePointsToLocalBareMirror(File _testGitDir) throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalBareMirror(File _testGitDir) + throws IOException, InterruptedException { assertAlternateFilePointsToLocalBareMirror(_testGitDir.getPath()); } From d3a6d569c41da8f51b2d1b61e0c4e8edf1eb3683 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 15 Jun 2023 00:25:52 +0200 Subject: [PATCH 123/132] Mark @Override methods like in master branch --- src/main/java/hudson/plugins/git/GitAPI.java | 2 ++ .../java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/main/java/hudson/plugins/git/GitAPI.java b/src/main/java/hudson/plugins/git/GitAPI.java index 506f0a6dbe..e3ef43d0eb 100644 --- a/src/main/java/hudson/plugins/git/GitAPI.java +++ b/src/main/java/hudson/plugins/git/GitAPI.java @@ -174,11 +174,13 @@ public GitClient subGit(String subdir) { } /** {@inheritDoc} */ + @Override public GitClient newGit(String somedir) { return Git.USE_CLI ? super.newGit(somedir) : jgit.newGit(somedir); } /** {@inheritDoc} */ + @Override public void setRemoteUrl(String name, String url) throws GitException, InterruptedException { if (Git.USE_CLI) { super.setRemoteUrl(name, url); diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java index b78f6d6e53..962342ca74 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/RemoteGitImpl.java @@ -343,16 +343,19 @@ public String getRemoteUrl(String name) throws GitException, InterruptedExceptio } /** {@inheritDoc} */ + @Override public Map getRemoteUrls() throws GitException, InterruptedException { return proxy.getRemoteUrls(); } /** {@inheritDoc} */ + @Override public Map getRemotePushUrls() throws GitException, InterruptedException { return proxy.getRemotePushUrls(); } /** {@inheritDoc} */ + @Override public void setRemoteUrl(String name, String url) throws GitException, InterruptedException { proxy.setRemoteUrl(name, url); } From 6840c75b75a14f999cd5ef37652a4372768fe6a8 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 15 Jun 2023 00:26:19 +0200 Subject: [PATCH 124/132] Change PrintWriter use of UTF8 like in master branch --- src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 44c95dac05..885d8a693c 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1601,7 +1601,7 @@ public void execute() throws GitException { String absoluteReference = objectsPath.getAbsolutePath().replace('\\', '/'); listener.getLogger().println("Using reference repository: " + reference); - try (PrintWriter w = new PrintWriter(alternates, "UTF-8")) { + try (PrintWriter w = new PrintWriter(alternates, StandardCharsets.UTF_8)) { // git implementations on windows also use w.print(absoluteReference); } From d8ddcdb22e8b17f4f05a3de29d0a5ede74b97321 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 15 Jun 2023 00:30:01 +0200 Subject: [PATCH 125/132] GitClient.java: minor fix to javadoc for newGit() --- src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java index 54c0ac5644..8ad8795f4a 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java @@ -724,7 +724,7 @@ Map getRemoteSymbolicReferences(String remoteRepoUrl, String pat /** * newGit. * - * @return a IGitAPI implementation to manage another git repository + * @return an {@link IGitAPI} implementation to manage another git repository * with same general settings and implementation as the current one. * @param somedir a {@link java.lang.String} object. */ From 896c2c6169e9059be273a025e6a2a7362a35fcf9 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 15 Jun 2023 00:30:37 +0200 Subject: [PATCH 126/132] LegacyCompatibleGitAPIImpl: import DigestUtils to use by short reference --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 9e60a3dc6b..55ff32c1b9 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.commons.codec.digest.DigestUtils; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; @@ -629,7 +630,7 @@ public SimpleEntry> getSubmodulesUrls( arrDirnames.add(referenceBaseDirAbs + "/" + needleNormBasename + ".git"); } - needleSha = org.apache.commons.codec.digest.DigestUtils.sha256Hex(needleNorm); + needleSha = DigestUtils.sha256Hex(needleNorm); arrDirnames.add(referenceBaseDirAbs + "/" + needleSha); arrDirnames.add(referenceBaseDirAbs + "/" + needleSha + ".git"); @@ -1108,7 +1109,7 @@ && isParameterizedReferenceRepository(reference) // This may be the more portable solution with regard to filesystems referenceExpanded = reference.replaceAll( "\\$\\{GIT_URL_SHA256\\}$", - org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + DigestUtils.sha256Hex(urlNormalized)); } else if (reference.endsWith("/${GIT_URL_SHA256_FALLBACK}")) { // The safest option - fall back to parent directory if // the expanded one does not have git repo data right now: @@ -1116,7 +1117,7 @@ && isParameterizedReferenceRepository(reference) // repository into smaller chunks without breaking builds. referenceExpanded = reference.replaceAll( "\\$\\{GIT_URL_SHA256_FALLBACK\\}$", - org.apache.commons.codec.digest.DigestUtils.sha256Hex(urlNormalized)); + DigestUtils.sha256Hex(urlNormalized)); if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null ) { From 62355d5e13ff236051b7b6a99a83497456af38ec Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 15 Jun 2023 00:31:18 +0200 Subject: [PATCH 127/132] GitClientCloneTest: drop use of FileUtils.readFileToString() like in master branch --- .../org/jenkinsci/plugins/gitclient/GitClientCloneTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index fa82bf2c99..61deef1667 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -24,7 +24,6 @@ import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; @@ -715,7 +714,7 @@ private void assertAlternateFilePointsToLocalMirror(String _testGitDir, String _ new File(_testGitDir, alternates), is(anExistingFile())); final String expectedContent = _testAltDir.replace("\\", "/") + "/objects"; - final String actualContent = FileUtils.readFileToString(new File(_testGitDir, alternates), "UTF-8"); + final String actualContent = Files.readString(testGitDir.toPath().resolve(alternates), StandardCharsets.UTF_8); assertThat("Alternates file content", actualContent, is(expectedContent)); final File alternatesDir = new File(actualContent); assertThat(alternatesDir, is(anExistingDirectory())); From 2c10b594e2fdc2a834fbb61b3bbe2af5cc5eac1e Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Thu, 15 Jun 2023 01:19:58 +0200 Subject: [PATCH 128/132] spotless is clueless Promulgates inconsistent coding style and breaks "logical" markup of messages defined on multiple lines (like "item " + javavar) which it fails to keep together, etc. Unavoidable evil I guess, but hopefully someone can configure it. --- .../plugins/gitclient/CliGitAPIImpl.java | 6 +- .../plugins/gitclient/JGitAPIImpl.java | 3 +- .../gitclient/LegacyCompatibleGitAPIImpl.java | 126 ++++++------------ .../plugins/gitclient/GitClientCloneTest.java | 22 ++- 4 files changed, 53 insertions(+), 104 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index a65efcffc0..298ea0aeed 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -1597,11 +1597,9 @@ public void execute() throws GitException, InterruptedException { expRef += " (expanded from " + ref + ")"; } if (!referencePath.exists()) { - listener.getLogger() - .println("[WARNING] Reference path does not exist: " + expRef); + listener.getLogger().println("[WARNING] Reference path does not exist: " + expRef); } else if (!referencePath.isDirectory()) { - listener.getLogger() - .println("[WARNING] Reference path is not a directory: " + expRef); + listener.getLogger().println("[WARNING] Reference path is not a directory: " + expRef); } else { args.add("--reference", referencePath.getPath()); } diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 885d8a693c..17fa0c9c7e 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -1583,8 +1583,7 @@ public void execute() throws GitException { } if (!referencePath.exists()) { - listener.getLogger() - .println("[WARNING] Reference path does not exist: " + reference); + listener.getLogger().println("[WARNING] Reference path does not exist: " + reference); } else if (!referencePath.isDirectory()) { listener.getLogger() .println("[WARNING] Reference path is not a directory: " + reference); diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 55ff32c1b9..f85a8cc1f3 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -468,8 +468,7 @@ public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { } LOGGER.log( - Level.FINEST, - "normalizeGitUrl('" + url + "', " + checkLocalPaths.toString() + ") => " + urlNormalized); + Level.FINEST, "normalizeGitUrl('" + url + "', " + checkLocalPaths.toString() + ") => " + urlNormalized); return urlNormalized; } @@ -636,8 +635,8 @@ public SimpleEntry> getSubmodulesUrls( LOGGER.log( Level.FINE, - "getSubmodulesUrls(): looking at basename-like subdirs under base refrepo '" - + referenceBaseDirAbs + "', per arrDirnames: " + arrDirnames.toString()); + "getSubmodulesUrls(): looking at basename-like subdirs under base refrepo '" + referenceBaseDirAbs + + "', per arrDirnames: " + arrDirnames.toString()); for (String dirname : arrDirnames) { f = new File(dirname); @@ -666,8 +665,8 @@ public SimpleEntry> getSubmodulesUrls( String uriNorm = normalizeGitUrl(uri, true); LOGGER.log( Level.FINE, - "getSubmodulesUrls(): checking uri='" + uri - + "' (uriNorm='" + uriNorm + "') vs needle"); + "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + + "') vs needle"); if (needleNorm.equals(uriNorm) || needle.equals(uri)) { result = new LinkedHashSet<>(); result.add(new String[] {dirname, uri, uriNorm, remoteName}); @@ -815,9 +814,7 @@ public SimpleEntry> getSubmodulesUrls( for (String dirname : arrDirnames) { // Note that here dirnames deal in absolutes f = new File(dirname); - LOGGER.log( - Level.FINEST, - "getSubmodulesUrls(): probing dir '" + dirname + "' if it exists"); + LOGGER.log(Level.FINEST, "getSubmodulesUrls(): probing dir '" + dirname + "' if it exists"); if (f.exists() && f.isDirectory()) { // No checks for ".git" or "objects" this time, already checked above // by getObjectsFile(). Probably should not check exists/dir either, @@ -845,12 +842,11 @@ public SimpleEntry> getSubmodulesUrls( String uriNorm = normalizeGitUrl(uri, true); LOGGER.log( Level.FINE, - "getSubmodulesUrls(): checking uri='" + uri - + "' (uriNorm='" + uriNorm + "') vs needle"); + "getSubmodulesUrls(): checking uri='" + uri + "' (uriNorm='" + uriNorm + + "') vs needle"); if (needle != null && needleNorm != null - && (needleNorm.equals(uriNorm) || needle.equals(uri)) - ) { + && (needleNorm.equals(uriNorm) || needle.equals(uri))) { result = new LinkedHashSet<>(); result.add(new String[] {dirname, uri, uriNorm, remoteName}); return new SimpleEntry<>(true, result); @@ -875,24 +871,21 @@ public SimpleEntry> getSubmodulesUrls( // Here is a good spot to recurse this routine into a // subdir that is already a known git workspace, to // add its data to list and/or return a found needle. - LOGGER.log( - Level.FINE, - "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); - SimpleEntry> subEntriesRet = - getSubmodulesUrls(dirname, needle, false); + LOGGER.log(Level.FINE, "getSubmodulesUrls(): recursing into dir '" + dirname + "'..."); + SimpleEntry> subEntriesRet = getSubmodulesUrls(dirname, needle, false); Boolean subEntriesExactMatched = subEntriesRet.getKey(); LinkedHashSet subEntries = subEntriesRet.getValue(); LOGGER.log( Level.FINE, - "getSubmodulesUrls(): returned from recursing into dir '" + dirname - + "' with " + subEntries.size() + " found mappings"); + "getSubmodulesUrls(): returned from recursing into dir '" + dirname + "' with " + + subEntries.size() + " found mappings"); if (!subEntries.isEmpty()) { if (needle != null && subEntriesExactMatched) { // We found nothing... until now! Bubble it up! LOGGER.log( Level.FINE, - "getSubmodulesUrls(): got an exact needle match from recursing into dir '" - + dirname + "': " + subEntries.iterator().next()[0]); + "getSubmodulesUrls(): got an exact needle match from recursing into dir '" + dirname + + "': " + subEntries.iterator().next()[0]); return subEntriesRet; } // ...else collect results to inspect and/or propagate later @@ -1061,25 +1054,17 @@ public File findParameterizedReferenceRepository(String reference, String url) { // Note: Initially we expect the reference to be a realistic dirname // with a special suffix to substitute after the logic below, so the // referencePath for that verbatim funny string should not exist now: - if (!referencePath.exists() - && isParameterizedReferenceRepository(reference) - && url != null - && !url.isEmpty() - ) { + if (!referencePath.exists() && isParameterizedReferenceRepository(reference) && url != null && !url.isEmpty()) { // Drop the trailing keyword to know the root refrepo dirname String referenceBaseDir = reference.replaceAll("/\\$\\{GIT_[^\\}]*\\}$", ""); File referenceBaseDirFile = new File(referenceBaseDir); if (!referenceBaseDirFile.exists()) { - LOGGER.log( - Level.WARNING, - "Base Git reference directory " + referenceBaseDir + " does not exist"); + LOGGER.log(Level.WARNING, "Base Git reference directory " + referenceBaseDir + " does not exist"); return null; } if (!referenceBaseDirFile.isDirectory()) { - LOGGER.log( - Level.WARNING, - "Base Git reference directory " + referenceBaseDir + " is not a directory"); + LOGGER.log(Level.WARNING, "Base Git reference directory " + referenceBaseDir + " is not a directory"); return null; } @@ -1107,26 +1092,21 @@ && isParameterizedReferenceRepository(reference) String referenceExpanded = null; if (reference.endsWith("/${GIT_URL_SHA256}")) { // This may be the more portable solution with regard to filesystems - referenceExpanded = reference.replaceAll( - "\\$\\{GIT_URL_SHA256\\}$", - DigestUtils.sha256Hex(urlNormalized)); + referenceExpanded = + reference.replaceAll("\\$\\{GIT_URL_SHA256\\}$", DigestUtils.sha256Hex(urlNormalized)); } else if (reference.endsWith("/${GIT_URL_SHA256_FALLBACK}")) { // The safest option - fall back to parent directory if // the expanded one does not have git repo data right now: // it allows to gradually convert a big combined reference // repository into smaller chunks without breaking builds. - referenceExpanded = reference.replaceAll( - "\\$\\{GIT_URL_SHA256_FALLBACK\\}$", - DigestUtils.sha256Hex(urlNormalized)); - if (getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null - ) { + referenceExpanded = + reference.replaceAll("\\$\\{GIT_URL_SHA256_FALLBACK\\}$", DigestUtils.sha256Hex(urlNormalized)); + if (getObjectsFile(referenceExpanded) == null && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } } else if (reference.endsWith("/${GIT_URL_BASENAME}") - || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}") - ) { + || reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { // This may be the more portable solution with regard to filesystems // First try with original user-provided casing of the URL (if local // dirs were cloned manually) @@ -1140,17 +1120,12 @@ && getObjectsFile(referenceExpanded + ".git") == null needleBasename = needleBasename.replaceAll(".git$", ""); if (reference.endsWith("/${GIT_URL_BASENAME}")) { - referenceExpanded = reference.replaceAll( - "\\$\\{GIT_URL_BASENAME\\}$", - needleBasename); + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", needleBasename); } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { - referenceExpanded = reference.replaceAll( - "\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", - needleBasename); + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", needleBasename); if (url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null - ) { + && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory (only if we do not check urlNormalized separately below) referenceExpanded = referenceBaseDir; } @@ -1158,8 +1133,7 @@ && getObjectsFile(referenceExpanded + ".git") == null if (!url.equals(urlNormalized) && getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null - ) { + && getObjectsFile(referenceExpanded + ".git") == null) { // Retry with automation-ready normalized URL sep = urlNormalized.lastIndexOf("/"); if (sep < 0) { @@ -1170,24 +1144,17 @@ && getObjectsFile(referenceExpanded + ".git") == null needleBasename = needleBasename.replaceAll(".git$", ""); if (reference.endsWith("/${GIT_URL_BASENAME}")) { - referenceExpanded = reference.replaceAll( - "\\$\\{GIT_URL_BASENAME\\}$", - needleBasename); + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME\\}$", needleBasename); } else { // if (reference.endsWith("/${GIT_URL_BASENAME_FALLBACK}")) { - referenceExpanded = reference.replaceAll( - "\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", - needleBasename); + referenceExpanded = reference.replaceAll("\\$\\{GIT_URL_BASENAME_FALLBACK\\}$", needleBasename); if (getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null - ) { + && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } } } - } else if (reference.endsWith("/${GIT_SUBMODULES}") - || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") - ) { + } else if (reference.endsWith("/${GIT_SUBMODULES}") || reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { // Here we employ git submodules - so we can reliably match // remote URLs (easily multiple) to particular modules, and // yet keep separate git index directories per module with @@ -1231,19 +1198,16 @@ && getObjectsFile(referenceExpanded + ".git") == null } LOGGER.log( Level.FINE, - "findParameterizedReferenceRepository(): got referenceExpanded='" - + referenceExpanded + "' from subEntries"); + "findParameterizedReferenceRepository(): got referenceExpanded='" + referenceExpanded + + "' from subEntries"); if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null - ) { + && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } } else { - LOGGER.log( - Level.FINE, - "findParameterizedReferenceRepository(): got no subEntries"); + LOGGER.log(Level.FINE, "findParameterizedReferenceRepository(): got no subEntries"); // If there is no hit, the non-fallback mode suggests a new // directory name to host the submodule (same rules as for // the refrepo forks' co-hosting friendly basename search), @@ -1258,18 +1222,12 @@ && getObjectsFile(referenceExpanded + ".git") == null needleBasename = needleBasename.replaceAll(".git$", ""); if (reference.endsWith("/${GIT_SUBMODULES}")) { - referenceExpanded = reference.replaceAll( - "\\$\\{GIT_SUBMODULES\\}$", - needleBasename); - } - else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { - referenceExpanded = reference.replaceAll( - "\\$\\{GIT_SUBMODULES\\}$", - needleBasename); + referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", needleBasename); + } else { // if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}")) { + referenceExpanded = reference.replaceAll("\\$\\{GIT_SUBMODULES\\}$", needleBasename); if (reference.endsWith("/${GIT_SUBMODULES_FALLBACK}") && getObjectsFile(referenceExpanded) == null - && getObjectsFile(referenceExpanded + ".git") == null - ) { + && getObjectsFile(referenceExpanded + ".git") == null) { // chop it off, use main directory referenceExpanded = referenceBaseDir; } @@ -1288,8 +1246,8 @@ && getObjectsFile(referenceExpanded + ".git") == null LOGGER.log( Level.INFO, - "After resolving the parameterized Git reference repository, " - + "decided to use '" + reference + "' directory for URL '" + url + "'"); + "After resolving the parameterized Git reference repository, " + "decided to use '" + reference + + "' directory for URL '" + url + "'"); } // if referencePath is the replaceable token and not existing directory if (!referencePath.exists() && !reference.endsWith(".git")) { diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index 61deef1667..b33c475369 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -363,8 +363,7 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce // the provided string, as we check in log below } - System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" - + "wsRefrepo='" + wsRefrepo); + System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + "wsRefrepo='" + wsRefrepo); testGitClient .clone_() @@ -398,7 +397,7 @@ public void test_clone_reference_parameterized_sha256() throws Exception, IOExce assertBranchesExist(testGitClient.getBranches(), "master"); assertNoObjectsInRepository(); } // else Skip: Missing if clone failed - currently would, - // with bogus path above and not pre-created path structure + // with bogus path above and not pre-created path structure } @Test @@ -433,8 +432,7 @@ public void test_clone_reference_parameterized_sha256_fallback() // the provided string, as we check in log below } - System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" - + "wsRefrepo='" + wsRefrepo); + System.err.println("wsRefrepoBase='" + wsRefrepoBase + "'\n" + "wsRefrepo='" + wsRefrepo); testGitClient .clone_() @@ -470,7 +468,7 @@ public void test_clone_reference_parameterized_sha256_fallback() assertBranchesExist(testGitClient.getBranches(), "master"); assertNoObjectsInRepository(); } // else Skip: Missing if clone failed - currently would, - // with bogus path above and not pre-created path structure + // with bogus path above and not pre-created path structure } private static final String SRC_DIR = (new File(".")).getAbsolutePath(); @@ -658,13 +656,11 @@ private void assertNoObjectsInRepository() { } // Most tests use this method, expecting a non-bare repo - private void assertAlternateFilePointsToLocalMirror() - throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalMirror() throws IOException, InterruptedException { assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir); } - private void assertAlternateFilePointsToLocalWorkspaceMirror() - throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalWorkspaceMirror() throws IOException, InterruptedException { assertAlternateFilePointsToLocalWorkspaceMirror(testGitDir); } @@ -686,13 +682,11 @@ private void assertAlternateFilePointsToLocalWorkspaceMirror(String _testGitDir, } // Similar for bare repos, without ".git/" dir - private void assertAlternateFilePointsToLocalBareMirror() - throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalBareMirror() throws IOException, InterruptedException { assertAlternateFilePointsToLocalBareMirror(testGitDir); } - private void assertAlternateFilePointsToLocalBareMirror(File _testGitDir) - throws IOException, InterruptedException { + private void assertAlternateFilePointsToLocalBareMirror(File _testGitDir) throws IOException, InterruptedException { assertAlternateFilePointsToLocalBareMirror(_testGitDir.getPath()); } From 4e1f3abb56c42f6b161f9002ebc8c493fa190c2b Mon Sep 17 00:00:00 2001 From: Mark Waite Date: Wed, 13 Sep 2023 06:10:50 -0600 Subject: [PATCH 129/132] Remove redundant null check --- .../java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index f1871b4f17..3aba010838 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -1577,10 +1577,7 @@ public void execute() throws GitException, InterruptedException { listener.error("Invalid repository for " + sModuleName); throw new GitException("Invalid repository for " + sModuleName); } - String strURIish = null; - if (urIish != null) { - strURIish = urIish.toPrivateString(); - } + String strURIish = urIish.toPrivateString(); if (isParameterizedReferenceRepository(ref)) { File referencePath = findParameterizedReferenceRepository(ref, strURIish); From f844f2397fccf654e689863bcf1b52d043d98e9d Mon Sep 17 00:00:00 2001 From: Mark Waite Date: Wed, 13 Sep 2023 06:31:49 -0600 Subject: [PATCH 130/132] Resolve spotbugs warnings for toString() and unread vars --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 49 +++++++------------ 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index f85a8cc1f3..0e9da74446 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -237,8 +237,7 @@ public static File getObjectsFile(File reference) { */ LOGGER.log( Level.FINEST, - "getObjectsFile(): found an fGit '" - + fGit.getAbsolutePath().toString() + "' which is a directory"); + "getObjectsFile(): found an fGit '" + fGit.getAbsolutePath() + "' which is a directory"); } else { // If ".git" FS object exists and is a not-empty file (and // is not a dir), then its content may point to some other @@ -249,8 +248,7 @@ public static File getObjectsFile(File reference) { // "gitdir: ../.git/modules/childRepoName" LOGGER.log( Level.FINEST, - "getObjectsFile(): found an fGit '" - + fGit.getAbsolutePath().toString() + "' which is NOT a directory"); + "getObjectsFile(): found an fGit '" + fGit.getAbsolutePath() + "' which is NOT a directory"); BufferedReader reader = null; try { String line; @@ -265,9 +263,9 @@ public static File getObjectsFile(File reference) { LOGGER.log( Level.FINE, "getObjectsFile(): while looking for 'gitdir:' in '" - + fGit.getAbsolutePath().toString() + + fGit.getAbsolutePath() + "', found reference to objects which should be at: '" - + objects.getAbsolutePath().toString() + "'"); + + objects.getAbsolutePath() + "'"); // Note: we don't use getCanonicalPath() here to avoid further filesystem // access and possible exceptions (the getAbsolutePath() is about string // processing), but callers might benefit from canonicising and ensuring @@ -280,21 +278,20 @@ public static File getObjectsFile(File reference) { } LOGGER.log( Level.FINEST, - "getObjectsFile(): while looking for 'gitdir:' in '" - + fGit.getAbsolutePath().toString() + "', ignoring line: " + line); + "getObjectsFile(): while looking for 'gitdir:' in '" + fGit.getAbsolutePath() + + "', ignoring line: " + line); } } if (objects == null) { LOGGER.log( Level.WARNING, - "getObjectsFile(): failed to parse '" - + fGit.getAbsolutePath().toString() + "': did not contain a 'gitdir:' entry"); + "getObjectsFile(): failed to parse '" + fGit.getAbsolutePath() + + "': did not contain a 'gitdir:' entry"); } } catch (IOException e) { LOGGER.log( Level.SEVERE, - "getObjectsFile(): failed to parse '" - + fGit.getAbsolutePath().toString() + "': " + e.toString()); + "getObjectsFile(): failed to parse '" + fGit.getAbsolutePath() + "': " + e.toString()); objects = null; } try { @@ -304,15 +301,12 @@ public static File getObjectsFile(File reference) { } catch (IOException e) { LOGGER.log( Level.SEVERE, - "getObjectsFile(): failed to close file after parsing '" - + fGit.getAbsolutePath().toString() + "': " + e.toString()); + "getObjectsFile(): failed to close file after parsing '" + fGit.getAbsolutePath() + "': " + + e.toString()); } } } else { - LOGGER.log( - Level.FINEST, - "getObjectsFile(): did not find any checked-out '" - + fGit.getAbsolutePath().toString() + "'"); + LOGGER.log(Level.FINEST, "getObjectsFile(): did not find any checked-out '" + fGit.getAbsolutePath() + "'"); } if (objects == null || !objects.isDirectory()) { @@ -329,19 +323,15 @@ public static File getObjectsFile(File reference) { if (objects.isDirectory()) { LOGGER.log( Level.FINEST, - "getObjectsFile(): found a bare '" - + objects.getAbsolutePath().toString() + "' which is a directory"); + "getObjectsFile(): found a bare '" + objects.getAbsolutePath() + "' which is a directory"); } else { LOGGER.log( Level.FINEST, - "getObjectsFile(): found a bare '" - + objects.getAbsolutePath().toString() + "' which is NOT a directory"); + "getObjectsFile(): found a bare '" + objects.getAbsolutePath() + + "' which is NOT a directory"); } } else { - LOGGER.log( - Level.FINEST, - "getObjectsFile(): did not find any bare '" - + objects.getAbsolutePath().toString() + "'"); + LOGGER.log(Level.FINEST, "getObjectsFile(): did not find any bare '" + objects.getAbsolutePath() + "'"); } } @@ -526,10 +516,7 @@ public SimpleEntry> getSubmodulesUrls( // We want to hit same dirs only once, so canonicize paths below String referenceBaseDirAbs; try { - referenceBaseDirAbs = new File(referenceBaseDir) - .getAbsoluteFile() - .getCanonicalPath() - .toString(); + referenceBaseDirAbs = new File(referenceBaseDir).getAbsoluteFile().getCanonicalPath(); } catch (IOException e) { // Access error while dereferencing some parent?.. referenceBaseDirAbs = new File(referenceBaseDir).getAbsoluteFile().toString(); @@ -930,9 +917,7 @@ public SimpleEntry> getSubmodulesUrls( // Iterating to filter suggestions in order of original // directory-walk prioritization under current reference String dirName = subEntry[0]; - String uri = subEntry[1]; String uriNorm = subEntry[2]; - String remoteName = subEntry[3]; Integer sep; String uriNormBasename; String dirBasename; From 43c38bafcfb6b7505b856f6c91b11e096dd50ad0 Mon Sep 17 00:00:00 2001 From: Mark Waite Date: Wed, 13 Sep 2023 06:33:01 -0600 Subject: [PATCH 131/132] Resolve spotbugs warnings on toLowerCase Use English locale as a UTF-8 capable locale --- .../plugins/gitclient/LegacyCompatibleGitAPIImpl.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 0e9da74446..19027814bc 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; @@ -400,7 +401,7 @@ public static Boolean isParameterizedReferenceRepository(String reference) { value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "Path operations below intentionally use absolute '/' in some cases") public static String normalizeGitUrl(String url, Boolean checkLocalPaths) { - String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(); + String urlNormalized = url.replaceAll("/*$", "").replaceAll(".git$", "").toLowerCase(Locale.ENGLISH); if (!url.contains("://")) { if (!url.startsWith("/") && !url.startsWith(".")) { // Not an URL with schema, not an absolute or relative pathname @@ -605,7 +606,7 @@ public SimpleEntry> getSubmodulesUrls( // FS object. arrDirnames.add(referenceBaseDirAbs + "/" + needleBasename); arrDirnames.add(referenceBaseDirAbs + "/" + needleBasename + ".git"); - needleBasenameLC = needleBasename.toLowerCase(); + needleBasenameLC = needleBasename.toLowerCase(Locale.ENGLISH); if (!needleBasenameLC.equals(needleBasename)) { // Retry with lowercased dirname arrDirnames.add(referenceBaseDirAbs + "/" + needleBasenameLC); From 65b55c5d5b50f2cedaf9fc92c651a77e44debe18 Mon Sep 17 00:00:00 2001 From: Mark Waite Date: Wed, 13 Sep 2023 06:49:51 -0600 Subject: [PATCH 132/132] Assure stream is closed on exception --- .../gitclient/LegacyCompatibleGitAPIImpl.java | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java index 19027814bc..5f2c4617fe 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/LegacyCompatibleGitAPIImpl.java @@ -250,10 +250,9 @@ public static File getObjectsFile(File reference) { LOGGER.log( Level.FINEST, "getObjectsFile(): found an fGit '" + fGit.getAbsolutePath() + "' which is NOT a directory"); - BufferedReader reader = null; - try { + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(new FileInputStream(fGit), "UTF-8"))) { String line; - reader = new BufferedReader(new InputStreamReader(new FileInputStream(fGit), "UTF-8")); while ((line = reader.readLine()) != null) { String[] parts = line.split(":", 2); if (parts.length >= 2) { @@ -295,16 +294,6 @@ public static File getObjectsFile(File reference) { "getObjectsFile(): failed to parse '" + fGit.getAbsolutePath() + "': " + e.toString()); objects = null; } - try { - if (reader != null) { - reader.close(); - } - } catch (IOException e) { - LOGGER.log( - Level.SEVERE, - "getObjectsFile(): failed to close file after parsing '" + fGit.getAbsolutePath() + "': " - + e.toString()); - } } } else { LOGGER.log(Level.FINEST, "getObjectsFile(): did not find any checked-out '" + fGit.getAbsolutePath() + "'");