SOLR-17602: Per-Module Dependency Locking

.gitattributes

@@ -1,5 +1,6 @@
-# Ignore all differences in line endings for the lock file.
+# Ignore all differences in line endings for lock files.
 versions.lock text eol=lf
+**/*.lockfile text eol=lf
 
 # Gradle files are always in LF.
 *.gradle          text eol=lf

.github/labeler.yml

@@ -2,10 +2,11 @@
 dependencies:
   - changed-files:
       - any-glob-to-any-file:
-        - gradle/libs.versions.toml # Solr 10+
-        - versions.props # Solr < v10
-        - versions.lock
-        - solr/licenses/**
+        - "gradle/libs.versions.toml" # Solr 10+
+        - "**/*.lockfile"
+        - "versions.props" # Solr < v10
+        - "versions.lock"
+        - "solr/licenses/**"
 
 # Add 'documentation' label to any changes within ref-guide or dev-docs
 documentation:

.github/renovate.json

@@ -4,9 +4,9 @@
   "enabled": true,
   "dependencyDashboard": false,
   "enabledManagers": ["gradle", "github-actions"],
-  "includePaths": ["versions.*", "build.gradle", ".github/workflows/*"],
+  "includePaths": ["gradle/libs.versions.toml", "versions.*", "build.gradle", ".github/workflows/*"],
   "postUpgradeTasks": {
-    "commands": ["./gradlew updateLicenses"],
+    "commands": ["./gradlew resolveAndLockAll", "./gradlew updateLicenses"],
     "fileFilters": ["solr/licenses/*.sha1"],
     "executionMode": "branch"
   },

.github/workflows/bin-solr-test.yml

@@ -38,7 +38,7 @@ jobs:
       with:
         path: |
           ~/.gradle/caches
-        key: ${{ runner.os }}-gradle-binsolr-${{ hashFiles('versions.lock') }}
+        key: ${{ runner.os }}-gradle-binsolr-${{ hashFiles('**/*.lockfile') }}
         restore-keys: |
           ${{ runner.os }}-gradle-binsolr-
           ${{ runner.os }}-gradle-

.github/workflows/docker-test.yml

@@ -42,7 +42,7 @@ jobs:
       with:
         path: |
           ~/.gradle/caches
-        key: ${{ runner.os }}-gradle-docker-${{ hashFiles('versions.lock') }}
+        key: ${{ runner.os }}-gradle-docker-${{ hashFiles('**/*.lockfile') }}
         restore-keys: |
           ${{ runner.os }}-gradle-docker-
           ${{ runner.os }}-gradle-

.github/workflows/gradle-precommit.yml

@@ -36,7 +36,7 @@ jobs:
       with:
         path: |
           ~/.gradle/caches
-        key: ${{ runner.os }}-gradle-precommit-${{ hashFiles('versions.lock') }}
+        key: ${{ runner.os }}-gradle-precommit-${{ hashFiles('**/*.lockfile') }}
         restore-keys: |
           ${{ runner.os }}-gradle-precommit-
           ${{ runner.os }}-gradle-

.github/workflows/solrj-test.yml

@@ -35,7 +35,7 @@ jobs:
       with:
         path: |
           ~/.gradle/caches
-        key: ${{ runner.os }}-gradle-solrj-${{ hashFiles('versions.lock') }}
+        key: ${{ runner.os }}-gradle-solrj-${{ hashFiles('**/*.lockfile') }}
         restore-keys: |
           ${{ runner.os }}-gradle-solrj-
           ${{ runner.os }}-gradle-

NOTICE.txt

@@ -1,6 +1,6 @@
 ==============================================================
  Apache Solr
- Copyright 2006-2024 The Apache Software Foundation
+ Copyright 2006-2025 The Apache Software Foundation
 ==============================================================
 
 This product includes software developed at

dev-docs/dependency-upgrades.adoc

@@ -30,43 +30,40 @@ In order to upgrade a dependency, you need to run through a number of steps:
 
 1. Identify the available versions from e.g. https://search.maven.org[Maven Central]
 2. Update the version in `gradle/libs.versions.toml` file
-3. Run `./gradlew writeLocks` to re-generate `versions.lock`. Note that this may cause a cascading effect where
+3. Run `./gradlew resolveAndLockAll` to re-generate lockfiles. Note that this may cause a cascading effect where
    the locked version of other dependencies also change.
 4. In case of a conflict, resolve the conflict according to `help/dependencies.txt`
-5. Check if there are any constraints that are obsolete after the dependency update
-6. Update the license and notice files of the changed dependencies. See `help/dependencies.txt` for
-   details.
-7. Run `./gradlew updateLicenses` to re-generate SHA1 checksums of the new jar files.
-8. Once in a while, a new version of a dependency will transitively bring in brand-new dependencies.
+5. Update the license and notice files of the changed dependencies. See `help/dependencies.txt` for details.
+6. Run `./gradlew updateLicenses` to re-generate SHA1 checksums of the new jar files.
+7. Once in a while, a new version of a dependency will transitively bring in brand-new dependencies.
    You'll need to decide whether to keep or exclude them. See `help/dependencies.txt` for details.
 
-=== Reviewing Constraints
+=== Constraints and Version Alignment
 
-The constraints are defined in gradle/validation/dependencies.gradle. There, if the updated dependency is listed,
-the constraint can be reviewed, updated or removed.
+To sync the version of direct and transitive dependencies across the project, we iterate in the `:platform` module
+over the libraries defined in `gradle/libs.version.toml` and add them as constraints. Then, we use the module in
+main modules like `:solr:api` and `:solr:core` and transitively pass down to all other modules the constraints.
 
-The constraints fall into two "groups". In the first group there are dependency constraints from dependencies
-that our project directly includes and require version alignment to sync the versions across all transitive
-dependencies. In the second group are dependencies that are only present as transitive dependencies.
-There, we try to follow the convention to provide additional information with "which dependencies use what version",
-so that the next person reviewing the constraint does not have to look it up. However, this is quite time-consuming
-to analyze the dependencies and therefore subject to change.
+If a new module does not depend on another module that already includes `:platform` as a platform dependency, it should
+explicitly add it to sync the versions with the rest of the project. `:solr:server` is one case where this is necessary.
 
-In order to review a constraint, you have to check if the updated dependency is mentioned in any of the constraints,
-either as a reason for another dependency constraint or as the constraint's dependency. Removing temporarily
-a constraint, the task writeLocks will fail if the constraint is still required.
+=== Addressing Security Vulnerabilities
 
-This process and the constraints of dependencies.gradle are not optimal, as it is quite time-consuming and not obvious
-by just looking at it. We just haven't found yet a more efficient way to maintain these constraints.
+When it comes to security vulnerabilities that are found in direct or transitive dependencies, the recommended way to
+address them is to update the specific library if there is a new release that solves this issue. For both direct and
+transitive dependencies, we simply have to update the version as described above.
 
-== Renovate bot Pull Requests
+In case it is a transitive dependency that is not directly used, you can simply add it to `libs.versions.toml` as you
+would with any other dependency. The dependency resolution approach defined in `:platform` will handle the rest.
+Don't forget to add a `# @keep` note with a reference to the vulnerable version and CVE that is fixed with the explicit
+definition of the library and new version. This way it is easier to keep track of unreferenced dependencies in our
+libraries toml file, and we can clean them up once the libraries using the modules are updated.
 
-The renovate bot may be replaced in the future with dependabot and this section may only be relevant for older
-versions (<10.0). See https://lists.apache.org/thread/1sb9ttv3lp57z2yod1htx1fykp5sj73z for updates.
+== Renovate bot Pull Requests
 
 A member of the Solr community operates a Github bot running https://github.com/renovatebot/renovate[Renovate], which
 files Pull Requests to Solr with dependency upgrade proposals. The PRs are labeled `dependencies` and do include
-changes resulting from `./gradlew writeLocks` and `updateLicenses`.
+changes resulting from the gradle tasks `resolveAndLockAll` and `updateLicenses`.
 
 Community members and committers can then review, and if manual changes are needed, help bring the PR to completion.
 For many dependencies, a changelog is included in the PR text, which may help guide the upgrade decision.
@@ -78,9 +75,13 @@ that will get its own separate Pull Request, so you can choose.
 If an upgrade is decided, simply merge (and backport) the PR. To skip an upgrade, close the PR. If a PR is left open,
 it will be re-used and auto updated whenever a newer patch- or minor version gets available. Thus, one can reduce
 churn from frequently-updated dependencies by delaying merge until a few weeks before a new release. One can also
-choose to change to a less frequent schedule or disable the bot, by editing `renovate.json`
+choose to change to a less frequent schedule or disable the bot, by editing `renovate.json`.
+
+Please note that Solr version prior to 10.X use a versions resolution plugin that uses `versions.lock` instead of
+`libs.version.toml`. Therefore, changes cannot be backported via cherry-pick.
 
 === Configuring renovate.json
+
 While the bot runs on a https://github.com/solrbot/renovate-github-action[GitHub repo external to the project],
 the bot behavior can be tailored by editing `.github/renovate.json` in this project.
 See https://docs.renovatebot.com[Renovatebot docs] for available options.

dev-docs/lucene-upgrade.md

@@ -27,17 +27,17 @@ Create a new branch locally e.g. `git checkout -b lucene940 -t origin/main` for
 
 ## Build
 
-### `versions.props` update
+### `gradle/libs.versions.toml` update
 
 ```
 - org.apache.lucene:*=9.3.0
 + org.apache.lucene:*=9.4.0
 ```
 
-### `versions.lock` update
+### lockfiles update
 
 ```
-gradlew --write-locks
+gradlew :resolveAndLockAll
 ```
 
 ### `solr/licenses` update

dev-tools/scripts/README.md

@@ -111,7 +111,7 @@ of the other tools in this folder.
 
     optional arguments:
       -h, --help         show this help message and exit
-      -l LUCENE_VERSION  Optional lucene version. By default will read versions.props
+      -l LUCENE_VERSION  Optional lucene version. By default will read gradle/libs.versions.toml
 
 ### releasedJirasRegex.py
 

dev-tools/scripts/addVersion.py

@@ -23,7 +23,7 @@
 import argparse
 import re
 from configparser import ConfigParser, ExtendedInterpolation
-from textwrap import dedent
+
 
 def update_changes(filename, new_version, init_changes, headers):
   print('  adding new section to %s...' % filename, end='', flush=True)
@@ -115,7 +115,7 @@ def check_lucene_match_version_tests():
 def read_config(current_version, current_lucene_version):
   parser = argparse.ArgumentParser(description='Add a new version to CHANGES, to Version.java, build.gradle and solrconfig.xml files')
   parser.add_argument('version', type=Version.parse, help='New Solr version')
-  parser.add_argument('-l', dest='lucene_version', type=Version.parse, help='Optional lucene version. By default will read versions.props')
+  parser.add_argument('-l', dest='lucene_version', type=Version.parse, help='Optional lucene version. By default will read gradle/libs.versions.toml')
   newconf = parser.parse_args()
   if not newconf.lucene_version:
     newconf.lucene_version = current_lucene_version

dev-tools/scripts/parseContributorsFromChanges.py

@@ -0,0 +1,63 @@
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import sys
+import re
+from collections import defaultdict
+
+# Read data from standard input
+data = sys.stdin.read()
+
+# Replace all carriage return line feed (Windows) with line feed
+data = data.replace('\r\n', '\n')
+
+# Replace all carriage return (Mac OS before X) with line feed
+data = data.replace('\r', '\n')
+
+# Split data at blank lines
+paras = data.split('\n\n')
+
+# Initialize a default dictionary to store contributors and their counts
+contributors = defaultdict(int)
+
+# Regular expression to find the attribution in parentheses at the end of a line
+pattern = re.compile(r"\(([^()]*)\)$")
+
+for para in paras:
+  # Normalize whitespace (replace all whitespace with a single space)
+  para = re.sub('\s+', ' ', para).strip()
+  #print(f'> {para}')
+
+  # Find all contributors in the line
+  match = pattern.search(para.strip())
+  if match:
+    attribution = match.group(1)
+    # might have a "via" committer; we only want the author here
+    attribution = attribution.split(" via ")[0] # keep left side
+    # Split the contributors by comma and strip whitespace
+    for contributor in attribution.split(','):
+      contributor = contributor.strip()
+      contributors[contributor] += 1
+
+del contributors['solrbot']
+
+sorted_contributors = sorted(contributors.items(), key=lambda item: item[1], reverse=True)
+
+# Print the contributors and their counts
+for contributor, count in sorted_contributors:
+  print(f'{contributor}: {count}')
+
+print('\n\nThanks to all contributors!: ')
+print(', '.join([contributor for contributor, count in sorted_contributors]))

dev-tools/scripts/scriptutil.py

@@ -14,13 +14,16 @@
 # limitations under the License.
 
 import argparse
+import os
 import re
 import subprocess
 import sys
-import os
-from enum import Enum
 import time
-import urllib.request, urllib.error, urllib.parse
+import urllib.error
+import urllib.parse
+import urllib.request
+from enum import Enum
+
 
 class Version(object):
   def __init__(self, major, minor, bugfix, prerelease):
@@ -179,7 +182,7 @@ def attemptDownload(urlString, fileName):
 
 version_prop_re = re.compile(r'baseVersion\s*=\s*([\'"])(.*)\1')
 
-lucene_version_prop_re = re.compile(r'org\.apache\.lucene:\*=(.*?)\n')
+lucene_version_prop_re = re.compile(r'^apache-lucene\s*=\s*"([a-zA-Z0-9\.\-]+)"')
 
 def find_current_version():
   script_path = os.path.dirname(os.path.realpath(__file__))
@@ -190,7 +193,7 @@ def find_current_version():
 def find_current_lucene_version():
   script_path = os.path.dirname(os.path.realpath(__file__))
   top_level_dir = os.path.join(os.path.abspath("%s/" % script_path), os.path.pardir, os.path.pardir)
-  versions_file = open('%s/versions.props' % top_level_dir).read()
+  versions_file = open('%s/gradle/libs.versions.toml' % top_level_dir).read()
   return lucene_version_prop_re.search(versions_file).group(1).strip()