Added non-conflicting hash for install files (#1454)

Author: MarconZet

BUILD

@@ -6,6 +6,7 @@ exports_files([
     "defs.bzl",
     "extensions.bzl",
     "specs.bzl",
+    "maven_install.json",
 ])
 
 licenses(["notice"])  # Apache 2.0

maven_install.json

@@ -1,7 +1,36 @@
 {
   "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL",
-  "__INPUT_ARTIFACTS_HASH": 1827703030,
-  "__RESOLVED_ARTIFACTS_HASH": -1607032127,
+  "__INPUT_ARTIFACTS_HASH": {
+    "com.google.code.gson:gson": -1146526807,
+    "com.google.errorprone:error_prone_annotations": -571311395,
+    "com.google.guava:guava": -1756621521,
+    "com.google.protobuf:protobuf-java": -154599721,
+    "com.google.protobuf:protobuf-java-util": -1033086717,
+    "io.netty:netty-tcnative-boringssl-static": -1979050407,
+    "org.hamcrest:hamcrest-core": 466791695,
+    "repositories": -1949687017
+  },
+  "__RESOLVED_ARTIFACTS_HASH": {
+    "com.google.code.findbugs:jsr305": 870839855,
+    "com.google.code.gson:gson": 50257904,
+    "com.google.errorprone:error_prone_annotations": 1527418394,
+    "com.google.guava:failureaccess": 1715931538,
+    "com.google.guava:guava": -1587873388,
+    "com.google.guava:listenablefuture": 1079558157,
+    "com.google.j2objc:j2objc-annotations": -1008747351,
+    "com.google.protobuf:protobuf-java": 657144169,
+    "com.google.protobuf:protobuf-java-util": 291568724,
+    "io.netty:netty-tcnative-boringssl-static": 786460467,
+    "io.netty:netty-tcnative-boringssl-static:jar:linux-aarch_64": -151974322,
+    "io.netty:netty-tcnative-boringssl-static:jar:linux-x86_64": -1831640381,
+    "io.netty:netty-tcnative-boringssl-static:jar:osx-aarch_64": -1661340718,
+    "io.netty:netty-tcnative-boringssl-static:jar:osx-x86_64": 2101324017,
+    "io.netty:netty-tcnative-boringssl-static:jar:windows-x86_64": 889950966,
+    "io.netty:netty-tcnative-classes": 1239547355,
+    "org.hamcrest:hamcrest": -533823501,
+    "org.hamcrest:hamcrest-core": 511008887,
+    "org.jspecify:jspecify": 117231129
+  },
   "conflict_resolution": {
     "com.google.errorprone:error_prone_annotations:2.23.0": "com.google.errorprone:error_prone_annotations:2.41.0"
   },
@@ -260,5 +289,5 @@
     ]
   },
   "services": {},
-  "version": "2"
+  "version": "3"
 }

private/extensions/maven.bzl

@@ -14,7 +14,7 @@ load("//private/rules:coursier.bzl", "DEFAULT_AAR_IMPORT_LABEL", "coursier_fetch
 load("//private/rules:maven_version.bzl", "compare_maven_versions")
 load("//private/rules:unpinned_maven_pin_command_alias.bzl", "unpinned_maven_pin_command_alias")
 load("//private/rules:v1_lock_file.bzl", "v1_lock_file")
-load("//private/rules:v2_lock_file.bzl", "v2_lock_file")
+load("//private/rules:v3_lock_file.bzl", "v2_lock_file", "v3_lock_file")
 load(":download_pinned_deps.bzl", "download_pinned_deps")
 
 DEFAULT_REPOSITORIES = [
@@ -755,12 +755,15 @@ def maven_impl(mctx):
                     "artifacts": {},
                     "dependencies": {},
                     "repositories": {},
-                    "version": "2",
+                    "version": "3",
                 }
             else:
                 lock_file = json.decode(lock_file_content)
 
-            if v2_lock_file.is_valid_lock_file(lock_file):
+            if v3_lock_file.is_valid_lock_file(lock_file):
+                artifacts = v3_lock_file.get_artifacts(lock_file)
+                importer = v3_lock_file
+            elif v2_lock_file.is_valid_lock_file(lock_file):
                 artifacts = v2_lock_file.get_artifacts(lock_file)
                 importer = v2_lock_file
             elif v1_lock_file.is_valid_lock_file(lock_file):

private/rules/coursier.bzl

@@ -33,7 +33,7 @@ load(
 load("//private/lib:coordinates.bzl", "to_key", "unpack_coordinates")
 load("//private/lib:urls.bzl", "remove_auth_from_url")
 load("//private/rules:v1_lock_file.bzl", "v1_lock_file")
-load("//private/rules:v2_lock_file.bzl", "v2_lock_file")
+load("//private/rules:v3_lock_file.bzl", "v2_lock_file", "v3_lock_file")
 
 _BUILD = """
 # package(default_visibility = [{visibilities}])  # https://github.com/bazelbuild/bazel/issues/13681
@@ -341,6 +341,13 @@ def _stable_artifact(artifact):
     keys = sorted(parsed.keys())
     return ":".join(["%s=%s" % (key, parsed[key]) for key in keys])
 
+def _add_to_hash_dictionary(dictionary, artifact, salt):
+    artifact_dict = json.decode(artifact)
+    key = artifact_dict["group"] + ":" + artifact_dict["artifact"]
+    value = dictionary.get(key, [])
+    value.append(hash(_stable_artifact(artifact) + salt))
+    dictionary[key] = value
+
 # Compute a signature of the list of artifacts that will be used to build
 # the dependency tree. This is used as a check to see whether the dependency
 # tree needs to be repinned.
@@ -359,24 +366,37 @@ def compute_dependency_inputs_signature(boms = [], artifacts = [], repositories
     artifact_inputs = []
     excluded_artifact_inputs = []
 
+    all_hashes = dict()
+
     if boms and len(boms):
         for bom in sorted(boms):
             artifact_inputs.append(_stable_artifact(bom))
+            _add_to_hash_dictionary(all_hashes, bom, "bom")
 
     for artifact in sorted(artifacts):
         artifact_inputs.append(_stable_artifact(artifact))
+        _add_to_hash_dictionary(all_hashes, artifact, "artifact")
 
     for artifact in sorted(excluded_artifacts):
         excluded_artifact_inputs.append(_stable_artifact(artifact))
+        _add_to_hash_dictionary(all_hashes, artifact, "excluded_artifact")
 
     v1_sig = hash(repr(sorted(artifact_inputs))) ^ hash(repr(sorted(repositories)))
 
     hash_parts = [sorted(artifact_inputs), sorted(repositories), sorted(excluded_artifact_inputs)]
-    current_version_sig = 0
+    v2_sig = 0
     for part in hash_parts:
-        current_version_sig ^= hash(repr(part))
+        v2_sig ^= hash(repr(part))
+
+    for k, v in all_hashes.items():
+        if len(v) == 1:
+            all_hashes[k] = v[0]
+        else:
+            all_hashes[k] = hash(repr(sorted(v)))
 
-    return (current_version_sig, [v1_sig])
+    all_hashes["repositories"] = hash(repr(sorted(repositories)))
+
+    return (all_hashes, [v1_sig, v2_sig])
 
 def get_netrc_lines_from_entries(netrc_entries):
     netrc_lines = []
@@ -540,21 +560,26 @@ def _pinned_coursier_fetch_impl(repository_ctx):
             "artifacts": {},
             "dependencies": {},
             "repositories": {},
-            "version": "2",
+            "version": "3",
         }
     else:
         maven_install_json_content = json.decode(lock_file_content)
 
-    if v1_lock_file.is_valid_lock_file(maven_install_json_content):
+    if v3_lock_file.is_valid_lock_file(maven_install_json_content):
+        importer = v3_lock_file
+    elif v2_lock_file.is_valid_lock_file(maven_install_json_content):
+        importer = v2_lock_file
+    elif v1_lock_file.is_valid_lock_file(maven_install_json_content):
         importer = v1_lock_file
+    else:
+        fail("Unable to read lock file: %s" % repository_ctx.attr.maven_install_json)
+
+    # Check if using the most recent lock file format.
+    if importer != v3_lock_file:
         print_if_not_repinning(
             repository_ctx,
             "Lock file should be updated. Please run `REPIN=1 bazel run @unpinned_%s//:pin`" % repository_ctx.name,
         )
-    elif v2_lock_file.is_valid_lock_file(maven_install_json_content):
-        importer = v2_lock_file
-    else:
-        fail("Unable to read lock file: %s" % repository_ctx.attr.maven_install_json)
 
     # Validation steps for maven_install.json.
 
@@ -612,10 +637,10 @@ def _pinned_coursier_fetch_impl(repository_ctx):
             )
         elif computed_artifacts_hash != input_artifacts_hash:
             if _get_fail_if_repin_required(repository_ctx):
-                fail("%s_install.json contains an invalid input signature (expected %s and got %s) and must be regenerated. " % (
+                to_print = importer.print_friendly_hash_difference(input_artifacts_hash, computed_artifacts_hash)
+                fail("%s_install.json contains an invalid input signature (%s) and must be regenerated. " % (
                          user_provided_name,
-                         input_artifacts_hash,
-                         computed_artifacts_hash,
+                         to_print,
                      ) +
                      "This typically happens when the maven_install artifacts have been changed but not repinned. " +
                      "PLEASE DO NOT MODIFY THIS FILE DIRECTLY! To generate a new " +
@@ -641,11 +666,12 @@ def _pinned_coursier_fetch_impl(repository_ctx):
         # Then, validate that the signature provided matches the contents of the dependency_tree.
         # This is to stop users from manually modifying maven_install.json.
         if _get_fail_if_repin_required(repository_ctx):
+            computed_hash = importer.compute_lock_file_hash(maven_install_json_content)
+            to_print = importer.print_friendly_hash_difference(dep_tree_signature, computed_hash)
             fail(
-                "%s_install.json contains an invalid signature (expected %s and got %s) and may be corrupted. " % (
+                "%s_install.json contains an invalid signature (%s) and may be corrupted. " % (
                     user_provided_name,
-                    dep_tree_signature,
-                    importer.compute_lock_file_hash(maven_install_json_content),
+                    to_print,
                 ) +
                 "PLEASE DO NOT MODIFY THIS FILE DIRECTLY! To generate a new " +
                 "%s_install.json and re-pin the artifacts, follow these steps: \n\n" % user_provided_name +
@@ -1415,7 +1441,7 @@ def _coursier_fetch_impl(repository_ctx):
 
     repository_ctx.file(
         "unsorted_deps.json",
-        content = v2_lock_file.render_lock_file(
+        content = v3_lock_file.render_lock_file(
             lock_file_contents,
             inputs_hash,
         ),
@@ -1424,7 +1450,7 @@ def _coursier_fetch_impl(repository_ctx):
     repository_ctx.report_progress("Generating BUILD targets..")
     (generated_imports, jar_versionless_target_labels) = parser.generate_imports(
         repository_ctx = repository_ctx,
-        dependencies = v2_lock_file.get_artifacts(lock_file_contents),
+        dependencies = v3_lock_file.get_artifacts(lock_file_contents),
         explicit_artifacts = {
             a["group"] + ":" + a["artifact"] + (":" + a["classifier"] if "classifier" in a else ""): True
             for a in artifacts

private/rules/pin_dependencies.bzl

@@ -15,7 +15,7 @@ load("//private/rules:coursier.bzl", "compute_dependency_inputs_signature")
 
 _TEMPLATE = """#!/usr/bin/env bash
 
-{resolver_cmd} --jvm_flags={jvm_flags} --argsfile {config} --input_hash '{input_hash}' --output {output}{dependency_index_output}
+{resolver_cmd} --jvm_flags={jvm_flags} --argsfile {config} --input-hash-path '{input_hash_path}' --output {output}{dependency_index_output}
 """
 
 def _stringify_exclusions(exclusions):
@@ -63,13 +63,19 @@ def _pin_dependencies_impl(ctx):
         content = json.encode_indent(config, indent = "  "),
     )
 
-    input_hash = compute_dependency_inputs_signature(
+    input_hash, _ = compute_dependency_inputs_signature(
         boms = ctx.attr.boms,
         artifacts = ctx.attr.artifacts,
         repositories = ctx.attr.repositories,
         excluded_artifacts = ctx.attr.excluded_artifacts,
     )
 
+    hash_file = ctx.actions.declare_file("%s-input-hash.json" % ctx.label.name)
+    ctx.actions.write(
+        hash_file,
+        content = json.encode_indent(input_hash, indent = "  "),
+    )
+
     dependency_index_output = ""
     if ctx.attr.dependency_index:
         dependency_index_output = " --dependency-index-output $BUILD_WORKSPACE_DIRECTORY/" + ctx.attr.dependency_index
@@ -79,7 +85,7 @@ def _pin_dependencies_impl(ctx):
         script,
         _TEMPLATE.format(
             config = config_file.short_path,
-            input_hash = input_hash[0],
+            input_hash_path = hash_file.short_path,
             resolver_cmd = ctx.executable.resolver.short_path,
             output = "$BUILD_WORKSPACE_DIRECTORY/" + ctx.attr.lock_file,
             dependency_index_output = dependency_index_output,
@@ -92,7 +98,7 @@ def _pin_dependencies_impl(ctx):
         DefaultInfo(
             executable = script,
             files = depset([script, config_file]),
-            runfiles = ctx.runfiles(files = [script, config_file]).merge(ctx.attr.resolver[DefaultInfo].default_runfiles),
+            runfiles = ctx.runfiles(files = [script, config_file, hash_file]).merge(ctx.attr.resolver[DefaultInfo].default_runfiles),
         ),
     ]
 

private/rules/v1_lock_file.bzl

@@ -42,6 +42,11 @@ def _get_lock_file_hash(lock_file_contents):
     dep_tree = lock_file_contents.get("dependency_tree", {})
     return dep_tree.get("__RESOLVED_ARTIFACTS_HASH")
 
+def _print_friendly_hash_difference(old_hash, new_hash):
+    if old_hash == new_hash:
+        return ""
+    return "expected %s and got %s" % (old_hash, new_hash)
+
 # The representation of a Windows path when read from the parsed Coursier JSON
 # is delimited by 4 back slashes. Replace them with 1 forward slash.
 def _normalize_to_unix_path(path):
@@ -138,6 +143,7 @@ v1_lock_file = struct(
     is_valid_lock_file = _is_valid_lock_file,
     get_input_artifacts_hash = _get_input_artifacts_hash,
     get_lock_file_hash = _get_lock_file_hash,
+    print_friendly_hash_difference = _print_friendly_hash_difference,
     compute_lock_file_hash = _compute_lock_file_hash,
     get_artifacts = _get_artifacts,
     get_netrc_entries = _get_netrc_entries,