From fa45261c85987fa78f2bb469a5d92b63b8ec120e Mon Sep 17 00:00:00 2001 From: Zain Asgar Date: Mon, 19 Apr 2021 20:24:28 -0700 Subject: [PATCH] Remove bcc submodule Summary: This moves BCC to be a git repo in bazel. Note: we need to use it as a git module because bcc has submodules. Test Plan: existing Reviewers: michelle, oazizi, vihang, #third_party_approvers Reviewed By: vihang, #third_party_approvers Differential Revision: https://phab.corp.pixielabs.ai/D8291 GitOrigin-RevId: c8017456e1ae60275906679195e7a862e7fde41d --- .gitmodules | 3 -- BUILD.bazel | 8 ----- bazel/external/bcc.BUILD | 49 ++++++++++++++++++++++++++++++ bazel/external/bpftrace.BUILD | 4 +-- bazel/repositories.bzl | 35 ++++++++++++++++++++- bazel/repository_locations.bzl | 8 +++++ src/stirling/bpf_tools/BUILD.bazel | 2 +- third_party/BUILD.bazel | 6 ---- third_party/foreign_cc/BUILD.bazel | 23 -------------- tools/licenses/BUILD.bazel | 16 ---------- 10 files changed, 94 insertions(+), 60 deletions(-) create mode 100644 bazel/external/bcc.BUILD diff --git a/.gitmodules b/.gitmodules index 0aacb1fb8ca..e69de29bb2d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "third_party/bcc"] - path = third_party/bcc - url = git@github.com:pixie-labs/bcc.git diff --git a/BUILD.bazel b/BUILD.bazel index f9d2541d07b..ecaa2858c08 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -196,14 +196,6 @@ list_pl_deps( name = "pl_3p_deps", ) -genrule( - name = "pl_3p_submodules", - srcs = [".gitmodules"], - outs = ["pl_3p_submodules.out"], - cmd = "git config --file $< --get-regexp url | cut -d' ' -f2 > $@", - visibility = ["//visibility:public"], -) - genrule( name = "pl_3p_go_sum", srcs = ["go.sum"], diff --git a/bazel/external/bcc.BUILD b/bazel/external/bcc.BUILD new file mode 100644 index 00000000000..58e49e560cb --- /dev/null +++ b/bazel/external/bcc.BUILD @@ -0,0 +1,49 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +load("@rules_foreign_cc//foreign_cc:defs.bzl", "cmake") + +licenses(["notice"]) + +filegroup( + name = "bcc_source", + srcs = glob(["**/*"]), +) + +cmake( + name = "bcc", + generate_crosstool_file = True, + lib_name = "libbcc_static", + lib_source = ":bcc_source", + # These link opts are dependencies of bcc. + linkopts = [ + # ELF binary parsing. + "-lelf", + ], + make_commands = [ + "make -j$(nproc) -C src/cc install", + ], + out_static_libs = [ + "libapi-static.a", + "libbcc.a", + "libbcc_bpf.a", + "libbcc-loader-static.a", + "libb_frontend.a", + "libclang_frontend.a", + "libusdt-static.a", + ], + visibility = ["//visibility:public"], +) diff --git a/bazel/external/bpftrace.BUILD b/bazel/external/bpftrace.BUILD index decd1f69894..ee3343e37ec 100644 --- a/bazel/external/bpftrace.BUILD +++ b/bazel/external/bpftrace.BUILD @@ -97,7 +97,7 @@ cc_library( deps = [ ":bpftrace-parser", "@com_llvm_lib//:llvm", - "@px//third_party/foreign_cc:bcc", + "@com_github_iovisor_bcc//:bcc", ], ) @@ -123,7 +123,7 @@ cc_library( ], tags = ["linux_only"], deps = [ - "@px//third_party/foreign_cc:bcc", + "@com_github_iovisor_bcc//:bcc", ], ) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 68165dc5d19..911d7103db0 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -14,8 +14,9 @@ # # SPDX-License-Identifier: Apache-2.0 +load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load(":repository_locations.bzl", "REPOSITORY_LOCATIONS") +load(":repository_locations.bzl", "GIT_REPOSITORY_LOCATIONS", "REPOSITORY_LOCATIONS") # Make all contents of an external repository accessible under a filegroup. # Used for external HTTP archives, e.g. cares. @@ -43,6 +44,30 @@ def _repo_impl(name, **kwargs): **kwargs ) +def _git_repo_impl(name, **kwargs): + # `existing_rule_keys` contains the names of repositories that have already + # been defined in the Bazel workspace. By skipping repos with existing keys, + # users can override dependency versions by using standard Bazel repository + # rules in their WORKSPACE files. + existing_rule_keys = native.existing_rules().keys() + if name in existing_rule_keys: + # This repository has already been defined, probably because the user + # wants to override the version. Do nothing. + return + + location = GIT_REPOSITORY_LOCATIONS[name] + + # HTTP tarball at a given URL. Add a BUILD file if requested. + new_git_repository( + name = name, + remote = location["remote"], + commit = location["commit"], + init_submodules = True, + recursive_init_submodules = True, + shallow_since = location.get("shallow_since", ""), + **kwargs + ) + # For bazel repos do not require customization. def _bazel_repo(name, **kwargs): _repo_impl(name, **kwargs) @@ -83,6 +108,8 @@ def _cc_deps(): _include_all_repo("com_github_libuv_libuv", patches = ["//bazel/external:libuv.patch"], patch_args = ["-p1"]) _include_all_repo("com_github_libarchive_libarchive") + _git_repo_impl("com_github_iovisor_bcc", build_file = "//bazel/external:bcc.BUILD") + _repo_impl("com_github_apache_arrow", build_file = "//bazel/external:arrow.BUILD") _repo_impl("com_github_ariafallah_csv_parser", build_file = "//bazel/external:csv_parser.BUILD") _repo_impl("com_github_arun11299_cpp_jwt", build_file = "//bazel/external:cpp_jwt.BUILD") @@ -119,6 +146,12 @@ def list_pl_deps(name): best_url = url repo_urls.append(best_url) + for repo_name, repo_config in GIT_REPOSITORY_LOCATIONS.items(): + remote = repo_config["remote"] + if remote.endswith(".git"): + remote = remote[:-len(".git")] + repo_urls.append(remote) + native.genrule( name = name, outs = ["{}.out".format(name)], diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index e247560f557..378c663d2e8 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -270,3 +270,11 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/pixie-labs/bpftrace/archive/da17ebda1090b5e09dc89d69a3afdb580d486670.tar.gz"], ), ) + +GIT_REPOSITORY_LOCATIONS = dict( + com_github_iovisor_bcc = dict( + remote = "https://github.com/pixie-labs/bcc.git", + commit = "c50098cadc54a602a7e423ae958dca8562ee2cf9", + shallow_since = "1618606641 -0700", + ), +) diff --git a/src/stirling/bpf_tools/BUILD.bazel b/src/stirling/bpf_tools/BUILD.bazel index 82ee436cb84..3a3517cca0f 100644 --- a/src/stirling/bpf_tools/BUILD.bazel +++ b/src/stirling/bpf_tools/BUILD.bazel @@ -40,7 +40,7 @@ pl_cc_library( "//src/stirling/bpf_tools/bcc_bpf:task_struct_mem_read", ] + select({ "@bazel_tools//src/conditions:linux_x86_64": [ - "//third_party/foreign_cc:bcc", + "@com_github_iovisor_bcc//:bcc", "@com_github_iovisor_bpftrace//:bpftrace", "//:llvm", ], diff --git a/third_party/BUILD.bazel b/third_party/BUILD.bazel index 32d323103b2..bcaba742cdd 100644 --- a/third_party/BUILD.bazel +++ b/third_party/BUILD.bazel @@ -18,12 +18,6 @@ load("@rules_cc//cc:defs.bzl", "cc_library") licenses(["notice"]) -filegroup( - name = "bcc_source", - srcs = glob(["bcc/**"]), - visibility = ["//visibility:public"], -) - cc_library( name = "clang_tidy_stub", srcs = ["clang_tidy_stub.cc"], diff --git a/third_party/foreign_cc/BUILD.bazel b/third_party/foreign_cc/BUILD.bazel index 8634637a7fc..4cf081dfc31 100644 --- a/third_party/foreign_cc/BUILD.bazel +++ b/third_party/foreign_cc/BUILD.bazel @@ -96,26 +96,3 @@ cmake( lib_name = "libarchive", lib_source = "@com_github_libarchive_libarchive//:all", ) -cmake( - name = "bcc", - generate_crosstool_file = True, - lib_name = "libbcc_static", - lib_source = "//third_party:bcc_source", - # These link opts are dependencies of bcc. - linkopts = [ - # ELF binary parsing. - "-lelf", - ], - make_commands = [ - "make -j$(nproc) -C src/cc install", - ], - out_static_libs = [ - "libapi-static.a", - "libbcc.a", - "libbcc_bpf.a", - "libbcc-loader-static.a", - "libb_frontend.a", - "libclang_frontend.a", - "libusdt-static.a", - ], -) diff --git a/tools/licenses/BUILD.bazel b/tools/licenses/BUILD.bazel index bfd076f41b6..806d6d4fa2b 100644 --- a/tools/licenses/BUILD.bazel +++ b/tools/licenses/BUILD.bazel @@ -62,20 +62,6 @@ fetch_licenses( use_pkg_dev_go = True, ) -fetch_licenses( - name = "submodule_licenses", - src = "//:pl_3p_submodules", - disallow_missing = select({ - "//bazel:stamped": True, - "//conditions:default": False, - }), - fetch_tool = ":fetch_licenses", - manual_licenses = "manual_licenses.json", - oauth_token = credential, - out_found = "submodule_licenses.json", - out_missing = "submodule_licenses_missing.json", -) - fetch_licenses( name = "deps_licenses", src = "//:pl_3p_deps", @@ -94,7 +80,6 @@ genrule( name = "all_licenses", srcs = [ "go_licenses.json", - "submodule_licenses.json", "deps_licenses.json", "//src/ui:npm_licenses", ], @@ -104,7 +89,6 @@ genrule( cmd = """ python3 $(location combine_licenses.py) \ $(location go_licenses.json) \ - $(location submodule_licenses.json) \ $(location deps_licenses.json) \ $(location //src/ui:npm_licenses) \ --output $(location all_licenses.json)