Enable easier local development of external dependencies.

Summary:
There was a lot of pain making changes to bcc, because bazel would rebuild all of bcc and all of bpftrace everytime you made a change.
This diff allows people to locally develop external dependencies and build their artifacts themselves. This way people can take advantage of make/cmake incremental builds while developing locally.
By default, this diff doesn't change the build process at all. But if lines in `repository_locations.bzl` are uncommented, then people can use the new local behaviour for bcc and bpftrace (also if desired other repos, but they aren't setup in this diff).

To achieve this, I add a bazel rule called `local_cc` that mimicks a foreign_cc rule, but instead of running make or cmake, it just copies the artifacts/includes from the specified directories.

This does mean people have to setup the build environment for the external dependencies themselves (when running with the new local repositories, obviously default builds don't have this requirement), but I think people will be happy to do that in exchange for not waiting 3-5 minutes for bcc and bpftrace to rebuild.

Test Plan: Skaffolded with local changes to bcc and bpftrace, and saw that the changes were in effect on the cluster. Skaffolded with the local_repositories commented out, and saw a normal build without the changes to bcc, as expected.

Reviewers: #stirling, vihang, zasgar, #third_party_approvers, yzhao

Reviewed By: #stirling, zasgar, #third_party_approvers, yzhao

Subscribers: yzhao

Signed-off-by: James Bartlett <[email protected]>

Differential Revision: https://phab.corp.pixielabs.ai/D10583

GitOrigin-RevId: 6b033ef

bazel/external/local_dev/BUILD.bazel

@@ -0,0 +1,15 @@
+# Copyright 2018- The Pixie Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0

bazel/external/local_dev/bcc.BUILD

@@ -0,0 +1,52 @@
+# Copyright 2018- The Pixie Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+load("@//bazel:local_cc.bzl", "local_cc")
+
+licenses(["notice"])
+
+filegroup(
+    name = "bcc_source",
+    srcs = glob(["**"]),
+)
+
+# This rule is a hack so that local development of bcc can be done without bazel rerunning the full bcc build each time. It assumes
+# that the following commands have been run in the local bcc directory:
+#   mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=install ..
+#   make install.
+# Then anytime you update bcc sources, you have to run `make install` in the bcc build dir, and then run bazel build.
+# Since bpftrace uses bcc as a dependency, if you want to get the benefits of the local incremental builds, you have to build both
+# bcc and bpftrace locally.
+local_cc(
+    name = "bcc",
+    install_prefix = "build/install",
+    lib_source = ":bcc_source",
+    linkopts = [
+        # ELF binary parsing.
+        "-lelf",
+    ],
+    out_include_dir = "include",
+    out_lib_dir = "lib",
+    out_static_libs = [
+        "libapi-static.a",
+        "libbcc.a",
+        "libbcc_bpf.a",
+        "libbcc-loader-static.a",
+        "libb_frontend.a",
+        "libclang_frontend.a",
+    ],
+    visibility = ["//visibility:public"],
+)

bazel/external/local_dev/bpftrace.BUILD

@@ -0,0 +1,64 @@
+# Copyright 2018- The Pixie Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+load("@//bazel:local_cc.bzl", "local_cc")
+
+licenses(["notice"])
+
+filegroup(
+    name = "bpftrace_source",
+    srcs = glob(["**"]),
+)
+
+# In order to get the same build as the production bpftrace build, you should run the following commands in the bpftrace local repo:
+#   mkdir build && cd build
+#   cmake -DCMAKE_INSTALL_PREFIX=install -DBUILD_TESTING=OFF -DENABLE_BFD_DISABLE=OFF -DENABLE_LIBDW=OFF -DENABLE_MAN=OFF
+#       -DLIBBCC_BPF_LIBRARIES=$BCC_INSTALL/lib/libbcc_bpf.a -DLIBBCC_INCLUDE_DIRS=$BCC_INSTALL/include
+#       -DLIBBCC_LIBRARIES=$BCC_INSTALL/lib/libbcc.a
+#       -DLIBBCC_LOADER_LIBRARY_STATIC=$BCC_INSTALL/lib/libbcc-loader-static.a -DCMAKE_BUILD_TYPE=Release ..
+#   make install
+# Everytime you make a local change to bcc and/or bpftrace you have to run make install in the local bcc repo and
+# then the local bpftrace repo.
+# Note: you may need to install the ubuntu package `libcereal-dev` to get it to work.
+local_cc(
+    name = "bpftrace",
+    install_prefix = "build/install",
+    lib_source = ":bpftrace_source",
+    linkopts = [
+        "-lelf",
+        "-lz",
+    ],
+    out_include_dir = "include",
+    out_lib_dir = "lib",
+    out_static_libs = [
+        "libbpftrace.a",
+        "libaot.a",
+        "libast.a",
+        "libruntime.a",
+        "libbpforc.a",
+        "libast_defs.a",
+        "libparser.a",
+        "libresources.a",
+        "libarch.a",
+        "libcxxdemangler_stdlib.a",
+        "libcxxdemangler_llvm.a",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_github_USCiLab_cereal//:cereal",
+        "@com_github_iovisor_bcc//:bcc",
+    ],
+)

bazel/local_cc.bzl

@@ -0,0 +1,75 @@
+# Copyright 2018- The Pixie Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+load("@rules_foreign_cc//foreign_cc/private:detect_root.bzl", "detect_root")
+load("@rules_foreign_cc//foreign_cc/private:framework.bzl", "CC_EXTERNAL_RULE_ATTRIBUTES", "CC_EXTERNAL_RULE_FRAGMENTS", "cc_external_rule_impl", "create_attrs")
+
+def _cp(_from, _to, flags = ""):
+    return "cp {} {} {}".format(flags, _from, _to)
+
+def _create_configure_script(configureParameters):
+    ctx = configureParameters.ctx
+    attrs = configureParameters.attrs
+    inputs = configureParameters.inputs
+
+    root = detect_root(ctx.attr.lib_source)
+    lib_name = attrs.lib_name or ctx.attr.name
+
+    install_prefix_path = "$EXT_BUILD_ROOT/{}/{}".format(root, ctx.attr.install_prefix)
+    install_dir = "$INSTALLDIR"
+
+    script = []
+
+    # Copy includes
+    from_path = "/".join([install_prefix_path, attrs.out_include_dir, "*"])
+    to_path = "/".join([install_dir, attrs.out_include_dir])
+    script.append(_cp(from_path, to_path, flags = "-r"))
+
+    # Copy libs
+    for out_lib in attrs.out_static_libs:
+        from_path = "/".join([install_prefix_path, attrs.out_lib_dir, out_lib])
+        to_path = "/".join([install_dir, attrs.out_lib_dir, out_lib])
+        script.append(_cp(from_path, to_path))
+    return script
+
+def _local_cc_impl(ctx):
+    attrs = create_attrs(ctx.attr, configure_name = "localcopy", create_configure_script = _create_configure_script)
+
+    return cc_external_rule_impl(ctx, attrs)
+
+attrs = dict(CC_EXTERNAL_RULE_ATTRIBUTES)
+attrs.update({
+    "install_prefix": attr.string(
+        mandatory = True,
+        doc = "Path to the install directory of the library (i.e. directory containing lib/ , include/ etc).",
+    ),
+})
+
+local_cc = rule(
+    attrs = attrs,
+    doc = (
+        "Almost drop in replacement for any `rules_foreign_cc` rule, that will just pull artifacts from a local " +
+        "make/cmake/etc build. This is useful for local development of external CC dependencies, because it means " +
+        "bazel doesn't have to rebuild the cmake/make dependency from scratch everytime. Instead the user is free to " +
+        "build the artifacts themselves and take advantage of incremental builds that way."
+    ),
+    fragments = CC_EXTERNAL_RULE_FRAGMENTS,
+    implementation = _local_cc_impl,
+    toolchains = [
+        "@rules_foreign_cc//foreign_cc/private/framework:shell_toolchain",
+        "@bazel_tools//tools/cpp:toolchain_type",
+    ],
+)

bazel/repositories.bzl

@@ -16,7 +16,7 @@
 
 load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-load(":repository_locations.bzl", "GIT_REPOSITORY_LOCATIONS", "REPOSITORY_LOCATIONS")
+load(":repository_locations.bzl", "GIT_REPOSITORY_LOCATIONS", "LOCAL_REPOSITORY_LOCATIONS", "REPOSITORY_LOCATIONS")
 
 # Make all contents of an external repository accessible under a filegroup.
 # Used for external HTTP archives, e.g. cares.
@@ -68,9 +68,31 @@ def _git_repo_impl(name, **kwargs):
         **kwargs
     )
 
+def _local_repo_impl(name, **kwargs):
+    # `existing_rule_keys` contains the names of repositories that have already
+    # been defined in the Bazel workspace. By skipping repos with existing keys,
+    # users can override dependency versions by using standard Bazel repository
+    # rules in their WORKSPACE files.
+    existing_rule_keys = native.existing_rules().keys()
+    if name in existing_rule_keys:
+        # This repository has already been defined, probably because the user
+        # wants to override the version. Do nothing.
+        return
+
+    location = LOCAL_REPOSITORY_LOCATIONS[name]
+
+    native.new_local_repository(
+        name = name,
+        path = location["path"],
+        **kwargs
+    )
+
 def _git_repo(name, **kwargs):
     _git_repo_impl(name, **kwargs)
 
+def _local_repo(name, **kwargs):
+    _local_repo_impl(name, **kwargs)
+
 # For bazel repos do not require customization.
 def _bazel_repo(name, **kwargs):
     _http_archive_repo_impl(name, **kwargs)
@@ -139,6 +161,9 @@ def _cc_deps():
     _bazel_repo("com_github_h2o_picohttpparser", build_file = "//bazel/external:picohttpparser.BUILD")
     _bazel_repo("com_github_opentelemetry_proto", build_file = "//bazel/external:opentelemetry.BUILD")
 
+    # Uncomment these to develop bcc and/or bpftrace locally. Should also comment out the corresponding _git_repo lines.
+    # _local_repo("com_github_iovisor_bcc", build_file = "//bazel/external/local_dev:bcc.BUILD")
+    # _local_repo("com_github_iovisor_bpftrace", build_file = "//bazel/external/local_dev:bpftrace.BUILD")
     _git_repo("com_github_iovisor_bcc", build_file = "//bazel/external:bcc.BUILD")
     _git_repo("com_github_iovisor_bpftrace", build_file = "//bazel/external:bpftrace.BUILD")
 

bazel/repository_locations.bzl

@@ -319,6 +319,7 @@ REPOSITORY_LOCATIONS = dict(
 # To use a local repo for local development, change `remote` to a file path.
 #   ex: remote = "/home/user/src/pixie-io/bcc"
 # Then change the local repo, commit the change, and replace `commit` with your new commit.
+# See LOCAL_REPOSITORY_LOCATIONS for an alternative approach.
 GIT_REPOSITORY_LOCATIONS = dict(
     com_github_iovisor_bcc = dict(
         remote = "https://github.com/pixie-io/bcc.git",
@@ -336,3 +337,20 @@ GIT_REPOSITORY_LOCATIONS = dict(
         shallow_since = "1638898188 -0800",
     ),
 )
+
+# To use a local repo for local development, update the path to point to your local repo.
+#   ex: path = "/home/user/pixie-io/bcc"
+# then uncomment the lines with `_local_repo(name_of_repo_you_care_about, ...)` in `repositories.bzl` and
+# comment out the corresponding lines with `_git_repo(name_of_repo_you_care_about, ...)`.
+# Note that if you do this, you have to handle the building of these repos' artifacts yourself.
+# See `bazel/external/local_dev` for more info about the right cmake commands for building these repos yourself.
+# WARNING: doing this has some downsides, so don't do it for production builds. For instance,
+# cflags and other settings set by bazel (eg -O3) won't be used, since you have to do the building manually.
+LOCAL_REPOSITORY_LOCATIONS = dict(
+    com_github_iovisor_bcc = dict(
+        path = "/home/user/pixie-io/bcc",
+    ),
+    com_github_iovisor_bpftrace = dict(
+        path = "/home/user/pixie-io/bpftrace",
+    ),
+)