index_build: Support custom compiler toolchains

- Use CLANG_TOOLCHAIN and CLANG_VERSION environment variables to determine this.
- Use a more robust approach to wrapping the compiler by creating a "shadow"
  toolchain at `/opt/toolchain` with `bin` and `lib` symlinked in from
  CLANG_TOOLCHAIN (except for `clang` and `clang++`). `clang` and `clang++` are
  symlinked from our `clang_wrapper.py`
- Allow -ffile-compilation-dir=. and -fdebug-compilation-dir=. Chrome GN builds
  use these and they're OK.
- Move -gen-cdb-fragment-path injection into the clang wrapper instead,
  so we don't have to rely on this being propagated properly from
  index_build.py. This makes V8 indexing work, enough if other flags are not
  propagated yet.

PiperOrigin-RevId: 801938479

Author: OSS-Fuzz Team

infra/base-images/base-builder/indexer/clang_wrapper.py

@@ -46,35 +46,40 @@
 # When we notice a project using these flags,
 # we should figure out how to handle them.
 _DISALLOWED_CLANG_FLAGS = (
-    "-fdebug-compilation-dir=",
     "-fdebug-prefix-map=",
-    "-ffile-compilation-dir=",
     "-ffile-prefix-map=",
 )
 
+# Chromium GN builds use these flags with a period to make paths relative to
+# the out directory. This is OK.
+_ALLOWED_CLANG_FLAGS_ONLY_WITH_PERIOD = (
+    "-fdebug-compilation-dir=",
+    "-ffile-compilation-dir=",
+)
+
 SRC = Path(os.getenv("SRC", "/src"))
 # On OSS-Fuzz build infra, $OUT is not /out.
 OUT = Path(os.getenv("OUT", "/out"))
 INDEXES_PATH = Path(os.getenv("INDEXES_PATH", "/indexes"))
 FUZZER_ENGINE = os.getenv("LIB_FUZZING_ENGINE", "/usr/lib/libFuzzingEngine.a")
 
 
-def rewrite_argv0(argv: Sequence[str]) -> list[str]:
+def rewrite_argv0(argv: Sequence[str], clang_toolchain: str) -> list[str]:
   """Rewrite argv[0] to point to the real clang location."""
   # We do this because we've set PATH to our wrapper.
-  rewritten = [os.path.join("/usr/local/bin/", os.path.basename(argv[0]))]
+  rewritten = [os.path.join(clang_toolchain, "bin", os.path.basename(argv[0]))]
   rewritten.extend(argv[1:])
   return rewritten
 
 
-def execute(argv: Sequence[str]) -> None:
-  argv = rewrite_argv0(argv)
+def execute(argv: Sequence[str], clang_toolchain: str) -> None:
+  argv = rewrite_argv0(argv, clang_toolchain)
   print("About to execute...", argv)
   os.execv(argv[0], tuple(argv))
 
 
-def run(argv: Sequence[str]) -> None:
-  argv = rewrite_argv0(argv)
+def run(argv: Sequence[str], clang_toolchain: str) -> None:
+  argv = rewrite_argv0(argv, clang_toolchain)
   print("About to run...", argv)
   ret = subprocess.run(argv, check=False)
   if ret.returncode != 0:
@@ -371,7 +376,17 @@ def check_fuzzing_engine_and_fix_argv(argv: MutableSequence[str]) -> bool:
 
 def _has_disallowed_clang_flags(argv: Sequence[str]) -> bool:
   """Checks if the command line arguments contain disallowed flags."""
-  return any(arg.startswith(_DISALLOWED_CLANG_FLAGS) for arg in argv)
+  if any(arg.startswith(_DISALLOWED_CLANG_FLAGS) for arg in argv):
+    return True
+
+  if any(
+      arg.startswith(_ALLOWED_CLANG_FLAGS_ONLY_WITH_PERIOD)
+      and not arg.endswith("=.")
+      for arg in argv
+  ):
+    return True
+
+  return False
 
 
 @dataclasses.dataclass(frozen=True)
@@ -400,7 +415,16 @@ def _filter_compile_commands(
   unused_cc_paths = set()
 
   for compile_command in compile_commands:
-    cc_path = Path(compile_command["directory"]) / compile_command["file"]
+    if (
+        "-ffile-compilation-dir=." in compile_command["arguments"]
+        or "-fdebug-compilation-dir=." in compile_command["arguments"]
+    ):
+      # Handle build systems that make their debug paths relative.
+      directory = Path(".")
+    else:
+      directory = Path(compile_command["directory"])
+
+    cc_path = Path(directory / compile_command["file"])
     if cc_path in cu_paths:
       filtered_compile_commands.append(compile_command)
       used_cu_paths.add(cc_path)
@@ -489,10 +513,27 @@ def main(argv: list[str]) -> None:
   if _has_disallowed_clang_flags(argv):
     raise ValueError("Disallowed clang flags found, aborting.")
 
+  # TODO: b/441872725 - Migrate more flags to be appended in the clang wrapper
+  # instead.
+  cdb_path = index_build.OUT / "cdb"
+  argv.extend(("-gen-cdb-fragment-path", cdb_path.as_posix()))
+  argv.extend((
+      "-isystem",
+      (
+          f"{compile_settings.clang_toolchain}/lib/clang/"
+          f"{compile_settings.clang_version}"
+      ),
+      "-resource-dir",
+      (
+          f"{compile_settings.clang_toolchain}/lib/clang/"
+          f"{compile_settings.clang_version}"
+      ),
+  ))
+
   if "-E" in argv:
     # Preprocessor-only invocation.
     modified_argv = remove_flag_and_value(argv, "-gen-cdb-fragment-path")
-    execute(modified_argv)
+    execute(modified_argv, compile_settings.clang_toolchain)
 
   fuzzing_engine_in_argv = check_fuzzing_engine_and_fix_argv(argv)
   indexer_targets: list[str] = [
@@ -502,29 +543,24 @@ def main(argv: list[str]) -> None:
   # If we are linking, collect the relevant flags and dependencies.
   output_file = get_flag_value(argv, "-o")
   if not output_file:
-    execute(argv)  # Missing output file
+    execute(argv, compile_settings.clang_toolchain)  # Missing output file
 
   output_file = Path(output_file)
 
   if output_file.name.endswith(".o"):
-    execute(argv)  # Not a real linker command
+    execute(argv, compile_settings.clang_toolchain)  # Not a real linker command
 
   if indexer_targets:
     if output_file.name not in indexer_targets:
       # Not a relevant linker command
       print(f"Not indexing as {output_file} is not in the allowlist")
-      execute(argv)
+      execute(argv, compile_settings.clang_toolchain)
   elif not fuzzing_engine_in_argv:
     # Not a fuzz target.
-    execute(argv)
+    execute(argv, compile_settings.clang_toolchain)
 
   print(f"Linking {argv}")
 
-  cdb_path = get_flag_value(argv, "-gen-cdb-fragment-path")
-  assert cdb_path, f"Missing Compile Directory Path: {argv}"
-
-  cdb_path = Path(cdb_path)
-
   # We can now run the linker and look at the output of some files.
   dependency_file = (cdb_path / output_file.name).with_suffix(".deps")
   why_extract_file = (cdb_path / output_file.name).with_suffix(".why_extract")
@@ -535,7 +571,7 @@ def main(argv: list[str]) -> None:
   # We force lld, but it doesn't include this dir by default.
   argv.append("-L/usr/local/lib")
   argv.append("-Qunused-arguments")
-  run(argv)
+  run(argv, compile_settings.clang_toolchain)
 
   build_id = get_build_id(output_file)
   assert build_id is not None

infra/base-images/base-builder/indexer/index_build.py

@@ -44,15 +44,20 @@
 _LD_BINARY = 'ld-linux-x86-64.so.2'
 _LD_PATH = Path('/lib64') / _LD_BINARY
 _LLVM_READELF_PATH = '/usr/local/bin/llvm-readelf'
-_CLANG_VERSION = '18'
 
 DEFAULT_COVERAGE_FLAGS = '-fsanitize-coverage=bb,no-prune,trace-pc-guard'
 DEFAULT_FUZZING_ENGINE = 'fuzzing_engine.cc'
 
+_CLANG_VERSION = os.getenv('CLANG_VERSION', '18')
+_CLANG_TOOLCHAIN = Path(os.getenv('CLANG_TOOLCHAIN', '/usr/local'))
+_TOOLCHAIN_WITH_WRAPPER = Path('/opt/toolchain')
+
+INDEXER_DIR = Path(__file__).parent
+
 # Some build systems isolate the compiler environment from the parent process,
 # so we can't always rely on using environment variables to pass settings to the
 # wrapper. Get around this by writing to a file instead.
-COMPILE_SETTINGS_PATH = Path(__file__).parent / 'compile_settings.json'
+COMPILE_SETTINGS_PATH = INDEXER_DIR / 'compile_settings.json'
 
 EXTRA_CFLAGS = (
     '-fno-omit-frame-pointer '
@@ -61,16 +66,15 @@
     '-fsanitize=address '
     '-Wno-invalid-offsetof '
     '{coverage_flags} '
-    f'-gen-cdb-fragment-path {OUT}/cdb '
     '-Qunused-arguments '
-    f'-isystem /usr/local/lib/clang/{_CLANG_VERSION} '
-    f'-resource-dir /usr/local/lib/clang/{_CLANG_VERSION} '
 )
 
 
 @dataclasses.dataclass(slots=True, frozen=True)
 class CompileSettings:
   coverage_flags: str
+  clang_toolchain: str
+  clang_version: str
 
 
 def read_compile_settings() -> CompileSettings:
@@ -100,26 +104,38 @@ def set_env_vars(coverage_flags: str):
   os.environ['CC'] = 'clang'
   os.environ['COMPILING_PROJECT'] = 'True'
   # Force users of clang to use our wrapper. This fixes e.g. libcups.
-  os.environ['PATH'] = f"/opt/indexer:{os.environ.get('PATH')}"
+  os.environ['PATH'] = (
+      f"{_TOOLCHAIN_WITH_WRAPPER / 'bin'}:{os.environ.get('PATH')}"
+  )
 
   existing_cflags = os.environ.get('CFLAGS', '')
   extra_cflags = EXTRA_CFLAGS.format(coverage_flags=coverage_flags)
   os.environ['CFLAGS'] = f'{existing_cflags} {extra_cflags}'.strip()
 
 
 def set_up_wrapper_dir():
-  """Set up symlinks to everything in /usr/local/bin/.
+  """Sets up a shadow toolchain.
 
-  Do this so build systems that snoop around clang's directory don't explode.
+  This sets up our clang wrapper for clang/clang++ and symlinks everything else
+  to point to the real toolchain.
   """
-  real_dir = '/usr/local/bin'
-  indexer_dir = '/opt/indexer'
-  for name in os.listdir():
-    src = os.path.join(real_dir, name)
-    dst = os.path.join(indexer_dir, name)
-    if name not in {'clang', 'clang++'}:
+  if _TOOLCHAIN_WITH_WRAPPER.exists():
+    shutil.rmtree(_TOOLCHAIN_WITH_WRAPPER)
+  _TOOLCHAIN_WITH_WRAPPER.mkdir(parents=True)
+
+  # Set up symlinks to every binary except for clang.
+  wrapper_bin_dir = _TOOLCHAIN_WITH_WRAPPER / 'bin'
+  wrapper_bin_dir.mkdir()
+  for name in os.listdir(_CLANG_TOOLCHAIN / 'bin'):
+    if name in ('clang', 'clang++'):
       continue
-    os.symlink(src, dst)
+
+    os.symlink(_CLANG_TOOLCHAIN / 'bin' / name, wrapper_bin_dir / name)
+  os.symlink(_CLANG_TOOLCHAIN / 'lib', _TOOLCHAIN_WITH_WRAPPER / 'lib')
+
+  # Set up our compiler wrappers.
+  os.symlink(INDEXER_DIR / 'clang_wrapper.py', wrapper_bin_dir / 'clang')
+  os.symlink(INDEXER_DIR / 'clang_wrapper.py', wrapper_bin_dir / 'clang++')
 
 
 @dataclasses.dataclass(slots=True, frozen=True)
@@ -334,11 +350,17 @@ def build_project(
   if binaries_only:
     os.environ['INDEXER_BINARIES_ONLY'] = '1'
 
-  write_compile_settings(CompileSettings(coverage_flags=coverage_flags))
+  write_compile_settings(
+      CompileSettings(
+          coverage_flags=coverage_flags,
+          clang_toolchain=_CLANG_TOOLCHAIN.as_posix(),
+          clang_version=_CLANG_VERSION,
+      )
+  )
 
   fuzzing_engine_dir = copy_fuzzing_engine(fuzzing_engine)
   build_fuzzing_engine_command = [
-      '/opt/indexer/clang++',
+      f'{_CLANG_TOOLCHAIN}/bin/clang++',
       '-c',
       '-Wall',
       '-Wextra',
@@ -352,16 +374,16 @@ def build_project(
       '-gen-cdb-fragment-path',
       f'{OUT}/cdb',
       '-Qunused-arguments',
-      f'-isystem /usr/local/lib/clang/{_CLANG_VERSION}',
+      f'-isystem {_CLANG_TOOLCHAIN}/lib/clang/{_CLANG_VERSION}',
       '/usr/lib/gcc/x86_64-linux-gnu/9/../../../../include/c++/9',
       '-I',
       '/usr/lib/gcc/x86_64-linux-gnu/9/../../../../include/x86_64-linux-gnu/c++/9',
       '-I',
       '/usr/lib/gcc/x86_64-linux-gnu/9/../../../../include/c++/9/backward',
       '-I',
-      f'/usr/local/lib/clang/{_CLANG_VERSION}/include',
+      f'{_CLANG_TOOLCHAIN}/lib/clang/{_CLANG_VERSION}/include',
       '-I',
-      '/usr/local/include',
+      f'{_CLANG_TOOLCHAIN}/include',
       '-I',
       '/usr/include/x86_64-linux-gnu',
       '-I',
@@ -379,7 +401,6 @@ def build_project(
   if os.path.exists(lib_fuzzing_engine):
     os.remove(lib_fuzzing_engine)
   os.symlink('/opt/indexer/fuzzing_engine.a', lib_fuzzing_engine)
-  set_up_wrapper_dir()
 
   compile_command = ['/usr/local/bin/compile']
   if compile_args:
@@ -804,6 +825,8 @@ def main():
   SNAPSHOT_DIR.mkdir(exist_ok=True)
   # We don't have an existing /out dir on oss-fuzz's build infra.
   OUT.mkdir(parents=True, exist_ok=True)
+
+  set_up_wrapper_dir()
   build_project(
       None if args.targets_all_index else targets_to_index,
       args.compile_arg,