Reland "[libFuzzer] Support using libc++"

petrhosek · petrhosek · commit eac2b47b9fcd · 2018-01-17T20:39:14.000Z
This is needed in case the users of libFuzzer use libc++ in their code, which the fuzz target (libFuzzer) will be linked against. When libc++ source is available, we build a private version of it and link it against libFuzzer which allows using the same static library against codebases which use both libc++ and libstdc++. Differential Revision: https://reviews.llvm.org/D37631 llvm-svn: 322755
diff --git a/compiler-rt/lib/fuzzer/CMakeLists.txt b/compiler-rt/lib/fuzzer/CMakeLists.txt
@@ -33,6 +33,10 @@ CHECK_CXX_SOURCE_COMPILES("
 
 set(LIBFUZZER_CFLAGS ${SANITIZER_COMMON_CFLAGS})
 
+if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND COMPILER_RT_LIBCXX_PATH)
+  list(APPEND LIBFUZZER_CFLAGS -nostdinc++ -D_LIBCPP_ABI_VERSION=__Fuzzer)
+endif()
+
 append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG -fno-omit-frame-pointer LIBFUZZER_CFLAGS)
 
 if (CMAKE_CXX_FLAGS MATCHES "fsanitize-coverage")
@@ -75,6 +79,38 @@ add_compiler_rt_runtime(clang_rt.fuzzer_no_main
   CFLAGS ${LIBFUZZER_CFLAGS}
   PARENT_TARGET fuzzer)
 
+if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND COMPILER_RT_LIBCXX_PATH)
+  macro(partially_link_libcxx name dir arch)
+    set(cxx_${arch}_merge_dir "${CMAKE_CURRENT_BINARY_DIR}/cxx_${arch}_merge.dir")
+    file(MAKE_DIRECTORY ${cxx_${arch}_merge_dir})
+    add_custom_command(TARGET clang_rt.${name}-${arch} POST_BUILD
+      COMMAND ${CMAKE_LINKER} --whole-archive "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" --no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o
+      COMMAND ${CMAKE_OBJCOPY} --localize-hidden ${name}.o
+      COMMAND ${CMAKE_COMMAND} -E remove "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>"
+      COMMAND ${CMAKE_AR} qcs "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" ${name}.o
+      WORKING_DIRECTORY ${cxx_${arch}_merge_dir}
+    )
+  endmacro()
+
+  foreach(arch ${FUZZER_SUPPORTED_ARCH})
+    get_target_flags_for_arch(${arch} TARGET_CFLAGS)
+    set(LIBCXX_${arch}_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libcxx_fuzzer_${arch})
+    add_custom_libcxx(libcxx_fuzzer_${arch} ${LIBCXX_${arch}_PREFIX}
+      CFLAGS ${TARGET_CFLAGS}
+             -D_LIBCPP_ABI_VERSION=__Fuzzer
+             -D_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS=1
+             -fvisibility=hidden
+      CMAKE_ARGS -DLIBCXX_ENABLE_EXCEPTIONS=OFF
+                 -DLIBCXX_CXX_ABI=none)
+    target_compile_options(RTfuzzer.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)
+    add_dependencies(RTfuzzer.${arch} libcxx_fuzzer_${arch})
+    target_compile_options(RTfuzzer_main.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)
+    add_dependencies(RTfuzzer_main.${arch} libcxx_fuzzer_${arch})
+    partially_link_libcxx(fuzzer_no_main ${LIBCXX_${arch}_PREFIX} ${arch})
+    partially_link_libcxx(fuzzer ${LIBCXX_${arch}_PREFIX} ${arch})
+  endforeach()
+endif()
+
 if(COMPILER_RT_INCLUDE_TESTS)
   add_subdirectory(tests)
 endif()
diff --git a/compiler-rt/lib/fuzzer/FuzzerInterface.h b/compiler-rt/lib/fuzzer/FuzzerInterface.h
@@ -30,35 +30,39 @@ extern "C" {
 // Executes the code under test with [Data, Data+Size) as the input.
 // libFuzzer will invoke this function *many* times with different inputs.
 // Must return 0.
-int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size);
+__attribute__((visibility("default"))) int
+LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size);
 
 // Optional user-provided initialization function.
 // If provided, this function will be called by libFuzzer once at startup.
 // It may read and modify argc/argv.
 // Must return 0.
-int LLVMFuzzerInitialize(int *argc, char ***argv);
+__attribute__((visibility("default"))) int LLVMFuzzerInitialize(int *argc,
+                                                                char ***argv);
 
 // Optional user-provided custom mutator.
 // Mutates raw data in [Data, Data+Size) inplace.
 // Returns the new size, which is not greater than MaxSize.
 // Given the same Seed produces the same mutation.
-size_t LLVMFuzzerCustomMutator(uint8_t *Data, size_t Size, size_t MaxSize,
-                               unsigned int Seed);
+__attribute__((visibility("default"))) size_t
+LLVMFuzzerCustomMutator(uint8_t *Data, size_t Size, size_t MaxSize,
+                        unsigned int Seed);
 
 // Optional user-provided custom cross-over function.
 // Combines pieces of Data1 & Data2 together into Out.
 // Returns the new size, which is not greater than MaxOutSize.
 // Should produce the same mutation given the same Seed.
-size_t LLVMFuzzerCustomCrossOver(const uint8_t *Data1, size_t Size1,
-                                 const uint8_t *Data2, size_t Size2,
-                                 uint8_t *Out, size_t MaxOutSize,
-                                 unsigned int Seed);
+__attribute__((visibility("default"))) size_t
+LLVMFuzzerCustomCrossOver(const uint8_t *Data1, size_t Size1,
+                          const uint8_t *Data2, size_t Size2, uint8_t *Out,
+                          size_t MaxOutSize, unsigned int Seed);
 
 // Experimental, may go away in future.
 // libFuzzer-provided function to be used inside LLVMFuzzerCustomMutator.
 // Mutates raw data in [Data, Data+Size) inplace.
 // Returns the new size, which is not greater than MaxSize.
-size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize);
+__attribute__((visibility("default"))) size_t
+LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize);
 
 #ifdef __cplusplus
 }  // extern "C"
diff --git a/compiler-rt/lib/fuzzer/FuzzerLoop.cpp b/compiler-rt/lib/fuzzer/FuzzerLoop.cpp
@@ -826,13 +826,15 @@ void Fuzzer::AnnounceOutput(const uint8_t *Data, size_t Size) {
 
 extern "C" {
 
-size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) {
+__attribute__((visibility("default"))) size_t
+LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) {
   assert(fuzzer::F);
   return fuzzer::F->GetMD().DefaultMutate(Data, Size, MaxSize);
 }
 
 // Experimental
-void LLVMFuzzerAnnounceOutput(const uint8_t *Data, size_t Size) {
+__attribute__((visibility("default"))) void
+LLVMFuzzerAnnounceOutput(const uint8_t *Data, size_t Size) {
   assert(fuzzer::F);
   fuzzer::F->AnnounceOutput(Data, Size);
 }
diff --git a/compiler-rt/lib/fuzzer/FuzzerMain.cpp b/compiler-rt/lib/fuzzer/FuzzerMain.cpp
@@ -16,6 +16,6 @@ extern "C" {
 int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size);
 }  // extern "C"
 
-int main(int argc, char **argv) {
+__attribute__((visibility("default"))) int main(int argc, char **argv) {
   return fuzzer::FuzzerDriver(&argc, &argv, LLVMFuzzerTestOneInput);
 }
diff --git a/compiler-rt/lib/fuzzer/tests/CMakeLists.txt b/compiler-rt/lib/fuzzer/tests/CMakeLists.txt
@@ -18,6 +18,10 @@ else()
   list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -lstdc++ -lpthread)
 endif()
 
+if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND COMPILER_RT_LIBCXX_PATH)
+  list(APPEND LIBFUZZER_UNITTEST_CFLAGS -nostdinc++ -D_LIBCPP_ABI_VERSION=__Fuzzer)
+endif()
+
 foreach(arch ${FUZZER_SUPPORTED_ARCH})
   set(LIBFUZZER_TEST_RUNTIME RTFuzzerTest.${arch})
   if(APPLE)
@@ -33,14 +37,20 @@ foreach(arch ${FUZZER_SUPPORTED_ARCH})
     ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
     FOLDER "Compiler-RT Runtime tests")
 
+  if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND COMPILER_RT_LIBCXX_PATH)
+    set(LIBFUZZER_TEST_RUNTIME_DEPS libcxx_fuzzer_${arch})
+    set(LIBFUZZER_TEST_RUNTIME_CFLAGS -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)
+    set(LIBFUZZER_TEST_RUNTIME_LINK_FLAGS ${LIBCXX_${arch}_PREFIX}/lib/libc++.a)
+  endif()
+
   set(FuzzerTestObjects)
   generate_compiler_rt_tests(FuzzerTestObjects
     FuzzerUnitTests "Fuzzer-${arch}-Test" ${arch}
     SOURCES FuzzerUnittest.cpp ${COMPILER_RT_GTEST_SOURCE}
     RUNTIME ${LIBFUZZER_TEST_RUNTIME}
-    DEPS gtest
-    CFLAGS ${LIBFUZZER_UNITTEST_CFLAGS}
-    LINK_FLAGS ${LIBFUZZER_UNITTEST_LINK_FLAGS})
+    DEPS gtest ${LIBFUZZER_TEST_RUNTIME_DEPS}
+    CFLAGS ${LIBFUZZER_UNITTEST_CFLAGS} ${LIBFUZZER_TEST_RUNTIME_CFLAGS}
+    LINK_FLAGS ${LIBFUZZER_UNITTEST_LINK_FLAGS} ${LIBFUZZER_TEST_RUNTIME_LINK_FLAGS})
   set_target_properties(FuzzerUnitTests PROPERTIES
     RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
 endforeach()
diff --git a/compiler-rt/test/fuzzer/CMakeLists.txt b/compiler-rt/test/fuzzer/CMakeLists.txt
@@ -8,36 +8,65 @@ if(COMPILER_RT_INCLUDE_TESTS)
   list(APPEND LIBFUZZER_TEST_DEPS FuzzerUnitTests)
 endif()
 
-set(LIBFUZZER_TESTSUITES)
+set(EXCLUDE_FROM_ALL ON)
 
+add_custom_target(check-fuzzer)
 
 if(COMPILER_RT_INCLUDE_TESTS)
   # libFuzzer unit tests.
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/unit/lit.site.cfg.in
     ${CMAKE_CURRENT_BINARY_DIR}/unit/lit.site.cfg)
-  list(APPEND LIBFUZZER_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/unit)
+  add_lit_testsuite(check-fuzzer-unit "Running Fuzzer unit tests"
+    ${CMAKE_CURRENT_BINARY_DIR}/unit
+    DEPENDS ${LIBFUZZER_TEST_DEPS})
+  set_target_properties(check-fuzzer-unit PROPERTIES FOLDER "Compiler-RT Tests")
+  add_dependencies(check-fuzzer check-fuzzer-unit)
 endif()
 
-foreach(arch ${FUZZER_SUPPORTED_ARCH})
-  set(LIBFUZZER_TEST_COMPILER ${COMPILER_RT_TEST_COMPILER})
-  get_test_cc_for_arch(${arch} LIBFUZZER_TEST_COMPILER LIBFUZZER_TEST_FLAGS)
+macro(test_fuzzer stdlib)
+  cmake_parse_arguments(TEST "" "" "DEPS" ${ARGN})
+  string(REPLACE "+" "x" stdlib_name ${stdlib})
+  string(REPLACE "-" ";" stdlib_list ${stdlib_name})
+  set(STDLIB_CAPITALIZED "")
+  foreach(part IN LISTS stdlib_list)
+    string(SUBSTRING ${part} 0 1 first_letter)
+    string(TOUPPER ${first_letter} first_letter)
+    string(REGEX REPLACE "^.(.*)" "${first_letter}\\1" part "${part}")
+    set(STDLIB_CAPITALIZED "${STDLIB_CAPITALIZED}${part}")
+  endforeach()
+  foreach(arch ${FUZZER_SUPPORTED_ARCH})
+    set(LIBFUZZER_TEST_COMPILER ${COMPILER_RT_TEST_COMPILER})
+    get_test_cc_for_arch(${arch} LIBFUZZER_TEST_COMPILER LIBFUZZER_TEST_FLAGS)
 
-  string(TOUPPER ${arch} ARCH_UPPER_CASE)
-  set(CONFIG_NAME ${ARCH_UPPER_CASE}${OS_NAME}Config)
+    set(LIBFUZZER_TEST_STDLIB ${stdlib})
 
-  # LIT-based libFuzzer tests.
-  configure_lit_site_cfg(
-    ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
-    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg
-    )
-  list(APPEND LIBFUZZER_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME})
+    string(TOUPPER ${arch} ARCH_UPPER_CASE)
+    set(CONFIG_NAME ${ARCH_UPPER_CASE}${STDLIB_CAPITALIZED}${OS_NAME}Config)
 
-endforeach()
+    # LIT-based libFuzzer tests.
+    configure_lit_site_cfg(
+      ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
+      ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg
+      )
 
-set(EXCLUDE_FROM_ALL ON)
+    add_lit_testsuite(check-fuzzer-${stdlib_name} "Running Fuzzer ${stdlib} tests"
+      ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}
+      DEPENDS ${LIBFUZZER_TEST_DEPS})
+    if(TEST_DEPS)
+      add_dependencies(check-fuzzer-${stdlib_name} ${TEST_DEPS})
+    endif()
+    set_target_properties(check-fuzzer-${stdlib_name} PROPERTIES FOLDER "Compiler-RT Tests")
+    add_dependencies(check-fuzzer check-fuzzer-${stdlib_name})
+  endforeach()
+endmacro()
 
-add_lit_testsuite(check-fuzzer "Running Fuzzer tests"
-  ${LIBFUZZER_TESTSUITES}
-  DEPENDS ${LIBFUZZER_TEST_DEPS})
-set_target_properties(check-fuzzer PROPERTIES FOLDER "Compiler-RT Tests")
+test_fuzzer("default")
+if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux")
+  if(TARGET cxx_shared)
+    test_fuzzer("libc++" DEPS cxx_shared)
+  endif()
+  if(TARGET cxx_static)
+    test_fuzzer("static-libc++" DEPS cxx_static)
+  endif()
+endif()
diff --git a/compiler-rt/test/fuzzer/lit.cfg b/compiler-rt/test/fuzzer/lit.cfg
@@ -7,6 +7,8 @@ config.test_format = lit.formats.ShTest(True)
 config.suffixes = ['.test']
 config.test_source_root = os.path.dirname(__file__)
 
+config.environment['LD_LIBRARY_PATH'] = config.llvm_library_dir
+
 # Choose between lit's internal shell pipeline runner and a real shell.  If
 # LIT_USE_INTERNAL_SHELL is in the environment, we use that as an override.
 use_lit_shell = os.environ.get("LIT_USE_INTERNAL_SHELL")
@@ -51,8 +53,13 @@ config.substitutions.append(('%libfuzzer_src', libfuzzer_src_root))
 
 def generate_compiler_cmd(is_cpp=True, fuzzer_enabled=True):
   compiler_cmd = config.c_compiler
-  link_cmd = '-lc++' if any(x in config.target_triple for x in ('darwin', 'freebsd')) else '-lstdc++'
-  std_cmd = '-std=c++11' if is_cpp else ''
+  if config.clang and config.stdlib == 'libc++':
+    link_cmd = '-stdlib=libc++'
+  elif config.clang and config.stdlib == 'static-libc++':
+    link_cmd = '-stdlib=libc++ -lc++abi -static-libstdc++'
+  else:
+    link_cmd = '-lc++' if any(x in config.target_triple for x in ('darwin', 'freebsd')) else '-lstdc++'
+  std_cmd = '--driver-mode=g++ -std=c++11' if is_cpp else ''
   sanitizers = ['address']
   if fuzzer_enabled:
     sanitizers.append('fuzzer')
diff --git a/compiler-rt/test/fuzzer/lit.site.cfg.in b/compiler-rt/test/fuzzer/lit.site.cfg.in
@@ -5,9 +5,11 @@ config.test_exec_root = "@CMAKE_CURRENT_BINARY_DIR@"
 config.cpp_compiler = "@LIBFUZZER_TEST_COMPILER@"
 config.target_flags = "@LIBFUZZER_TEST_FLAGS@"
 config.c_compiler = "@LIBFUZZER_TEST_COMPILER@"
+config.stdlib = "@LIBFUZZER_TEST_STDLIB@"
 
 config.osx_sysroot_flag = "@OSX_SYSROOT_FLAG@"
 config.cmake_binary_dir = "@CMAKE_BINARY_DIR@"
+config.llvm_library_dir = "@LLVM_LIBRARY_DIR@"
 config.target_triple = "@TARGET_TRIPLE@"
 
 # Load common config for all compiler-rt lit tests.

Original file line number	Diff line number	Diff line change
`@@ -16,6 +16,6 @@ extern "C" {`
`16`	`16`	`int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size);`
`17`	`17`	`} // extern "C"`
`18`	`18`
`19`		`-int main(int argc, char **argv) {`
	`19`	`+__attribute__((visibility("default"))) int main(int argc, char **argv) {`
`20`	`20`	`return fuzzer::FuzzerDriver(&argc, &argv, LLVMFuzzerTestOneInput);`
`21`	`21`	`}`