Bug 1635762 - Switch libFuzzer upstream to git and add local patches. r=truber

Differential Revision: https://phabricator.services.mozilla.com/D74040

tools/fuzzing/interface/FuzzingInterface.h

@@ -12,6 +12,10 @@
 
 #include <fstream>
 
+#ifdef LIBFUZZER
+#  include "FuzzerExtFunctions.h"
+#endif
+
 #include "FuzzerRegistry.h"
 #include "mozilla/Assertions.h"
 

tools/fuzzing/libfuzzer/FuzzerClangCounters.cpp

@@ -1,49 +0,0 @@
-//===- FuzzerExtraCounters.cpp - Extra coverage counters ------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-// Coverage counters from Clang's SourceBasedCodeCoverage.
-//===----------------------------------------------------------------------===//
-
-// Support for SourceBasedCodeCoverage is experimental:
-// * Works only for the main binary, not DSOs yet.
-// * Works only on Linux.
-// * Does not implement print_pcs/print_coverage yet.
-// * Is not fully evaluated for performance and sensitivity.
-//   We expect large performance drop due to 64-bit counters,
-//   and *maybe* better sensitivity due to more fine-grained counters.
-//   Preliminary comparison on a single benchmark (RE2) shows
-//   a bit worse sensitivity though.
-
-#include "FuzzerDefs.h"
-
-#if LIBFUZZER_LINUX
-__attribute__((weak)) extern uint64_t __start___llvm_prf_cnts;
-__attribute__((weak)) extern uint64_t __stop___llvm_prf_cnts;
-namespace fuzzer {
-uint64_t *ClangCountersBegin() { return &__start___llvm_prf_cnts; }
-uint64_t *ClangCountersEnd() { return &__stop___llvm_prf_cnts; }
-}  // namespace fuzzer
-#else
-// TODO: Implement on Mac (if the data shows it's worth it).
-//__attribute__((visibility("hidden")))
-//extern uint64_t CountersStart __asm("section$start$__DATA$__llvm_prf_cnts");
-//__attribute__((visibility("hidden")))
-//extern uint64_t CountersEnd __asm("section$end$__DATA$__llvm_prf_cnts");
-namespace fuzzer {
-uint64_t *ClangCountersBegin() { return nullptr; }
-uint64_t *ClangCountersEnd() { return  nullptr; }
-}  // namespace fuzzer
-#endif
-
-namespace fuzzer {
-ATTRIBUTE_NO_SANITIZE_ALL
-void ClearClangCounters() {  // hand-written memset, don't asan-ify.
-  for (auto P = ClangCountersBegin(); P < ClangCountersEnd(); P++)
-    *P = 0;
-}
-}

tools/fuzzing/libfuzzer/FuzzerDriver.cpp

@@ -565,7 +565,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   using namespace fuzzer;
   assert(argc && argv && "Argument pointers cannot be nullptr");
   std::string Argv0((*argv)[0]);
-  EF = new ExternalFunctions();
+  if (!EF)
+    EF = new ExternalFunctions();
   if (EF->LLVMFuzzerInitialize)
     EF->LLVMFuzzerInitialize(argc, argv);
   if (EF->__msan_scoped_disable_interceptor_checks)

tools/fuzzing/libfuzzer/FuzzerExtFunctionsDlsymWin.cpp

@@ -1,62 +0,0 @@
-//===- FuzzerExtFunctionsDlsymWin.cpp - Interface to external functions ---===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-// Implementation using dynamic loading for Windows.
-//===----------------------------------------------------------------------===//
-#include "FuzzerDefs.h"
-#if LIBFUZZER_WINDOWS
-
-#include "FuzzerExtFunctions.h"
-#include "FuzzerIO.h"
-#include "Windows.h"
-
-// This must be included after Windows.h.
-#include "Psapi.h"
-
-namespace fuzzer {
-
-ExternalFunctions::ExternalFunctions() {
-  HMODULE Modules[1024];
-  DWORD BytesNeeded;
-  HANDLE CurrentProcess = GetCurrentProcess();
-
-  if (!EnumProcessModules(CurrentProcess, Modules, sizeof(Modules),
-                          &BytesNeeded)) {
-    Printf("EnumProcessModules failed (error: %d).\n", GetLastError());
-    exit(1);
-  }
-
-  if (sizeof(Modules) < BytesNeeded) {
-    Printf("Error: the array is not big enough to hold all loaded modules.\n");
-    exit(1);
-  }
-
-  for (size_t i = 0; i < (BytesNeeded / sizeof(HMODULE)); i++)
-  {
-    FARPROC Fn;
-#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN)                            \
-    if (this->NAME == nullptr) {                                               \
-      Fn = GetProcAddress(Modules[i], #NAME);                                  \
-      if (Fn == nullptr)                                                       \
-         Fn = GetProcAddress(Modules[i], #NAME "__dll");                       \
-      this->NAME = (decltype(ExternalFunctions::NAME)) Fn;                     \
-    }
-#include "FuzzerExtFunctions.def"
-#undef EXT_FUNC
-  }
-
-#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN)                            \
-  if (this->NAME == nullptr && WARN)                                           \
-    Printf("WARNING: Failed to find function \"%s\".\n", #NAME);
-#include "FuzzerExtFunctions.def"
-#undef EXT_FUNC
-}
-
-} // namespace fuzzer
-
-#endif // LIBFUZZER_WINDOWS

tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeakAlias.cpp

@@ -1,56 +0,0 @@
-//===- FuzzerExtFunctionsWeakAlias.cpp - Interface to external functions --===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-// Implementation using weak aliases. Works for Windows.
-//===----------------------------------------------------------------------===//
-#include "FuzzerDefs.h"
-#if LIBFUZZER_WINDOWS
-
-#include "FuzzerExtFunctions.h"
-#include "FuzzerIO.h"
-
-using namespace fuzzer;
-
-extern "C" {
-// Declare these symbols as weak to allow them to be optionally defined.
-#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN)                            \
-  RETURN_TYPE NAME##Def FUNC_SIG {                                             \
-    Printf("ERROR: Function \"%s\" not defined.\n", #NAME);                    \
-    exit(1);                                                                   \
-  }                                                                            \
-  RETURN_TYPE NAME FUNC_SIG __attribute__((weak, alias(#NAME "Def")));
-
-#include "FuzzerExtFunctions.def"
-
-#undef EXT_FUNC
-}
-
-template <typename T>
-static T *GetFnPtr(T *Fun, T *FunDef, const char *FnName, bool WarnIfMissing) {
-  if (Fun == FunDef) {
-    if (WarnIfMissing)
-      Printf("WARNING: Failed to find function \"%s\".\n", FnName);
-    return nullptr;
-  }
-  return Fun;
-}
-
-namespace fuzzer {
-
-ExternalFunctions::ExternalFunctions() {
-#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN)                            \
-  this->NAME = GetFnPtr<decltype(::NAME)>(::NAME, ::NAME##Def, #NAME, WARN);
-
-#include "FuzzerExtFunctions.def"
-
-#undef EXT_FUNC
-}
-
-} // namespace fuzzer
-
-#endif // LIBFUZZER_WINDOWS

tools/fuzzing/libfuzzer/FuzzerInternal.h

@@ -67,7 +67,7 @@ public:
   static void StaticFileSizeExceedCallback();
   static void StaticGracefulExitCallback();
 
-  void ExecuteCallback(const uint8_t *Data, size_t Size);
+  int ExecuteCallback(const uint8_t *Data, size_t Size);
   bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false,
               InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr);
 

tools/fuzzing/libfuzzer/FuzzerLoop.cpp

@@ -450,7 +450,9 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile,
   if (!Size)
     return false;
 
-  ExecuteCallback(Data, Size);
+  if (ExecuteCallback(Data, Size) > 0) {
+    return false;
+  }
 
   UniqFeatureSetTmp.clear();
   size_t FoundUniqFeaturesOfII = 0;
@@ -507,7 +509,7 @@ static bool LooseMemeq(const uint8_t *A, const uint8_t *B, size_t Size) {
          !memcmp(A + Size - Limit / 2, B + Size - Limit / 2, Limit / 2);
 }
 
-void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) {
+int Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) {
   TPC.RecordInitialStack();
   TotalNumberOfRuns++;
   assert(InFuzzingThread());
@@ -520,23 +522,24 @@ void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) {
   if (CurrentUnitData && CurrentUnitData != Data)
     memcpy(CurrentUnitData, Data, Size);
   CurrentUnitSize = Size;
+  int Res = 0;
   {
     ScopedEnableMsanInterceptorChecks S;
     AllocTracer.Start(Options.TraceMalloc);
     UnitStartTime = system_clock::now();
     TPC.ResetMaps();
     RunningUserCallback = true;
-    int Res = CB(DataCopy, Size);
+    Res = CB(DataCopy, Size);
     RunningUserCallback = false;
     UnitStopTime = system_clock::now();
-    (void)Res;
-    assert(Res == 0);
+    assert(Res >= 0);
     HasMoreMallocsThanFrees = AllocTracer.Stop();
   }
   if (!LooseMemeq(DataCopy, Data, Size))
     CrashOnOverwrittenData();
   CurrentUnitSize = 0;
   delete[] DataCopy;
+  return Res;
 }
 
 std::string Fuzzer::WriteToOutputCorpus(const Unit &U) {
@@ -660,6 +663,10 @@ void Fuzzer::MutateAndTestOne() {
                                   II.DataFlowTraceForFocusFunction);
     else
       NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen);
+
+    if (!NewSize)
+      continue;
+
     assert(NewSize > 0 && "Mutator returned empty unit");
     assert(NewSize <= CurrentMaxMutationLen && "Mutator return oversized unit");
     Size = NewSize;
@@ -828,7 +835,9 @@ void Fuzzer::MinimizeCrashLoop(const Unit &U) {
     memcpy(CurrentUnitData, U.data(), U.size());
     for (int i = 0; i < Options.MutateDepth; i++) {
       size_t NewSize = MD.Mutate(CurrentUnitData, U.size(), MaxMutationLen);
-      assert(NewSize > 0 && NewSize <= MaxMutationLen);
+      assert(NewSize <= MaxMutationLen);
+      if (!NewSize)
+        continue;
       ExecuteCallback(CurrentUnitData, NewSize);
       PrintPulseAndReportSlowInput(CurrentUnitData, NewSize);
       TryDetectingAMemoryLeak(CurrentUnitData, NewSize,

tools/fuzzing/libfuzzer/FuzzerMerge.cpp

@@ -224,7 +224,9 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) {
     OF.flush();  // Flush is important since Command::Execute may crash.
     // Run.
     TPC.ResetMaps();
-    ExecuteCallback(U.data(), U.size());
+    if (ExecuteCallback(U.data(), U.size()) > 0) {
+      continue;
+    }
     // Collect coverage. We are iterating over the files in this order:
     // * First, files in the initial corpus ordered by size, smallest first.
     // * Then, all other files, smallest first.

tools/fuzzing/libfuzzer/FuzzerShmem.h

@@ -1,69 +0,0 @@
-//===- FuzzerShmem.h - shared memory interface ------------------*- C++ -* ===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-// SharedMemoryRegion
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_FUZZER_SHMEM_H
-#define LLVM_FUZZER_SHMEM_H
-
-#include <algorithm>
-#include <cstring>
-#include <string>
-
-#include "FuzzerDefs.h"
-
-namespace fuzzer {
-
-class SharedMemoryRegion {
- public:
-  bool Create(const char *Name);
-  bool Open(const char *Name);
-  bool Destroy(const char *Name);
-  uint8_t *GetData() { return Data; }
-  void PostServer() {Post(0);}
-  void WaitServer() {Wait(0);}
-  void PostClient() {Post(1);}
-  void WaitClient() {Wait(1);}
-
-  size_t WriteByteArray(const uint8_t *Bytes, size_t N) {
-    assert(N <= kShmemSize - sizeof(N));
-    memcpy(GetData(), &N, sizeof(N));
-    memcpy(GetData() + sizeof(N), Bytes, N);
-    assert(N == ReadByteArraySize());
-    return N;
-  }
-  size_t ReadByteArraySize() {
-    size_t Res;
-    memcpy(&Res, GetData(), sizeof(Res));
-    return Res;
-  }
-  uint8_t *GetByteArray() { return GetData() + sizeof(size_t); }
-
-  bool IsServer() const { return Data && IAmServer; }
-  bool IsClient() const { return Data && !IAmServer; }
-
-private:
-
-  static const size_t kShmemSize = 1 << 22;
-  bool IAmServer;
-  std::string Path(const char *Name);
-  std::string SemName(const char *Name, int Idx);
-  void Post(int Idx);
-  void Wait(int Idx);
-
-  bool Map(int fd);
-  uint8_t *Data = nullptr;
-  void *Semaphore[2];
-};
-
-extern SharedMemoryRegion SMR;
-
-}  // namespace fuzzer
-
-#endif  // LLVM_FUZZER_SHMEM_H

tools/fuzzing/libfuzzer/FuzzerShmemFuchsia.cpp

@@ -1,38 +0,0 @@
-//===- FuzzerShmemPosix.cpp - Posix shared memory ---------------*- C++ -* ===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-// SharedMemoryRegion.  For Fuchsia, this is just stubs as equivalence servers
-// are not currently supported.
-//===----------------------------------------------------------------------===//
-#include "FuzzerDefs.h"
-
-#if LIBFUZZER_FUCHSIA
-
-#include "FuzzerShmem.h"
-
-namespace fuzzer {
-
-bool SharedMemoryRegion::Create(const char *Name) {
-  return false;
-}
-
-bool SharedMemoryRegion::Open(const char *Name) {
-  return false;
-}
-
-bool SharedMemoryRegion::Destroy(const char *Name) {
-  return false;
-}
-
-void SharedMemoryRegion::Post(int Idx) {}
-
-void SharedMemoryRegion::Wait(int Idx) {}
-
-}  // namespace fuzzer
-
-#endif  // LIBFUZZER_FUCHSIA

tools/fuzzing/libfuzzer/FuzzerShmemPosix.cpp

@@ -1,103 +0,0 @@
-//===- FuzzerShmemPosix.cpp - Posix shared memory ---------------*- C++ -* ===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-// SharedMemoryRegion
-//===----------------------------------------------------------------------===//
-#include "FuzzerDefs.h"
-#if LIBFUZZER_POSIX
-
-#include "FuzzerIO.h"
-#include "FuzzerShmem.h"
-
-#include <errno.h>
-#include <fcntl.h>
-#include <semaphore.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-namespace fuzzer {
-
-std::string SharedMemoryRegion::Path(const char *Name) {
-  return DirPlusFile(TmpDir(), Name);
-}
-
-std::string SharedMemoryRegion::SemName(const char *Name, int Idx) {
-  std::string Res(Name);
-  return Res + (char)('0' + Idx);
-}
-
-bool SharedMemoryRegion::Map(int fd) {
-  Data =
-      (uint8_t *)mmap(0, kShmemSize, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0);
-  if (Data == (uint8_t*)-1)
-    return false;
-  return true;
-}
-
-bool SharedMemoryRegion::Create(const char *Name) {
-  int fd = open(Path(Name).c_str(), O_CREAT | O_RDWR, 0777);
-  if (fd < 0) return false;
-  if (ftruncate(fd, kShmemSize) < 0) return false;
-  if (!Map(fd))
-    return false;
-  for (int i = 0; i < 2; i++) {
-    sem_unlink(SemName(Name, i).c_str());
-    Semaphore[i] = sem_open(SemName(Name, i).c_str(), O_CREAT, 0644, 0);
-    if (Semaphore[i] == (void *)-1)
-      return false;
-  }
-  IAmServer = true;
-  return true;
-}
-
-bool SharedMemoryRegion::Open(const char *Name) {
-  int fd = open(Path(Name).c_str(), O_RDWR);
-  if (fd < 0) return false;
-  struct stat stat_res;
-  if (0 != fstat(fd, &stat_res))
-    return false;
-  assert(stat_res.st_size == kShmemSize);
-  if (!Map(fd))
-    return false;
-  for (int i = 0; i < 2; i++) {
-    Semaphore[i] = sem_open(SemName(Name, i).c_str(), 0);
-    if (Semaphore[i] == (void *)-1)
-      return false;
-  }
-  IAmServer = false;
-  return true;
-}
-
-bool SharedMemoryRegion::Destroy(const char *Name) {
-  return 0 == unlink(Path(Name).c_str());
-}
-
-void SharedMemoryRegion::Post(int Idx) {
-  assert(Idx == 0 || Idx == 1);
-  sem_post((sem_t*)Semaphore[Idx]);
-}
-
-void SharedMemoryRegion::Wait(int Idx) {
-  assert(Idx == 0 || Idx == 1);
-  for (int i = 0; i < 10 && sem_wait((sem_t*)Semaphore[Idx]); i++) {
-    // sem_wait may fail if interrupted by a signal.
-    sleep(i);
-    if (i)
-      Printf("%s: sem_wait[%d] failed %s\n", i < 9 ? "WARNING" : "ERROR", i,
-             strerror(errno));
-    if (i == 9) abort();
-  }
-}
-
-}  // namespace fuzzer
-
-#endif  // LIBFUZZER_POSIX

tools/fuzzing/libfuzzer/FuzzerShmemWindows.cpp

@@ -1,64 +0,0 @@
-//===- FuzzerShmemWindows.cpp - Posix shared memory -------------*- C++ -* ===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-// SharedMemoryRegion
-//===----------------------------------------------------------------------===//
-#include "FuzzerDefs.h"
-#if LIBFUZZER_WINDOWS
-
-#include "FuzzerIO.h"
-#include "FuzzerShmem.h"
-
-#include <fcntl.h>
-#include <stdio.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-
-namespace fuzzer {
-
-std::string SharedMemoryRegion::Path(const char *Name) {
-  return DirPlusFile(TmpDir(), Name);
-}
-
-std::string SharedMemoryRegion::SemName(const char *Name, int Idx) {
-  std::string Res(Name);
-  return Res + (char)('0' + Idx);
-}
-
-bool SharedMemoryRegion::Map(int fd) {
-  assert(0 && "UNIMPLEMENTED");
-  return false;
-}
-
-bool SharedMemoryRegion::Create(const char *Name) {
-  assert(0 && "UNIMPLEMENTED");
-  return false;
-}
-
-bool SharedMemoryRegion::Open(const char *Name) {
-  assert(0 && "UNIMPLEMENTED");
-  return false;
-}
-
-bool SharedMemoryRegion::Destroy(const char *Name) {
-  assert(0 && "UNIMPLEMENTED");
-  return false;
-}
-
-void SharedMemoryRegion::Post(int Idx) {
-  assert(0 && "UNIMPLEMENTED");
-}
-
-void SharedMemoryRegion::Wait(int Idx) {
-  Semaphore[1] = nullptr;
-  assert(0 && "UNIMPLEMENTED");
-}
-
-}  // namespace fuzzer
-
-#endif  // LIBFUZZER_WINDOWS

tools/fuzzing/libfuzzer/clone_libfuzzer.sh

@@ -1,31 +1,35 @@
-#!/bin/bash
+#!/bin/bash -e
 
 # Optionally get revision from cmd line
-# Changelog: https://reviews.llvm.org/source/compiler-rt/history/compiler-rt/trunk/lib/fuzzer/
-[ $1 ] && REVISION=$1 || REVISION=356365
+[ $1 ] && REVISION=$1 || REVISION=c815210013f27cfac07d6b53b47e8ac53e86afa3
 
 mkdir tmp
-svn co -qr $REVISION https://llvm.org/svn/llvm-project/compiler-rt/trunk tmp || exit
+git clone --single-branch --no-checkout --shallow-since "2019-03-01" https://github.com/llvm/llvm-project tmp
 
-if [ $1 ]; then
-  # libFuzzer source files
-  CPPS=($(ls -rv tmp/lib/fuzzer/*.cpp))
-  CPPS=(${CPPS[@]##*/})
-  CPPS=(${CPPS[@]##FuzzerMain*}) # ignored
+(cd tmp && git reset --hard $REVISION)
 
-  # Update SOURCES entries
-  sed -e "/^SOURCES/,/^]/ {/'/d}" -i moz.build
-  for CPP in ${CPPS[@]}; do sed -e "/^SOURCES/ a \\\t'${CPP}'," -i moz.build; done
+# libFuzzer source files
+CPPS=($(ls -rv tmp/compiler-rt/lib/fuzzer/*.cpp))
+CPPS=(${CPPS[@]##*/})
+CPPS=(${CPPS[@]##FuzzerMain*}) # ignored
 
-  # Remove previous files
-  rm *.{cpp,h,def}
-fi
+# Update SOURCES entries
+sed -e "/^SOURCES/,/^]/ {/'/d}" -i moz.build
+for CPP in ${CPPS[@]}; do sed -e "/^SOURCES/ a \\    '${CPP}'," -i moz.build; done
+
+# Remove previous files
+rm *.{cpp,h,def}
 
 # Copy files
-cp tmp/lib/fuzzer/*.{cpp,h,def} .
+cp tmp/compiler-rt/lib/fuzzer/*.{cpp,h,def} .
+
+# Apply local patches
+for patch in patches/*.patch
+do
+  patch -p4 < $patch
+done
 
 # Remove the temporary directory
 rm -Rf tmp/
 
-[ $1 ] && echo "Updated libFuzzer to ${REVISION}"
-
+echo "Updated libFuzzer to ${REVISION}"

tools/fuzzing/libfuzzer/moz.build

@@ -8,17 +8,16 @@ Library('fuzzer')
 
 EXPORTS += [
     'FuzzerDefs.h',
+    'FuzzerExtFunctions.def',
+    'FuzzerExtFunctions.h',
 ]
 
 SOURCES += [
-    'FuzzerClangCounters.cpp',
     'FuzzerCrossOver.cpp',
     'FuzzerDataFlowTrace.cpp',
     'FuzzerDriver.cpp',
     'FuzzerExtFunctionsDlsym.cpp',
-    'FuzzerExtFunctionsDlsymWin.cpp',
     'FuzzerExtFunctionsWeak.cpp',
-    'FuzzerExtFunctionsWeakAlias.cpp',
     'FuzzerExtFunctionsWindows.cpp',
     'FuzzerExtraCounters.cpp',
     'FuzzerFork.cpp',
@@ -29,16 +28,13 @@ SOURCES += [
     'FuzzerMerge.cpp',
     'FuzzerMutate.cpp',
     'FuzzerSHA1.cpp',
-    'FuzzerShmemFuchsia.cpp',
-    'FuzzerShmemPosix.cpp',
-    'FuzzerShmemWindows.cpp',
     'FuzzerTracePC.cpp',
     'FuzzerUtil.cpp',
     'FuzzerUtilDarwin.cpp',
     'FuzzerUtilFuchsia.cpp',
     'FuzzerUtilLinux.cpp',
     'FuzzerUtilPosix.cpp',
-    'FuzzerUtilWindows.cpp'
+    'FuzzerUtilWindows.cpp',
 ]
 
 if CONFIG['CC_TYPE'] == 'clang':

tools/fuzzing/libfuzzer/patches/10-ef-runtime.patch

@@ -0,0 +1,31 @@
+# HG changeset patch
+# User Christian Holler <choller@mozilla.com>
+# Date 1586344236 -7200
+#      Wed Apr 08 13:10:36 2020 +0200
+# Node ID ab10fba76a52a5b205038150ad198dfc4583c566
+# Parent  5f0446b8b0b20f432fa5dc016b781270c469fe83
+[libFuzzer] Allow external functions to be defined at runtime
+
+diff --git a/tools/fuzzing/libfuzzer/FuzzerDriver.cpp b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp
+--- a/tools/fuzzing/libfuzzer/FuzzerDriver.cpp
++++ b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp
+@@ -560,17 +560,18 @@ int AnalyzeDictionary(Fuzzer *F, const V
+   Printf("###### End of useless dictionary elements. ######\n");
+   return 0;
+ }
+ 
+ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
+   using namespace fuzzer;
+   assert(argc && argv && "Argument pointers cannot be nullptr");
+   std::string Argv0((*argv)[0]);
+-  EF = new ExternalFunctions();
++  if (!EF)
++    EF = new ExternalFunctions();
+   if (EF->LLVMFuzzerInitialize)
+     EF->LLVMFuzzerInitialize(argc, argv);
+   if (EF->__msan_scoped_disable_interceptor_checks)
+     EF->__msan_scoped_disable_interceptor_checks();
+   const Vector<std::string> Args(*argv, *argv + *argc);
+   assert(!Args.empty());
+   ProgName = new std::string(Args[0]);
+   if (Argv0 != *ProgName) {

tools/fuzzing/libfuzzer/patches/11-callback-rv.patch

@@ -0,0 +1,130 @@
+# HG changeset patch
+# User Christian Holler <choller@mozilla.com>
+# Date 1586345242 -7200
+#      Wed Apr 08 13:27:22 2020 +0200
+# Node ID 142bb91a6f14a721fea81c75d18da92edb6e6ea3
+# Parent  ab10fba76a52a5b205038150ad198dfc4583c566
+[libFuzzer] Change libFuzzer callback contract to allow positive return values
+
+diff --git a/tools/fuzzing/libfuzzer/FuzzerInternal.h b/tools/fuzzing/libfuzzer/FuzzerInternal.h
+--- a/tools/fuzzing/libfuzzer/FuzzerInternal.h
++++ b/tools/fuzzing/libfuzzer/FuzzerInternal.h
+@@ -62,17 +62,17 @@ public:
+ 
+   static void StaticAlarmCallback();
+   static void StaticCrashSignalCallback();
+   static void StaticExitCallback();
+   static void StaticInterruptCallback();
+   static void StaticFileSizeExceedCallback();
+   static void StaticGracefulExitCallback();
+ 
+-  void ExecuteCallback(const uint8_t *Data, size_t Size);
++  int ExecuteCallback(const uint8_t *Data, size_t Size);
+   bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false,
+               InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr);
+ 
+   // Merge Corpora[1:] into Corpora[0].
+   void Merge(const Vector<std::string> &Corpora);
+   void CrashResistantMergeInternalStep(const std::string &ControlFilePath);
+   MutationDispatcher &GetMD() { return MD; }
+   void PrintFinalStats();
+diff --git a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp
+--- a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp
++++ b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp
+@@ -445,17 +445,19 @@ void Fuzzer::PrintPulseAndReportSlowInpu
+   }
+ }
+ 
+ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile,
+                     InputInfo *II, bool *FoundUniqFeatures) {
+   if (!Size)
+     return false;
+ 
+-  ExecuteCallback(Data, Size);
++  if (ExecuteCallback(Data, Size) > 0) {
++    return false;
++  }
+ 
+   UniqFeatureSetTmp.clear();
+   size_t FoundUniqFeaturesOfII = 0;
+   size_t NumUpdatesBefore = Corpus.NumFeatureUpdates();
+   TPC.CollectFeatures([&](size_t Feature) {
+     if (Corpus.AddFeature(Feature, Size, Options.Shrink))
+       UniqFeatureSetTmp.push_back(Feature);
+     if (Options.ReduceInputs && II)
+@@ -502,46 +504,47 @@ static bool LooseMemeq(const uint8_t *A,
+   const size_t Limit = 64;
+   if (Size <= 64)
+     return !memcmp(A, B, Size);
+   // Compare first and last Limit/2 bytes.
+   return !memcmp(A, B, Limit / 2) &&
+          !memcmp(A + Size - Limit / 2, B + Size - Limit / 2, Limit / 2);
+ }
+ 
+-void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) {
++int Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) {
+   TPC.RecordInitialStack();
+   TotalNumberOfRuns++;
+   assert(InFuzzingThread());
+   // We copy the contents of Unit into a separate heap buffer
+   // so that we reliably find buffer overflows in it.
+   uint8_t *DataCopy = new uint8_t[Size];
+   memcpy(DataCopy, Data, Size);
+   if (EF->__msan_unpoison)
+     EF->__msan_unpoison(DataCopy, Size);
+   if (CurrentUnitData && CurrentUnitData != Data)
+     memcpy(CurrentUnitData, Data, Size);
+   CurrentUnitSize = Size;
++  int Res = 0;
+   {
+     ScopedEnableMsanInterceptorChecks S;
+     AllocTracer.Start(Options.TraceMalloc);
+     UnitStartTime = system_clock::now();
+     TPC.ResetMaps();
+     RunningUserCallback = true;
+-    int Res = CB(DataCopy, Size);
++    Res = CB(DataCopy, Size);
+     RunningUserCallback = false;
+     UnitStopTime = system_clock::now();
+-    (void)Res;
+-    assert(Res == 0);
++    assert(Res >= 0);
+     HasMoreMallocsThanFrees = AllocTracer.Stop();
+   }
+   if (!LooseMemeq(DataCopy, Data, Size))
+     CrashOnOverwrittenData();
+   CurrentUnitSize = 0;
+   delete[] DataCopy;
++  return Res;
+ }
+ 
+ std::string Fuzzer::WriteToOutputCorpus(const Unit &U) {
+   if (Options.OnlyASCII)
+     assert(IsASCII(U));
+   if (Options.OutputCorpus.empty())
+     return "";
+   std::string Path = DirPlusFile(Options.OutputCorpus, Hash(U));
+diff --git a/tools/fuzzing/libfuzzer/FuzzerMerge.cpp b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp
+--- a/tools/fuzzing/libfuzzer/FuzzerMerge.cpp
++++ b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp
+@@ -219,17 +219,19 @@ void Fuzzer::CrashResistantMergeInternal
+       U.shrink_to_fit();
+     }
+     std::ostringstream StartedLine;
+     // Write the pre-run marker.
+     OF << "STARTED " << i << " " << U.size() << "\n";
+     OF.flush();  // Flush is important since Command::Execute may crash.
+     // Run.
+     TPC.ResetMaps();
+-    ExecuteCallback(U.data(), U.size());
++    if (ExecuteCallback(U.data(), U.size()) > 0) {
++      continue;
++    }
+     // Collect coverage. We are iterating over the files in this order:
+     // * First, files in the initial corpus ordered by size, smallest first.
+     // * Then, all other files, smallest first.
+     // So it makes no sense to record all features for all files, instead we
+     // only record features that were not seen before.
+     Set<size_t> UniqFeatures;
+     TPC.CollectFeatures([&](size_t Feature) {
+       if (AllFeatures.insert(Feature).second)

tools/fuzzing/libfuzzer/patches/12-custom-mutator-fail.patch

@@ -0,0 +1,53 @@
+# HG changeset patch
+# User Christian Holler <choller@mozilla.com>
+# Date 1586345312 -7200
+#      Wed Apr 08 13:28:32 2020 +0200
+# Node ID 169280b21031865a2fb217af759ad3124dd87ae2
+# Parent  8244be44ec9b52eb81b56e39d04c97a1e4eb13ab
+[libFuzzer] Allow custom mutators to fail
+
+diff --git a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp
+--- a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp
++++ b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp
+@@ -659,16 +659,20 @@ void Fuzzer::MutateAndTestOne() {
+     MaybeExitGracefully();
+     size_t NewSize = 0;
+     if (II.HasFocusFunction && !II.DataFlowTraceForFocusFunction.empty() &&
+         Size <= CurrentMaxMutationLen)
+       NewSize = MD.MutateWithMask(CurrentUnitData, Size, Size,
+                                   II.DataFlowTraceForFocusFunction);
+     else
+       NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen);
++
++    if (!NewSize)
++      continue;
++
+     assert(NewSize > 0 && "Mutator returned empty unit");
+     assert(NewSize <= CurrentMaxMutationLen && "Mutator return oversized unit");
+     Size = NewSize;
+     II.NumExecutedMutations++;
+ 
+     bool FoundUniqFeatures = false;
+     bool NewCov = RunOne(CurrentUnitData, Size, /*MayDeleteFile=*/true, &II,
+                          &FoundUniqFeatures);
+@@ -827,17 +831,19 @@ void Fuzzer::Loop(const Vector<std::stri
+ void Fuzzer::MinimizeCrashLoop(const Unit &U) {
+   if (U.size() <= 1)
+     return;
+   while (!TimedOut() && TotalNumberOfRuns < Options.MaxNumberOfRuns) {
+     MD.StartMutationSequence();
+     memcpy(CurrentUnitData, U.data(), U.size());
+     for (int i = 0; i < Options.MutateDepth; i++) {
+       size_t NewSize = MD.Mutate(CurrentUnitData, U.size(), MaxMutationLen);
+-      assert(NewSize > 0 && NewSize <= MaxMutationLen);
++      assert(NewSize <= MaxMutationLen);
++      if (!NewSize)
++        continue;
+       ExecuteCallback(CurrentUnitData, NewSize);
+       PrintPulseAndReportSlowInput(CurrentUnitData, NewSize);
+       TryDetectingAMemoryLeak(CurrentUnitData, NewSize,
+                               /*DuringInitialCorpusExecution*/ false);
+     }
+   }
+ }
+ 

tools/fuzzing/libfuzzer/patches/13-unused-write.patch

@@ -0,0 +1,80 @@
+diff --git a/tools/fuzzing/libfuzzer/FuzzerIO.cpp b/tools/fuzzing/libfuzzer/FuzzerIO.cpp
+--- a/tools/fuzzing/libfuzzer/FuzzerIO.cpp
++++ b/tools/fuzzing/libfuzzer/FuzzerIO.cpp
+@@ -3,16 +3,17 @@
+ // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ // See https://llvm.org/LICENSE.txt for license information.
+ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ //
+ //===----------------------------------------------------------------------===//
+ // IO functions.
+ //===----------------------------------------------------------------------===//
+ 
++#include "mozilla/Unused.h"
+ #include "FuzzerDefs.h"
+ #include "FuzzerExtFunctions.h"
+ #include "FuzzerIO.h"
+ #include "FuzzerUtil.h"
+ #include <algorithm>
+ #include <cstdarg>
+ #include <fstream>
+ #include <iterator>
+@@ -59,17 +60,17 @@ std::string FileToString(const std::stri
+ void CopyFileToErr(const std::string &Path) {
+   Printf("%s", FileToString(Path).c_str());
+ }
+ 
+ void WriteToFile(const Unit &U, const std::string &Path) {
+   // Use raw C interface because this function may be called from a sig handler.
+   FILE *Out = fopen(Path.c_str(), "w");
+   if (!Out) return;
+-  fwrite(U.data(), sizeof(U[0]), U.size(), Out);
++  mozilla::Unused << fwrite(U.data(), sizeof(U[0]), U.size(), Out);
+   fclose(Out);
+ }
+ 
+ void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V,
+                             long *Epoch, size_t MaxSize, bool ExitOnError) {
+   long E = Epoch ? *Epoch : 0;
+   Vector<std::string> Files;
+   ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true);
+diff --git a/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp b/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp
+--- a/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp
++++ b/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp
+@@ -2,16 +2,17 @@
+ //
+ // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ // See https://llvm.org/LICENSE.txt for license information.
+ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ //
+ //===----------------------------------------------------------------------===//
+ // IO functions implementation using Posix API.
+ //===----------------------------------------------------------------------===//
++#include "mozilla/Unused.h"
+ #include "FuzzerDefs.h"
+ #if LIBFUZZER_POSIX || LIBFUZZER_FUCHSIA
+ 
+ #include "FuzzerExtFunctions.h"
+ #include "FuzzerIO.h"
+ #include <cstdarg>
+ #include <cstdio>
+ #include <dirent.h>
+@@ -154,17 +155,17 @@ bool IsInterestingCoverageFile(const std
+   if (FileName.find("/usr/include/") != std::string::npos)
+     return false;
+   if (FileName == "<null>")
+     return false;
+   return true;
+ }
+ 
+ void RawPrint(const char *Str) {
+-  write(2, Str, strlen(Str));
++  mozilla::Unused << write(2, Str, strlen(Str));
+ }
+ 
+ void MkDir(const std::string &Path) {
+   mkdir(Path.c_str(), 0700);
+ }
+ 
+ void RmDir(const std::string &Path) {
+   rmdir(Path.c_str());

tools/fuzzing/libfuzzer/patches/14-explicit-allocator.patch

@@ -0,0 +1,22 @@
+diff --git a/tools/fuzzing/libfuzzer/FuzzerDefs.h b/tools/fuzzing/libfuzzer/FuzzerDefs.h
+--- a/tools/fuzzing/libfuzzer/FuzzerDefs.h
++++ b/tools/fuzzing/libfuzzer/FuzzerDefs.h
+@@ -174,17 +174,17 @@ extern ExternalFunctions *EF;
+ // We are using a custom allocator to give a different symbol name to STL
+ // containers in order to avoid ODR violations.
+ template<typename T>
+   class fuzzer_allocator: public std::allocator<T> {
+     public:
+       fuzzer_allocator() = default;
+ 
+       template<class U>
+-      fuzzer_allocator(const fuzzer_allocator<U>&) {}
++      explicit fuzzer_allocator(const fuzzer_allocator<U>&) {}
+ 
+       template<class Other>
+       struct rebind { typedef fuzzer_allocator<Other> other;  };
+   };
+ 
+ template<typename T>
+ using Vector = std::vector<T, fuzzer_allocator<T>>;
+