From 53f361c0072aab72ddc81ff5fa25583a4fe45613 Mon Sep 17 00:00:00 2001 From: Christian Holler Date: Mon, 11 May 2020 12:45:07 +0000 Subject: [PATCH] Bug 1635762 - Switch libFuzzer upstream to git and add local patches. r=truber Differential Revision: https://phabricator.services.mozilla.com/D74040 --- tools/fuzzing/interface/FuzzingInterface.h | 4 + .../fuzzing/libfuzzer/FuzzerClangCounters.cpp | 49 ------- tools/fuzzing/libfuzzer/FuzzerDriver.cpp | 3 +- .../libfuzzer/FuzzerExtFunctionsDlsymWin.cpp | 62 --------- .../libfuzzer/FuzzerExtFunctionsWeakAlias.cpp | 56 -------- tools/fuzzing/libfuzzer/FuzzerInternal.h | 2 +- tools/fuzzing/libfuzzer/FuzzerLoop.cpp | 21 ++- tools/fuzzing/libfuzzer/FuzzerMerge.cpp | 4 +- tools/fuzzing/libfuzzer/FuzzerShmem.h | 69 ---------- .../fuzzing/libfuzzer/FuzzerShmemFuchsia.cpp | 38 ----- tools/fuzzing/libfuzzer/FuzzerShmemPosix.cpp | 103 -------------- .../fuzzing/libfuzzer/FuzzerShmemWindows.cpp | 64 --------- tools/fuzzing/libfuzzer/clone_libfuzzer.sh | 40 +++--- tools/fuzzing/libfuzzer/moz.build | 10 +- .../libfuzzer/patches/10-ef-runtime.patch | 31 +++++ .../libfuzzer/patches/11-callback-rv.patch | 130 ++++++++++++++++++ .../patches/12-custom-mutator-fail.patch | 53 +++++++ .../libfuzzer/patches/13-unused-write.patch | 80 +++++++++++ .../patches/14-explicit-allocator.patch | 22 +++ 19 files changed, 366 insertions(+), 475 deletions(-) delete mode 100644 tools/fuzzing/libfuzzer/FuzzerClangCounters.cpp delete mode 100644 tools/fuzzing/libfuzzer/FuzzerExtFunctionsDlsymWin.cpp delete mode 100644 tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeakAlias.cpp delete mode 100644 tools/fuzzing/libfuzzer/FuzzerShmem.h delete mode 100644 tools/fuzzing/libfuzzer/FuzzerShmemFuchsia.cpp delete mode 100644 tools/fuzzing/libfuzzer/FuzzerShmemPosix.cpp delete mode 100644 tools/fuzzing/libfuzzer/FuzzerShmemWindows.cpp create mode 100644 tools/fuzzing/libfuzzer/patches/10-ef-runtime.patch create mode 100644 tools/fuzzing/libfuzzer/patches/11-callback-rv.patch create mode 100644 tools/fuzzing/libfuzzer/patches/12-custom-mutator-fail.patch create mode 100644 tools/fuzzing/libfuzzer/patches/13-unused-write.patch create mode 100644 tools/fuzzing/libfuzzer/patches/14-explicit-allocator.patch diff --git a/tools/fuzzing/interface/FuzzingInterface.h b/tools/fuzzing/interface/FuzzingInterface.h index 5c13184d6380..ad49ed3c01ee 100644 --- a/tools/fuzzing/interface/FuzzingInterface.h +++ b/tools/fuzzing/interface/FuzzingInterface.h @@ -12,6 +12,10 @@ #include +#ifdef LIBFUZZER +# include "FuzzerExtFunctions.h" +#endif + #include "FuzzerRegistry.h" #include "mozilla/Assertions.h" diff --git a/tools/fuzzing/libfuzzer/FuzzerClangCounters.cpp b/tools/fuzzing/libfuzzer/FuzzerClangCounters.cpp deleted file mode 100644 index f69e922cf004..000000000000 --- a/tools/fuzzing/libfuzzer/FuzzerClangCounters.cpp +++ /dev/null @@ -1,49 +0,0 @@ -//===- FuzzerExtraCounters.cpp - Extra coverage counters ------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// Coverage counters from Clang's SourceBasedCodeCoverage. -//===----------------------------------------------------------------------===// - -// Support for SourceBasedCodeCoverage is experimental: -// * Works only for the main binary, not DSOs yet. -// * Works only on Linux. -// * Does not implement print_pcs/print_coverage yet. -// * Is not fully evaluated for performance and sensitivity. -// We expect large performance drop due to 64-bit counters, -// and *maybe* better sensitivity due to more fine-grained counters. -// Preliminary comparison on a single benchmark (RE2) shows -// a bit worse sensitivity though. - -#include "FuzzerDefs.h" - -#if LIBFUZZER_LINUX -__attribute__((weak)) extern uint64_t __start___llvm_prf_cnts; -__attribute__((weak)) extern uint64_t __stop___llvm_prf_cnts; -namespace fuzzer { -uint64_t *ClangCountersBegin() { return &__start___llvm_prf_cnts; } -uint64_t *ClangCountersEnd() { return &__stop___llvm_prf_cnts; } -} // namespace fuzzer -#else -// TODO: Implement on Mac (if the data shows it's worth it). -//__attribute__((visibility("hidden"))) -//extern uint64_t CountersStart __asm("section$start$__DATA$__llvm_prf_cnts"); -//__attribute__((visibility("hidden"))) -//extern uint64_t CountersEnd __asm("section$end$__DATA$__llvm_prf_cnts"); -namespace fuzzer { -uint64_t *ClangCountersBegin() { return nullptr; } -uint64_t *ClangCountersEnd() { return nullptr; } -} // namespace fuzzer -#endif - -namespace fuzzer { -ATTRIBUTE_NO_SANITIZE_ALL -void ClearClangCounters() { // hand-written memset, don't asan-ify. - for (auto P = ClangCountersBegin(); P < ClangCountersEnd(); P++) - *P = 0; -} -} diff --git a/tools/fuzzing/libfuzzer/FuzzerDriver.cpp b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp index 9c99d5ffb5e1..a116aca9dcb8 100644 --- a/tools/fuzzing/libfuzzer/FuzzerDriver.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp @@ -565,7 +565,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { using namespace fuzzer; assert(argc && argv && "Argument pointers cannot be nullptr"); std::string Argv0((*argv)[0]); - EF = new ExternalFunctions(); + if (!EF) + EF = new ExternalFunctions(); if (EF->LLVMFuzzerInitialize) EF->LLVMFuzzerInitialize(argc, argv); if (EF->__msan_scoped_disable_interceptor_checks) diff --git a/tools/fuzzing/libfuzzer/FuzzerExtFunctionsDlsymWin.cpp b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsDlsymWin.cpp deleted file mode 100644 index 321b3ec5d414..000000000000 --- a/tools/fuzzing/libfuzzer/FuzzerExtFunctionsDlsymWin.cpp +++ /dev/null @@ -1,62 +0,0 @@ -//===- FuzzerExtFunctionsDlsymWin.cpp - Interface to external functions ---===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// Implementation using dynamic loading for Windows. -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" -#if LIBFUZZER_WINDOWS - -#include "FuzzerExtFunctions.h" -#include "FuzzerIO.h" -#include "Windows.h" - -// This must be included after Windows.h. -#include "Psapi.h" - -namespace fuzzer { - -ExternalFunctions::ExternalFunctions() { - HMODULE Modules[1024]; - DWORD BytesNeeded; - HANDLE CurrentProcess = GetCurrentProcess(); - - if (!EnumProcessModules(CurrentProcess, Modules, sizeof(Modules), - &BytesNeeded)) { - Printf("EnumProcessModules failed (error: %d).\n", GetLastError()); - exit(1); - } - - if (sizeof(Modules) < BytesNeeded) { - Printf("Error: the array is not big enough to hold all loaded modules.\n"); - exit(1); - } - - for (size_t i = 0; i < (BytesNeeded / sizeof(HMODULE)); i++) - { - FARPROC Fn; -#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - if (this->NAME == nullptr) { \ - Fn = GetProcAddress(Modules[i], #NAME); \ - if (Fn == nullptr) \ - Fn = GetProcAddress(Modules[i], #NAME "__dll"); \ - this->NAME = (decltype(ExternalFunctions::NAME)) Fn; \ - } -#include "FuzzerExtFunctions.def" -#undef EXT_FUNC - } - -#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - if (this->NAME == nullptr && WARN) \ - Printf("WARNING: Failed to find function \"%s\".\n", #NAME); -#include "FuzzerExtFunctions.def" -#undef EXT_FUNC -} - -} // namespace fuzzer - -#endif // LIBFUZZER_WINDOWS diff --git a/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeakAlias.cpp b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeakAlias.cpp deleted file mode 100644 index e10f7b4dcac2..000000000000 --- a/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeakAlias.cpp +++ /dev/null @@ -1,56 +0,0 @@ -//===- FuzzerExtFunctionsWeakAlias.cpp - Interface to external functions --===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// Implementation using weak aliases. Works for Windows. -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" -#if LIBFUZZER_WINDOWS - -#include "FuzzerExtFunctions.h" -#include "FuzzerIO.h" - -using namespace fuzzer; - -extern "C" { -// Declare these symbols as weak to allow them to be optionally defined. -#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - RETURN_TYPE NAME##Def FUNC_SIG { \ - Printf("ERROR: Function \"%s\" not defined.\n", #NAME); \ - exit(1); \ - } \ - RETURN_TYPE NAME FUNC_SIG __attribute__((weak, alias(#NAME "Def"))); - -#include "FuzzerExtFunctions.def" - -#undef EXT_FUNC -} - -template -static T *GetFnPtr(T *Fun, T *FunDef, const char *FnName, bool WarnIfMissing) { - if (Fun == FunDef) { - if (WarnIfMissing) - Printf("WARNING: Failed to find function \"%s\".\n", FnName); - return nullptr; - } - return Fun; -} - -namespace fuzzer { - -ExternalFunctions::ExternalFunctions() { -#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ - this->NAME = GetFnPtr(::NAME, ::NAME##Def, #NAME, WARN); - -#include "FuzzerExtFunctions.def" - -#undef EXT_FUNC -} - -} // namespace fuzzer - -#endif // LIBFUZZER_WINDOWS diff --git a/tools/fuzzing/libfuzzer/FuzzerInternal.h b/tools/fuzzing/libfuzzer/FuzzerInternal.h index f20dae014145..3921add0a807 100644 --- a/tools/fuzzing/libfuzzer/FuzzerInternal.h +++ b/tools/fuzzing/libfuzzer/FuzzerInternal.h @@ -67,7 +67,7 @@ public: static void StaticFileSizeExceedCallback(); static void StaticGracefulExitCallback(); - void ExecuteCallback(const uint8_t *Data, size_t Size); + int ExecuteCallback(const uint8_t *Data, size_t Size); bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false, InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr); diff --git a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp index b86512b12558..dc82f7138dff 100644 --- a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp @@ -450,7 +450,9 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, if (!Size) return false; - ExecuteCallback(Data, Size); + if (ExecuteCallback(Data, Size) > 0) { + return false; + } UniqFeatureSetTmp.clear(); size_t FoundUniqFeaturesOfII = 0; @@ -507,7 +509,7 @@ static bool LooseMemeq(const uint8_t *A, const uint8_t *B, size_t Size) { !memcmp(A + Size - Limit / 2, B + Size - Limit / 2, Limit / 2); } -void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { +int Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { TPC.RecordInitialStack(); TotalNumberOfRuns++; assert(InFuzzingThread()); @@ -520,23 +522,24 @@ void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { if (CurrentUnitData && CurrentUnitData != Data) memcpy(CurrentUnitData, Data, Size); CurrentUnitSize = Size; + int Res = 0; { ScopedEnableMsanInterceptorChecks S; AllocTracer.Start(Options.TraceMalloc); UnitStartTime = system_clock::now(); TPC.ResetMaps(); RunningUserCallback = true; - int Res = CB(DataCopy, Size); + Res = CB(DataCopy, Size); RunningUserCallback = false; UnitStopTime = system_clock::now(); - (void)Res; - assert(Res == 0); + assert(Res >= 0); HasMoreMallocsThanFrees = AllocTracer.Stop(); } if (!LooseMemeq(DataCopy, Data, Size)) CrashOnOverwrittenData(); CurrentUnitSize = 0; delete[] DataCopy; + return Res; } std::string Fuzzer::WriteToOutputCorpus(const Unit &U) { @@ -660,6 +663,10 @@ void Fuzzer::MutateAndTestOne() { II.DataFlowTraceForFocusFunction); else NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen); + + if (!NewSize) + continue; + assert(NewSize > 0 && "Mutator returned empty unit"); assert(NewSize <= CurrentMaxMutationLen && "Mutator return oversized unit"); Size = NewSize; @@ -828,7 +835,9 @@ void Fuzzer::MinimizeCrashLoop(const Unit &U) { memcpy(CurrentUnitData, U.data(), U.size()); for (int i = 0; i < Options.MutateDepth; i++) { size_t NewSize = MD.Mutate(CurrentUnitData, U.size(), MaxMutationLen); - assert(NewSize > 0 && NewSize <= MaxMutationLen); + assert(NewSize <= MaxMutationLen); + if (!NewSize) + continue; ExecuteCallback(CurrentUnitData, NewSize); PrintPulseAndReportSlowInput(CurrentUnitData, NewSize); TryDetectingAMemoryLeak(CurrentUnitData, NewSize, diff --git a/tools/fuzzing/libfuzzer/FuzzerMerge.cpp b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp index dace45ece1d2..86433ded36b1 100644 --- a/tools/fuzzing/libfuzzer/FuzzerMerge.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp @@ -224,7 +224,9 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { OF.flush(); // Flush is important since Command::Execute may crash. // Run. TPC.ResetMaps(); - ExecuteCallback(U.data(), U.size()); + if (ExecuteCallback(U.data(), U.size()) > 0) { + continue; + } // Collect coverage. We are iterating over the files in this order: // * First, files in the initial corpus ordered by size, smallest first. // * Then, all other files, smallest first. diff --git a/tools/fuzzing/libfuzzer/FuzzerShmem.h b/tools/fuzzing/libfuzzer/FuzzerShmem.h deleted file mode 100644 index 53568e0acb69..000000000000 --- a/tools/fuzzing/libfuzzer/FuzzerShmem.h +++ /dev/null @@ -1,69 +0,0 @@ -//===- FuzzerShmem.h - shared memory interface ------------------*- C++ -* ===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// SharedMemoryRegion -//===----------------------------------------------------------------------===// - -#ifndef LLVM_FUZZER_SHMEM_H -#define LLVM_FUZZER_SHMEM_H - -#include -#include -#include - -#include "FuzzerDefs.h" - -namespace fuzzer { - -class SharedMemoryRegion { - public: - bool Create(const char *Name); - bool Open(const char *Name); - bool Destroy(const char *Name); - uint8_t *GetData() { return Data; } - void PostServer() {Post(0);} - void WaitServer() {Wait(0);} - void PostClient() {Post(1);} - void WaitClient() {Wait(1);} - - size_t WriteByteArray(const uint8_t *Bytes, size_t N) { - assert(N <= kShmemSize - sizeof(N)); - memcpy(GetData(), &N, sizeof(N)); - memcpy(GetData() + sizeof(N), Bytes, N); - assert(N == ReadByteArraySize()); - return N; - } - size_t ReadByteArraySize() { - size_t Res; - memcpy(&Res, GetData(), sizeof(Res)); - return Res; - } - uint8_t *GetByteArray() { return GetData() + sizeof(size_t); } - - bool IsServer() const { return Data && IAmServer; } - bool IsClient() const { return Data && !IAmServer; } - -private: - - static const size_t kShmemSize = 1 << 22; - bool IAmServer; - std::string Path(const char *Name); - std::string SemName(const char *Name, int Idx); - void Post(int Idx); - void Wait(int Idx); - - bool Map(int fd); - uint8_t *Data = nullptr; - void *Semaphore[2]; -}; - -extern SharedMemoryRegion SMR; - -} // namespace fuzzer - -#endif // LLVM_FUZZER_SHMEM_H diff --git a/tools/fuzzing/libfuzzer/FuzzerShmemFuchsia.cpp b/tools/fuzzing/libfuzzer/FuzzerShmemFuchsia.cpp deleted file mode 100644 index e9ce50c2ac8a..000000000000 --- a/tools/fuzzing/libfuzzer/FuzzerShmemFuchsia.cpp +++ /dev/null @@ -1,38 +0,0 @@ -//===- FuzzerShmemPosix.cpp - Posix shared memory ---------------*- C++ -* ===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// SharedMemoryRegion. For Fuchsia, this is just stubs as equivalence servers -// are not currently supported. -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" - -#if LIBFUZZER_FUCHSIA - -#include "FuzzerShmem.h" - -namespace fuzzer { - -bool SharedMemoryRegion::Create(const char *Name) { - return false; -} - -bool SharedMemoryRegion::Open(const char *Name) { - return false; -} - -bool SharedMemoryRegion::Destroy(const char *Name) { - return false; -} - -void SharedMemoryRegion::Post(int Idx) {} - -void SharedMemoryRegion::Wait(int Idx) {} - -} // namespace fuzzer - -#endif // LIBFUZZER_FUCHSIA diff --git a/tools/fuzzing/libfuzzer/FuzzerShmemPosix.cpp b/tools/fuzzing/libfuzzer/FuzzerShmemPosix.cpp deleted file mode 100644 index 50cdcfb509dc..000000000000 --- a/tools/fuzzing/libfuzzer/FuzzerShmemPosix.cpp +++ /dev/null @@ -1,103 +0,0 @@ -//===- FuzzerShmemPosix.cpp - Posix shared memory ---------------*- C++ -* ===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// SharedMemoryRegion -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" -#if LIBFUZZER_POSIX - -#include "FuzzerIO.h" -#include "FuzzerShmem.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace fuzzer { - -std::string SharedMemoryRegion::Path(const char *Name) { - return DirPlusFile(TmpDir(), Name); -} - -std::string SharedMemoryRegion::SemName(const char *Name, int Idx) { - std::string Res(Name); - return Res + (char)('0' + Idx); -} - -bool SharedMemoryRegion::Map(int fd) { - Data = - (uint8_t *)mmap(0, kShmemSize, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0); - if (Data == (uint8_t*)-1) - return false; - return true; -} - -bool SharedMemoryRegion::Create(const char *Name) { - int fd = open(Path(Name).c_str(), O_CREAT | O_RDWR, 0777); - if (fd < 0) return false; - if (ftruncate(fd, kShmemSize) < 0) return false; - if (!Map(fd)) - return false; - for (int i = 0; i < 2; i++) { - sem_unlink(SemName(Name, i).c_str()); - Semaphore[i] = sem_open(SemName(Name, i).c_str(), O_CREAT, 0644, 0); - if (Semaphore[i] == (void *)-1) - return false; - } - IAmServer = true; - return true; -} - -bool SharedMemoryRegion::Open(const char *Name) { - int fd = open(Path(Name).c_str(), O_RDWR); - if (fd < 0) return false; - struct stat stat_res; - if (0 != fstat(fd, &stat_res)) - return false; - assert(stat_res.st_size == kShmemSize); - if (!Map(fd)) - return false; - for (int i = 0; i < 2; i++) { - Semaphore[i] = sem_open(SemName(Name, i).c_str(), 0); - if (Semaphore[i] == (void *)-1) - return false; - } - IAmServer = false; - return true; -} - -bool SharedMemoryRegion::Destroy(const char *Name) { - return 0 == unlink(Path(Name).c_str()); -} - -void SharedMemoryRegion::Post(int Idx) { - assert(Idx == 0 || Idx == 1); - sem_post((sem_t*)Semaphore[Idx]); -} - -void SharedMemoryRegion::Wait(int Idx) { - assert(Idx == 0 || Idx == 1); - for (int i = 0; i < 10 && sem_wait((sem_t*)Semaphore[Idx]); i++) { - // sem_wait may fail if interrupted by a signal. - sleep(i); - if (i) - Printf("%s: sem_wait[%d] failed %s\n", i < 9 ? "WARNING" : "ERROR", i, - strerror(errno)); - if (i == 9) abort(); - } -} - -} // namespace fuzzer - -#endif // LIBFUZZER_POSIX diff --git a/tools/fuzzing/libfuzzer/FuzzerShmemWindows.cpp b/tools/fuzzing/libfuzzer/FuzzerShmemWindows.cpp deleted file mode 100644 index d330ebf4fd07..000000000000 --- a/tools/fuzzing/libfuzzer/FuzzerShmemWindows.cpp +++ /dev/null @@ -1,64 +0,0 @@ -//===- FuzzerShmemWindows.cpp - Posix shared memory -------------*- C++ -* ===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// SharedMemoryRegion -//===----------------------------------------------------------------------===// -#include "FuzzerDefs.h" -#if LIBFUZZER_WINDOWS - -#include "FuzzerIO.h" -#include "FuzzerShmem.h" - -#include -#include -#include -#include - -namespace fuzzer { - -std::string SharedMemoryRegion::Path(const char *Name) { - return DirPlusFile(TmpDir(), Name); -} - -std::string SharedMemoryRegion::SemName(const char *Name, int Idx) { - std::string Res(Name); - return Res + (char)('0' + Idx); -} - -bool SharedMemoryRegion::Map(int fd) { - assert(0 && "UNIMPLEMENTED"); - return false; -} - -bool SharedMemoryRegion::Create(const char *Name) { - assert(0 && "UNIMPLEMENTED"); - return false; -} - -bool SharedMemoryRegion::Open(const char *Name) { - assert(0 && "UNIMPLEMENTED"); - return false; -} - -bool SharedMemoryRegion::Destroy(const char *Name) { - assert(0 && "UNIMPLEMENTED"); - return false; -} - -void SharedMemoryRegion::Post(int Idx) { - assert(0 && "UNIMPLEMENTED"); -} - -void SharedMemoryRegion::Wait(int Idx) { - Semaphore[1] = nullptr; - assert(0 && "UNIMPLEMENTED"); -} - -} // namespace fuzzer - -#endif // LIBFUZZER_WINDOWS diff --git a/tools/fuzzing/libfuzzer/clone_libfuzzer.sh b/tools/fuzzing/libfuzzer/clone_libfuzzer.sh index 890eeec01dad..2131cd4fbbac 100755 --- a/tools/fuzzing/libfuzzer/clone_libfuzzer.sh +++ b/tools/fuzzing/libfuzzer/clone_libfuzzer.sh @@ -1,31 +1,35 @@ -#!/bin/bash +#!/bin/bash -e # Optionally get revision from cmd line -# Changelog: https://reviews.llvm.org/source/compiler-rt/history/compiler-rt/trunk/lib/fuzzer/ -[ $1 ] && REVISION=$1 || REVISION=356365 +[ $1 ] && REVISION=$1 || REVISION=c815210013f27cfac07d6b53b47e8ac53e86afa3 mkdir tmp -svn co -qr $REVISION https://llvm.org/svn/llvm-project/compiler-rt/trunk tmp || exit +git clone --single-branch --no-checkout --shallow-since "2019-03-01" https://github.com/llvm/llvm-project tmp -if [ $1 ]; then - # libFuzzer source files - CPPS=($(ls -rv tmp/lib/fuzzer/*.cpp)) - CPPS=(${CPPS[@]##*/}) - CPPS=(${CPPS[@]##FuzzerMain*}) # ignored +(cd tmp && git reset --hard $REVISION) - # Update SOURCES entries - sed -e "/^SOURCES/,/^]/ {/'/d}" -i moz.build - for CPP in ${CPPS[@]}; do sed -e "/^SOURCES/ a \\\t'${CPP}'," -i moz.build; done +# libFuzzer source files +CPPS=($(ls -rv tmp/compiler-rt/lib/fuzzer/*.cpp)) +CPPS=(${CPPS[@]##*/}) +CPPS=(${CPPS[@]##FuzzerMain*}) # ignored - # Remove previous files - rm *.{cpp,h,def} -fi +# Update SOURCES entries +sed -e "/^SOURCES/,/^]/ {/'/d}" -i moz.build +for CPP in ${CPPS[@]}; do sed -e "/^SOURCES/ a \\ '${CPP}'," -i moz.build; done + +# Remove previous files +rm *.{cpp,h,def} # Copy files -cp tmp/lib/fuzzer/*.{cpp,h,def} . +cp tmp/compiler-rt/lib/fuzzer/*.{cpp,h,def} . + +# Apply local patches +for patch in patches/*.patch +do + patch -p4 < $patch +done # Remove the temporary directory rm -Rf tmp/ -[ $1 ] && echo "Updated libFuzzer to ${REVISION}" - +echo "Updated libFuzzer to ${REVISION}" diff --git a/tools/fuzzing/libfuzzer/moz.build b/tools/fuzzing/libfuzzer/moz.build index d9c3a4231468..a04169df57b0 100644 --- a/tools/fuzzing/libfuzzer/moz.build +++ b/tools/fuzzing/libfuzzer/moz.build @@ -8,17 +8,16 @@ Library('fuzzer') EXPORTS += [ 'FuzzerDefs.h', + 'FuzzerExtFunctions.def', + 'FuzzerExtFunctions.h', ] SOURCES += [ - 'FuzzerClangCounters.cpp', 'FuzzerCrossOver.cpp', 'FuzzerDataFlowTrace.cpp', 'FuzzerDriver.cpp', 'FuzzerExtFunctionsDlsym.cpp', - 'FuzzerExtFunctionsDlsymWin.cpp', 'FuzzerExtFunctionsWeak.cpp', - 'FuzzerExtFunctionsWeakAlias.cpp', 'FuzzerExtFunctionsWindows.cpp', 'FuzzerExtraCounters.cpp', 'FuzzerFork.cpp', @@ -29,16 +28,13 @@ SOURCES += [ 'FuzzerMerge.cpp', 'FuzzerMutate.cpp', 'FuzzerSHA1.cpp', - 'FuzzerShmemFuchsia.cpp', - 'FuzzerShmemPosix.cpp', - 'FuzzerShmemWindows.cpp', 'FuzzerTracePC.cpp', 'FuzzerUtil.cpp', 'FuzzerUtilDarwin.cpp', 'FuzzerUtilFuchsia.cpp', 'FuzzerUtilLinux.cpp', 'FuzzerUtilPosix.cpp', - 'FuzzerUtilWindows.cpp' + 'FuzzerUtilWindows.cpp', ] if CONFIG['CC_TYPE'] == 'clang': diff --git a/tools/fuzzing/libfuzzer/patches/10-ef-runtime.patch b/tools/fuzzing/libfuzzer/patches/10-ef-runtime.patch new file mode 100644 index 000000000000..31cfb413c55d --- /dev/null +++ b/tools/fuzzing/libfuzzer/patches/10-ef-runtime.patch @@ -0,0 +1,31 @@ +# HG changeset patch +# User Christian Holler +# Date 1586344236 -7200 +# Wed Apr 08 13:10:36 2020 +0200 +# Node ID ab10fba76a52a5b205038150ad198dfc4583c566 +# Parent 5f0446b8b0b20f432fa5dc016b781270c469fe83 +[libFuzzer] Allow external functions to be defined at runtime + +diff --git a/tools/fuzzing/libfuzzer/FuzzerDriver.cpp b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp +--- a/tools/fuzzing/libfuzzer/FuzzerDriver.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp +@@ -560,17 +560,18 @@ int AnalyzeDictionary(Fuzzer *F, const V + Printf("###### End of useless dictionary elements. ######\n"); + return 0; + } + + int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { + using namespace fuzzer; + assert(argc && argv && "Argument pointers cannot be nullptr"); + std::string Argv0((*argv)[0]); +- EF = new ExternalFunctions(); ++ if (!EF) ++ EF = new ExternalFunctions(); + if (EF->LLVMFuzzerInitialize) + EF->LLVMFuzzerInitialize(argc, argv); + if (EF->__msan_scoped_disable_interceptor_checks) + EF->__msan_scoped_disable_interceptor_checks(); + const Vector Args(*argv, *argv + *argc); + assert(!Args.empty()); + ProgName = new std::string(Args[0]); + if (Argv0 != *ProgName) { diff --git a/tools/fuzzing/libfuzzer/patches/11-callback-rv.patch b/tools/fuzzing/libfuzzer/patches/11-callback-rv.patch new file mode 100644 index 000000000000..297985df600b --- /dev/null +++ b/tools/fuzzing/libfuzzer/patches/11-callback-rv.patch @@ -0,0 +1,130 @@ +# HG changeset patch +# User Christian Holler +# Date 1586345242 -7200 +# Wed Apr 08 13:27:22 2020 +0200 +# Node ID 142bb91a6f14a721fea81c75d18da92edb6e6ea3 +# Parent ab10fba76a52a5b205038150ad198dfc4583c566 +[libFuzzer] Change libFuzzer callback contract to allow positive return values + +diff --git a/tools/fuzzing/libfuzzer/FuzzerInternal.h b/tools/fuzzing/libfuzzer/FuzzerInternal.h +--- a/tools/fuzzing/libfuzzer/FuzzerInternal.h ++++ b/tools/fuzzing/libfuzzer/FuzzerInternal.h +@@ -62,17 +62,17 @@ public: + + static void StaticAlarmCallback(); + static void StaticCrashSignalCallback(); + static void StaticExitCallback(); + static void StaticInterruptCallback(); + static void StaticFileSizeExceedCallback(); + static void StaticGracefulExitCallback(); + +- void ExecuteCallback(const uint8_t *Data, size_t Size); ++ int ExecuteCallback(const uint8_t *Data, size_t Size); + bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false, + InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr); + + // Merge Corpora[1:] into Corpora[0]. + void Merge(const Vector &Corpora); + void CrashResistantMergeInternalStep(const std::string &ControlFilePath); + MutationDispatcher &GetMD() { return MD; } + void PrintFinalStats(); +diff --git a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp +--- a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp +@@ -445,17 +445,19 @@ void Fuzzer::PrintPulseAndReportSlowInpu + } + } + + bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, + InputInfo *II, bool *FoundUniqFeatures) { + if (!Size) + return false; + +- ExecuteCallback(Data, Size); ++ if (ExecuteCallback(Data, Size) > 0) { ++ return false; ++ } + + UniqFeatureSetTmp.clear(); + size_t FoundUniqFeaturesOfII = 0; + size_t NumUpdatesBefore = Corpus.NumFeatureUpdates(); + TPC.CollectFeatures([&](size_t Feature) { + if (Corpus.AddFeature(Feature, Size, Options.Shrink)) + UniqFeatureSetTmp.push_back(Feature); + if (Options.ReduceInputs && II) +@@ -502,46 +504,47 @@ static bool LooseMemeq(const uint8_t *A, + const size_t Limit = 64; + if (Size <= 64) + return !memcmp(A, B, Size); + // Compare first and last Limit/2 bytes. + return !memcmp(A, B, Limit / 2) && + !memcmp(A + Size - Limit / 2, B + Size - Limit / 2, Limit / 2); + } + +-void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { ++int Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { + TPC.RecordInitialStack(); + TotalNumberOfRuns++; + assert(InFuzzingThread()); + // We copy the contents of Unit into a separate heap buffer + // so that we reliably find buffer overflows in it. + uint8_t *DataCopy = new uint8_t[Size]; + memcpy(DataCopy, Data, Size); + if (EF->__msan_unpoison) + EF->__msan_unpoison(DataCopy, Size); + if (CurrentUnitData && CurrentUnitData != Data) + memcpy(CurrentUnitData, Data, Size); + CurrentUnitSize = Size; ++ int Res = 0; + { + ScopedEnableMsanInterceptorChecks S; + AllocTracer.Start(Options.TraceMalloc); + UnitStartTime = system_clock::now(); + TPC.ResetMaps(); + RunningUserCallback = true; +- int Res = CB(DataCopy, Size); ++ Res = CB(DataCopy, Size); + RunningUserCallback = false; + UnitStopTime = system_clock::now(); +- (void)Res; +- assert(Res == 0); ++ assert(Res >= 0); + HasMoreMallocsThanFrees = AllocTracer.Stop(); + } + if (!LooseMemeq(DataCopy, Data, Size)) + CrashOnOverwrittenData(); + CurrentUnitSize = 0; + delete[] DataCopy; ++ return Res; + } + + std::string Fuzzer::WriteToOutputCorpus(const Unit &U) { + if (Options.OnlyASCII) + assert(IsASCII(U)); + if (Options.OutputCorpus.empty()) + return ""; + std::string Path = DirPlusFile(Options.OutputCorpus, Hash(U)); +diff --git a/tools/fuzzing/libfuzzer/FuzzerMerge.cpp b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp +--- a/tools/fuzzing/libfuzzer/FuzzerMerge.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp +@@ -219,17 +219,19 @@ void Fuzzer::CrashResistantMergeInternal + U.shrink_to_fit(); + } + std::ostringstream StartedLine; + // Write the pre-run marker. + OF << "STARTED " << i << " " << U.size() << "\n"; + OF.flush(); // Flush is important since Command::Execute may crash. + // Run. + TPC.ResetMaps(); +- ExecuteCallback(U.data(), U.size()); ++ if (ExecuteCallback(U.data(), U.size()) > 0) { ++ continue; ++ } + // Collect coverage. We are iterating over the files in this order: + // * First, files in the initial corpus ordered by size, smallest first. + // * Then, all other files, smallest first. + // So it makes no sense to record all features for all files, instead we + // only record features that were not seen before. + Set UniqFeatures; + TPC.CollectFeatures([&](size_t Feature) { + if (AllFeatures.insert(Feature).second) diff --git a/tools/fuzzing/libfuzzer/patches/12-custom-mutator-fail.patch b/tools/fuzzing/libfuzzer/patches/12-custom-mutator-fail.patch new file mode 100644 index 000000000000..def78ff9209b --- /dev/null +++ b/tools/fuzzing/libfuzzer/patches/12-custom-mutator-fail.patch @@ -0,0 +1,53 @@ +# HG changeset patch +# User Christian Holler +# Date 1586345312 -7200 +# Wed Apr 08 13:28:32 2020 +0200 +# Node ID 169280b21031865a2fb217af759ad3124dd87ae2 +# Parent 8244be44ec9b52eb81b56e39d04c97a1e4eb13ab +[libFuzzer] Allow custom mutators to fail + +diff --git a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp +--- a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp +@@ -659,16 +659,20 @@ void Fuzzer::MutateAndTestOne() { + MaybeExitGracefully(); + size_t NewSize = 0; + if (II.HasFocusFunction && !II.DataFlowTraceForFocusFunction.empty() && + Size <= CurrentMaxMutationLen) + NewSize = MD.MutateWithMask(CurrentUnitData, Size, Size, + II.DataFlowTraceForFocusFunction); + else + NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen); ++ ++ if (!NewSize) ++ continue; ++ + assert(NewSize > 0 && "Mutator returned empty unit"); + assert(NewSize <= CurrentMaxMutationLen && "Mutator return oversized unit"); + Size = NewSize; + II.NumExecutedMutations++; + + bool FoundUniqFeatures = false; + bool NewCov = RunOne(CurrentUnitData, Size, /*MayDeleteFile=*/true, &II, + &FoundUniqFeatures); +@@ -827,17 +831,19 @@ void Fuzzer::Loop(const Vector 0 && NewSize <= MaxMutationLen); ++ assert(NewSize <= MaxMutationLen); ++ if (!NewSize) ++ continue; + ExecuteCallback(CurrentUnitData, NewSize); + PrintPulseAndReportSlowInput(CurrentUnitData, NewSize); + TryDetectingAMemoryLeak(CurrentUnitData, NewSize, + /*DuringInitialCorpusExecution*/ false); + } + } + } + diff --git a/tools/fuzzing/libfuzzer/patches/13-unused-write.patch b/tools/fuzzing/libfuzzer/patches/13-unused-write.patch new file mode 100644 index 000000000000..a5d2fae2ade8 --- /dev/null +++ b/tools/fuzzing/libfuzzer/patches/13-unused-write.patch @@ -0,0 +1,80 @@ +diff --git a/tools/fuzzing/libfuzzer/FuzzerIO.cpp b/tools/fuzzing/libfuzzer/FuzzerIO.cpp +--- a/tools/fuzzing/libfuzzer/FuzzerIO.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerIO.cpp +@@ -3,16 +3,17 @@ + // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + // See https://llvm.org/LICENSE.txt for license information. + // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + // + //===----------------------------------------------------------------------===// + // IO functions. + //===----------------------------------------------------------------------===// + ++#include "mozilla/Unused.h" + #include "FuzzerDefs.h" + #include "FuzzerExtFunctions.h" + #include "FuzzerIO.h" + #include "FuzzerUtil.h" + #include + #include + #include + #include +@@ -59,17 +60,17 @@ std::string FileToString(const std::stri + void CopyFileToErr(const std::string &Path) { + Printf("%s", FileToString(Path).c_str()); + } + + void WriteToFile(const Unit &U, const std::string &Path) { + // Use raw C interface because this function may be called from a sig handler. + FILE *Out = fopen(Path.c_str(), "w"); + if (!Out) return; +- fwrite(U.data(), sizeof(U[0]), U.size(), Out); ++ mozilla::Unused << fwrite(U.data(), sizeof(U[0]), U.size(), Out); + fclose(Out); + } + + void ReadDirToVectorOfUnits(const char *Path, Vector *V, + long *Epoch, size_t MaxSize, bool ExitOnError) { + long E = Epoch ? *Epoch : 0; + Vector Files; + ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true); +diff --git a/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp b/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp +--- a/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp +@@ -2,16 +2,17 @@ + // + // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + // See https://llvm.org/LICENSE.txt for license information. + // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + // + //===----------------------------------------------------------------------===// + // IO functions implementation using Posix API. + //===----------------------------------------------------------------------===// ++#include "mozilla/Unused.h" + #include "FuzzerDefs.h" + #if LIBFUZZER_POSIX || LIBFUZZER_FUCHSIA + + #include "FuzzerExtFunctions.h" + #include "FuzzerIO.h" + #include + #include + #include +@@ -154,17 +155,17 @@ bool IsInterestingCoverageFile(const std + if (FileName.find("/usr/include/") != std::string::npos) + return false; + if (FileName == "") + return false; + return true; + } + + void RawPrint(const char *Str) { +- write(2, Str, strlen(Str)); ++ mozilla::Unused << write(2, Str, strlen(Str)); + } + + void MkDir(const std::string &Path) { + mkdir(Path.c_str(), 0700); + } + + void RmDir(const std::string &Path) { + rmdir(Path.c_str()); diff --git a/tools/fuzzing/libfuzzer/patches/14-explicit-allocator.patch b/tools/fuzzing/libfuzzer/patches/14-explicit-allocator.patch new file mode 100644 index 000000000000..9293b904a594 --- /dev/null +++ b/tools/fuzzing/libfuzzer/patches/14-explicit-allocator.patch @@ -0,0 +1,22 @@ +diff --git a/tools/fuzzing/libfuzzer/FuzzerDefs.h b/tools/fuzzing/libfuzzer/FuzzerDefs.h +--- a/tools/fuzzing/libfuzzer/FuzzerDefs.h ++++ b/tools/fuzzing/libfuzzer/FuzzerDefs.h +@@ -174,17 +174,17 @@ extern ExternalFunctions *EF; + // We are using a custom allocator to give a different symbol name to STL + // containers in order to avoid ODR violations. + template + class fuzzer_allocator: public std::allocator { + public: + fuzzer_allocator() = default; + + template +- fuzzer_allocator(const fuzzer_allocator&) {} ++ explicit fuzzer_allocator(const fuzzer_allocator&) {} + + template + struct rebind { typedef fuzzer_allocator other; }; + }; + + template + using Vector = std::vector>; +