From 2df205e3f0e771b397719ca1ebe2aa4f1b10313d Mon Sep 17 00:00:00 2001 From: Mihai Alexandru Michis Date: Mon, 11 May 2020 17:00:53 +0300 Subject: [PATCH] Backed out changeset 14b092e7a974 (bug 1635762) for causing failures in browser_Telemetry_timestamp_test.js CLOSED TREE --- tools/fuzzing/interface/FuzzingInterface.h | 4 - .../fuzzing/libfuzzer/FuzzerClangCounters.cpp | 49 +++++++ tools/fuzzing/libfuzzer/FuzzerDriver.cpp | 3 +- .../libfuzzer/FuzzerExtFunctionsDlsymWin.cpp | 62 +++++++++ .../libfuzzer/FuzzerExtFunctionsWeakAlias.cpp | 56 ++++++++ tools/fuzzing/libfuzzer/FuzzerInternal.h | 2 +- tools/fuzzing/libfuzzer/FuzzerLoop.cpp | 21 +-- tools/fuzzing/libfuzzer/FuzzerMerge.cpp | 4 +- tools/fuzzing/libfuzzer/FuzzerShmem.h | 69 ++++++++++ .../fuzzing/libfuzzer/FuzzerShmemFuchsia.cpp | 38 +++++ tools/fuzzing/libfuzzer/FuzzerShmemPosix.cpp | 103 ++++++++++++++ .../fuzzing/libfuzzer/FuzzerShmemWindows.cpp | 64 +++++++++ tools/fuzzing/libfuzzer/clone_libfuzzer.sh | 40 +++--- tools/fuzzing/libfuzzer/moz.build | 10 +- .../libfuzzer/patches/10-ef-runtime.patch | 31 ----- .../libfuzzer/patches/11-callback-rv.patch | 130 ------------------ .../patches/12-custom-mutator-fail.patch | 53 ------- .../libfuzzer/patches/13-unused-write.patch | 80 ----------- .../patches/14-explicit-allocator.patch | 22 --- 19 files changed, 475 insertions(+), 366 deletions(-) create mode 100644 tools/fuzzing/libfuzzer/FuzzerClangCounters.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerExtFunctionsDlsymWin.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeakAlias.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerShmem.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerShmemFuchsia.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerShmemPosix.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerShmemWindows.cpp delete mode 100644 tools/fuzzing/libfuzzer/patches/10-ef-runtime.patch delete mode 100644 tools/fuzzing/libfuzzer/patches/11-callback-rv.patch delete mode 100644 tools/fuzzing/libfuzzer/patches/12-custom-mutator-fail.patch delete mode 100644 tools/fuzzing/libfuzzer/patches/13-unused-write.patch delete mode 100644 tools/fuzzing/libfuzzer/patches/14-explicit-allocator.patch diff --git a/tools/fuzzing/interface/FuzzingInterface.h b/tools/fuzzing/interface/FuzzingInterface.h index ad49ed3c01ee..5c13184d6380 100644 --- a/tools/fuzzing/interface/FuzzingInterface.h +++ b/tools/fuzzing/interface/FuzzingInterface.h @@ -12,10 +12,6 @@ #include -#ifdef LIBFUZZER -# include "FuzzerExtFunctions.h" -#endif - #include "FuzzerRegistry.h" #include "mozilla/Assertions.h" diff --git a/tools/fuzzing/libfuzzer/FuzzerClangCounters.cpp b/tools/fuzzing/libfuzzer/FuzzerClangCounters.cpp new file mode 100644 index 000000000000..f69e922cf004 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerClangCounters.cpp @@ -0,0 +1,49 @@ +//===- FuzzerExtraCounters.cpp - Extra coverage counters ------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Coverage counters from Clang's SourceBasedCodeCoverage. +//===----------------------------------------------------------------------===// + +// Support for SourceBasedCodeCoverage is experimental: +// * Works only for the main binary, not DSOs yet. +// * Works only on Linux. +// * Does not implement print_pcs/print_coverage yet. +// * Is not fully evaluated for performance and sensitivity. +// We expect large performance drop due to 64-bit counters, +// and *maybe* better sensitivity due to more fine-grained counters. +// Preliminary comparison on a single benchmark (RE2) shows +// a bit worse sensitivity though. + +#include "FuzzerDefs.h" + +#if LIBFUZZER_LINUX +__attribute__((weak)) extern uint64_t __start___llvm_prf_cnts; +__attribute__((weak)) extern uint64_t __stop___llvm_prf_cnts; +namespace fuzzer { +uint64_t *ClangCountersBegin() { return &__start___llvm_prf_cnts; } +uint64_t *ClangCountersEnd() { return &__stop___llvm_prf_cnts; } +} // namespace fuzzer +#else +// TODO: Implement on Mac (if the data shows it's worth it). +//__attribute__((visibility("hidden"))) +//extern uint64_t CountersStart __asm("section$start$__DATA$__llvm_prf_cnts"); +//__attribute__((visibility("hidden"))) +//extern uint64_t CountersEnd __asm("section$end$__DATA$__llvm_prf_cnts"); +namespace fuzzer { +uint64_t *ClangCountersBegin() { return nullptr; } +uint64_t *ClangCountersEnd() { return nullptr; } +} // namespace fuzzer +#endif + +namespace fuzzer { +ATTRIBUTE_NO_SANITIZE_ALL +void ClearClangCounters() { // hand-written memset, don't asan-ify. + for (auto P = ClangCountersBegin(); P < ClangCountersEnd(); P++) + *P = 0; +} +} diff --git a/tools/fuzzing/libfuzzer/FuzzerDriver.cpp b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp index a116aca9dcb8..9c99d5ffb5e1 100644 --- a/tools/fuzzing/libfuzzer/FuzzerDriver.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp @@ -565,8 +565,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { using namespace fuzzer; assert(argc && argv && "Argument pointers cannot be nullptr"); std::string Argv0((*argv)[0]); - if (!EF) - EF = new ExternalFunctions(); + EF = new ExternalFunctions(); if (EF->LLVMFuzzerInitialize) EF->LLVMFuzzerInitialize(argc, argv); if (EF->__msan_scoped_disable_interceptor_checks) diff --git a/tools/fuzzing/libfuzzer/FuzzerExtFunctionsDlsymWin.cpp b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsDlsymWin.cpp new file mode 100644 index 000000000000..321b3ec5d414 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsDlsymWin.cpp @@ -0,0 +1,62 @@ +//===- FuzzerExtFunctionsDlsymWin.cpp - Interface to external functions ---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Implementation using dynamic loading for Windows. +//===----------------------------------------------------------------------===// +#include "FuzzerDefs.h" +#if LIBFUZZER_WINDOWS + +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" +#include "Windows.h" + +// This must be included after Windows.h. +#include "Psapi.h" + +namespace fuzzer { + +ExternalFunctions::ExternalFunctions() { + HMODULE Modules[1024]; + DWORD BytesNeeded; + HANDLE CurrentProcess = GetCurrentProcess(); + + if (!EnumProcessModules(CurrentProcess, Modules, sizeof(Modules), + &BytesNeeded)) { + Printf("EnumProcessModules failed (error: %d).\n", GetLastError()); + exit(1); + } + + if (sizeof(Modules) < BytesNeeded) { + Printf("Error: the array is not big enough to hold all loaded modules.\n"); + exit(1); + } + + for (size_t i = 0; i < (BytesNeeded / sizeof(HMODULE)); i++) + { + FARPROC Fn; +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + if (this->NAME == nullptr) { \ + Fn = GetProcAddress(Modules[i], #NAME); \ + if (Fn == nullptr) \ + Fn = GetProcAddress(Modules[i], #NAME "__dll"); \ + this->NAME = (decltype(ExternalFunctions::NAME)) Fn; \ + } +#include "FuzzerExtFunctions.def" +#undef EXT_FUNC + } + +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + if (this->NAME == nullptr && WARN) \ + Printf("WARNING: Failed to find function \"%s\".\n", #NAME); +#include "FuzzerExtFunctions.def" +#undef EXT_FUNC +} + +} // namespace fuzzer + +#endif // LIBFUZZER_WINDOWS diff --git a/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeakAlias.cpp b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeakAlias.cpp new file mode 100644 index 000000000000..e10f7b4dcac2 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeakAlias.cpp @@ -0,0 +1,56 @@ +//===- FuzzerExtFunctionsWeakAlias.cpp - Interface to external functions --===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Implementation using weak aliases. Works for Windows. +//===----------------------------------------------------------------------===// +#include "FuzzerDefs.h" +#if LIBFUZZER_WINDOWS + +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" + +using namespace fuzzer; + +extern "C" { +// Declare these symbols as weak to allow them to be optionally defined. +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + RETURN_TYPE NAME##Def FUNC_SIG { \ + Printf("ERROR: Function \"%s\" not defined.\n", #NAME); \ + exit(1); \ + } \ + RETURN_TYPE NAME FUNC_SIG __attribute__((weak, alias(#NAME "Def"))); + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +} + +template +static T *GetFnPtr(T *Fun, T *FunDef, const char *FnName, bool WarnIfMissing) { + if (Fun == FunDef) { + if (WarnIfMissing) + Printf("WARNING: Failed to find function \"%s\".\n", FnName); + return nullptr; + } + return Fun; +} + +namespace fuzzer { + +ExternalFunctions::ExternalFunctions() { +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + this->NAME = GetFnPtr(::NAME, ::NAME##Def, #NAME, WARN); + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +} + +} // namespace fuzzer + +#endif // LIBFUZZER_WINDOWS diff --git a/tools/fuzzing/libfuzzer/FuzzerInternal.h b/tools/fuzzing/libfuzzer/FuzzerInternal.h index 3921add0a807..f20dae014145 100644 --- a/tools/fuzzing/libfuzzer/FuzzerInternal.h +++ b/tools/fuzzing/libfuzzer/FuzzerInternal.h @@ -67,7 +67,7 @@ public: static void StaticFileSizeExceedCallback(); static void StaticGracefulExitCallback(); - int ExecuteCallback(const uint8_t *Data, size_t Size); + void ExecuteCallback(const uint8_t *Data, size_t Size); bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false, InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr); diff --git a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp index dc82f7138dff..b86512b12558 100644 --- a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp @@ -450,9 +450,7 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, if (!Size) return false; - if (ExecuteCallback(Data, Size) > 0) { - return false; - } + ExecuteCallback(Data, Size); UniqFeatureSetTmp.clear(); size_t FoundUniqFeaturesOfII = 0; @@ -509,7 +507,7 @@ static bool LooseMemeq(const uint8_t *A, const uint8_t *B, size_t Size) { !memcmp(A + Size - Limit / 2, B + Size - Limit / 2, Limit / 2); } -int Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { +void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { TPC.RecordInitialStack(); TotalNumberOfRuns++; assert(InFuzzingThread()); @@ -522,24 +520,23 @@ int Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { if (CurrentUnitData && CurrentUnitData != Data) memcpy(CurrentUnitData, Data, Size); CurrentUnitSize = Size; - int Res = 0; { ScopedEnableMsanInterceptorChecks S; AllocTracer.Start(Options.TraceMalloc); UnitStartTime = system_clock::now(); TPC.ResetMaps(); RunningUserCallback = true; - Res = CB(DataCopy, Size); + int Res = CB(DataCopy, Size); RunningUserCallback = false; UnitStopTime = system_clock::now(); - assert(Res >= 0); + (void)Res; + assert(Res == 0); HasMoreMallocsThanFrees = AllocTracer.Stop(); } if (!LooseMemeq(DataCopy, Data, Size)) CrashOnOverwrittenData(); CurrentUnitSize = 0; delete[] DataCopy; - return Res; } std::string Fuzzer::WriteToOutputCorpus(const Unit &U) { @@ -663,10 +660,6 @@ void Fuzzer::MutateAndTestOne() { II.DataFlowTraceForFocusFunction); else NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen); - - if (!NewSize) - continue; - assert(NewSize > 0 && "Mutator returned empty unit"); assert(NewSize <= CurrentMaxMutationLen && "Mutator return oversized unit"); Size = NewSize; @@ -835,9 +828,7 @@ void Fuzzer::MinimizeCrashLoop(const Unit &U) { memcpy(CurrentUnitData, U.data(), U.size()); for (int i = 0; i < Options.MutateDepth; i++) { size_t NewSize = MD.Mutate(CurrentUnitData, U.size(), MaxMutationLen); - assert(NewSize <= MaxMutationLen); - if (!NewSize) - continue; + assert(NewSize > 0 && NewSize <= MaxMutationLen); ExecuteCallback(CurrentUnitData, NewSize); PrintPulseAndReportSlowInput(CurrentUnitData, NewSize); TryDetectingAMemoryLeak(CurrentUnitData, NewSize, diff --git a/tools/fuzzing/libfuzzer/FuzzerMerge.cpp b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp index 86433ded36b1..dace45ece1d2 100644 --- a/tools/fuzzing/libfuzzer/FuzzerMerge.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp @@ -224,9 +224,7 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { OF.flush(); // Flush is important since Command::Execute may crash. // Run. TPC.ResetMaps(); - if (ExecuteCallback(U.data(), U.size()) > 0) { - continue; - } + ExecuteCallback(U.data(), U.size()); // Collect coverage. We are iterating over the files in this order: // * First, files in the initial corpus ordered by size, smallest first. // * Then, all other files, smallest first. diff --git a/tools/fuzzing/libfuzzer/FuzzerShmem.h b/tools/fuzzing/libfuzzer/FuzzerShmem.h new file mode 100644 index 000000000000..53568e0acb69 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerShmem.h @@ -0,0 +1,69 @@ +//===- FuzzerShmem.h - shared memory interface ------------------*- C++ -* ===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// SharedMemoryRegion +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_SHMEM_H +#define LLVM_FUZZER_SHMEM_H + +#include +#include +#include + +#include "FuzzerDefs.h" + +namespace fuzzer { + +class SharedMemoryRegion { + public: + bool Create(const char *Name); + bool Open(const char *Name); + bool Destroy(const char *Name); + uint8_t *GetData() { return Data; } + void PostServer() {Post(0);} + void WaitServer() {Wait(0);} + void PostClient() {Post(1);} + void WaitClient() {Wait(1);} + + size_t WriteByteArray(const uint8_t *Bytes, size_t N) { + assert(N <= kShmemSize - sizeof(N)); + memcpy(GetData(), &N, sizeof(N)); + memcpy(GetData() + sizeof(N), Bytes, N); + assert(N == ReadByteArraySize()); + return N; + } + size_t ReadByteArraySize() { + size_t Res; + memcpy(&Res, GetData(), sizeof(Res)); + return Res; + } + uint8_t *GetByteArray() { return GetData() + sizeof(size_t); } + + bool IsServer() const { return Data && IAmServer; } + bool IsClient() const { return Data && !IAmServer; } + +private: + + static const size_t kShmemSize = 1 << 22; + bool IAmServer; + std::string Path(const char *Name); + std::string SemName(const char *Name, int Idx); + void Post(int Idx); + void Wait(int Idx); + + bool Map(int fd); + uint8_t *Data = nullptr; + void *Semaphore[2]; +}; + +extern SharedMemoryRegion SMR; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_SHMEM_H diff --git a/tools/fuzzing/libfuzzer/FuzzerShmemFuchsia.cpp b/tools/fuzzing/libfuzzer/FuzzerShmemFuchsia.cpp new file mode 100644 index 000000000000..e9ce50c2ac8a --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerShmemFuchsia.cpp @@ -0,0 +1,38 @@ +//===- FuzzerShmemPosix.cpp - Posix shared memory ---------------*- C++ -* ===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// SharedMemoryRegion. For Fuchsia, this is just stubs as equivalence servers +// are not currently supported. +//===----------------------------------------------------------------------===// +#include "FuzzerDefs.h" + +#if LIBFUZZER_FUCHSIA + +#include "FuzzerShmem.h" + +namespace fuzzer { + +bool SharedMemoryRegion::Create(const char *Name) { + return false; +} + +bool SharedMemoryRegion::Open(const char *Name) { + return false; +} + +bool SharedMemoryRegion::Destroy(const char *Name) { + return false; +} + +void SharedMemoryRegion::Post(int Idx) {} + +void SharedMemoryRegion::Wait(int Idx) {} + +} // namespace fuzzer + +#endif // LIBFUZZER_FUCHSIA diff --git a/tools/fuzzing/libfuzzer/FuzzerShmemPosix.cpp b/tools/fuzzing/libfuzzer/FuzzerShmemPosix.cpp new file mode 100644 index 000000000000..50cdcfb509dc --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerShmemPosix.cpp @@ -0,0 +1,103 @@ +//===- FuzzerShmemPosix.cpp - Posix shared memory ---------------*- C++ -* ===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// SharedMemoryRegion +//===----------------------------------------------------------------------===// +#include "FuzzerDefs.h" +#if LIBFUZZER_POSIX + +#include "FuzzerIO.h" +#include "FuzzerShmem.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fuzzer { + +std::string SharedMemoryRegion::Path(const char *Name) { + return DirPlusFile(TmpDir(), Name); +} + +std::string SharedMemoryRegion::SemName(const char *Name, int Idx) { + std::string Res(Name); + return Res + (char)('0' + Idx); +} + +bool SharedMemoryRegion::Map(int fd) { + Data = + (uint8_t *)mmap(0, kShmemSize, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0); + if (Data == (uint8_t*)-1) + return false; + return true; +} + +bool SharedMemoryRegion::Create(const char *Name) { + int fd = open(Path(Name).c_str(), O_CREAT | O_RDWR, 0777); + if (fd < 0) return false; + if (ftruncate(fd, kShmemSize) < 0) return false; + if (!Map(fd)) + return false; + for (int i = 0; i < 2; i++) { + sem_unlink(SemName(Name, i).c_str()); + Semaphore[i] = sem_open(SemName(Name, i).c_str(), O_CREAT, 0644, 0); + if (Semaphore[i] == (void *)-1) + return false; + } + IAmServer = true; + return true; +} + +bool SharedMemoryRegion::Open(const char *Name) { + int fd = open(Path(Name).c_str(), O_RDWR); + if (fd < 0) return false; + struct stat stat_res; + if (0 != fstat(fd, &stat_res)) + return false; + assert(stat_res.st_size == kShmemSize); + if (!Map(fd)) + return false; + for (int i = 0; i < 2; i++) { + Semaphore[i] = sem_open(SemName(Name, i).c_str(), 0); + if (Semaphore[i] == (void *)-1) + return false; + } + IAmServer = false; + return true; +} + +bool SharedMemoryRegion::Destroy(const char *Name) { + return 0 == unlink(Path(Name).c_str()); +} + +void SharedMemoryRegion::Post(int Idx) { + assert(Idx == 0 || Idx == 1); + sem_post((sem_t*)Semaphore[Idx]); +} + +void SharedMemoryRegion::Wait(int Idx) { + assert(Idx == 0 || Idx == 1); + for (int i = 0; i < 10 && sem_wait((sem_t*)Semaphore[Idx]); i++) { + // sem_wait may fail if interrupted by a signal. + sleep(i); + if (i) + Printf("%s: sem_wait[%d] failed %s\n", i < 9 ? "WARNING" : "ERROR", i, + strerror(errno)); + if (i == 9) abort(); + } +} + +} // namespace fuzzer + +#endif // LIBFUZZER_POSIX diff --git a/tools/fuzzing/libfuzzer/FuzzerShmemWindows.cpp b/tools/fuzzing/libfuzzer/FuzzerShmemWindows.cpp new file mode 100644 index 000000000000..d330ebf4fd07 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerShmemWindows.cpp @@ -0,0 +1,64 @@ +//===- FuzzerShmemWindows.cpp - Posix shared memory -------------*- C++ -* ===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// SharedMemoryRegion +//===----------------------------------------------------------------------===// +#include "FuzzerDefs.h" +#if LIBFUZZER_WINDOWS + +#include "FuzzerIO.h" +#include "FuzzerShmem.h" + +#include +#include +#include +#include + +namespace fuzzer { + +std::string SharedMemoryRegion::Path(const char *Name) { + return DirPlusFile(TmpDir(), Name); +} + +std::string SharedMemoryRegion::SemName(const char *Name, int Idx) { + std::string Res(Name); + return Res + (char)('0' + Idx); +} + +bool SharedMemoryRegion::Map(int fd) { + assert(0 && "UNIMPLEMENTED"); + return false; +} + +bool SharedMemoryRegion::Create(const char *Name) { + assert(0 && "UNIMPLEMENTED"); + return false; +} + +bool SharedMemoryRegion::Open(const char *Name) { + assert(0 && "UNIMPLEMENTED"); + return false; +} + +bool SharedMemoryRegion::Destroy(const char *Name) { + assert(0 && "UNIMPLEMENTED"); + return false; +} + +void SharedMemoryRegion::Post(int Idx) { + assert(0 && "UNIMPLEMENTED"); +} + +void SharedMemoryRegion::Wait(int Idx) { + Semaphore[1] = nullptr; + assert(0 && "UNIMPLEMENTED"); +} + +} // namespace fuzzer + +#endif // LIBFUZZER_WINDOWS diff --git a/tools/fuzzing/libfuzzer/clone_libfuzzer.sh b/tools/fuzzing/libfuzzer/clone_libfuzzer.sh index 2131cd4fbbac..890eeec01dad 100755 --- a/tools/fuzzing/libfuzzer/clone_libfuzzer.sh +++ b/tools/fuzzing/libfuzzer/clone_libfuzzer.sh @@ -1,35 +1,31 @@ -#!/bin/bash -e +#!/bin/bash # Optionally get revision from cmd line -[ $1 ] && REVISION=$1 || REVISION=c815210013f27cfac07d6b53b47e8ac53e86afa3 +# Changelog: https://reviews.llvm.org/source/compiler-rt/history/compiler-rt/trunk/lib/fuzzer/ +[ $1 ] && REVISION=$1 || REVISION=356365 mkdir tmp -git clone --single-branch --no-checkout --shallow-since "2019-03-01" https://github.com/llvm/llvm-project tmp +svn co -qr $REVISION https://llvm.org/svn/llvm-project/compiler-rt/trunk tmp || exit -(cd tmp && git reset --hard $REVISION) +if [ $1 ]; then + # libFuzzer source files + CPPS=($(ls -rv tmp/lib/fuzzer/*.cpp)) + CPPS=(${CPPS[@]##*/}) + CPPS=(${CPPS[@]##FuzzerMain*}) # ignored -# libFuzzer source files -CPPS=($(ls -rv tmp/compiler-rt/lib/fuzzer/*.cpp)) -CPPS=(${CPPS[@]##*/}) -CPPS=(${CPPS[@]##FuzzerMain*}) # ignored + # Update SOURCES entries + sed -e "/^SOURCES/,/^]/ {/'/d}" -i moz.build + for CPP in ${CPPS[@]}; do sed -e "/^SOURCES/ a \\\t'${CPP}'," -i moz.build; done -# Update SOURCES entries -sed -e "/^SOURCES/,/^]/ {/'/d}" -i moz.build -for CPP in ${CPPS[@]}; do sed -e "/^SOURCES/ a \\ '${CPP}'," -i moz.build; done - -# Remove previous files -rm *.{cpp,h,def} + # Remove previous files + rm *.{cpp,h,def} +fi # Copy files -cp tmp/compiler-rt/lib/fuzzer/*.{cpp,h,def} . - -# Apply local patches -for patch in patches/*.patch -do - patch -p4 < $patch -done +cp tmp/lib/fuzzer/*.{cpp,h,def} . # Remove the temporary directory rm -Rf tmp/ -echo "Updated libFuzzer to ${REVISION}" +[ $1 ] && echo "Updated libFuzzer to ${REVISION}" + diff --git a/tools/fuzzing/libfuzzer/moz.build b/tools/fuzzing/libfuzzer/moz.build index a04169df57b0..d9c3a4231468 100644 --- a/tools/fuzzing/libfuzzer/moz.build +++ b/tools/fuzzing/libfuzzer/moz.build @@ -8,16 +8,17 @@ Library('fuzzer') EXPORTS += [ 'FuzzerDefs.h', - 'FuzzerExtFunctions.def', - 'FuzzerExtFunctions.h', ] SOURCES += [ + 'FuzzerClangCounters.cpp', 'FuzzerCrossOver.cpp', 'FuzzerDataFlowTrace.cpp', 'FuzzerDriver.cpp', 'FuzzerExtFunctionsDlsym.cpp', + 'FuzzerExtFunctionsDlsymWin.cpp', 'FuzzerExtFunctionsWeak.cpp', + 'FuzzerExtFunctionsWeakAlias.cpp', 'FuzzerExtFunctionsWindows.cpp', 'FuzzerExtraCounters.cpp', 'FuzzerFork.cpp', @@ -28,13 +29,16 @@ SOURCES += [ 'FuzzerMerge.cpp', 'FuzzerMutate.cpp', 'FuzzerSHA1.cpp', + 'FuzzerShmemFuchsia.cpp', + 'FuzzerShmemPosix.cpp', + 'FuzzerShmemWindows.cpp', 'FuzzerTracePC.cpp', 'FuzzerUtil.cpp', 'FuzzerUtilDarwin.cpp', 'FuzzerUtilFuchsia.cpp', 'FuzzerUtilLinux.cpp', 'FuzzerUtilPosix.cpp', - 'FuzzerUtilWindows.cpp', + 'FuzzerUtilWindows.cpp' ] if CONFIG['CC_TYPE'] == 'clang': diff --git a/tools/fuzzing/libfuzzer/patches/10-ef-runtime.patch b/tools/fuzzing/libfuzzer/patches/10-ef-runtime.patch deleted file mode 100644 index 31cfb413c55d..000000000000 --- a/tools/fuzzing/libfuzzer/patches/10-ef-runtime.patch +++ /dev/null @@ -1,31 +0,0 @@ -# HG changeset patch -# User Christian Holler -# Date 1586344236 -7200 -# Wed Apr 08 13:10:36 2020 +0200 -# Node ID ab10fba76a52a5b205038150ad198dfc4583c566 -# Parent 5f0446b8b0b20f432fa5dc016b781270c469fe83 -[libFuzzer] Allow external functions to be defined at runtime - -diff --git a/tools/fuzzing/libfuzzer/FuzzerDriver.cpp b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp ---- a/tools/fuzzing/libfuzzer/FuzzerDriver.cpp -+++ b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp -@@ -560,17 +560,18 @@ int AnalyzeDictionary(Fuzzer *F, const V - Printf("###### End of useless dictionary elements. ######\n"); - return 0; - } - - int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { - using namespace fuzzer; - assert(argc && argv && "Argument pointers cannot be nullptr"); - std::string Argv0((*argv)[0]); -- EF = new ExternalFunctions(); -+ if (!EF) -+ EF = new ExternalFunctions(); - if (EF->LLVMFuzzerInitialize) - EF->LLVMFuzzerInitialize(argc, argv); - if (EF->__msan_scoped_disable_interceptor_checks) - EF->__msan_scoped_disable_interceptor_checks(); - const Vector Args(*argv, *argv + *argc); - assert(!Args.empty()); - ProgName = new std::string(Args[0]); - if (Argv0 != *ProgName) { diff --git a/tools/fuzzing/libfuzzer/patches/11-callback-rv.patch b/tools/fuzzing/libfuzzer/patches/11-callback-rv.patch deleted file mode 100644 index 297985df600b..000000000000 --- a/tools/fuzzing/libfuzzer/patches/11-callback-rv.patch +++ /dev/null @@ -1,130 +0,0 @@ -# HG changeset patch -# User Christian Holler -# Date 1586345242 -7200 -# Wed Apr 08 13:27:22 2020 +0200 -# Node ID 142bb91a6f14a721fea81c75d18da92edb6e6ea3 -# Parent ab10fba76a52a5b205038150ad198dfc4583c566 -[libFuzzer] Change libFuzzer callback contract to allow positive return values - -diff --git a/tools/fuzzing/libfuzzer/FuzzerInternal.h b/tools/fuzzing/libfuzzer/FuzzerInternal.h ---- a/tools/fuzzing/libfuzzer/FuzzerInternal.h -+++ b/tools/fuzzing/libfuzzer/FuzzerInternal.h -@@ -62,17 +62,17 @@ public: - - static void StaticAlarmCallback(); - static void StaticCrashSignalCallback(); - static void StaticExitCallback(); - static void StaticInterruptCallback(); - static void StaticFileSizeExceedCallback(); - static void StaticGracefulExitCallback(); - -- void ExecuteCallback(const uint8_t *Data, size_t Size); -+ int ExecuteCallback(const uint8_t *Data, size_t Size); - bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false, - InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr); - - // Merge Corpora[1:] into Corpora[0]. - void Merge(const Vector &Corpora); - void CrashResistantMergeInternalStep(const std::string &ControlFilePath); - MutationDispatcher &GetMD() { return MD; } - void PrintFinalStats(); -diff --git a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp ---- a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp -+++ b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp -@@ -445,17 +445,19 @@ void Fuzzer::PrintPulseAndReportSlowInpu - } - } - - bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, - InputInfo *II, bool *FoundUniqFeatures) { - if (!Size) - return false; - -- ExecuteCallback(Data, Size); -+ if (ExecuteCallback(Data, Size) > 0) { -+ return false; -+ } - - UniqFeatureSetTmp.clear(); - size_t FoundUniqFeaturesOfII = 0; - size_t NumUpdatesBefore = Corpus.NumFeatureUpdates(); - TPC.CollectFeatures([&](size_t Feature) { - if (Corpus.AddFeature(Feature, Size, Options.Shrink)) - UniqFeatureSetTmp.push_back(Feature); - if (Options.ReduceInputs && II) -@@ -502,46 +504,47 @@ static bool LooseMemeq(const uint8_t *A, - const size_t Limit = 64; - if (Size <= 64) - return !memcmp(A, B, Size); - // Compare first and last Limit/2 bytes. - return !memcmp(A, B, Limit / 2) && - !memcmp(A + Size - Limit / 2, B + Size - Limit / 2, Limit / 2); - } - --void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { -+int Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { - TPC.RecordInitialStack(); - TotalNumberOfRuns++; - assert(InFuzzingThread()); - // We copy the contents of Unit into a separate heap buffer - // so that we reliably find buffer overflows in it. - uint8_t *DataCopy = new uint8_t[Size]; - memcpy(DataCopy, Data, Size); - if (EF->__msan_unpoison) - EF->__msan_unpoison(DataCopy, Size); - if (CurrentUnitData && CurrentUnitData != Data) - memcpy(CurrentUnitData, Data, Size); - CurrentUnitSize = Size; -+ int Res = 0; - { - ScopedEnableMsanInterceptorChecks S; - AllocTracer.Start(Options.TraceMalloc); - UnitStartTime = system_clock::now(); - TPC.ResetMaps(); - RunningUserCallback = true; -- int Res = CB(DataCopy, Size); -+ Res = CB(DataCopy, Size); - RunningUserCallback = false; - UnitStopTime = system_clock::now(); -- (void)Res; -- assert(Res == 0); -+ assert(Res >= 0); - HasMoreMallocsThanFrees = AllocTracer.Stop(); - } - if (!LooseMemeq(DataCopy, Data, Size)) - CrashOnOverwrittenData(); - CurrentUnitSize = 0; - delete[] DataCopy; -+ return Res; - } - - std::string Fuzzer::WriteToOutputCorpus(const Unit &U) { - if (Options.OnlyASCII) - assert(IsASCII(U)); - if (Options.OutputCorpus.empty()) - return ""; - std::string Path = DirPlusFile(Options.OutputCorpus, Hash(U)); -diff --git a/tools/fuzzing/libfuzzer/FuzzerMerge.cpp b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp ---- a/tools/fuzzing/libfuzzer/FuzzerMerge.cpp -+++ b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp -@@ -219,17 +219,19 @@ void Fuzzer::CrashResistantMergeInternal - U.shrink_to_fit(); - } - std::ostringstream StartedLine; - // Write the pre-run marker. - OF << "STARTED " << i << " " << U.size() << "\n"; - OF.flush(); // Flush is important since Command::Execute may crash. - // Run. - TPC.ResetMaps(); -- ExecuteCallback(U.data(), U.size()); -+ if (ExecuteCallback(U.data(), U.size()) > 0) { -+ continue; -+ } - // Collect coverage. We are iterating over the files in this order: - // * First, files in the initial corpus ordered by size, smallest first. - // * Then, all other files, smallest first. - // So it makes no sense to record all features for all files, instead we - // only record features that were not seen before. - Set UniqFeatures; - TPC.CollectFeatures([&](size_t Feature) { - if (AllFeatures.insert(Feature).second) diff --git a/tools/fuzzing/libfuzzer/patches/12-custom-mutator-fail.patch b/tools/fuzzing/libfuzzer/patches/12-custom-mutator-fail.patch deleted file mode 100644 index def78ff9209b..000000000000 --- a/tools/fuzzing/libfuzzer/patches/12-custom-mutator-fail.patch +++ /dev/null @@ -1,53 +0,0 @@ -# HG changeset patch -# User Christian Holler -# Date 1586345312 -7200 -# Wed Apr 08 13:28:32 2020 +0200 -# Node ID 169280b21031865a2fb217af759ad3124dd87ae2 -# Parent 8244be44ec9b52eb81b56e39d04c97a1e4eb13ab -[libFuzzer] Allow custom mutators to fail - -diff --git a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp ---- a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp -+++ b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp -@@ -659,16 +659,20 @@ void Fuzzer::MutateAndTestOne() { - MaybeExitGracefully(); - size_t NewSize = 0; - if (II.HasFocusFunction && !II.DataFlowTraceForFocusFunction.empty() && - Size <= CurrentMaxMutationLen) - NewSize = MD.MutateWithMask(CurrentUnitData, Size, Size, - II.DataFlowTraceForFocusFunction); - else - NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen); -+ -+ if (!NewSize) -+ continue; -+ - assert(NewSize > 0 && "Mutator returned empty unit"); - assert(NewSize <= CurrentMaxMutationLen && "Mutator return oversized unit"); - Size = NewSize; - II.NumExecutedMutations++; - - bool FoundUniqFeatures = false; - bool NewCov = RunOne(CurrentUnitData, Size, /*MayDeleteFile=*/true, &II, - &FoundUniqFeatures); -@@ -827,17 +831,19 @@ void Fuzzer::Loop(const Vector 0 && NewSize <= MaxMutationLen); -+ assert(NewSize <= MaxMutationLen); -+ if (!NewSize) -+ continue; - ExecuteCallback(CurrentUnitData, NewSize); - PrintPulseAndReportSlowInput(CurrentUnitData, NewSize); - TryDetectingAMemoryLeak(CurrentUnitData, NewSize, - /*DuringInitialCorpusExecution*/ false); - } - } - } - diff --git a/tools/fuzzing/libfuzzer/patches/13-unused-write.patch b/tools/fuzzing/libfuzzer/patches/13-unused-write.patch deleted file mode 100644 index a5d2fae2ade8..000000000000 --- a/tools/fuzzing/libfuzzer/patches/13-unused-write.patch +++ /dev/null @@ -1,80 +0,0 @@ -diff --git a/tools/fuzzing/libfuzzer/FuzzerIO.cpp b/tools/fuzzing/libfuzzer/FuzzerIO.cpp ---- a/tools/fuzzing/libfuzzer/FuzzerIO.cpp -+++ b/tools/fuzzing/libfuzzer/FuzzerIO.cpp -@@ -3,16 +3,17 @@ - // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. - // See https://llvm.org/LICENSE.txt for license information. - // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - // - //===----------------------------------------------------------------------===// - // IO functions. - //===----------------------------------------------------------------------===// - -+#include "mozilla/Unused.h" - #include "FuzzerDefs.h" - #include "FuzzerExtFunctions.h" - #include "FuzzerIO.h" - #include "FuzzerUtil.h" - #include - #include - #include - #include -@@ -59,17 +60,17 @@ std::string FileToString(const std::stri - void CopyFileToErr(const std::string &Path) { - Printf("%s", FileToString(Path).c_str()); - } - - void WriteToFile(const Unit &U, const std::string &Path) { - // Use raw C interface because this function may be called from a sig handler. - FILE *Out = fopen(Path.c_str(), "w"); - if (!Out) return; -- fwrite(U.data(), sizeof(U[0]), U.size(), Out); -+ mozilla::Unused << fwrite(U.data(), sizeof(U[0]), U.size(), Out); - fclose(Out); - } - - void ReadDirToVectorOfUnits(const char *Path, Vector *V, - long *Epoch, size_t MaxSize, bool ExitOnError) { - long E = Epoch ? *Epoch : 0; - Vector Files; - ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true); -diff --git a/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp b/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp ---- a/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp -+++ b/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp -@@ -2,16 +2,17 @@ - // - // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. - // See https://llvm.org/LICENSE.txt for license information. - // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - // - //===----------------------------------------------------------------------===// - // IO functions implementation using Posix API. - //===----------------------------------------------------------------------===// -+#include "mozilla/Unused.h" - #include "FuzzerDefs.h" - #if LIBFUZZER_POSIX || LIBFUZZER_FUCHSIA - - #include "FuzzerExtFunctions.h" - #include "FuzzerIO.h" - #include - #include - #include -@@ -154,17 +155,17 @@ bool IsInterestingCoverageFile(const std - if (FileName.find("/usr/include/") != std::string::npos) - return false; - if (FileName == "") - return false; - return true; - } - - void RawPrint(const char *Str) { -- write(2, Str, strlen(Str)); -+ mozilla::Unused << write(2, Str, strlen(Str)); - } - - void MkDir(const std::string &Path) { - mkdir(Path.c_str(), 0700); - } - - void RmDir(const std::string &Path) { - rmdir(Path.c_str()); diff --git a/tools/fuzzing/libfuzzer/patches/14-explicit-allocator.patch b/tools/fuzzing/libfuzzer/patches/14-explicit-allocator.patch deleted file mode 100644 index 9293b904a594..000000000000 --- a/tools/fuzzing/libfuzzer/patches/14-explicit-allocator.patch +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/tools/fuzzing/libfuzzer/FuzzerDefs.h b/tools/fuzzing/libfuzzer/FuzzerDefs.h ---- a/tools/fuzzing/libfuzzer/FuzzerDefs.h -+++ b/tools/fuzzing/libfuzzer/FuzzerDefs.h -@@ -174,17 +174,17 @@ extern ExternalFunctions *EF; - // We are using a custom allocator to give a different symbol name to STL - // containers in order to avoid ODR violations. - template - class fuzzer_allocator: public std::allocator { - public: - fuzzer_allocator() = default; - - template -- fuzzer_allocator(const fuzzer_allocator&) {} -+ explicit fuzzer_allocator(const fuzzer_allocator&) {} - - template - struct rebind { typedef fuzzer_allocator other; }; - }; - - template - using Vector = std::vector>; -