handler: Add report upload.

The handler is now capable of uploading crash reports from the database.
At present, only one upload attempt is made, and the report will be
moved to “completed” in the database after the attempt, regardless of
whether it succeeded or failed.

The handler also has support to push annotations from its command line
into the process annotations map of each crash report it writes. This is
intended to set basic information about each crash report, such as the
product ID and version. Each potentially crashy process can’t be relied
on to maintain this information on their own.

With this change, Crashpad is now 100% capable of running a handler that
maintains a database and uploads crash reports to a Breakpad-type server
such that Breakpad properly interprets the reports. This is all possible
from the command line.

TEST=run_with_crashpad --handler crashpad_handler \
         -a --database=/tmp/crashpad_db \
         -a --url=https://clients2.google.com/cr/staging_report \
         -a --annotation=prod=crashpad \
         -a --annotation=ver=0.6.0 \
         crashy_program

R=rsesek@chromium.org

Review URL: https://codereview.chromium.org/982613002

handler/mac/crash_report_exception_handler.cc

@@ -59,9 +59,11 @@ class CallErrorWritingCrashReport {
 
 CrashReportExceptionHandler::CrashReportExceptionHandler(
     CrashReportDatabase* database,
-    CrashReportUploadThread* upload_thread)
+    CrashReportUploadThread* upload_thread,
+    const std::map<std::string, std::string>* process_annotations)
     : database_(database),
-      upload_thread_(upload_thread) {
+      upload_thread_(upload_thread),
+      process_annotations_(process_annotations) {
 }
 
 CrashReportExceptionHandler::~CrashReportExceptionHandler() {
@@ -108,6 +110,18 @@ kern_return_t CrashReportExceptionHandler::CatchMachException(
     return KERN_FAILURE;
   }
 
+  if (!process_snapshot.InitializeException(thread,
+                                            exception,
+                                            code,
+                                            code_count,
+                                            *flavor,
+                                            old_state,
+                                            old_state_count)) {
+    return KERN_FAILURE;
+  }
+
+  process_snapshot.SetAnnotationsSimpleMap(*process_annotations_);
+
   CrashReportDatabase::NewReport* new_report;
   CrashReportDatabase::OperationStatus database_status =
       database_->PrepareNewCrashReport(&new_report);

handler/mac/crash_report_exception_handler.h

@@ -17,6 +17,9 @@
 
 #include <mach/mach.h>
 
+#include <map>
+#include <string>
+
 #include "base/basictypes.h"
 #include "client/crash_report_database.h"
 #include "handler/mac/crash_report_upload_thread.h"
@@ -33,8 +36,20 @@ class CrashReportExceptionHandler : public UniversalMachExcServer::Interface {
   //! \param[in] database The database to store crash reports in. Weak.
   //! \param[in] upload_thread The upload thread to notify when a new crash
   //!     report is written into \a database.
-  CrashReportExceptionHandler(CrashReportDatabase* database,
-                              CrashReportUploadThread* upload_thread);
+  //! \param[in] process_annotations A map of annotations to insert as
+  //!     process-level annotations into each crash report that is written. Do
+  //!     not confuse this with module-level annotations, which are under the
+  //!     control of the crashing process, and are used to implement Chrome’s
+  //!     “crash keys.” Process-level annotations are those that are beyond the
+  //!     control of the crashing process, which must reliably be set even if
+  //!     the process crashes before it’s able to establish its own annotations.
+  //!     To interoperate with Breakpad servers, the recommended practice is to
+  //!     specify values for the `"prod"` and `"ver"` keys as process
+  //!     annotations.
+  CrashReportExceptionHandler(
+      CrashReportDatabase* database,
+      CrashReportUploadThread* upload_thread,
+      const std::map<std::string, std::string>* process_annotations);
 
   ~CrashReportExceptionHandler();
 
@@ -61,6 +76,7 @@ class CrashReportExceptionHandler : public UniversalMachExcServer::Interface {
  private:
   CrashReportDatabase* database_;  // weak
   CrashReportUploadThread* upload_thread_;  // weak
+  const std::map<std::string, std::string>* process_annotations_;  // weak
 
   DISALLOW_COPY_AND_ASSIGN(CrashReportExceptionHandler);
 };

handler/mac/crash_report_upload_thread.cc

@@ -16,14 +16,119 @@
 
 #include <errno.h>
 
+#include <map>
 #include <vector>
+#include <utility>
 
 #include "base/logging.h"
+#include "base/memory/scoped_ptr.h"
+#include "snapshot/minidump/process_snapshot_minidump.h"
+#include "snapshot/module_snapshot.h"
+#include "util/file/file_reader.h"
+#include "util/net/http_body.h"
+#include "util/net/http_multipart_builder.h"
+#include "util/net/http_transport.h"
 
 namespace crashpad {
 
-CrashReportUploadThread::CrashReportUploadThread(CrashReportDatabase* database)
-    : database_(database),
+namespace {
+
+// Given a minidump file readable by |minidump_file_reader|, returns a map of
+// key-value pairs to use as HTTP form parameters for upload to a Breakpad
+// server. The map is built by combining the process simple annotations map with
+// each module’s simple annotations map. In the case of duplicate keys, the map
+// will retain the first value found for any key, and will log a warning about
+// discarded values. Each module’s annotations vector is also examined and built
+// into a single string value, with distinct elements separated by newlines, and
+// stored at the key named “list_annotations”, which supersedes any other key
+// found by that name.
+//
+// In the event of an error reading the minidump file, a message will be logged.
+std::map<std::string, std::string> BreakpadHTTPFormParametersFromMinidump(
+    FileReader* minidump_file_reader) {
+  ProcessSnapshotMinidump minidump_process_snapshot;
+  if (!minidump_process_snapshot.Initialize(minidump_file_reader)) {
+    return std::map<std::string, std::string>();
+  }
+
+  std::map<std::string, std::string> parameters =
+      minidump_process_snapshot.AnnotationsSimpleMap();
+
+  std::string list_annotations;
+  for (const ModuleSnapshot* module : minidump_process_snapshot.Modules()) {
+    for (const auto& kv : module->AnnotationsSimpleMap()) {
+      if (parameters.find(kv.first) != parameters.end()) {
+        LOG(WARNING) << "duplicate key " << kv.first << ", discarding value "
+                     << kv.second;
+      } else {
+        parameters.insert(kv);
+      }
+    }
+
+    for (std::string annotation : module->AnnotationsVector()) {
+      list_annotations.append(annotation);
+      list_annotations.append("\n");
+    }
+  }
+
+  if (!list_annotations.empty()) {
+    // Remove the final newline character.
+    list_annotations.pop_back();
+
+    const char kListAnnotationsKey[] = "list_annotations";
+    auto it = parameters.find(kListAnnotationsKey);
+    if (it != parameters.end()) {
+      LOG(WARNING) << "duplicate key " << kListAnnotationsKey
+                   << ", discarding value " << it->second;
+      it->second = list_annotations;
+    } else {
+      parameters.insert(std::make_pair(kListAnnotationsKey, list_annotations));
+    }
+  }
+
+  return parameters;
+}
+
+// Calls CrashReportDatabase::RecordUploadAttempt() with |successful| set to
+// false upon destruction unless disarmed by calling Fire() or Disarm(). Fire()
+// triggers an immediate call. Armed upon construction.
+class CallRecordUploadAttempt {
+ public:
+  CallRecordUploadAttempt(CrashReportDatabase* database,
+                          const CrashReportDatabase::Report* report)
+      : database_(database),
+        report_(report) {
+  }
+
+  ~CallRecordUploadAttempt() {
+    Fire();
+  }
+
+  void Fire() {
+    if (report_) {
+      database_->RecordUploadAttempt(report_, false, std::string());
+    }
+
+    Disarm();
+  }
+
+  void Disarm() {
+    report_ = nullptr;
+  }
+
+ private:
+  CrashReportDatabase* database_;  // weak
+  const CrashReportDatabase::Report* report_;  // weak
+
+  DISALLOW_COPY_AND_ASSIGN(CallRecordUploadAttempt);
+};
+
+}  // namespace
+
+CrashReportUploadThread::CrashReportUploadThread(CrashReportDatabase* database,
+                                                 const std::string& url)
+    : url_(url),
+      database_(database),
       semaphore_(0),
       thread_(0),
       running_(false) {
@@ -104,8 +209,99 @@ void CrashReportUploadThread::ProcessPendingReports() {
 
 void CrashReportUploadThread::ProcessPendingReport(
     const CrashReportDatabase::Report& report) {
-  // TODO(mark): Actually upload the report, if uploads are enabled.
-  database_->SkipReportUpload(report.uuid);
+  // TODO(mark): Allow uploads to be disabled.
+  // TODO(mark): Rate-limit uploads.
+
+  const CrashReportDatabase::Report* upload_report;
+  CrashReportDatabase::OperationStatus status =
+      database_->GetReportForUploading(report.uuid, &upload_report);
+  switch (status) {
+    case CrashReportDatabase::kNoError:
+      break;
+
+    case CrashReportDatabase::kBusyError:
+      return;
+
+    case CrashReportDatabase::kReportNotFound:
+    case CrashReportDatabase::kFileSystemError:
+    case CrashReportDatabase::kDatabaseError:
+      // In these cases, SkipReportUpload() might not work either, but it’s best
+      // to at least try to get the report out of the way.
+      database_->SkipReportUpload(report.uuid);
+      return;
+  }
+
+  CallRecordUploadAttempt call_record_upload_attempt(database_, upload_report);
+
+  std::string response_body;
+  UploadResult upload_result = UploadReport(upload_report, &response_body);
+  switch (upload_result) {
+    case UploadResult::kSuccess:
+      call_record_upload_attempt.Disarm();
+      database_->RecordUploadAttempt(upload_report, true, response_body);
+      break;
+    case UploadResult::kPermanentFailure:
+    case UploadResult::kRetry:
+      call_record_upload_attempt.Fire();
+
+      // TODO(mark): Deal with retries properly: don’t call SkipReportUplaod()
+      // if the result was kRetry and the report hasn’t already been retried
+      // too many times.
+      database_->SkipReportUpload(report.uuid);
+      break;
+  }
+}
+
+CrashReportUploadThread::UploadResult CrashReportUploadThread::UploadReport(
+    const CrashReportDatabase::Report* report,
+    std::string* response_body) {
+  std::map<std::string, std::string> parameters;
+
+  {
+    FileReader minidump_file_reader;
+    if (!minidump_file_reader.Open(report->file_path)) {
+      // If the minidump file can’t be opened, all hope is lost.
+      return UploadResult::kPermanentFailure;
+    }
+
+    // If the minidump file could be opened, ignore any errors that might occur
+    // when attempting to interpret it. This may result in its being uploaded
+    // with few or no parameters, but as long as there’s a dump file, the server
+    // can decide what to do with it.
+    parameters = BreakpadHTTPFormParametersFromMinidump(&minidump_file_reader);
+  }
+
+  HTTPMultipartBuilder http_multipart_builder;
+
+  const char kMinidumpKey[] = "upload_file_minidump";
+
+  for (const auto& kv : parameters) {
+    if (kv.first == kMinidumpKey) {
+      LOG(WARNING) << "reserved key " << kv.first << ", discarding value "
+                   << kv.second;
+    } else {
+      http_multipart_builder.SetFormData(kv.first, kv.second);
+    }
+  }
+
+  http_multipart_builder.SetFileAttachment(kMinidumpKey,
+                                           report->file_path.BaseName().value(),
+                                           report->file_path,
+                                           "application/octet-stream");
+
+  // TODO(mark): There should be a timeout option for upload.
+  scoped_ptr<HTTPTransport> http_transport(HTTPTransport::Create());
+  http_transport->SetURL(url_);
+  HTTPHeaders::value_type content_type =
+      http_multipart_builder.GetContentType();
+  http_transport->SetHeader(content_type.first, content_type.second);
+  http_transport->SetBodyStream(http_multipart_builder.GetBodyStream().Pass());
+
+  if (!http_transport->ExecuteSynchronously(response_body)) {
+    return UploadResult::kRetry;
+  }
+
+  return UploadResult::kSuccess;
 }
 
 // static

handler/mac/crash_report_upload_thread.h

@@ -19,6 +19,8 @@
 
 #include <pthread.h>
 
+#include <string>
+
 #include "client/crash_report_database.h"
 #include "util/synchronization/semaphore.h"
 
@@ -39,7 +41,12 @@ namespace crashpad {
 //! processes.
 class CrashReportUploadThread {
  public:
-  explicit CrashReportUploadThread(CrashReportDatabase* database);
+  //! \brief Constructs a new object.
+  //!
+  //! \param[in] database The database to upload crash reports from.
+  //! \param[in] url The URL of the server to upload crash reports to.
+  CrashReportUploadThread(CrashReportDatabase* database,
+                          const std::string& url);
   ~CrashReportUploadThread();
 
   //! \brief Starts a dedicated upload thread, which executes ThreadMain().
@@ -69,6 +76,26 @@ class CrashReportUploadThread {
   void ReportPending();
 
  private:
+  //! \brief The result code from UploadReport().
+  enum class UploadResult {
+    //! \brief The crash report was uploaded successfully.
+    kSuccess,
+
+    //! \brief The crash report upload failed in such a way that recovery is
+    //!     impossible.
+    //!
+    //! No further upload attempts should be made for the report.
+    kPermanentFailure,
+
+    //! \brief The crash report upload failed, but it might succeed again if
+    //!     retried in the future.
+    //!
+    //! If the report has not already been retried too many times, the caller
+    //! may arrange to call UploadReport() for the report again in the future,
+    //! after a suitable delay.
+    kRetry,
+  };
+
   //! \brief Calls ProcessPendingReports() in response to ReportPending() having
   //!     been called on any thread, as well as periodically on a timer.
   void ThreadMain();
@@ -81,15 +108,31 @@ class CrashReportUploadThread {
   //!
   //! \param[in] report The crash report to process.
   //!
-  //! If report upload is enabled, this method attempts to upload \a report. If
-  //! the upload is successful, the report will be marked as “completed” in the
-  //! database. If the upload fails and more retries are desired, the report’s
-  //! upload-attempt count and last-upload-attempt time will be updated in the
-  //! database and it will remain in the “pending” state. If the upload fails
-  //! and no more retries are desired, or report upload is disabled, it will be
-  //! marked as “completed” in the database without ever having been uploaded.
+  //! If report upload is enabled, this method attempts to upload \a report by
+  //! calling UplaodReport(). If the upload is successful, the report will be
+  //! marked as “completed” in the database. If the upload fails and more
+  //! retries are desired, the report’s upload-attempt count and
+  //! last-upload-attempt time will be updated in the database and it will
+  //! remain in the “pending” state. If the upload fails and no more retries are
+  //! desired, or report upload is disabled, it will be marked as “completed” in
+  //! the database without ever having been uploaded.
   void ProcessPendingReport(const CrashReportDatabase::Report& report);
 
+  //! \brief Attempts to upload a crash report.
+  //!
+  //! \param[in] report The report to upload. The caller is responsible for
+  //!     calling CrashReportDatabase::GetReportForUploading() before calling
+  //!     this method, and for calling
+  //!     CrashReportDatabase::RecordUploadAttempt() after calling this method.
+  //! \param[out] response_body If the upload attempt is successful, this will
+  //!     be set to the response body sent by the server. Breakpad-type servers
+  //!     provide the crash ID assigned by the server in the response body.
+  //!
+  //! \return A member of UploadResult indicating the result of the upload
+  //!    attempt.
+  UploadResult UploadReport(const CrashReportDatabase::Report* report,
+                            std::string* response_body);
+
   //! \brief Cals ThreadMain().
   //!
   //! \param[in] arg A pointer to the object on which to invoke ThreadMain().
@@ -97,6 +140,7 @@ class CrashReportUploadThread {
   //! \return `nullptr`.
   static void* RunThreadMain(void* arg);
 
+  std::string url_;
   CrashReportDatabase* database_;  // weak
   Semaphore semaphore_;  // TODO(mark): Use a condition variable instead?
   pthread_t thread_;

handler/mac/main.cc

@@ -16,7 +16,9 @@
 #include <libgen.h>
 #include <stdlib.h>
 
+#include <map>
 #include <string>
+#include <utility>
 
 #include "base/files/file_path.h"
 #include "base/logging.h"
@@ -34,15 +36,29 @@
 namespace crashpad {
 namespace {
 
+bool SplitString(const std::string& string,
+                 std::string* left,
+                 std::string* right) {
+  size_t equals_pos = string.find('=');
+  if (equals_pos == 0 || equals_pos == std::string::npos) {
+    return false;
+  }
+
+  left->assign(string, 0, equals_pos);
+  right->assign(string, equals_pos + 1, std::string::npos);
+  return true;
+}
+
 void Usage(const std::string& me) {
   fprintf(stderr,
 "Usage: %s [OPTION]...\n"
 "Crashpad's exception handler server.\n"
 "\n"
-"      --database=PATH    store the crash report database at PATH\n"
-"      --handshake-fd=FD  establish communication with the client over FD\n"
-"      --help             display this help and exit\n"
-"      --version          output version information and exit\n",
+"      --annotation=KEY=VALUE  set a process annotation in each crash report\n"
+"      --database=PATH         store the crash report database at PATH\n"
+"      --handshake-fd=FD       establish communication with the client over FD\n"
+"      --help                  display this help and exit\n"
+"      --version               output version information and exit\n",
           me.c_str());
   ToolSupport::UsageTail(me);
 }
@@ -53,8 +69,10 @@ int HandlerMain(int argc, char* argv[]) {
   enum OptionFlags {
     // Long options without short equivalents.
     kOptionLastChar = 255,
+    kOptionAnnotation,
     kOptionDatabase,
     kOptionHandshakeFD,
+    kOptionURL,
 
     // Standard options.
     kOptionHelp = -2,
@@ -62,14 +80,18 @@ int HandlerMain(int argc, char* argv[]) {
   };
 
   struct {
+    std::map<std::string, std::string> annotations;
+    std::string url;
     const char* database;
     int handshake_fd;
   } options = {};
   options.handshake_fd = -1;
 
   const struct option long_options[] = {
+      {"annotation", required_argument, nullptr, kOptionAnnotation},
       {"database", required_argument, nullptr, kOptionDatabase},
       {"handshake-fd", required_argument, nullptr, kOptionHandshakeFD},
+      {"url", required_argument, nullptr, kOptionURL},
       {"help", no_argument, nullptr, kOptionHelp},
       {"version", no_argument, nullptr, kOptionVersion},
       {nullptr, 0, nullptr, 0},
@@ -78,10 +100,28 @@ int HandlerMain(int argc, char* argv[]) {
   int opt;
   while ((opt = getopt_long(argc, argv, "", long_options, nullptr)) != -1) {
     switch (opt) {
-      case kOptionDatabase:
+      case kOptionAnnotation: {
+        std::string key;
+        std::string value;
+        if (!SplitString(optarg, &key, &value)) {
+          ToolSupport::UsageHint(me, "--annotation requires KEY=VALUE");
+          return EXIT_FAILURE;
+        }
+        auto it = options.annotations.find(key);
+        if (it != options.annotations.end()) {
+          LOG(WARNING) << "duplicate key " << key << ", discarding value "
+                       << it->second;
+          it->second = value;
+        } else {
+          options.annotations.insert(std::make_pair(key, value));
+        }
+        break;
+      }
+      case kOptionDatabase: {
         options.database = optarg;
         break;
-      case kOptionHandshakeFD:
+      }
+      case kOptionHandshakeFD: {
         if (!StringToNumber(optarg, &options.handshake_fd) ||
             options.handshake_fd < 0) {
           ToolSupport::UsageHint(me,
@@ -89,15 +129,23 @@ int HandlerMain(int argc, char* argv[]) {
           return EXIT_FAILURE;
         }
         break;
-      case kOptionHelp:
+      }
+      case kOptionURL: {
+        options.url = optarg;
+        break;
+      }
+      case kOptionHelp: {
         Usage(me);
         return EXIT_SUCCESS;
-      case kOptionVersion:
+      }
+      case kOptionVersion: {
         ToolSupport::Version(me);
         return EXIT_SUCCESS;
-      default:
+      }
+      default: {
         ToolSupport::UsageHint(me, nullptr);
         return EXIT_FAILURE;
+      }
     }
   }
   argc -= optind;
@@ -132,10 +180,11 @@ int HandlerMain(int argc, char* argv[]) {
     return EXIT_FAILURE;
   }
 
-  CrashReportUploadThread upload_thread(database.get());
+  CrashReportUploadThread upload_thread(database.get(), options.url);
   upload_thread.Start();
 
-  CrashReportExceptionHandler exception_handler(database.get(), &upload_thread);
+  CrashReportExceptionHandler exception_handler(
+      database.get(), &upload_thread, &options.annotations);
 
   exception_handler_server.Run(&exception_handler);
 

util/file/file_reader.cc

@@ -69,9 +69,7 @@ FileReader::FileReader()
 FileReader::~FileReader() {
 }
 
-bool FileReader::Open(const base::FilePath& path,
-                      FileWriteMode write_mode,
-                      FilePermissions permissions) {
+bool FileReader::Open(const base::FilePath& path) {
   CHECK(!file_.is_valid());
   file_.reset(LoggingOpenFileForRead(path));
   if (!file_.is_valid()) {

util/file/file_reader.h

@@ -110,9 +110,7 @@ class FileReader : public FileReaderInterface {
   //!
   //! \note After a successful call, this method cannot be called again until
   //!     after Close().
-  bool Open(const base::FilePath& path,
-            FileWriteMode write_mode,
-            FilePermissions permissions);
+  bool Open(const base::FilePath& path);
 
   //! \brief Wraps CheckedCloseHandle().
   //!