Add Extract-Based Log Searching

Summary: Some log readers care about the substring rather than the entire line, so fetching an entire line is optional.

Reviewed By: nqmtuan, marekcirkos

Differential Revision: D3035471

fb-gh-sync-id: d7727817f650a4c87dbfff59be6b7fe6136693de
shipit-source-id: d7727817f650a4c87dbfff59be6b7fe6136693de

FBControlCore/Diagnostics/FBLogSearch.h

@@ -52,14 +52,15 @@
 /**
  Constructs a Batch Log Search for the provided mapping of log names to predicates.
  The provided mapping is an NSDictionary where:
- - The keys are an NSArray of NSStrings of the names of the Logs to search. An Empty list means that the value will apply to all predicates.
+ - The keys are the names of the Diagnostics to search. The empty string matches against all input diagnostics.
- - The values are an NSArray of FBLogSearchPredicates of the predicates to search the named logs with.
+ - The values are an NSArray of FBLogSearchPredicates of the predicates to search the the diagnostic with.
 
  @param mapping the mapping to search with.
+ @param lines YES to include the full line in the output, NO for the matched substring.
  @param error an error out for any error in the mapping format.
  @return an FBBatchLogSearch instance if the mapping is valid, nil otherwise.
  */
-+ (instancetype)withMapping:(NSDictionary *)mapping error:(NSError **)error;
++ (instancetype)withMapping:(NSDictionary *)mapping lines:(BOOL)lines error:(NSError **)error;
 
 /**
  Runs the Reciever over an array of Diagnostics.
@@ -77,9 +78,10 @@
 
  @param diagnostics an NSArray of FBDiagnostics to search.
  @param predicate a Log Search Predicate to search with.
+ @param lines YES to include the full line in the output, NO for the matched substring.
  @return an NSDictionary mapping log names to the matching lines that were found in the diagnostics.
  */
-+ (NSDictionary *)searchDiagnostics:(NSArray *)diagnostics withPredicate:(FBLogSearchPredicate *)predicate;
++ (NSDictionary *)searchDiagnostics:(NSArray *)diagnostics withPredicate:(FBLogSearchPredicate *)predicate lines:(BOOL)lines;
 
 @end
 

FBControlCore/Diagnostics/FBLogSearch.m

@@ -45,7 +45,7 @@
   }
   NSRange range = line.length ? NSMakeRange(0, line.length - 1) : NSMakeRange(0, 0);
   NSTextCheckingResult *result = [self.regularExpression firstMatchInString:line options:0 range:range];
-  if (result.range.location == NSNotFound) {
+  if (!result || result.range.location == NSNotFound || result.range.length < 1) {
     return nil;
   }
   return [line substringWithRange:result.range];
@@ -215,7 +215,7 @@
 {
   NSRegularExpression *regex = [NSRegularExpression
     regularExpressionWithPattern:pattern
-    options:0
+    options:NSRegularExpressionAnchorsMatchLines
     error:nil];
   return [[FBLogSearchPredicate_Regex alloc] initWithRegularExpression:regex];
 }
@@ -299,6 +299,7 @@
 @interface FBBatchLogSearch ()
 
 @property (nonatomic, copy, readonly) NSDictionary *mapping;
+@property (nonatomic, assign, readonly) BOOL lines;
 
 @end
 
@@ -306,25 +307,18 @@
 
 #pragma mark Initializers
 
-+ (instancetype)withMapping:(NSDictionary *)mapping error:(NSError **)error
++ (instancetype)withMapping:(NSDictionary *)mapping lines:(BOOL)lines error:(NSError **)error
 {
-  for (id key in mapping.allKeys) {
+  if (![FBCollectionInformation isDictionaryHeterogeneous:mapping keyClass:NSString.class valueClass:NSArray.class]) {
-    if (![key isKindOfClass:NSArray.class]) {
+   return [[FBControlCoreError describeFormat:@"%@ is not an dictionary<string, string>", mapping] fail:error];
-      return [[FBControlCoreError describeFormat:@"%@ key is not an array", key] fail:error];
-    }
-    if (![FBCollectionInformation isArrayHeterogeneous:key withClass:NSString.class]) {
-      return [[FBControlCoreError describeFormat:@"%@ key is not an array of strings", key] fail:error];
-    }
   }
+
   for (id value in mapping.allValues) {
-    if (![value isKindOfClass:NSArray.class]) {
-      return [[FBControlCoreError describeFormat:@"%@ value is not an array", value] fail:error];
-    }
     if (![FBCollectionInformation isArrayHeterogeneous:value withClass:FBLogSearchPredicate.class]) {
       return [[FBControlCoreError describeFormat:@"%@ value is not an array of log search predicates", value] fail:error];
     }
   }
-  return [[FBBatchLogSearch alloc] initWithMapping:mapping];
+  return [[FBBatchLogSearch alloc] initWithMapping:mapping lines:lines];
 }
 
 + (instancetype)inflateFromJSON:(NSDictionary *)json error:(NSError **)error
@@ -332,14 +326,20 @@
   if (![json isKindOfClass:NSDictionary.class]) {
     return [[FBControlCoreError describeFormat:@"%@ is not a dictionary", json] fail:error];
   }
-  if (![FBCollectionInformation isDictionaryHeterogeneous:json keyClass:NSArray.class valueClass:NSArray.class]) {
+  NSNumber *lines = json[@"lines"];
-    return [[FBControlCoreError describeFormat:@"%@ is not a dictionary of <array, array>", json] fail:error];
+  if (![lines isKindOfClass:NSNumber.class]) {
+    return [[FBControlCoreError describeFormat:@"%@ is not a number for 'lines'", lines] fail:error];
   }
 
-  NSMutableDictionary *mapping = [NSMutableDictionary dictionary];
+  NSDictionary *jsonMapping = json[@"mapping"];
-  for (NSString *key in json.allKeys) {
+  if (![FBCollectionInformation isDictionaryHeterogeneous:jsonMapping keyClass:NSString.class valueClass:NSArray.class]) {
+    return [[FBControlCoreError describeFormat:@"%@ is not a dictionary of <array, array>", jsonMapping] fail:error];
+  }
+
+  NSMutableDictionary *predicateMapping = [NSMutableDictionary dictionary];
+  for (NSString *key in jsonMapping.allKeys) {
     NSMutableArray *predicates = [NSMutableArray array];
-    for (NSDictionary *predicateJSON in json[key]) {
+    for (NSDictionary *predicateJSON in jsonMapping[key]) {
       FBLogSearchPredicate *predicate = [FBLogSearchPredicate inflateFromJSON:predicateJSON error:error];
       if (!predicate) {
         return [[FBControlCoreError describeFormat:@"%@ is not a predicate", predicateJSON] fail:error];
@@ -347,12 +347,12 @@
       [predicates addObject:predicate];
     }
 
-    mapping[key] = [predicates copy];
+    predicateMapping[key] = [predicates copy];
   }
-  return [self withMapping:[mapping copy] error:error];
+  return [self withMapping:[predicateMapping copy] lines:lines.boolValue error:error];
 }
 
-- (instancetype)initWithMapping:(NSDictionary *)mapping
+- (instancetype)initWithMapping:(NSDictionary *)mapping lines:(BOOL)lines
 {
   self = [super init];
   if (!self) {
@@ -360,6 +360,7 @@
   }
 
   _mapping = mapping;
+  _lines = lines;
 
   return self;
 }
@@ -374,6 +375,7 @@
   }
 
   _mapping = [coder decodeObjectForKey:NSStringFromSelector(@selector(mapping))];
+  _lines = [coder decodeBoolForKey:NSStringFromSelector(@selector(lines))];
 
   return self;
 }
@@ -381,24 +383,28 @@
 - (void)encodeWithCoder:(NSCoder *)coder
 {
   [coder encodeObject:self.mapping forKey:NSStringFromSelector(@selector(mapping))];
+  [coder encodeBool:self.lines forKey:NSStringFromSelector(@selector(lines))];
 }
 
 #pragma mark NSCopying
 
 - (instancetype)copyWithZone:(NSZone *)zone
 {
-  return [[FBBatchLogSearch alloc] initWithMapping:self.mapping];
+  return [[FBBatchLogSearch alloc] initWithMapping:self.mapping lines:self.lines];
 }
 
 #pragma mark FBJSONSerializationDescribeable Implementation
 
 - (id)jsonSerializableRepresentation
 {
-  NSMutableDictionary *dictionary = [NSMutableDictionary dictionary];
+  NSMutableDictionary *mappingDictionary = [NSMutableDictionary dictionary];
   for (NSArray *key in self.mapping) {
-    dictionary[key] = [self.mapping[key] valueForKey:@"jsonSerializableRepresentation"];
+    mappingDictionary[key] = [self.mapping[key] valueForKey:@"jsonSerializableRepresentation"];
   }
-  return [dictionary copy];
+  return @{
+    @"lines" : @(self.lines),
+    @"mapping" : [mappingDictionary copy],
+  };
 }
 
 #pragma mark FBDebugDescribeable Implementation
@@ -429,12 +435,12 @@
     return NO;
   }
 
-  return [self.mapping isEqualToDictionary:object.mapping];
+  return self.lines == object.lines && [self.mapping isEqualToDictionary:object.mapping];
 }
 
 - (NSUInteger)hash
 {
-  return self.mapping.hash;
+  return (NSUInteger) self.lines ^ self.mapping.hash;
 }
 
 #pragma mark Public API
@@ -448,36 +454,35 @@
 
   // Construct and NSArray<FBLogSearch> instances
   NSMutableArray *searchers = [NSMutableArray array];
-  for (NSArray *nameArray in self.mapping.allKeys) {
+  for (NSString *diagnosticName in self.mapping.allKeys) {
-    NSArray *predicates = self.mapping[nameArray];
+    NSArray *predicates = self.mapping[diagnosticName];
 
-    if ([nameArray isEqualToArray:@[]]) {
+    if ([diagnosticName isEqualToString:@""]) {
       for (FBDiagnostic *diagnostic in diagnostics) {
         for (FBLogSearchPredicate *predicate in predicates) {
           [searchers addObject:[FBLogSearch withDiagnostic:diagnostic predicate:predicate]];
         }
       }
     }
-    for (NSString *name in nameArray) {
+    FBDiagnostic *diagnostic = namesToDiagnostics[diagnosticName];
-      FBDiagnostic *diagnostic = namesToDiagnostics[name];
+    if (!diagnostic) {
-      if (!diagnostic) {
+      continue;
-        continue;
+    }
-      }
+    for (FBLogSearchPredicate *predicate in predicates) {
-      for (FBLogSearchPredicate *predicate in predicates) {
+      [searchers addObject:[FBLogSearch withDiagnostic:diagnostic predicate:predicate]];
-        [searchers addObject:[FBLogSearch withDiagnostic:diagnostic predicate:predicate]];
-      }
     }
   }
 
   // Perform the search, concurrently
+  BOOL lines = self.lines;
   NSArray *results = [FBConcurrentCollectionOperations
     mapFilter:[searchers copy]
     map:^ NSArray * (FBLogSearch *searcher) {
-      NSString *line = searcher.firstMatchingLine;
+      NSString *match = lines ? searcher.firstMatchingLine : searcher.firstMatch;
-      if (!line) {
+      if (!match) {
         return nil;
       }
-      return @[searcher.diagnostic.shortName, line];
+      return @[searcher.diagnostic.shortName, match];
     }
     predicate:FBConcurrentCollectionOperations.notNullPredicate];
 
@@ -497,9 +502,9 @@
   return [output copy];
 }
 
-+ (NSDictionary *)searchDiagnostics:(NSArray *)diagnostics withPredicate:(FBLogSearchPredicate *)predicate
++ (NSDictionary *)searchDiagnostics:(NSArray *)diagnostics withPredicate:(FBLogSearchPredicate *)predicate lines:(BOOL)lines
 {
-  return [[self withMapping:@{@[] : @[predicate]} error:nil] search:diagnostics];
+  return [[self withMapping:@{@[] : @[predicate]} lines:lines error:nil] search:diagnostics];
 }
 
 @end

FBControlCoreTests/Tests/FBLogSearchTests.m

@@ -59,7 +59,7 @@
   FBLogSearch *searcher = [FBLogSearch withDiagnostic:self.simulatorSystemLog predicate:[FBLogSearchPredicate regex:
     @"layer position \\d+ \\d+ bounds \\d+ \\d+ \\d+ \\d+"
   ]];
-  XCTAssertEqualObjects(searcher.firstMatch, @"layer position 375 667 bounds 0 0 750 1334");
+  XCTAssertEqualObjects(searcher.firstMatch, @"layer position 375 667 bounds 0 0 750 133");
   XCTAssertEqualObjects(searcher.firstMatchingLine, @"Mar  7 16:50:18 some-hostname backboardd[24912]: layer position 375 667 bounds 0 0 750 1334");
 }
 
@@ -85,62 +85,99 @@
 
 @end
 
-@interface FBBatchLogSearcherTests : XCTestCase
+@interface FBBatchLogSearcherTests : FBControlCoreValueTestCase
 
 @end
 
 @implementation FBBatchLogSearcherTests
 
-- (void)testBatchSearchFindsAcrossMultipleDiagnostics
+- (NSDictionary *)complexMapping
 {
-  NSDictionary *mapping = @{
+  return @{
-    @[@"simulator_system", @"tree"] : @[
+    @"simulator_system" : @[
-      [FBLogSearchPredicate substrings:@[@"Springboard", @"IOHIDSession", @"rect"]],
+      [FBLogSearchPredicate substrings:@[@"Springboard", @"SpringBoard", @"IOHIDSession"]],
-      [FBLogSearchPredicate regex:@"layer position \\d+ \\d+ bounds \\d+ \\d+ \\d+ \\d+"]
+      [FBLogSearchPredicate regex:@"layer position \\d+ \\d+ bounds \\d+ \\d+ \\d+ \\d+"],
-    ],
-    @[@"simulator_system"] : @[
       [FBLogSearchPredicate substrings:@[@"ADDING REMOTE com.apple.Maps"]],
-    ],
-    @[@"tree"] : @[
       [FBLogSearchPredicate regex:@"(ANIMPOSSIBLE|REGEAAAAAAAAA)"],
     ],
-    @[@"photo0"] : @[
+    @"tree" : @[
-      [FBLogSearchPredicate substrings:@[@"111", @"222"]],
+      [FBLogSearchPredicate substrings:@[@"Springboard", @"SpringBoard", @"IOHIDSession"]],
+      [FBLogSearchPredicate regex:@"(ANIMPOSSIBLE|REGEAAAAAAAAA)"],
+    ],
+    @"photo0" : @[
+      [FBLogSearchPredicate substrings:@[@"BAAAAAAAA"]],
     ]
   };
-  NSError *error = nil;
-  FBBatchLogSearch *batchSearch = [FBBatchLogSearch withMapping:mapping error:&error];
-  XCTAssertNil(error);
-  XCTAssertNotNil(batchSearch);
-  NSDictionary *results = [batchSearch search:@[
-    self.simulatorSystemLog,
-    self.treeJSONDiagnostic,
-    self.photoDiagnostic
-  ]];
-  XCTAssertNotNil(results);
-  XCTAssertEqual([results[@"simulator_system"] count], 3u);
-  XCTAssertEqual([results[@"tree"] count], 1u);
-  XCTAssertEqual([results[@"photo0"] count], 0u);
 }
 
-- (void)testSearchAllFindsAcrossAllDiagnostics
+- (NSDictionary *)searchAllMapping
 {
-  NSDictionary *mapping = @{@[] : @[
+  return @{@"" : @[
-    [FBLogSearchPredicate substrings:@[@"Springboard", @"IOHIDSession", @"rect"]],
+    [FBLogSearchPredicate substrings:@[@"Springboard", @"SpringBoard", @"IOHIDSession"]],
     [FBLogSearchPredicate regex:@"layer position \\d+ \\d+ bounds \\d+ \\d+ \\d+ \\d+"],
     [FBLogSearchPredicate substrings:@[@"ADDING REMOTE com.apple.Maps"]],
     [FBLogSearchPredicate regex:@"(ANIMPOSSIBLE|REGEAAAAAAAAA)"],
     [FBLogSearchPredicate substrings:@[@"111", @"222"]],
   ]};
-  NSError *error = nil;
+}
-  FBBatchLogSearch *batchSearch = [FBBatchLogSearch withMapping:mapping error:&error];
+
-  XCTAssertNil(error);
+- (NSArray *)diagnostics
-  XCTAssertNotNil(batchSearch);
+{
-  NSDictionary *results = [batchSearch search:@[
+  return @[
     self.simulatorSystemLog,
     self.treeJSONDiagnostic,
-    self.photoDiagnostic
+    self.photoDiagnostic,
-  ]];
+  ];
+}
+
+- (void)testValueSemantics
+{
+  NSArray *batches = @[
+    [FBBatchLogSearch withMapping:self.complexMapping lines:YES error:nil],
+    [FBBatchLogSearch withMapping:self.complexMapping lines:NO error:nil],
+    [FBBatchLogSearch withMapping:self.searchAllMapping lines:YES error:nil],
+    [FBBatchLogSearch withMapping:self.searchAllMapping lines:NO error:nil],
+  ];
+
+  [self assertEqualityOfCopy:batches];
+  [self assertUnarchiving:batches];
+  [self assertJSONSerialization:batches];
+  [self assertJSONDeserialization:batches];
+}
+
+
+- (void)testBatchSearchFindsLinesAcrossMultipleDiagnostics
+{
+  FBBatchLogSearch *batchSearch = [FBBatchLogSearch withMapping:self.complexMapping lines:YES error:nil];
+  NSDictionary *results = [batchSearch search:self.diagnostics];
+  XCTAssertNotNil(results);
+  XCTAssertEqual([results[@"simulator_system"] count], 3u);
+  XCTAssertEqual([results[@"tree"] count], 1u);
+  XCTAssertEqual([results[@"photo0"] count], 0u);
+
+  XCTAssertEqualObjects(results[@"simulator_system"][0], @"Mar  7 16:50:18 some-hostname backboardd[24912]: ____IOHIDSessionScheduleAsync_block_invoke: thread_id=0x700000323000");
+  XCTAssertEqualObjects(results[@"simulator_system"][1], @"Mar  7 16:50:18 some-hostname backboardd[24912]: layer position 375 667 bounds 0 0 750 1334");
+  XCTAssertEqualObjects(results[@"simulator_system"][2], @"Mar  7 16:50:21 some-hostname SpringBoard[24911]: ADDING REMOTE com.apple.Maps, <BBRemoteDataProvider 0x7fca290e3fc0; com.apple.Maps>");
+}
+
+- (void)testBatchSearchFindsExtractsAcrossMultipleDiagnostics
+{
+  FBBatchLogSearch *batchSearch = [FBBatchLogSearch withMapping:self.complexMapping lines:NO error:nil];
+  NSDictionary *results = [batchSearch search:self.diagnostics];
+  XCTAssertNotNil(results);
+  XCTAssertEqual([results[@"simulator_system"] count], 3u);
+  XCTAssertEqual([results[@"tree"] count], 1u);
+  XCTAssertEqual([results[@"photo0"] count], 0u);
+
+  XCTAssertEqualObjects(results[@"simulator_system"][0], @"IOHIDSession");
+  XCTAssertEqualObjects(results[@"simulator_system"][1], @"layer position 375 667 bounds 0 0 750 133");
+  XCTAssertEqualObjects(results[@"simulator_system"][2], @"ADDING REMOTE com.apple.Maps");
+}
+
+- (void)testSearchAllFindsAcrossAllDiagnostics
+{
+  FBBatchLogSearch *batchSearch = [FBBatchLogSearch withMapping:self.searchAllMapping lines:YES error:nil];
+  NSDictionary *results = [batchSearch search:self.diagnostics];
   XCTAssertNotNil(results);
   XCTAssertEqual([results[@"simulator_system"] count], 4u);
   XCTAssertEqual([results[@"tree"] count], 1u);