Don't set insertId when writing to BigQuery in ingestion-sink

This commit is contained in:
Daniel Thorn 2019-11-26 17:19:44 -08:00 коммит произвёл Jeff Klukas
Родитель 993f9de3b3
Коммит 7b719ca419
2 изменённых файлов: 3 добавлений и 4 удалений

Просмотреть файл

@ -6,7 +6,6 @@ import com.google.cloud.bigquery.InsertAllResponse;
import com.google.cloud.bigquery.TableId;
import com.google.common.annotations.VisibleForTesting;
import com.google.pubsub.v1.PubsubMessage;
import com.mozilla.telemetry.ingestion.core.Constant.Attribute;
import com.mozilla.telemetry.ingestion.core.util.Json;
import com.mozilla.telemetry.ingestion.sink.transform.PubsubMessageToObjectNode;
import com.mozilla.telemetry.ingestion.sink.transform.PubsubMessageToObjectNode.Format;
@ -96,8 +95,7 @@ public class BigQuery {
@Override
protected synchronized void write(PubsubMessage input) {
Map<String, Object> content = Json.asMap(encoder.apply(input));
Optional.ofNullable(input.getAttributesOrDefault(Attribute.DOCUMENT_ID, null))
.map(id -> builder.addRow(id, content)).orElseGet(() -> builder.addRow(content));
builder.addRow(content);
}
@Override

Просмотреть файл

@ -3,6 +3,7 @@ package com.mozilla.telemetry.ingestion.sink.io;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyLong;
@ -119,7 +120,7 @@ public class BigQueryTest {
List<InsertAllRequest.RowToInsert> rows = ((BigQuery.Write.Batch) output.batches
.get(BATCH_KEY)).builder.build().getRows();
assertEquals("id", rows.get(0).getId());
assertNull(rows.get(0).getId());
assertEquals(ImmutableMap.of("document_id", "id"), rows.get(0).getContent());
assertEquals(20, output.batches.get(BATCH_KEY).byteSize);
}