[googledatastore] Initial commit of the Google Datastore Client

Add a DB client for Google Cloud Datastore. This initial commit includes implementation of all CRUD operations (modulo scan), and integration into the rest of YCSB (pom.xml changes, etc)

Next few commits will include Scan/query support, single entity group support, and better documentation.

bin/ycsb

@@ -56,6 +56,7 @@ DATABASES = {
     "dynamodb"     : "com.yahoo.ycsb.db.DynamoDBClient",
     "elasticsearch": "com.yahoo.ycsb.db.ElasticSearchClient",
     "gemfire"      : "com.yahoo.ycsb.db.GemFireClient",
+    "googledatastore" : "com.yahoo.ycsb.db.GoogleDatastoreClient",
     "hbase094"     : "com.yahoo.ycsb.db.HBaseClient",
     "hbase098"     : "com.yahoo.ycsb.db.HBaseClient",
     "hbase10"      : "com.yahoo.ycsb.db.HBaseClient10",

distribution/pom.xml

@@ -74,6 +74,11 @@ LICENSE file.
       <artifactId>gemfire-binding</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>com.yahoo.ycsb</groupId>
+      <artifactId>googledatastore-binding</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>com.yahoo.ycsb</groupId>
       <artifactId>hbase094-binding</artifactId>

googledatastore/README.md

@@ -0,0 +1,40 @@
+<!--
+Copyright (c) 2010 Yahoo! Inc., 2012 - 2015 YCSB contributors.
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License. You
+may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied. See the License for the specific language governing
+permissions and limitations under the License. See accompanying
+LICENSE file.
+-->
+
+# Google Cloud Datastore Binding
+
+https://cloud.google.com/datastore/docs/concepts/overview?hl=en
+
+## Configure
+
+    YCSB_HOME - YCSB home directory
+    DATASTORE_HOME - Google Cloud Datastore YCSB client package files
+
+Please refer to https://github.com/brianfrankcooper/YCSB/wiki/Using-the-Database-Libraries for more information on setup.
+
+# Benchmark
+
+    $YCSB_HOME/bin/ycsb load googledatastore -P workloads/workloada -P googledatastore.properties
+    $YCSB_HOME/bin/ycsb run googledatastore -P workloads/workloada -P googledatastore.properties
+
+# Properties
+
+    $DATASTORE_HOME/conf/googledatastore.properties
+
+# FAQs
+

googledatastore/conf/googledatastore.properties

@@ -0,0 +1,22 @@
+# Copyright (c) 2015 YCSB contributors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License. You
+# may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License. See accompanying
+# LICENSE file.
+
+#
+# Sample property file for Google Cloud Datastore DB client 
+
+## Mandatory parameters
+
+## Optional parameters
+

googledatastore/pom.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- 
+Copyright (c) 2012 - 2015 YCSB contributors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License. You
+may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied. See the License for the specific language governing
+permissions and limitations under the License. See accompanying
+LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>com.yahoo.ycsb</groupId>
+    <artifactId>binding-parent</artifactId>
+    <version>0.6.0-SNAPSHOT</version>
+    <relativePath>../binding-parent</relativePath>
+  </parent>
+
+  <artifactId>googledatastore-binding</artifactId>
+  <name>Google Cloud Datastore Binding</name>
+  <url>https://github.com/GoogleCloudPlatform/google-cloud-datastore</url>
+  <dependencies>
+    <dependency>
+      <groupId>com.google.apis</groupId>
+      <artifactId>google-api-services-datastore-protobuf</artifactId>
+      <version>v1beta2-rev1-3.0.2</version>
+    </dependency>
+    <dependency>
+      <groupId>log4j</groupId>
+      <artifactId>log4j</artifactId>
+      <version>1.2.17</version>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.ycsb</groupId>
+      <artifactId>core</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+  </dependencies>
+</project>

googledatastore/src/main/java/com/yahoo/ycsb/db/GoogleDatastoreClient.java

@@ -0,0 +1,294 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package com.yahoo.ycsb.db;
+
+import com.google.api.client.auth.oauth2.Credential;
+import com.google.api.services.datastore.DatastoreV1.*;
+import com.google.api.services.datastore.DatastoreV1.CommitRequest.Mode;
+import com.google.api.services.datastore.DatastoreV1.ReadOptions.ReadConsistency;
+import com.google.api.services.datastore.client.Datastore;
+import com.google.api.services.datastore.client.DatastoreException;
+import com.google.api.services.datastore.client.DatastoreFactory;
+import com.google.api.services.datastore.client.DatastoreHelper;
+import com.google.api.services.datastore.client.DatastoreOptions;
+
+import com.yahoo.ycsb.ByteIterator;
+import com.yahoo.ycsb.DB;
+import com.yahoo.ycsb.DBException;
+import com.yahoo.ycsb.Status;
+import com.yahoo.ycsb.StringByteIterator;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.Vector;
+
+import javax.annotation.Nullable;
+
+/**
+ * Google Cloud Datastore Client for YCSB.
+ * Author: stfeng@
+ */
+
+public class GoogleDatastoreClient extends DB {
+  /**
+   * Defines a MutationType used in this class.
+   */
+  private enum MutationType {
+    UPSERT,
+    UPDATE,
+    DELETE
+  }
+
+  private static Logger logger =
+      Logger.getLogger(GoogleDatastoreClient.class);
+
+  // Read consistency defaults to "eventual" (this is the same as other
+  // DB client, such as DynamoDB). User can override this via configure.
+  private ReadConsistency readConsistency = ReadConsistency.EVENTUAL;
+
+  private Datastore datastore = null;
+
+  public GoogleDatastoreClient() {}
+
+  /**
+   * Initialize any state for this DB. Called once per DB instance; there is
+   * one DB instance per client thread.
+   */
+  @Override
+  public void init() throws DBException {
+    String debug = getProperties().getProperty("googledatastore.debug", null);
+    if (null != debug && "true".equalsIgnoreCase(debug)) {
+        logger.setLevel(Level.DEBUG);
+    }
+
+    // We need the following 3 essential properties to initialize datastore:
+    //
+    // - DatasetId,
+    // - Path to private key file,
+    // - Service account email address.
+    String datasetId = getProperties().getProperty(
+        "googledatastore.datasetId", null);
+    if (datasetId == null) {
+      throw new DBException(
+          "Required property \"datasetId\" missing.");
+    }
+
+    String privateKeyFile = getProperties().getProperty(
+        "googledatastore.privateKeyFile", null);
+    if (privateKeyFile == null) {
+      throw new DBException(
+          "Required property \"privateKeyFile\" missing.");
+    }
+
+    String serviceAccount = getProperties().getProperty(
+        "googledatastore.serviceAccount", null);
+    if (serviceAccount == null) {
+      throw new DBException(
+          "Required property \"serviceAccount\" missing.");
+    }
+
+    // Below are properties related to benchmarking.
+
+    String readConsistencyConfig = getProperties().getProperty(
+        "googledatastore.readConsistency", null);
+    if (readConsistencyConfig != null) {
+      try {
+        this.readConsistency = ReadConsistency.valueOf(
+            readConsistencyConfig.trim().toUpperCase());
+      } catch (IllegalArgumentException e) {
+        throw new DBException("Invalid read consistency specified: " +
+            readConsistencyConfig + ". Expecting STRONG or EVENTUAL.");
+      }
+    }
+
+    try {
+      // Setup the connection to Google Cloud Datastore with the credentials
+      // obtained from the configure.
+      DatastoreOptions.Builder options = new DatastoreOptions.Builder();
+      Credential credential = DatastoreHelper.getServiceAccountCredential(
+          serviceAccount, privateKeyFile);
+      logger.info("Using JWT Service Account credential.");
+      logger.info("DatasetID: " + datasetId + "\nService Account Email: " +
+          serviceAccount + "\nPrivate Key File Path: " + privateKeyFile);
+
+      datastore = DatastoreFactory.get().create(
+          options.credential(credential).dataset(datasetId).build());
+
+    } catch (GeneralSecurityException exception) {
+      throw new DBException("Security error connecting to the datastore: " +
+            exception.getMessage());
+
+    } catch (IOException exception) {
+      throw new DBException("I/O error connecting to the datastore: " +
+            exception.getMessage());
+    }
+
+    logger.info("Datastore client instance created:\n" +
+        datastore.toString());
+  }
+
+  @Override
+  public Status read(String table, String key, Set<String> fields,
+          HashMap<String, ByteIterator> result) {
+    LookupRequest.Builder lookupRequest = LookupRequest.newBuilder();
+    lookupRequest.addKey(buildPrimaryKey(table, key));
+    lookupRequest.getReadOptionsBuilder().setReadConsistency(
+        this.readConsistency);
+    // Note above, datastore lookupRequest always reads the entire entity, it
+    // does not support reading a subset of "fields" (properties) of an entity.
+
+    logger.debug("Built lookup request as:\n" + lookupRequest.toString());
+
+    LookupResponse response = null;
+    try {
+      response = datastore.lookup(lookupRequest.build());
+
+    } catch (DatastoreException exception) {
+      logger.error(
+          String.format("Datastore Exception when reading (%s): %s %s",
+              exception.getMessage(),
+              exception.getMethodName(),
+              exception.getCode()));
+
+      // DatastoreException.getCode() returns an HTTP response code which we
+      // will bubble up to the user as part of the YCSB Status "name".
+      return new Status("ERROR-" + exception.getCode(), exception.getMessage());
+    }
+
+    if (response.getFoundCount() == 0) {
+      return new Status("ERROR-404", "Not Found, key is: " + key);
+    }
+
+    Entity entity = response.getFound(0).getEntity();
+    logger.debug("Read entity: " + entity.toString());
+
+    Map<String, Value> properties = DatastoreHelper.getPropertyMap(entity);
+    Set<String> propertiesToReturn =
+        (fields == null ? properties.keySet() : fields);
+
+    for (String name : propertiesToReturn) {
+      if (properties.containsKey(name)) {
+        result.put(name, new StringByteIterator(properties.get(name)
+            .getStringValue()));
+      }
+    }
+
+    return Status.OK;
+  }
+
+  @Override
+  public Status scan(String table, String startkey, int recordcount,
+      Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
+      return Status.OK;
+  }
+
+  @Override
+  public Status update(String table, String key,
+      HashMap<String, ByteIterator> values) {
+
+    return doSingleItemMutation(table, key, values, MutationType.UPDATE);
+  }
+
+  @Override
+  public Status insert(String table, String key,
+      HashMap<String, ByteIterator> values) {
+    // Use Upsert to allow overwrite of existing key instead of failing the
+    // load (or run) just because the DB already has the key.
+    // This is the same behavior as what other DB does here (such as
+    // the DynamoDB client).
+    return doSingleItemMutation(table, key, values, MutationType.UPSERT);
+  }
+
+  @Override
+  public Status delete(String table, String key) {
+    return doSingleItemMutation(table, key, null, MutationType.DELETE);
+  }
+
+  private Key.Builder buildPrimaryKey(String table, String key) {
+    return Key.newBuilder().addPathElement(
+        Key.PathElement.newBuilder()
+        .setKind(table)
+        .setName(key));
+  }
+
+  private Status doSingleItemMutation(String table, String key,
+      @Nullable HashMap<String, ByteIterator> values,
+      MutationType mutationType) {
+    // First build the key.
+    Key.Builder datastoreKey = buildPrimaryKey(table, key);
+
+    // Build a commit request in non-transactional mode.
+    // Single item mutation to google datastore
+    // is always atomic and strongly consistent. Transaction is only necessary
+    // for multi-item mutation, or Read-modify-write operation.
+    CommitRequest.Builder commitRequest = CommitRequest.newBuilder();
+    commitRequest.setMode(Mode.NON_TRANSACTIONAL);
+
+    if (mutationType == MutationType.DELETE) {
+      commitRequest.getMutationBuilder().addDelete(datastoreKey);
+
+    } else {
+      // If this is not for delete, build the entity.
+      Entity.Builder entityBuilder = Entity.newBuilder();
+      entityBuilder.setKey(datastoreKey);
+      for (Entry<String, ByteIterator> val : values.entrySet()) {
+        entityBuilder.addProperty(Property.newBuilder()
+            .setName(val.getKey())
+            .setValue(Value.newBuilder()
+                .setStringValue(val.getValue().toString())));
+      }
+      Entity entity = entityBuilder.build();
+      logger.debug("entity built as: " + entity.toString());
+
+      if (mutationType == MutationType.UPSERT) {
+        commitRequest.getMutationBuilder().addUpsert(entity);
+      } else if (mutationType == MutationType.UPDATE){
+        commitRequest.getMutationBuilder().addUpdate(entity);
+      } else {
+        throw new RuntimeException("Impossible MutationType, code bug.");
+      }
+    }
+
+    try {
+      datastore.commit(commitRequest.build());
+      logger.debug("successfully committed.");
+
+    } catch (DatastoreException exception) {
+      // Catch all Datastore rpc errors.
+      // Log the exception, the name of the method called and the error code.
+      logger.error(
+          String.format("Datastore Exception when committing (%s): %s %s",
+              exception.getMessage(),
+              exception.getMethodName(),
+              exception.getCode()));
+
+      // DatastoreException.getCode() returns an HTTP response code which we
+      // will bubble up to the user as part of the YCSB Status "name".
+      return new Status("ERROR-" + exception.getCode(), exception.getMessage());
+    }
+
+    return Status.OK;
+ }
+}

googledatastore/src/main/resources/log4j.properties

@@ -0,0 +1,25 @@
+# Copyright (c) 2015 YCSB contributors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License. You
+# may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License. See accompanying
+# LICENSE file.
+
+#define the console appender
+log4j.appender.consoleAppender = org.apache.log4j.ConsoleAppender
+
+# now define the layout for the appender
+log4j.appender.consoleAppender.layout = org.apache.log4j.PatternLayout
+log4j.appender.consoleAppender.layout.ConversionPattern=%-4r [%t] %-5p %c %x -%m%n
+
+# now map our console appender as a root logger, means all log messages will go
+# to this appender
+log4j.rootLogger = INFO, consoleAppender

pom.xml

@@ -108,6 +108,7 @@ LICENSE file.
     <module>dynamodb</module>
     <module>elasticsearch</module>
     <module>gemfire</module>
+    <module>googledatastore</module>
     <module>hbase094</module>
     <module>hbase098</module>
     <module>hbase10</module>