Azure
/
YCSB
зеркало из https://github.com/Azure/YCSB.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

[solr7] Add support for Apache Solr 7 

closes #1415

Signed-off-by: Kevin Risden <krisden@apache.org>
This commit is contained in:
Mubashir Kazia 2020-03-17 18:08:00 -04:00 коммит произвёл Kevin Risden
Родитель 780aec9235
Коммит af47d643ca
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 040FAE3292C5F73F
15 изменённых файлов: 1033 добавлений и 4 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

3
bin/bindings.properties
Просмотреть файл

@ -1,5 +1,5 @@
#
# Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved.
# Copyright (c) 2012 - 2020 YCSB contributors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You
@ -71,6 +71,7 @@ rocksdb:site.ycsb.db.rocksdb.RocksDBClient
s3:site.ycsb.db.S3Client
solr:site.ycsb.db.solr.SolrClient
solr6:site.ycsb.db.solr6.SolrClient
solr7:site.ycsb.db.solr7.SolrClient
tarantool:site.ycsb.db.TarantoolClient
tablestore:site.ycsb.db.tablestore.TableStoreClient
voltdb:site.ycsb.db.voltdb.VoltClient4

3
bin/ycsb
Просмотреть файл

@ -1,6 +1,6 @@
#!/usr/bin/env python
#
# Copyright (c) 2012 - 2015 YCSB contributors. All rights reserved.
# Copyright (c) 2012 - 2020 YCSB contributors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You
@ -100,6 +100,7 @@ DATABASES = {
"s3" : "site.ycsb.db.S3Client",
"solr" : "site.ycsb.db.solr.SolrClient",
"solr6" : "site.ycsb.db.solr6.SolrClient",
"solr7" : "site.ycsb.db.solr7.SolrClient",
"tarantool" : "site.ycsb.db.TarantoolClient",
"tablestore" : "site.ycsb.db.tablestore.TableStoreClient"
}

7
distribution/pom.xml
Просмотреть файл

@ -1,5 +1,5 @@
<!--
Copyright (c) 2012 - 2015 YCSB contributors. All rights reserved.
Copyright (c) 2012 - 2020 YCSB contributors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You
@ -244,6 +244,11 @@ LICENSE file.
<artifactId>solr6-binding</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>site.ycsb</groupId>
<artifactId>solr7-binding</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>site.ycsb</groupId>
<artifactId>tarantool-binding</artifactId>

4
pom.xml
Просмотреть файл

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (c) 2012 - 2018 YCSB contributors. All rights reserved.
Copyright (c) 2012 - 2020 YCSB contributors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You
@ -145,6 +145,7 @@ LICENSE file.
<s3.version>1.10.20</s3.version>
<solr.version>5.5.3</solr.version>
<solr6.version>6.4.1</solr6.version>
<solr7.version>7.7.2</solr7.version>
<tarantool.version>1.6.5</tarantool.version>
<thrift.version>0.8.0</thrift.version>
<voldemort.version>0.81</voldemort.version>
@ -199,6 +200,7 @@ LICENSE file.
<module>s3</module>
<module>solr</module>
<module>solr6</module>
<module>solr7</module>
<module>tarantool</module>
<module>tablestore</module>
<!--<module>voldemort</module>-->

110
solr7/README.md Normal file
Просмотреть файл

@ -0,0 +1,110 @@
<!--
Copyright (c) 2020 YCSB contributors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You
may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing
permissions and limitations under the License. See accompanying
LICENSE file.
-->
## Quick Start
This section describes how to run YCSB on Solr running locally.
### 1. Set Up YCSB
Clone the YCSB git repository and compile:
git clone git://github.com/brianfrankcooper/YCSB.git
cd YCSB
mvn -pl site.ycsb:solr7-binding -am clean package
### 2. Set Up Solr
There must be a running Solr instance with a core/collection pre-defined and configured.
- See this [API](https://cwiki.apache.org/confluence/display/solr/CoreAdmin+API#CoreAdminAPI-CREATE) reference on how to create a core.
- See this [API](https://cwiki.apache.org/confluence/display/solr/Collections+API#CollectionsAPI-api1) reference on how to create a collection in SolrCloud mode.
The `conf/schema.xml` configuration file present in the core/collection just created must be configured to handle the expected field names during benchmarking.
Below illustrates a sample from a schema config file that matches the default field names used by the ycsb client:
<field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false"/>
<field name="field0" type="text_general" indexed="true" stored="true"/>
<field name="field1" type="text_general" indexed="true" stored="true"/>
<field name="field2" type="text_general" indexed="true" stored="true"/>
<field name="field3" type="text_general" indexed="true" stored="true"/>
<field name="field4" type="text_general" indexed="true" stored="true"/>
<field name="field5" type="text_general" indexed="true" stored="true"/>
<field name="field6" type="text_general" indexed="true" stored="true"/>
<field name="field7" type="text_general" indexed="true" stored="true"/>
<field name="field8" type="text_general" indexed="true" stored="true"/>
<field name="field9" type="text_general" indexed="true" stored="true"/>
If running in SolrCloud mode ensure there is an external Zookeeper cluster running.
- See [here](https://cwiki.apache.org/confluence/display/solr/Setting+Up+an+External+ZooKeeper+Ensemble) for details on how to set up an external Zookeeper cluster.
- See [here](https://cwiki.apache.org/confluence/display/solr/Using+ZooKeeper+to+Manage+Configuration+Files) for instructions on how to use Zookeeper to manage your core/collection configuration files.
### 3. Run YCSB
Now you are ready to run! First, load the data:
./bin/ycsb load solr7 -s -P workloads/workloada -p table=<core/collection name>
Then, run the workload:
./bin/ycsb run solr7 -s -P workloads/workloada -p table=<core/collection name>
For further configuration see below:
### Default Configuration Parameters
The default settings for the Solr node that is created is as follows:
- `solr.cloud`
- A Boolean value indicating if Solr is running in SolrCloud mode. If so there must be an external Zookeeper cluster running also.
- Default value is `false` and therefore expects solr to be running in stand-alone mode.
- `solr.base.url`
- The base URL in which to interface with a running Solr instance in stand-alone mode
- Default value is `http://localhost:8983/solr
- `solr.commit.within.time`
- The max time in ms to wait for a commit when in batch mode, ignored otherwise
- Default value is `1000ms`
- `solr.batch.mode`
- Indicates if inserts/updates/deletes should be commited in batches (frequency controlled by the `solr.commit.within.time` parameter) or commit 1 document at a time.
- Default value is `false`
- `solr.zookeeper.hosts`
- A list of comma seperated host:port pairs of Zookeeper nodes used to manage SolrCloud configurations.
- Must be passed when in [SolrCloud](https://cwiki.apache.org/confluence/display/solr/SolrCloud) mode.
- Default value is `localhost:2181`
### Custom Configuration
If you wish to customize the settings used to create the Solr node
you can created a new property file that contains your desired Solr
node settings and pass it in via the parameter to 'bin/ycsb' script. Note that
the default properties will be kept if you don't explicitly overwrite them.
Assuming that we have a properties file named "myproperties.data" that contains
custom Solr node configuration you can execute the following to
pass it into the Solr client:
./bin/ycsb run solr7 -P workloads/workloada -P myproperties.data -s
If you wish to use SolrCloud mode ensure a Solr cluster is running with an
external zookeeper cluster and an appropriate collection has been created.
Make sure to pass the following properties as parameters to 'bin/ycsb' script.
solr.cloud=true
solr.zookeeper.hosts=<zkHost2>:<zkPort1>,...,<zkHostN>:<zkPortN>

92
solr7/pom.xml Normal file
Просмотреть файл

@ -0,0 +1,92 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (c) 2020 YCSB contributors.
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You
may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing
permissions and limitations under the License. See accompanying
LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>site.ycsb</groupId>
<artifactId>binding-parent</artifactId>
<version>0.18.0-SNAPSHOT</version>
<relativePath>../binding-parent</relativePath>
</parent>
<artifactId>solr7-binding</artifactId>
<name>Solr 7 Binding</name>
<packaging>jar</packaging>
<properties>
<skipJDK9Tests>true</skipJDK9Tests>
<skipJDK10Tests>true</skipJDK10Tests>
<skipJDK11Tests>true</skipJDK11Tests>
</properties>
<dependencies>
<dependency>
<groupId>site.ycsb</groupId>
<artifactId>core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
<version>${solr7.version}</version>
</dependency>
<!-- commons-codec required for Solr Kerberos support -->
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.14</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.30</version>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-test-framework</artifactId>
<version>${solr7.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<profiles>
<!-- Solr 6+ requires JDK8 to run, so none of our tests
will work unless we're using jdk8.
-->
<profile>
<id>jdk8-tests</id>
<activation>
<jdk>1.8</jdk>
</activation>
<properties>
<skipTests>false</skipTests>
</properties>
</profile>
</profiles>
</project>

331
solr7/src/main/java/site/ycsb/db/solr7/SolrClient.java Normal file
Просмотреть файл

@ -0,0 +1,331 @@
/**
* Copyright (c) 2020 YCSB contributors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package site.ycsb.db.solr7;
import site.ycsb.ByteIterator;
import site.ycsb.DB;
import site.ycsb.DBException;
import site.ycsb.Status;
import site.ycsb.StringByteIterator;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import java.io.IOException;
import java.util.*;
import java.util.Map.Entry;
/**
* Solr client for YCSB framework.
*
* <p>
* Default properties to set:
* </p>
* <ul>
* See README.md
* </ul>
*
*/
public class SolrClient extends DB {
public static final String DEFAULT_CLOUD_MODE = "false";
public static final String DEFAULT_BATCH_MODE = "false";
public static final String DEFAULT_ZOOKEEPER_HOSTS = "localhost:2181";
public static final String DEFAULT_SOLR_BASE_URL = "http://localhost:8983/solr";
public static final String DEFAULT_COMMIT_WITHIN_TIME = "1000";
private org.apache.solr.client.solrj.SolrClient client;
private Integer commitTime;
private Boolean batchMode;
/**
* Initialize any state for this DB. Called once per DB instance; there is one DB instance per
* client thread.
*/
@Override
public void init() throws DBException {
Properties props = getProperties();
commitTime = Integer
.parseInt(props.getProperty("solr.commit.within.time", DEFAULT_COMMIT_WITHIN_TIME));
batchMode = Boolean.parseBoolean(props.getProperty("solr.batch.mode", DEFAULT_BATCH_MODE));
String jaasConfPath = props.getProperty("solr.jaas.conf.path");
if(jaasConfPath != null) {
System.setProperty("java.security.auth.login.config", jaasConfPath);
HttpClientUtil.setHttpClientBuilder(new Krb5HttpClientBuilder().getHttpClientBuilder(Optional.empty()));
}
// Check if Solr cluster is running in SolrCloud or Stand-alone mode
Boolean cloudMode = Boolean.parseBoolean(props.getProperty("solr.cloud", DEFAULT_CLOUD_MODE));
System.err.println("Solr Cloud Mode = " + cloudMode);
if (cloudMode) {
System.err.println("Solr Zookeeper Remote Hosts = "
+ props.getProperty("solr.zookeeper.hosts", DEFAULT_ZOOKEEPER_HOSTS));
client = new CloudSolrClient.Builder().withZkHost(
Arrays.asList(props.getProperty("solr.zookeeper.hosts", DEFAULT_ZOOKEEPER_HOSTS).split(","))).build();
} else {
client = new HttpSolrClient.Builder(props.getProperty("solr.base.url", DEFAULT_SOLR_BASE_URL)).build();
}
}
@Override
public void cleanup() throws DBException {
try {
client.close();
} catch (IOException e) {
throw new DBException(e);
}
}
/**
* Insert a record in the database. Any field/value pairs in the specified values HashMap will be
* written into the record with the specified record key.
*
* @param table
* The name of the table
* @param key
* The record key of the record to insert.
* @param values
* A HashMap of field/value pairs to insert in the record
* @return Zero on success, a non-zero error code on error. See this class's description for a
* discussion of error codes.
*/
@Override
public Status insert(String table, String key, Map<String, ByteIterator> values) {
try {
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", key);
for (Entry<String, String> entry : StringByteIterator.getStringMap(values).entrySet()) {
doc.addField(entry.getKey(), entry.getValue());
}
UpdateResponse response;
if (batchMode) {
response = client.add(table, doc, commitTime);
} else {
response = client.add(table, doc);
client.commit(table);
}
return checkStatus(response.getStatus());
} catch (IOException | SolrServerException e) {
e.printStackTrace();
}
return Status.ERROR;
}
/**
* Delete a record from the database.
*
* @param table
* The name of the table
* @param key
* The record key of the record to delete.
* @return Zero on success, a non-zero error code on error. See this class's description for a
* discussion of error codes.
*/
@Override
public Status delete(String table, String key) {
try {
UpdateResponse response;
if (batchMode) {
response = client.deleteById(table, key, commitTime);
} else {
response = client.deleteById(table, key);
client.commit(table);
}
return checkStatus(response.getStatus());
} catch (IOException | SolrServerException e) {
e.printStackTrace();
}
return Status.ERROR;
}
/**
* Read a record from the database. Each field/value pair from the result will be stored in a
* HashMap.
*
* @param table
* The name of the table
* @param key
* The record key of the record to read.
* @param fields
* The list of fields to read, or null for all of them
* @param result
* A HashMap of field/value pairs for the result
* @return Zero on success, a non-zero error code on error or "not found".
*/
@Override
public Status read(String table, String key, Set<String> fields,
Map<String, ByteIterator> result) {
try {
Boolean returnFields = false;
String[] fieldList = null;
if (fields != null) {
returnFields = true;
fieldList = fields.toArray(new String[fields.size()]);
}
SolrQuery query = new SolrQuery();
query.setQuery("id:" + key);
if (returnFields) {
query.setFields(fieldList);
}
final QueryResponse response = client.query(table, query);
SolrDocumentList results = response.getResults();
if ((results != null) && (results.getNumFound() > 0)) {
for (String field : results.get(0).getFieldNames()) {
result.put(field,
new StringByteIterator(String.valueOf(results.get(0).getFirstValue(field))));
}
}
return checkStatus(response.getStatus());
} catch (IOException | SolrServerException e) {
e.printStackTrace();
}
return Status.ERROR;
}
/**
* Update a record in the database. Any field/value pairs in the specified values HashMap will be
* written into the record with the specified record key, overwriting any existing values with the
* same field name.
*
* @param table
* The name of the table
* @param key
* The record key of the record to write.
* @param values
* A HashMap of field/value pairs to update in the record
* @return Zero on success, a non-zero error code on error. See this class's description for a
* discussion of error codes.
*/
@Override
public Status update(String table, String key, Map<String, ByteIterator> values) {
try {
SolrInputDocument updatedDoc = new SolrInputDocument();
updatedDoc.addField("id", key);
for (Entry<String, String> entry : StringByteIterator.getStringMap(values).entrySet()) {
updatedDoc.addField(entry.getKey(), Collections.singletonMap("set", entry.getValue()));
}
UpdateResponse writeResponse;
if (batchMode) {
writeResponse = client.add(table, updatedDoc, commitTime);
} else {
writeResponse = client.add(table, updatedDoc);
client.commit(table);
}
return checkStatus(writeResponse.getStatus());
} catch (IOException | SolrServerException e) {
e.printStackTrace();
}
return Status.ERROR;
}
/**
* Perform a range scan for a set of records in the database. Each field/value pair from the
* result will be stored in a HashMap.
*
* @param table
* The name of the table
* @param startkey
* The record key of the first record to read.
* @param recordcount
* The number of records to read
* @param fields
* The list of fields to read, or null for all of them
* @param result
* A Vector of HashMaps, where each HashMap is a set field/value pairs for one record
* @return Zero on success, a non-zero error code on error. See this class's description for a
* discussion of error codes.
*/
@Override
public Status scan(String table, String startkey, int recordcount, Set<String> fields,
Vector<HashMap<String, ByteIterator>> result) {
try {
Boolean returnFields = false;
String[] fieldList = null;
if (fields != null) {
returnFields = true;
fieldList = fields.toArray(new String[fields.size()]);
}
SolrQuery query = new SolrQuery();
query.setQuery("*:*");
query.setParam("fq", "id:[ " + startkey + " TO * ]");
if (returnFields) {
query.setFields(fieldList);
}
query.setRows(recordcount);
final QueryResponse response = client.query(table, query);
SolrDocumentList results = response.getResults();
HashMap<String, ByteIterator> entry;
for (SolrDocument hit : results) {
entry = new HashMap<>((int) results.getNumFound());
for (String field : hit.getFieldNames()) {
entry.put(field, new StringByteIterator(String.valueOf(hit.getFirstValue(field))));
}
result.add(entry);
}
return checkStatus(response.getStatus());
} catch (IOException | SolrServerException e) {
e.printStackTrace();
}
return Status.ERROR;
}
private Status checkStatus(int status) {
Status responseStatus;
switch (status) {
case 0:
responseStatus = Status.OK;
break;
case 400:
responseStatus = Status.BAD_REQUEST;
break;
case 403:
responseStatus = Status.FORBIDDEN;
break;
case 404:
responseStatus = Status.NOT_FOUND;
break;
case 500:
responseStatus = Status.ERROR;
break;
case 503:
responseStatus = Status.SERVICE_UNAVAILABLE;
break;
default:
responseStatus = Status.UNEXPECTED_STATE;
break;
}
return responseStatus;
}
}

23
solr7/src/main/java/site/ycsb/db/solr7/package-info.java Normal file
Просмотреть файл

@ -0,0 +1,23 @@
/*
* Copyright (c) 2020 YCSB contributors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
/**
* The YCSB binding for
* <a href="http://lucene.apache.org/solr/">Solr</a>.
*/
package site.ycsb.db.solr7;

22
solr7/src/main/resources/log4j.properties Normal file
Просмотреть файл

@ -0,0 +1,22 @@
# Copyright (c) 2020 YCSB contributors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You
# may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License. See accompanying
# LICENSE file.
# Root logger option
log4j.rootLogger=INFO, stderr
# Direct log messages to stderr
log4j.appender.stderr=org.apache.log4j.ConsoleAppender
log4j.appender.stderr.Target=System.err
log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
log4j.appender.stderr.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n

166
solr7/src/test/java/site/ycsb/db/solr7/SolrClientBaseTest.java Normal file
Просмотреть файл

@ -0,0 +1,166 @@
/**
* Copyright (c) 2020 YCSB contributors. All rights reserved.
* <p/>
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package site.ycsb.db.solr7;
import site.ycsb.ByteIterator;
import site.ycsb.DB;
import site.ycsb.Status;
import site.ycsb.StringByteIterator;
import site.ycsb.workloads.CoreWorkload;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.common.util.NamedList;
import org.junit.*;
import java.io.File;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Properties;
import java.util.Set;
import java.util.Vector;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
public abstract class SolrClientBaseTest {
protected static MiniSolrCloudCluster miniSolrCloudCluster;
private DB instance;
private final static HashMap<String, ByteIterator> MOCK_DATA;
protected final static String MOCK_TABLE = "ycsb";
private final static String MOCK_KEY0 = "0";
private final static String MOCK_KEY1 = "1";
private final static int NUM_RECORDS = 10;
private final static String FIELD_PREFIX = CoreWorkload.FIELD_NAME_PREFIX_DEFAULT;
static {
MOCK_DATA = new HashMap<>(NUM_RECORDS);
for (int i = 0; i < NUM_RECORDS; i++) {
MOCK_DATA.put(FIELD_PREFIX + i, new StringByteIterator("value" + i));
}
}
@BeforeClass
public static void onlyOnce() throws Exception {
Path miniSolrCloudClusterTempDirectory = Files.createTempDirectory("miniSolrCloudCluster");
miniSolrCloudClusterTempDirectory.toFile().deleteOnExit();
miniSolrCloudCluster = new MiniSolrCloudCluster(1, miniSolrCloudClusterTempDirectory, JettyConfig.builder().build());
// Upload Solr configuration
URL configDir = SolrClientBaseTest.class.getClassLoader().getResource("solr_config");
assertNotNull(configDir);
miniSolrCloudCluster.uploadConfigSet(Paths.get(configDir.toURI()), MOCK_TABLE);
}
@AfterClass
public static void destroy() throws Exception {
if(miniSolrCloudCluster != null) {
miniSolrCloudCluster.shutdown();
}
}
@Before
public void setup() throws Exception {
CollectionAdminRequest.createCollection(MOCK_TABLE, MOCK_TABLE, 1, 1)
.withProperty("solr.directoryFactory", "solr.StandardDirectoryFactory")
.process(miniSolrCloudCluster.getSolrClient());
miniSolrCloudCluster.waitForActiveCollection(MOCK_TABLE, 1, 1);
Thread.sleep(1000);
instance = getDB();
}
@After
public void tearDown() throws Exception {
if(miniSolrCloudCluster != null) {
CollectionAdminRequest.deleteCollection(MOCK_TABLE)
.processAndWait(miniSolrCloudCluster.getSolrClient(), 60);
Thread.sleep(1000);
}
}
@Test
public void testInsert() throws Exception {
Status result = instance.insert(MOCK_TABLE, MOCK_KEY0, MOCK_DATA);
assertEquals(Status.OK, result);
}
@Test
public void testDelete() throws Exception {
Status result = instance.delete(MOCK_TABLE, MOCK_KEY1);
assertEquals(Status.OK, result);
}
@Test
public void testRead() throws Exception {
Set<String> fields = MOCK_DATA.keySet();
HashMap<String, ByteIterator> resultParam = new HashMap<>(NUM_RECORDS);
Status result = instance.read(MOCK_TABLE, MOCK_KEY1, fields, resultParam);
assertEquals(Status.OK, result);
}
@Test
public void testUpdate() throws Exception {
HashMap<String, ByteIterator> newValues = new HashMap<>(NUM_RECORDS);
for (int i = 0; i < NUM_RECORDS; i++) {
newValues.put(FIELD_PREFIX + i, new StringByteIterator("newvalue" + i));
}
Status result = instance.update(MOCK_TABLE, MOCK_KEY1, newValues);
assertEquals(Status.OK, result);
//validate that the values changed
HashMap<String, ByteIterator> resultParam = new HashMap<>(NUM_RECORDS);
instance.read(MOCK_TABLE, MOCK_KEY1, MOCK_DATA.keySet(), resultParam);
for (int i = 0; i < NUM_RECORDS; i++) {
assertEquals("newvalue" + i, resultParam.get(FIELD_PREFIX + i).toString());
}
}
@Test
public void testScan() throws Exception {
Set<String> fields = MOCK_DATA.keySet();
Vector<HashMap<String, ByteIterator>> resultParam = new Vector<>(NUM_RECORDS);
Status result = instance.scan(MOCK_TABLE, MOCK_KEY1, NUM_RECORDS, fields, resultParam);
assertEquals(Status.OK, result);
}
/**
* Gets the test DB.
*
* @return The test DB.
*/
protected DB getDB() {
return getDB(new Properties());
}
/**
* Gets the test DB.
*
* @param props
* Properties to pass to the client.
* @return The test DB.
*/
protected abstract DB getDB(Properties props);
}

56
solr7/src/test/java/site/ycsb/db/solr7/SolrClientCloudTest.java Normal file
Просмотреть файл

@ -0,0 +1,56 @@
/**
* Copyright (c) 2020 YCSB contributors. All rights reserved.
* <p/>
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package site.ycsb.db.solr7;
import site.ycsb.DB;
import org.junit.After;
import java.util.Properties;
import static org.junit.Assume.assumeNoException;
public class SolrClientCloudTest extends SolrClientBaseTest {
private SolrClient instance;
@After
public void tearDown() throws Exception {
try {
if(instance != null) {
instance.cleanup();
}
} finally {
super.tearDown();
}
}
@Override
protected DB getDB(Properties props) {
instance = new SolrClient();
props.setProperty("solr.cloud", "true");
props.setProperty("solr.zookeeper.hosts", miniSolrCloudCluster.getSolrClient().getZkHost());
instance.setProperties(props);
try {
instance.init();
} catch (Exception error) {
assumeNoException(error);
}
return instance;
}
}

62
solr7/src/test/java/site/ycsb/db/solr7/SolrClientTest.java Normal file
Просмотреть файл

@ -0,0 +1,62 @@
/**
* Copyright (c) 2020 YCSB contributors. All rights reserved.
* <p/>
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package site.ycsb.db.solr7;
import site.ycsb.DB;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.junit.After;
import java.util.Properties;
import static org.junit.Assume.assumeNoException;
public class SolrClientTest extends SolrClientBaseTest {
private SolrClient instance;
@After
public void tearDown() throws Exception {
try {
if(instance != null) {
instance.cleanup();
}
} finally {
super.tearDown();
}
}
@Override
protected DB getDB(Properties props) {
instance = new SolrClient();
// Use the first Solr server in the cluster.
// Doesn't matter if there are more since requests will be forwarded properly by Solr.
JettySolrRunner jettySolrRunner = miniSolrCloudCluster.getJettySolrRunners().get(0);
String solrBaseUrl = String.format("http://localhost:%s%s", jettySolrRunner.getLocalPort(),
jettySolrRunner.getBaseUrl());
props.setProperty("solr.base.url", solrBaseUrl);
instance.setProperties(props);
try {
instance.init();
} catch (Exception error) {
assumeNoException(error);
}
return instance;
}
}

29
solr7/src/test/resources/log4j.properties Normal file
Просмотреть файл

@ -0,0 +1,29 @@
# Copyright (c) 2020 YCSB contributors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You
# may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License. See accompanying
# LICENSE file.
#
# Root logger option
log4j.rootLogger=INFO, stderr
log4j.appender.stderr=org.apache.log4j.ConsoleAppender
log4j.appender.stderr.target=System.err
log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
log4j.appender.stderr.layout.conversionPattern=%d{yyyy/MM/dd HH:mm:ss} %-5p %c %x - %m%n
# Suppress messages from ZooKeeper
log4j.logger.org.apache.zookeeper=ERROR
# Solr classes are too chatty in test at INFO
log4j.logger.org.apache.solr=ERROR
log4j.logger.org.eclipse.jetty=ERROR

66
solr7/src/test/resources/solr_config/schema.xml Normal file
Просмотреть файл

@ -0,0 +1,66 @@
<?xml version="1.0" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!--
Copied from Apache Solr 7.7.2 solr/solrj/src/test-files/solrj/solr/collection1/conf/schema.xml
Modified to only required types and fields for YCSB testing.
-->
<schema name="test" version="1.6">
<!-- field type definitions... note that the "name" attribute is
just a label to be used by field definitions. The "class"
attribute and any other attributes determine the real type and
behavior of the fieldType.
-->
<types>
<fieldType name="int" class="solr.IntPointField" docValues="true"/>
<fieldType name="long" class="solr.LongPointField" docValues="true"/>
<fieldType name="string" class="solr.StrField" sortMissingLast="true" docValues="true" />
<fieldType name="text" class="solr.TextField">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory"/>
<filter class="solr.PorterStemFilterFactory"/>
</analyzer>
</fieldType>
</types>
<fields>
<field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
<field name="text" type="text" indexed="true" stored="false"/>
<field name="_version_" type="long" indexed="false" stored="false"/>
<field name="field0" type="text" indexed="true" stored="true"/>
<field name="field1" type="text" indexed="true" stored="true"/>
<field name="field2" type="text" indexed="true" stored="true"/>
<field name="field3" type="text" indexed="true" stored="true"/>
<field name="field4" type="text" indexed="true" stored="true"/>
<field name="field5" type="text" indexed="true" stored="true"/>
<field name="field6" type="text" indexed="true" stored="true"/>
<field name="field7" type="text" indexed="true" stored="true"/>
<field name="field8" type="text" indexed="true" stored="true"/>
<field name="field9" type="text" indexed="true" stored="true"/>
</fields>
<uniqueKey>id</uniqueKey>
</schema>

63
solr7/src/test/resources/solr_config/solrconfig.xml Normal file
Просмотреть файл

@ -0,0 +1,63 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!--
Copied from Apache Solr 5.4.0 solr/src/test/resources/solr_config/solrconfig.xml
-->
<!--
This is a stripped down config file used for a simple example...
It is *not* a good example to work from.
-->
<config>
<luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<indexConfig>
<useCompoundFile>${useCompoundFile:false}</useCompoundFile>
</indexConfig>
<dataDir>${solr.data.dir:}</dataDir>
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.StandardDirectoryFactory}"/>
<updateHandler class="solr.DirectUpdateHandler2">
<updateLog>
<str name="dir">${solr.data.dir:}</str>
</updateLog>
</updateHandler>
<requestDispatcher handleSelect="true" >
<requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
</requestDispatcher>
<requestHandler name="standard" class="solr.StandardRequestHandler" default="true" />
<requestHandler name="/admin/ping" class="solr.PingRequestHandler">
<lst name="invariants">
<str name="q">*:*</str>
</lst>
<lst name="defaults">
<str name="echoParams">all</str>
</lst>
<str name="healthcheckFile">server-enabled.txt</str>
</requestHandler>
<!-- config for the admin interface -->
<admin>
<defaultQuery>solr</defaultQuery>
</admin>
</config>