Skip to content

Commit 1842761

Browse files
kimchykimchy
committed
add search stress test
1 parent 2460ee8 commit 1842761

File tree

1 file changed

+323
-0
lines changed

1 file changed

+323
-0
lines changed

‎modules/test/integration/src/test/java/org/elasticsearch/test/stress/search1/Search1StressTest.java

Lines changed: 323 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,323 @@
1+
/*
2+
* Licensed to Elastic Search and Shay Banon under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. Elastic Search licenses this
6+
* file to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.test.stress.search1;
21+
22+
import org.elasticsearch.action.search.SearchResponse;
23+
import org.elasticsearch.action.search.SearchType;
24+
import org.elasticsearch.client.action.search.SearchRequestBuilder;
25+
import org.elasticsearch.common.logging.ESLogger;
26+
import org.elasticsearch.common.logging.Loggers;
27+
import org.elasticsearch.common.settings.ImmutableSettings;
28+
import org.elasticsearch.common.settings.Settings;
29+
import org.elasticsearch.common.unit.TimeValue;
30+
import org.elasticsearch.common.util.concurrent.jsr166y.ThreadLocalRandom;
31+
import org.elasticsearch.common.xcontent.XContentBuilder;
32+
import org.elasticsearch.common.xcontent.XContentFactory;
33+
import org.elasticsearch.index.query.xcontent.QueryBuilders;
34+
import org.elasticsearch.node.Node;
35+
import org.elasticsearch.node.NodeBuilder;
36+
import org.elasticsearch.search.SearchHit;
37+
import org.elasticsearch.search.sort.SortOrder;
38+
39+
import java.util.Arrays;
40+
import java.util.concurrent.atomic.AtomicLong;
41+
42+
/**
43+
* @author kimchy (shay.banon)
44+
*/
45+
public class Search1StressTest {
46+
47+
private final ESLogger logger = Loggers.getLogger(getClass());
48+
49+
50+
private int numberOfNodes = 4;
51+
52+
private int indexers = 0;
53+
private TimeValue indexerThrottle = TimeValue.timeValueMillis(100);
54+
private int searchers = 0;
55+
private TimeValue searcherThrottle = TimeValue.timeValueMillis(20);
56+
private int numberOfIndices = 10;
57+
private int numberOfTypes = 4;
58+
private int numberOfValues = 20;
59+
private int numberOfHits = 300;
60+
61+
private Settings settings = ImmutableSettings.Builder.EMPTY_SETTINGS;
62+
63+
private TimeValue period = TimeValue.timeValueMinutes(20);
64+
65+
private AtomicLong indexCounter = new AtomicLong();
66+
private AtomicLong searchCounter = new AtomicLong();
67+
68+
69+
private Node client;
70+
71+
public Search1StressTest setNumberOfNodes(int numberOfNodes) {
72+
this.numberOfNodes = numberOfNodes;
73+
return this;
74+
}
75+
76+
public Search1StressTest setIndexers(int indexers) {
77+
this.indexers = indexers;
78+
return this;
79+
}
80+
81+
public Search1StressTest setIndexerThrottle(TimeValue indexerThrottle) {
82+
this.indexerThrottle = indexerThrottle;
83+
return this;
84+
}
85+
86+
public Search1StressTest setSearchers(int searchers) {
87+
this.searchers = searchers;
88+
return this;
89+
}
90+
91+
public Search1StressTest setSearcherThrottle(TimeValue searcherThrottle) {
92+
this.searcherThrottle = searcherThrottle;
93+
return this;
94+
}
95+
96+
public Search1StressTest setNumberOfIndices(int numberOfIndices) {
97+
this.numberOfIndices = numberOfIndices;
98+
return this;
99+
}
100+
101+
public Search1StressTest setNumberOfTypes(int numberOfTypes) {
102+
this.numberOfTypes = numberOfTypes;
103+
return this;
104+
}
105+
106+
public Search1StressTest setNumberOfValues(int numberOfValues) {
107+
this.numberOfValues = numberOfValues;
108+
return this;
109+
}
110+
111+
public Search1StressTest setNumberOfHits(int numberOfHits) {
112+
this.numberOfHits = numberOfHits;
113+
return this;
114+
}
115+
116+
public Search1StressTest setSettings(Settings settings) {
117+
this.settings = settings;
118+
return this;
119+
}
120+
121+
public Search1StressTest setPeriod(TimeValue period) {
122+
this.period = period;
123+
return this;
124+
}
125+
126+
private String nextIndex() {
127+
return "test" + Math.abs(ThreadLocalRandom.current().nextInt()) % numberOfIndices;
128+
}
129+
130+
private String nextType() {
131+
return "type" + Math.abs(ThreadLocalRandom.current().nextInt()) % numberOfTypes;
132+
}
133+
134+
private int nextNumValue() {
135+
return Math.abs(ThreadLocalRandom.current().nextInt()) % numberOfValues;
136+
}
137+
138+
private String nextFieldValue() {
139+
return "value" + Math.abs(ThreadLocalRandom.current().nextInt()) % numberOfValues;
140+
}
141+
142+
private class Searcher extends Thread {
143+
144+
volatile boolean close = false;
145+
146+
volatile boolean closed = false;
147+
148+
@Override public void run() {
149+
while (true) {
150+
if (close) {
151+
closed = true;
152+
return;
153+
}
154+
try {
155+
String indexName = nextIndex();
156+
SearchRequestBuilder builder = client.client().prepareSearch(indexName);
157+
if (ThreadLocalRandom.current().nextBoolean()) {
158+
builder.addSort("num", SortOrder.DESC);
159+
} else if (ThreadLocalRandom.current().nextBoolean()) {
160+
// add a _score based sorting, won't do any sorting, just to test...
161+
builder.addSort("_score", SortOrder.DESC);
162+
}
163+
if (ThreadLocalRandom.current().nextBoolean()) {
164+
builder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH);
165+
}
166+
int size = Math.abs(ThreadLocalRandom.current().nextInt()) % numberOfHits;
167+
builder.setSize(size);
168+
if (ThreadLocalRandom.current().nextBoolean()) {
169+
// update from
170+
builder.setFrom(size / 2);
171+
}
172+
String value = nextFieldValue();
173+
builder.setQuery(QueryBuilders.termQuery("field", value));
174+
searchCounter.incrementAndGet();
175+
SearchResponse searchResponse = builder.execute().actionGet();
176+
if (searchResponse.failedShards() > 0) {
177+
logger.warn("failed search " + Arrays.toString(searchResponse.shardFailures()));
178+
}
179+
// verify that all come from the requested index
180+
for (SearchHit hit : searchResponse.hits()) {
181+
if (!hit.shard().index().equals(indexName)) {
182+
logger.warn("got wrong index, asked for [{}], got [{}]", indexName, hit.shard().index());
183+
}
184+
}
185+
// verify that all has the relevant value
186+
for (SearchHit hit : searchResponse.hits()) {
187+
if (!value.equals(hit.sourceAsMap().get("field"))) {
188+
logger.warn("got wrong field, asked for [{}], got [{}]", value, hit.sourceAsMap().get("field"));
189+
}
190+
}
191+
Thread.sleep(searcherThrottle.millis());
192+
} catch (Exception e) {
193+
logger.warn("failed to search", e);
194+
}
195+
}
196+
}
197+
}
198+
199+
private class Indexer extends Thread {
200+
201+
volatile boolean close = false;
202+
203+
volatile boolean closed = false;
204+
205+
@Override public void run() {
206+
while (true) {
207+
if (close) {
208+
closed = true;
209+
return;
210+
}
211+
try {
212+
indexDoc();
213+
Thread.sleep(indexerThrottle.millis());
214+
} catch (Exception e) {
215+
logger.warn("failed to index / sleep", e);
216+
}
217+
}
218+
}
219+
}
220+
221+
private void indexDoc() throws Exception {
222+
XContentBuilder json = XContentFactory.jsonBuilder().startObject()
223+
.field("num", nextNumValue())
224+
.field("field", nextFieldValue());
225+
226+
json.endObject();
227+
228+
client.client().prepareIndex(nextIndex(), nextType())
229+
.setSource(json)
230+
.execute().actionGet();
231+
indexCounter.incrementAndGet();
232+
}
233+
234+
public void run() throws Exception {
235+
Node[] nodes = new Node[numberOfNodes];
236+
for (int i = 0; i < nodes.length; i++) {
237+
nodes[i] = NodeBuilder.nodeBuilder().settings(settings).node();
238+
}
239+
client = NodeBuilder.nodeBuilder().settings(settings).client(true).node();
240+
241+
for (int i = 0; i < numberOfIndices; i++) {
242+
client.client().admin().indices().prepareCreate("test" + i).execute().actionGet();
243+
}
244+
245+
Indexer[] indexerThreads = new Indexer[indexers];
246+
for (int i = 0; i < indexerThreads.length; i++) {
247+
indexerThreads[i] = new Indexer();
248+
}
249+
for (int i = 0; i < indexerThreads.length; i++) {
250+
indexerThreads[i].start();
251+
}
252+
253+
Thread.sleep(10000);
254+
255+
Searcher[] searcherThreads = new Searcher[searchers];
256+
for (int i = 0; i < searcherThreads.length; i++) {
257+
searcherThreads[i] = new Searcher();
258+
}
259+
for (int i = 0; i < searcherThreads.length; i++) {
260+
searcherThreads[i].start();
261+
}
262+
263+
long testStart = System.currentTimeMillis();
264+
265+
while (true) {
266+
Thread.sleep(5000);
267+
if ((System.currentTimeMillis() - testStart) > period.millis()) {
268+
break;
269+
}
270+
}
271+
272+
System.out.println("DONE, closing .....");
273+
274+
for (int i = 0; i < searcherThreads.length; i++) {
275+
searcherThreads[i].close = true;
276+
}
277+
278+
for (int i = 0; i < indexerThreads.length; i++) {
279+
indexerThreads[i].close = true;
280+
}
281+
282+
Thread.sleep(indexerThrottle.millis() + 10000);
283+
284+
for (int i = 0; i < searcherThreads.length; i++) {
285+
if (!searcherThreads[i].closed) {
286+
logger.warn("search thread not closed!");
287+
}
288+
}
289+
for (int i = 0; i < indexerThreads.length; i++) {
290+
if (!indexerThreads[i].closed) {
291+
logger.warn("index thread not closed!");
292+
}
293+
}
294+
295+
client.close();
296+
for (Node node : nodes) {
297+
node.close();
298+
}
299+
300+
System.out.println("********** DONE, indexed [" + indexCounter.get() + "], searched [" + searchCounter.get() + "]");
301+
}
302+
303+
public static void main(String[] args) throws Exception {
304+
Settings settings = ImmutableSettings.settingsBuilder()
305+
.put("gateway.type", "none")
306+
.build();
307+
308+
Search1StressTest test = new Search1StressTest()
309+
.setPeriod(TimeValue.timeValueMinutes(10))
310+
.setSettings(settings)
311+
.setNumberOfNodes(2)
312+
.setIndexers(2)
313+
.setIndexerThrottle(TimeValue.timeValueMillis(100))
314+
.setSearchers(10)
315+
.setSearcherThrottle(TimeValue.timeValueMillis(10))
316+
.setNumberOfIndices(10)
317+
.setNumberOfTypes(5)
318+
.setNumberOfValues(50)
319+
.setNumberOfHits(300);
320+
321+
test.run();
322+
}
323+
}

0 commit comments

Comments
 (0)