Skip to content

Commit 180d225

Browse files
kimchykimchy
committed
Percolator, closes elastic#624.
1 parent 2d180eb commit 180d225

File tree

40 files changed

+1754
-196
lines changed

40 files changed

+1754
-196
lines changed

‎modules/benchmark/micro/src/main/java/org/elasticsearch/benchmark/percolator/EmbeddedPercolatorBenchmarkTest.java

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.elasticsearch.benchmark.percolator;
2121

2222
import org.elasticsearch.common.StopWatch;
23+
import org.elasticsearch.common.inject.AbstractModule;
2324
import org.elasticsearch.common.inject.Injector;
2425
import org.elasticsearch.common.inject.ModulesBuilder;
2526
import org.elasticsearch.common.settings.ImmutableSettings;
@@ -33,8 +34,7 @@
3334
import org.elasticsearch.index.cache.IndexCacheModule;
3435
import org.elasticsearch.index.engine.IndexEngineModule;
3536
import org.elasticsearch.index.mapper.MapperServiceModule;
36-
import org.elasticsearch.index.percolator.PercolatorModule;
37-
import org.elasticsearch.index.percolator.PercolatorService;
37+
import org.elasticsearch.index.percolator.PercolatorExecutor;
3838
import org.elasticsearch.index.query.IndexQueryParserModule;
3939
import org.elasticsearch.index.settings.IndexSettingsModule;
4040
import org.elasticsearch.index.similarity.SimilarityModule;
@@ -69,30 +69,34 @@ public static void main(String[] args) throws Exception {
6969
new SimilarityModule(settings),
7070
new IndexQueryParserModule(settings),
7171
new IndexNameModule(index),
72-
new PercolatorModule()
72+
new AbstractModule() {
73+
@Override protected void configure() {
74+
bind(PercolatorExecutor.class).asEagerSingleton();
75+
}
76+
}
7377
).createInjector();
7478

75-
final PercolatorService percolatorService = injector.getInstance(PercolatorService.class);
79+
final PercolatorExecutor percolatorExecutor = injector.getInstance(PercolatorExecutor.class);
7680

77-
XContentBuilder doc = XContentFactory.jsonBuilder().startObject()
81+
XContentBuilder doc = XContentFactory.jsonBuilder().startObject().startObject("doc").startObject("type1")
7882
.field("field1", 1)
7983
.field("field2", "value")
8084
.field("field3", "the quick brown fox jumped over the lazy dog")
81-
.endObject();
85+
.endObject().endObject().endObject();
8286
final byte[] source = doc.copiedBytes();
8387

84-
PercolatorService.Response percolate = percolatorService.percolate(new PercolatorService.Request("type1", source));
88+
PercolatorExecutor.Response percolate = percolatorExecutor.percolate(new PercolatorExecutor.Request(source));
8589

8690
for (int i = 0; i < NUMBER_OF_QUERIES; i++) {
87-
percolatorService.addQuery("test" + i, termQuery("field3", "quick"));
91+
percolatorExecutor.addQuery("test" + i, termQuery("field3", "quick"));
8892
}
8993

9094

9195
System.out.println("Warming Up (1000)");
9296
StopWatch stopWatch = new StopWatch().start();
9397
System.out.println("Running " + 1000);
9498
for (long i = 0; i < 1000; i++) {
95-
percolate = percolatorService.percolate(new PercolatorService.Request("type1", source));
99+
percolate = percolatorExecutor.percolate(new PercolatorExecutor.Request(source));
96100
}
97101
System.out.println("[Warmup] Percolated in " + stopWatch.stop().totalTime() + " TP Millis " + (NUMBER_OF_ITERATIONS / stopWatch.totalTime().millisFrac()));
98102

@@ -103,7 +107,7 @@ public static void main(String[] args) throws Exception {
103107
threads[i] = new Thread(new Runnable() {
104108
@Override public void run() {
105109
for (long i = 0; i < NUMBER_OF_ITERATIONS; i++) {
106-
PercolatorService.Response percolate = percolatorService.percolate(new PercolatorService.Request("type1", source));
110+
PercolatorExecutor.Response percolate = percolatorExecutor.percolate(new PercolatorExecutor.Request(source));
107111
}
108112
latch.countDown();
109113
}

‎modules/benchmark/micro/src/main/java/org/elasticsearch/benchmark/percolator/SinglePercolatorStressBenchmark.java

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
/*
2+
* Licensed to Elastic Search and Shay Banon under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. Elastic Search licenses this
6+
* file to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.benchmark.percolator;
21+
22+
import org.elasticsearch.action.percolate.PercolateResponse;
23+
import org.elasticsearch.client.Client;
24+
import org.elasticsearch.common.StopWatch;
25+
import org.elasticsearch.common.settings.Settings;
26+
import org.elasticsearch.common.xcontent.XContentBuilder;
27+
import org.elasticsearch.node.Node;
28+
29+
import java.io.IOException;
30+
import java.util.concurrent.TimeUnit;
31+
32+
import static org.elasticsearch.client.Requests.*;
33+
import static org.elasticsearch.cluster.metadata.IndexMetaData.*;
34+
import static org.elasticsearch.common.settings.ImmutableSettings.*;
35+
import static org.elasticsearch.common.xcontent.XContentFactory.*;
36+
import static org.elasticsearch.index.query.xcontent.QueryBuilders.*;
37+
import static org.elasticsearch.node.NodeBuilder.*;
38+
39+
/**
40+
* @author kimchy (shay.banon)
41+
*/
42+
public class SinglePercolatorStressBenchmark {
43+
44+
public static void main(String[] args) throws Exception {
45+
Settings settings = settingsBuilder()
46+
.put("cluster.routing.schedule", 200, TimeUnit.MILLISECONDS)
47+
.put("gateway.type", "none")
48+
.put(SETTING_NUMBER_OF_SHARDS, 2)
49+
.put(SETTING_NUMBER_OF_REPLICAS, 1)
50+
.build();
51+
52+
Node[] nodes = new Node[2];
53+
for (int i = 0; i < nodes.length; i++) {
54+
nodes[i] = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "node" + i)).node();
55+
}
56+
57+
Node client = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "client")).client(true).node();
58+
59+
Client client1 = client.client();
60+
61+
client1.admin().indices().create(createIndexRequest("test")).actionGet();
62+
Thread.sleep(1000);
63+
64+
int COUNT = 200000;
65+
int QUERIES = 10;
66+
// register queries
67+
for (int i = 0; i < QUERIES; i++) {
68+
client1.prepareIndex("_percolator", "test", Integer.toString(i))
69+
.setSource(jsonBuilder().startObject()
70+
.field("query", termQuery("name", "value"))
71+
.endObject())
72+
.setRefresh(true)
73+
.execute().actionGet();
74+
}
75+
76+
StopWatch stopWatch = new StopWatch().start();
77+
System.out.println("Percolating [" + COUNT + "] ...");
78+
int i = 1;
79+
for (; i <= COUNT; i++) {
80+
PercolateResponse percolate = client1.preparePercolate("test").setSource(source(Integer.toString(i), "value"))
81+
.execute().actionGet();
82+
if (percolate.matches().size() != QUERIES) {
83+
System.err.println("No matching number of queries");
84+
}
85+
if ((i % 10000) == 0) {
86+
System.out.println("Percolated " + i + " took " + stopWatch.stop().lastTaskTime());
87+
stopWatch.start();
88+
}
89+
}
90+
System.out.println("Percolation took " + stopWatch.totalTime() + ", TPS " + (((double) COUNT) / stopWatch.totalTime().secondsFrac()));
91+
92+
client.close();
93+
94+
for (Node node : nodes) {
95+
node.close();
96+
}
97+
}
98+
99+
private static XContentBuilder source(String id, String nameValue) throws IOException {
100+
long time = System.currentTimeMillis();
101+
return jsonBuilder().startObject().startObject("doc").startObject("type1")
102+
.field("id", id)
103+
.field("numeric1", time)
104+
.field("numeric2", time)
105+
.field("numeric3", time)
106+
.field("numeric4", time)
107+
.field("numeric5", time)
108+
.field("numeric6", time)
109+
.field("numeric7", time)
110+
.field("numeric8", time)
111+
.field("numeric9", time)
112+
.field("numeric10", time)
113+
.field("name", nameValue)
114+
.endObject().endObject().endObject();
115+
}
116+
}

‎modules/elasticsearch/src/main/java/org/elasticsearch/action/TransportActionModule.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
import org.elasticsearch.action.get.TransportGetAction;
6060
import org.elasticsearch.action.index.TransportIndexAction;
6161
import org.elasticsearch.action.mlt.TransportMoreLikeThisAction;
62+
import org.elasticsearch.action.percolate.TransportPercolateAction;
6263
import org.elasticsearch.action.search.TransportSearchAction;
6364
import org.elasticsearch.action.search.TransportSearchScrollAction;
6465
import org.elasticsearch.action.search.type.*;
@@ -131,5 +132,7 @@ public class TransportActionModule extends AbstractModule {
131132
bind(TransportSearchScrollAction.class).asEagerSingleton();
132133

133134
bind(TransportMoreLikeThisAction.class).asEagerSingleton();
135+
136+
bind(TransportPercolateAction.class).asEagerSingleton();
134137
}
135138
}

‎modules/elasticsearch/src/main/java/org/elasticsearch/action/TransportActions.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ public class TransportActions {
4444

4545
public static final String MORE_LIKE_THIS = "indices/moreLikeThis";
4646

47+
public static final String PERCOLATE = "indices/percolate";
48+
4749
public static class Admin {
4850

4951
public static class Indices {

‎modules/elasticsearch/src/main/java/org/elasticsearch/action/percolate/PercolateRequest.java

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
/*
2+
* Licensed to Elastic Search and Shay Banon under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. Elastic Search licenses this
6+
* file to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.action.percolate;
21+
22+
import org.apache.lucene.util.UnicodeUtil;
23+
import org.elasticsearch.ElasticSearchGenerationException;
24+
import org.elasticsearch.action.ActionRequestValidationException;
25+
import org.elasticsearch.action.support.single.custom.SingleCustomOperationRequest;
26+
import org.elasticsearch.common.Required;
27+
import org.elasticsearch.common.Unicode;
28+
import org.elasticsearch.common.io.stream.StreamInput;
29+
import org.elasticsearch.common.io.stream.StreamOutput;
30+
import org.elasticsearch.common.xcontent.XContentBuilder;
31+
import org.elasticsearch.common.xcontent.XContentFactory;
32+
import org.elasticsearch.common.xcontent.XContentType;
33+
34+
import java.io.IOException;
35+
import java.util.Arrays;
36+
import java.util.Map;
37+
38+
import static org.elasticsearch.action.Actions.*;
39+
40+
/**
41+
* @author kimchy
42+
*/
43+
public class PercolateRequest extends SingleCustomOperationRequest {
44+
45+
private String index;
46+
47+
private byte[] source;
48+
private int sourceOffset;
49+
private int sourceLength;
50+
private boolean sourceUnsafe;
51+
52+
PercolateRequest() {
53+
54+
}
55+
56+
/**
57+
* Constructs a new percolate request.
58+
*
59+
* @param index The index name
60+
*/
61+
public PercolateRequest(String index) {
62+
this.index = index;
63+
}
64+
65+
public PercolateRequest index(String index) {
66+
this.index = index;
67+
return this;
68+
}
69+
70+
public String index() {
71+
return this.index;
72+
}
73+
74+
/**
75+
* Before we fork on a local thread, make sure we copy over the bytes if they are unsafe
76+
*/
77+
@Override public void beforeLocalFork() {
78+
source();
79+
}
80+
81+
public byte[] source() {
82+
if (sourceUnsafe || sourceOffset > 0) {
83+
source = Arrays.copyOfRange(source, sourceOffset, sourceOffset + sourceLength);
84+
sourceOffset = 0;
85+
sourceUnsafe = false;
86+
}
87+
return source;
88+
}
89+
90+
public byte[] unsafeSource() {
91+
return this.source;
92+
}
93+
94+
public int unsafeSourceOffset() {
95+
return this.sourceOffset;
96+
}
97+
98+
public int unsafeSourceLength() {
99+
return this.sourceLength;
100+
}
101+
102+
@Required public PercolateRequest source(Map source) throws ElasticSearchGenerationException {
103+
return source(source, XContentType.SMILE);
104+
}
105+
106+
@Required public PercolateRequest source(Map source, XContentType contentType) throws ElasticSearchGenerationException {
107+
try {
108+
XContentBuilder builder = XContentFactory.contentBuilder(contentType);
109+
builder.map(source);
110+
return source(builder);
111+
} catch (IOException e) {
112+
throw new ElasticSearchGenerationException("Failed to generate [" + source + "]", e);
113+
}
114+
}
115+
116+
@Required public PercolateRequest source(String source) {
117+
UnicodeUtil.UTF8Result result = Unicode.fromStringAsUtf8(source);
118+
this.source = result.result;
119+
this.sourceOffset = 0;
120+
this.sourceLength = result.length;
121+
this.sourceUnsafe = true;
122+
return this;
123+
}
124+
125+
@Required public PercolateRequest source(XContentBuilder sourceBuilder) {
126+
try {
127+
source = sourceBuilder.unsafeBytes();
128+
sourceOffset = 0;
129+
sourceLength = sourceBuilder.unsafeBytesLength();
130+
sourceUnsafe = true;
131+
} catch (IOException e) {
132+
throw new ElasticSearchGenerationException("Failed to generate [" + sourceBuilder + "]", e);
133+
}
134+
return this;
135+
}
136+
137+
public PercolateRequest source(byte[] source) {
138+
return source(source, 0, source.length);
139+
}
140+
141+
@Required public PercolateRequest source(byte[] source, int offset, int length) {
142+
return source(source, offset, length, false);
143+
}
144+
145+
@Required public PercolateRequest source(byte[] source, int offset, int length, boolean unsafe) {
146+
this.source = source;
147+
this.sourceOffset = offset;
148+
this.sourceLength = length;
149+
this.sourceUnsafe = unsafe;
150+
return this;
151+
}
152+
153+
@Override public ActionRequestValidationException validate() {
154+
ActionRequestValidationException validationException = super.validate();
155+
if (index == null) {
156+
validationException = addValidationError("index is missing", validationException);
157+
}
158+
if (source == null) {
159+
validationException = addValidationError("source is missing", validationException);
160+
}
161+
return validationException;
162+
}
163+
164+
@Override public void readFrom(StreamInput in) throws IOException {
165+
super.readFrom(in);
166+
index = in.readUTF();
167+
168+
sourceUnsafe = false;
169+
sourceOffset = 0;
170+
sourceLength = in.readVInt();
171+
source = new byte[sourceLength];
172+
in.readFully(source);
173+
}
174+
175+
@Override public void writeTo(StreamOutput out) throws IOException {
176+
super.writeTo(out);
177+
out.writeUTF(index);
178+
179+
out.writeVInt(sourceLength);
180+
out.writeBytes(source, sourceOffset, sourceLength);
181+
}
182+
}

0 commit comments

Comments
 (0)