Skip to content

Commit 5e8c6f2

Browse files
author
Matt Hartzler
committed
Wraps NodeClient to open/close indexes at the cluster level based on a LRU cache of configurable size
1 parent 8fef3df commit 5e8c6f2

File tree

5 files changed

+395
-0
lines changed

5 files changed

+395
-0
lines changed

‎plugins/client/lru/build.gradle

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
dependsOn(':elasticsearch')
2+
3+
apply plugin: 'java'
4+
apply plugin: 'maven'
5+
6+
archivesBaseName = "elasticsearch-client-lru"
7+
8+
explodedDistDir = new File(distsDir, 'exploded')
9+
10+
manifest.mainAttributes("Implementation-Title": "ElasticSearch::Plugins::LruClient", "Implementation-Version": rootProject.version, "Implementation-Date": buildTimeStr)
11+
12+
configurations.compile.transitive = true
13+
configurations.testCompile.transitive = true
14+
15+
// no need to use the resource dir
16+
sourceSets.main.resources.srcDirs 'src/main/java'
17+
sourceSets.test.resources.srcDirs 'src/test/java'
18+
19+
// add the source files to the dist jar
20+
//jar {
21+
// from sourceSets.main.allJava
22+
//}
23+
24+
configurations {
25+
dists
26+
distLib {
27+
visible = false
28+
transitive = false
29+
}
30+
}
31+
32+
dependencies {
33+
compile project(':elasticsearch')
34+
35+
testCompile project(':test-testng')
36+
testCompile('org.testng:testng:5.10:jdk15') { transitive = false }
37+
testCompile 'org.hamcrest:hamcrest-all:1.1'
38+
}
39+
40+
test {
41+
//useTestNG()
42+
//jmvArgs = ["-ea", "-Xmx1024m"]
43+
//suiteName = project.name
44+
//listeners = ["org.elasticsearch.util.testng.Listeners"]
45+
//systemProperties["es.test.log.conf"] = System.getProperty("es.test.log.conf", "log4j-gradle.properties")
46+
}
47+
48+
task explodedDist(dependsOn: [jar], description: 'Builds the plugin zip file') << {
49+
[explodedDistDir]*.mkdirs()
50+
51+
copy {
52+
from configurations.distLib
53+
into explodedDistDir
54+
}
55+
56+
// remove elasticsearch files (compile above adds the elasticsearch one)
57+
ant.delete { fileset(dir: explodedDistDir, includes: "elasticsearch-*.jar") }
58+
59+
copy {
60+
from libsDir
61+
into explodedDistDir
62+
}
63+
64+
ant.delete { fileset(dir: explodedDistDir, includes: "elasticsearch-*-javadoc.jar") }
65+
ant.delete { fileset(dir: explodedDistDir, includes: "elasticsearch-*-sources.jar") }
66+
}
67+
68+
task zip(type: Zip, dependsOn: ['explodedDist']) {
69+
from(explodedDistDir) {
70+
}
71+
}
72+
73+
task release(dependsOn: [zip]) << {
74+
ant.delete(dir: explodedDistDir)
75+
copy {
76+
from distsDir
77+
into(new File(rootProject.distsDir, "plugins"))
78+
}
79+
}
80+
81+
configurations {
82+
deployerJars
83+
}
84+
85+
dependencies {
86+
deployerJars "org.apache.maven.wagon:wagon-http:1.0-beta-2"
87+
}
88+
89+
task sourcesJar(type: Jar, dependsOn: classes) {
90+
classifier = 'sources'
91+
from sourceSets.main.allSource
92+
}
93+
94+
task javadocJar(type: Jar, dependsOn: javadoc) {
95+
classifier = 'javadoc'
96+
from javadoc.destinationDir
97+
}
98+
99+
artifacts {
100+
archives sourcesJar
101+
archives javadocJar
102+
}
103+
104+
uploadArchives {
105+
repositories.mavenDeployer {
106+
configuration = configurations.deployerJars
107+
repository(url: rootProject.mavenRepoUrl) {
108+
authentication(userName: rootProject.mavenRepoUser, password: rootProject.mavenRepoPass)
109+
}
110+
snapshotRepository(url: rootProject.mavenSnapshotRepoUrl) {
111+
authentication(userName: rootProject.mavenRepoUser, password: rootProject.mavenRepoPass)
112+
}
113+
114+
pom.project {
115+
inceptionYear '2010'
116+
name 'elasticsearch-plugins-lruclient'
117+
description 'Lru Client Plugin for ElasticSearch'
118+
licenses {
119+
license {
120+
name 'The Apache Software License, Version 2.0'
121+
url 'http://www.apache.org/licenses/LICENSE-2.0.txt'
122+
distribution 'repo'
123+
}
124+
}
125+
scm {
126+
connection 'git://github.com/elasticsearch/elasticsearch.git'
127+
developerConnection '[email protected]:elasticsearch/elasticsearch.git'
128+
url 'http://github.com/elasticsearch/elasticsearch'
129+
}
130+
}
131+
132+
pom.whenConfigured {pom ->
133+
pom.dependencies = pom.dependencies.findAll {dep -> dep.scope != 'test' } // removes the test scoped ones
134+
}
135+
}
136+
}

‎plugins/client/lru/src/main/java/es-plugin.properties

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
plugin=org.elasticsearch.plugin.lruclient.LruClientPlugin

‎plugins/client/lru/src/main/java/org/elasticsearch/plugin/lruclient/LruClient.java

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
/*
2+
* Licensed to Elastic Search and Shay Banon under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. Elastic Search licenses this
6+
* file to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.plugin.lruclient;
21+
22+
import java.util.Date;
23+
import java.util.LinkedHashMap;
24+
import java.util.Map.Entry;
25+
import org.elasticsearch.action.ActionFuture;
26+
import org.elasticsearch.action.ActionListener;
27+
import org.elasticsearch.action.bulk.BulkRequest;
28+
import org.elasticsearch.action.bulk.BulkResponse;
29+
import org.elasticsearch.action.bulk.TransportBulkAction;
30+
import org.elasticsearch.action.count.CountRequest;
31+
import org.elasticsearch.action.count.CountResponse;
32+
import org.elasticsearch.action.count.TransportCountAction;
33+
import org.elasticsearch.action.delete.DeleteRequest;
34+
import org.elasticsearch.action.delete.DeleteResponse;
35+
import org.elasticsearch.action.delete.TransportDeleteAction;
36+
import org.elasticsearch.action.deletebyquery.DeleteByQueryRequest;
37+
import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
38+
import org.elasticsearch.action.deletebyquery.TransportDeleteByQueryAction;
39+
import org.elasticsearch.action.get.GetRequest;
40+
import org.elasticsearch.action.get.GetResponse;
41+
import org.elasticsearch.action.get.TransportGetAction;
42+
import org.elasticsearch.action.index.IndexRequest;
43+
import org.elasticsearch.action.index.IndexResponse;
44+
import org.elasticsearch.action.index.TransportIndexAction;
45+
import org.elasticsearch.action.mlt.MoreLikeThisRequest;
46+
import org.elasticsearch.action.mlt.TransportMoreLikeThisAction;
47+
import org.elasticsearch.action.search.*;
48+
import org.elasticsearch.client.node.NodeAdminClient;
49+
import org.elasticsearch.client.node.NodeClient;
50+
import org.elasticsearch.common.inject.Inject;
51+
import org.elasticsearch.common.logging.ESLogger;
52+
import org.elasticsearch.common.logging.Loggers;
53+
import org.elasticsearch.common.settings.Settings;
54+
import org.elasticsearch.indices.IndexMissingException;
55+
import org.elasticsearch.threadpool.ThreadPool;
56+
57+
public class LruClient extends NodeClient {
58+
59+
private final ESLogger logger;
60+
private final LruCache cache;
61+
62+
private class LruCache extends LinkedHashMap<String, Date> {
63+
private int capacity = 1;
64+
public LruCache(int capacity) {
65+
super(capacity,0.75f,true);
66+
this.capacity = capacity;
67+
}
68+
69+
@Override
70+
protected boolean removeEldestEntry(Entry<String, Date> eldest) {
71+
if(size()>capacity) {
72+
logger.debug("LruCache over capacity: {} removing: {}", this.capacity, eldest.getKey());
73+
closeIndex(eldest.getKey());
74+
return true;
75+
}
76+
return false;
77+
}
78+
79+
}
80+
81+
@Inject public LruClient(Settings settings, ThreadPool threadPool, NodeAdminClient admin,
82+
TransportIndexAction indexAction, TransportDeleteAction deleteAction, TransportBulkAction bulkAction,
83+
TransportDeleteByQueryAction deleteByQueryAction, TransportGetAction getAction, TransportCountAction countAction,
84+
TransportSearchAction searchAction, TransportSearchScrollAction searchScrollAction,
85+
TransportMoreLikeThisAction moreLikeThisAction) {
86+
super(settings,threadPool,admin,indexAction,deleteAction,bulkAction,deleteByQueryAction,getAction,countAction,searchAction,searchScrollAction,moreLikeThisAction);
87+
this.logger = Loggers.getLogger(getClass());
88+
this.cache = new LruCache(settings.getAsInt("node.lrucache.size", 1));
89+
logger.debug("LruClient",new Exception("DEBUG"));
90+
}
91+
92+
private void ensureOpen(final String index) {
93+
if(!this.cache.containsKey(index)) {
94+
logger.debug("Index {} not found! Try to open...", index);
95+
// force in foreground...
96+
try {
97+
admin().indices().prepareOpen(index).execute().actionGet();
98+
}
99+
catch(IndexMissingException e) {
100+
// ignore
101+
logger.debug("Index {} does not exist! Can't open.", index);
102+
}
103+
}
104+
this.cache.put(index, new Date());
105+
}
106+
107+
private void ensureOpen(final String [] indices) {
108+
for(int i = 0; i<indices.length; i++)
109+
ensureOpen(indices[i]);
110+
}
111+
112+
private void closeIndex(final String index) {
113+
// let happen in the background
114+
logger.debug("Closing index: {} ...", index);
115+
admin().indices().prepareClose(index).execute().actionGet();
116+
}
117+
118+
@Override public void index(IndexRequest request, ActionListener<IndexResponse> listener) {
119+
logger.debug("index");
120+
ensureOpen(request.index());
121+
super.index(request,listener);
122+
}
123+
124+
@Override public ActionFuture<DeleteResponse> delete(DeleteRequest request) {
125+
logger.debug("delete");
126+
ensureOpen(request.index());
127+
return super.delete(request);
128+
}
129+
130+
@Override public void delete(DeleteRequest request, ActionListener<DeleteResponse> listener) {
131+
logger.debug("delete");
132+
ensureOpen(request.index());
133+
super.delete(request, listener);
134+
}
135+
136+
@Override public ActionFuture<BulkResponse> bulk(BulkRequest request) {
137+
logger.debug("bulk");
138+
return super.bulk(request);
139+
}
140+
141+
@Override public void bulk(BulkRequest request, ActionListener<BulkResponse> listener) {
142+
logger.debug("bulk");
143+
super.bulk(request,listener);
144+
}
145+
146+
@Override public ActionFuture<DeleteByQueryResponse> deleteByQuery(DeleteByQueryRequest request) {
147+
logger.debug("deleteByQuery");
148+
ensureOpen(request.indices());
149+
return super.deleteByQuery(request);
150+
}
151+
152+
@Override public void deleteByQuery(DeleteByQueryRequest request, ActionListener<DeleteByQueryResponse> listener) {
153+
logger.debug("deleteByQuery");
154+
ensureOpen(request.indices());
155+
super.deleteByQuery(request,listener);
156+
}
157+
158+
@Override public ActionFuture<GetResponse> get(GetRequest request) {
159+
logger.debug("get");
160+
ensureOpen(request.index());
161+
return super.get(request);
162+
}
163+
164+
@Override public void get(GetRequest request, ActionListener<GetResponse> listener) {
165+
logger.debug("get");
166+
ensureOpen(request.index());
167+
super.get(request, listener);
168+
}
169+
170+
@Override public ActionFuture<CountResponse> count(CountRequest request) {
171+
logger.debug("count");
172+
ensureOpen(request.indices());
173+
return super.count(request);
174+
}
175+
176+
@Override public void count(CountRequest request, ActionListener<CountResponse> listener) {
177+
logger.debug("count");
178+
ensureOpen(request.indices());
179+
super.count(request, listener);
180+
}
181+
182+
@Override public ActionFuture<SearchResponse> search(SearchRequest request) {
183+
logger.debug("search");
184+
ensureOpen(request.indices());
185+
return super.search(request);
186+
}
187+
188+
@Override public void search(SearchRequest request, ActionListener<SearchResponse> listener) {
189+
logger.debug("search");
190+
ensureOpen(request.indices());
191+
super.search(request, listener);
192+
}
193+
194+
@Override public ActionFuture<SearchResponse> searchScroll(SearchScrollRequest request) {
195+
logger.debug("searchScroll");
196+
return super.searchScroll(request);
197+
}
198+
199+
@Override public void searchScroll(SearchScrollRequest request, ActionListener<SearchResponse> listener) {
200+
logger.debug("searchScroll");
201+
super.searchScroll(request, listener);
202+
}
203+
204+
@Override public ActionFuture<SearchResponse> moreLikeThis(MoreLikeThisRequest request) {
205+
logger.debug("moreLikeThis");
206+
ensureOpen(request.index());
207+
return super.moreLikeThis(request);
208+
}
209+
210+
@Override public void moreLikeThis(MoreLikeThisRequest request, ActionListener<SearchResponse> listener) {
211+
logger.debug("moreLikeThis");
212+
ensureOpen(request.index());
213+
super.moreLikeThis(request, listener);
214+
}
215+
216+
}

‎plugins/client/lru/src/main/java/org/elasticsearch/plugin/lruclient/LruClientPlugin.java

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Licensed to Elastic Search and Shay Banon under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. Elastic Search licenses this
6+
* file to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.plugin.lruclient;
21+
22+
import java.util.Collection;
23+
import java.util.List;
24+
import org.elasticsearch.common.collect.Lists;
25+
import org.elasticsearch.common.inject.Module;
26+
import org.elasticsearch.plugins.AbstractPlugin;
27+
28+
/**
29+
* @author matt.hartzler
30+
*/
31+
public class LruClientPlugin extends AbstractPlugin {
32+
33+
@Override public String name() {
34+
return "lru-client";
35+
}
36+
37+
@Override public String description() {
38+
return "Lru Index Support via the NodeClient";
39+
}
40+
}

0 commit comments

Comments
 (0)