Skip to content

Commit e0831fa

Browse files
pnowak85xhaggi
pnowak85
authored and
xhaggi
committed
DATAES-525 - Fixed issue in scrolling not applying the search query.
The root cause of the deletion problem was the doScroll which did not apply the given query and therefor returned all entries from the index. The doScroll implementation has been fixed to apply the query and multiple unit tests have been added to ensure that the delete methods delete the desired documents only and leave the rest untouched. Also added unit tests for the scrolling to test it against real queries (not only matchAll). Original pull request: spring-projects#240
1 parent c77b543 commit e0831fa

File tree

2 files changed

+174
-0
lines changed

2 files changed

+174
-0
lines changed

‎src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchRestTemplate.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -979,6 +979,12 @@ private SearchResponse doScroll(SearchRequest request, SearchQuery searchQuery)
979979
Assert.notNull(searchQuery.getTypes(), "No type define for Query");
980980
Assert.notNull(searchQuery.getPageable(), "Query.pageable is required for scan & scroll");
981981

982+
if (searchQuery.getQuery() != null) {
983+
request.source().query(searchQuery.getQuery());
984+
} else {
985+
request.source().query(QueryBuilders.matchAllQuery());
986+
}
987+
982988
if (searchQuery.getFilter() != null) {
983989
request.source().postFilter(searchQuery.getFilter());
984990
}

‎src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplateTests.java

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
import java.util.List;
2424
import java.util.Map;
2525
import java.util.UUID;
26+
import java.util.stream.Collectors;
27+
2628
import org.apache.commons.lang.StringUtils;
2729
import org.assertj.core.util.Lists;
2830
import org.elasticsearch.action.get.MultiGetItemResponse;
@@ -2419,6 +2421,172 @@ public void shouldReturnMappingForGivenEntityClass() {
24192421
assertThat(((Map) ((Map) mapping.get("properties")).get("message")).get("type"), Matchers.<Object>is("text"));
24202422
}
24212423

2424+
@Test // DATAES-525
2425+
public void shouldDeleteOnlyDocumentsMatchedByDeleteQuery() {
2426+
List<IndexQuery> indexQueries = new ArrayList<>();
2427+
2428+
// given
2429+
// document to be deleted
2430+
String documentIdToDelete = UUID.randomUUID().toString();
2431+
indexQueries.add(getIndexQuery(SampleEntity.builder().id(documentIdToDelete).message("some message")
2432+
.version(System.currentTimeMillis()).build()));
2433+
2434+
// remaining document
2435+
String remainingDocumentId = UUID.randomUUID().toString();
2436+
indexQueries.add(getIndexQuery(SampleEntity.builder().id(remainingDocumentId).message("some other message")
2437+
.version(System.currentTimeMillis()).build()));
2438+
elasticsearchTemplate.bulkIndex(indexQueries);
2439+
elasticsearchTemplate.refresh(SampleEntity.class);
2440+
2441+
// when
2442+
DeleteQuery deleteQuery = new DeleteQuery();
2443+
deleteQuery.setQuery(idsQuery().addIds(documentIdToDelete));
2444+
elasticsearchTemplate.delete(deleteQuery, SampleEntity.class);
2445+
elasticsearchTemplate.refresh(SampleEntity.class);
2446+
2447+
// then
2448+
// document with id "remainingDocumentId" should still be indexed
2449+
SearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery()).build();
2450+
Page<SampleEntity> sampleEntities = elasticsearchTemplate.queryForPage(searchQuery, SampleEntity.class);
2451+
assertThat(sampleEntities.getTotalElements(), equalTo(1L));
2452+
assertThat(sampleEntities.getContent().get(0).getId(), is(remainingDocumentId));
2453+
}
2454+
2455+
@Test // DATAES-525
2456+
public void shouldDeleteOnlyDocumentsMatchedByCriteriaQuery() {
2457+
List<IndexQuery> indexQueries = new ArrayList<>();
2458+
2459+
// given
2460+
// document to be deleted
2461+
String documentIdToDelete = UUID.randomUUID().toString();
2462+
indexQueries.add(getIndexQuery(SampleEntity.builder().id(documentIdToDelete).message("some message")
2463+
.version(System.currentTimeMillis()).build()));
2464+
2465+
// remaining document
2466+
String remainingDocumentId = UUID.randomUUID().toString();
2467+
indexQueries.add(getIndexQuery(SampleEntity.builder().id(remainingDocumentId).message("some other message")
2468+
.version(System.currentTimeMillis()).build()));
2469+
elasticsearchTemplate.bulkIndex(indexQueries);
2470+
elasticsearchTemplate.refresh(SampleEntity.class);
2471+
2472+
// when
2473+
CriteriaQuery criteriaQuery = new CriteriaQuery(new Criteria("id").is(documentIdToDelete));
2474+
elasticsearchTemplate.delete(criteriaQuery, SampleEntity.class);
2475+
elasticsearchTemplate.refresh(SampleEntity.class);
2476+
2477+
// then
2478+
// document with id "remainingDocumentId" should still be indexed
2479+
SearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery()).build();
2480+
Page<SampleEntity> sampleEntities = elasticsearchTemplate.queryForPage(searchQuery, SampleEntity.class);
2481+
assertThat(sampleEntities.getTotalElements(), equalTo(1L));
2482+
assertThat(sampleEntities.getContent().get(0).getId(), is(remainingDocumentId));
2483+
}
2484+
2485+
@Test // DATAES-525
2486+
public void shouldDeleteDocumentForGivenIdOnly() {
2487+
List<IndexQuery> indexQueries = new ArrayList<>();
2488+
2489+
// given
2490+
// document to be deleted
2491+
String documentIdToDelete = UUID.randomUUID().toString();
2492+
indexQueries.add(getIndexQuery(SampleEntity.builder().id(documentIdToDelete).message("some message")
2493+
.version(System.currentTimeMillis()).build()));
2494+
2495+
// remaining document
2496+
String remainingDocumentId = UUID.randomUUID().toString();
2497+
indexQueries.add(getIndexQuery(SampleEntity.builder().id(remainingDocumentId).message("some other message")
2498+
.version(System.currentTimeMillis()).build()));
2499+
elasticsearchTemplate.bulkIndex(indexQueries);
2500+
elasticsearchTemplate.refresh(SampleEntity.class);
2501+
2502+
// when
2503+
elasticsearchTemplate.delete(SampleEntity.class, documentIdToDelete);
2504+
elasticsearchTemplate.refresh(SampleEntity.class);
2505+
2506+
// then
2507+
// document with id "remainingDocumentId" should still be indexed
2508+
SearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery()).build();
2509+
Page<SampleEntity> sampleEntities = elasticsearchTemplate.queryForPage(searchQuery, SampleEntity.class);
2510+
assertThat(sampleEntities.getTotalElements(), equalTo(1L));
2511+
assertThat(sampleEntities.getContent().get(0).getId(), is(remainingDocumentId));
2512+
}
2513+
2514+
@Test // DATAES-525
2515+
public void shouldApplyCriteriaQueryToScanAndScrollForGivenCriteriaQuery() {
2516+
// given
2517+
List<IndexQuery> indexQueries = new ArrayList<>();
2518+
indexQueries.add(getIndexQuery(SampleEntity.builder().id(UUID.randomUUID().toString())
2519+
.message("some message that should be found by the scroll query").version(System.currentTimeMillis())
2520+
.build()));
2521+
indexQueries.add(getIndexQuery(SampleEntity.builder().id(UUID.randomUUID().toString())
2522+
.message("some other message that should be found by the scroll query")
2523+
.version(System.currentTimeMillis()).build()));
2524+
String notFindableMessage = "this entity must not be found by the scroll query";
2525+
indexQueries.add(getIndexQuery(SampleEntity.builder().id(UUID.randomUUID().toString())
2526+
.message(notFindableMessage).version(System.currentTimeMillis()).build()));
2527+
2528+
elasticsearchTemplate.bulkIndex(indexQueries);
2529+
elasticsearchTemplate.refresh(SampleEntity.class);
2530+
2531+
// when
2532+
CriteriaQuery criteriaQuery = new CriteriaQuery(new Criteria("message").contains("message"));
2533+
criteriaQuery.addIndices(INDEX_NAME);
2534+
criteriaQuery.addTypes(TYPE_NAME);
2535+
criteriaQuery.setPageable(PageRequest.of(0, 10));
2536+
2537+
ScrolledPage<SampleEntity> scroll = (ScrolledPage<SampleEntity>) elasticsearchTemplate.startScroll(1000,
2538+
criteriaQuery, SampleEntity.class);
2539+
List<SampleEntity> sampleEntities = new ArrayList<>();
2540+
while (scroll.hasContent()) {
2541+
sampleEntities.addAll(scroll.getContent());
2542+
scroll = (ScrolledPage<SampleEntity>) elasticsearchTemplate.continueScroll(scroll.getScrollId(), 1000,
2543+
SampleEntity.class);
2544+
}
2545+
elasticsearchTemplate.clearScroll(scroll.getScrollId());
2546+
2547+
// then
2548+
assertThat(sampleEntities.size(), is(equalTo(2)));
2549+
assertThat(sampleEntities.stream().map(SampleEntity::getMessage).collect(Collectors.toList()),
2550+
not(contains(notFindableMessage)));
2551+
}
2552+
2553+
@Test // DATAES-525
2554+
public void shouldApplySearchQueryToScanAndScrollForGivenSearchQuery() {
2555+
// given
2556+
List<IndexQuery> indexQueries = new ArrayList<>();
2557+
indexQueries.add(getIndexQuery(SampleEntity.builder().id(UUID.randomUUID().toString())
2558+
.message("some message that should be found by the scroll query").version(System.currentTimeMillis())
2559+
.build()));
2560+
indexQueries.add(getIndexQuery(SampleEntity.builder().id(UUID.randomUUID().toString())
2561+
.message("some other message that should be found by the scroll query")
2562+
.version(System.currentTimeMillis()).build()));
2563+
String notFindableMessage = "this entity must not be found by the scroll query";
2564+
indexQueries.add(getIndexQuery(SampleEntity.builder().id(UUID.randomUUID().toString())
2565+
.message(notFindableMessage).version(System.currentTimeMillis()).build()));
2566+
2567+
elasticsearchTemplate.bulkIndex(indexQueries);
2568+
elasticsearchTemplate.refresh(SampleEntity.class);
2569+
2570+
// when
2571+
SearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchQuery("message", "message"))
2572+
.withIndices(INDEX_NAME).withTypes(TYPE_NAME).withPageable(PageRequest.of(0, 10)).build();
2573+
2574+
ScrolledPage<SampleEntity> scroll = (ScrolledPage<SampleEntity>) elasticsearchTemplate.startScroll(1000,
2575+
searchQuery, SampleEntity.class);
2576+
List<SampleEntity> sampleEntities = new ArrayList<>();
2577+
while (scroll.hasContent()) {
2578+
sampleEntities.addAll(scroll.getContent());
2579+
scroll = (ScrolledPage<SampleEntity>) elasticsearchTemplate.continueScroll(scroll.getScrollId(), 1000,
2580+
SampleEntity.class);
2581+
}
2582+
elasticsearchTemplate.clearScroll(scroll.getScrollId());
2583+
2584+
// then
2585+
assertThat(sampleEntities.size(), is(equalTo(2)));
2586+
assertThat(sampleEntities.stream().map(SampleEntity::getMessage).collect(Collectors.toList()),
2587+
not(contains(notFindableMessage)));
2588+
}
2589+
24222590
private IndexQuery getIndexQuery(SampleEntity sampleEntity) {
24232591
return new IndexQueryBuilder()
24242592
.withId(sampleEntity.getId())

0 commit comments

Comments
 (0)