Skip to content

Commit 23e1e9f

Browse files
kevinleturcakonczak
kevinleturc
authored and
akonczak
committed
DATAES-217 Propose scan method with a class in order to retrieve index and type from Document
1 parent 751302d commit 23e1e9f

File tree

3 files changed

+141
-33
lines changed

3 files changed

+141
-33
lines changed

‎src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchOperations.java

Lines changed: 40 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -484,23 +484,55 @@ public interface ElasticsearchOperations {
484484
/**
485485
* Returns scroll id for criteria query
486486
*
487-
* @param query
488-
* @param scrollTimeInMillis
489-
* @param noFields
490-
* @return
487+
* @param query The criteria query.
488+
* @param scrollTimeInMillis The time in millisecond for scroll feature
489+
* {@link org.elasticsearch.action.search.SearchRequestBuilder#setScroll(org.elasticsearch.common.unit.TimeValue)}.
490+
* @param noFields The no fields support
491+
* {@link org.elasticsearch.action.search.SearchRequestBuilder#setNoFields()}.
492+
* @return The scan id for input query.
491493
*/
492494
String scan(CriteriaQuery query, long scrollTimeInMillis, boolean noFields);
493495

496+
/**
497+
* Returns scroll id for criteria query
498+
*
499+
* @param query The criteria query.
500+
* @param scrollTimeInMillis The time in millisecond for scroll feature
501+
* {@link org.elasticsearch.action.search.SearchRequestBuilder#setScroll(org.elasticsearch.common.unit.TimeValue)}.
502+
* @param noFields The no fields support
503+
* {@link org.elasticsearch.action.search.SearchRequestBuilder#setNoFields()}.
504+
* @param clazz The class of entity to retrieve.
505+
* @param <T> The type of entity to retrieve.
506+
* @return The scan id for input query.
507+
*/
508+
<T> String scan(CriteriaQuery query, long scrollTimeInMillis, boolean noFields, Class<T> clazz);
509+
494510
/**
495511
* Returns scroll id for scan query
496512
*
497-
* @param query
498-
* @param scrollTimeInMillis
499-
* @param noFields
500-
* @return
513+
* @param query The search query.
514+
* @param scrollTimeInMillis The time in millisecond for scroll feature
515+
* {@link org.elasticsearch.action.search.SearchRequestBuilder#setScroll(org.elasticsearch.common.unit.TimeValue)}.
516+
* @param noFields The no fields support
517+
* {@link org.elasticsearch.action.search.SearchRequestBuilder#setNoFields()}.
518+
* @return The scan id for input query.
501519
*/
502520
String scan(SearchQuery query, long scrollTimeInMillis, boolean noFields);
503521

522+
/**
523+
* Returns scroll id for scan query
524+
*
525+
* @param query The search query.
526+
* @param scrollTimeInMillis The time in millisecond for scroll feature
527+
* {@link org.elasticsearch.action.search.SearchRequestBuilder#setScroll(org.elasticsearch.common.unit.TimeValue)}.
528+
* @param noFields The no fields support
529+
* {@link org.elasticsearch.action.search.SearchRequestBuilder#setNoFields()}.
530+
* @param clazz The class of entity to retrieve.
531+
* @param <T> The type of entity to retrieve.
532+
* @return The scan id for input query.
533+
*/
534+
<T> String scan(SearchQuery query, long scrollTimeInMillis, boolean noFields, Class<T> clazz);
535+
504536
/**
505537
* Scrolls the results for give scroll id
506538
*

‎src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplate.java

Lines changed: 43 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -337,8 +337,7 @@ public <T> FacetedPage<T> queryForPage(StringQuery query, Class<T> clazz, Search
337337
@Override
338338
public <T> CloseableIterator<T> stream(CriteriaQuery query, Class<T> clazz) {
339339
final long scrollTimeInMillis = TimeValue.timeValueMinutes(1).millis();
340-
setPersistentEntityIndexAndType(query, clazz);
341-
final String initScrollId = scan(query, scrollTimeInMillis, false);
340+
final String initScrollId = scan(query, scrollTimeInMillis, false, clazz);
342341
return doStream(initScrollId, scrollTimeInMillis, clazz, resultsMapper);
343342
}
344343

@@ -350,8 +349,7 @@ public <T> CloseableIterator<T> stream(SearchQuery query, Class<T> clazz) {
350349
@Override
351350
public <T> CloseableIterator<T> stream(SearchQuery query, final Class<T> clazz, final SearchResultMapper mapper) {
352351
final long scrollTimeInMillis = TimeValue.timeValueMinutes(1).millis();
353-
setPersistentEntityIndexAndType(query, clazz);
354-
final String initScrollId = scan(query, scrollTimeInMillis, false);
352+
final String initScrollId = scan(query, scrollTimeInMillis, false, clazz);
355353
return doStream(initScrollId, scrollTimeInMillis, clazz, mapper);
356354
}
357355

@@ -675,13 +673,52 @@ public <T> void delete(CriteriaQuery criteriaQuery, Class<T> clazz) {
675673

676674
@Override
677675
public String scan(CriteriaQuery criteriaQuery, long scrollTimeInMillis, boolean noFields) {
676+
return doScan(prepareScan(criteriaQuery, scrollTimeInMillis, noFields), criteriaQuery);
677+
}
678+
679+
@Override
680+
public <T> String scan(CriteriaQuery criteriaQuery, long scrollTimeInMillis, boolean noFields, Class<T> clazz) {
681+
return doScan(prepareScan(criteriaQuery, scrollTimeInMillis, noFields, clazz), criteriaQuery);
682+
}
683+
684+
@Override
685+
public String scan(SearchQuery searchQuery, long scrollTimeInMillis, boolean noFields) {
686+
return doScan(prepareScan(searchQuery, scrollTimeInMillis, noFields), searchQuery);
687+
}
688+
689+
@Override
690+
public <T> String scan(SearchQuery searchQuery, long scrollTimeInMillis, boolean noFields, Class<T> clazz) {
691+
return doScan(prepareScan(searchQuery, scrollTimeInMillis, noFields, clazz), searchQuery);
692+
}
693+
694+
private <T> SearchRequestBuilder prepareScan(Query query, long scrollTimeInMillis, boolean noFields, Class<T> clazz) {
695+
setPersistentEntityIndexAndType(query, clazz);
696+
return prepareScan(query, scrollTimeInMillis, noFields);
697+
}
698+
699+
private SearchRequestBuilder prepareScan(Query query, long scrollTimeInMillis, boolean noFields) {
700+
SearchRequestBuilder requestBuilder = client.prepareSearch(toArray(query.getIndices())).setSearchType(SCAN)
701+
.setTypes(toArray(query.getTypes()))
702+
.setScroll(TimeValue.timeValueMillis(scrollTimeInMillis)).setFrom(0)
703+
.setSize(query.getPageable().getPageSize());
704+
705+
if (isNotEmpty(query.getFields())) {
706+
requestBuilder.addFields(toArray(query.getFields()));
707+
}
708+
709+
if (noFields) {
710+
requestBuilder.setNoFields();
711+
}
712+
return requestBuilder;
713+
}
714+
715+
private String doScan(SearchRequestBuilder requestBuilder, CriteriaQuery criteriaQuery) {
678716
Assert.notNull(criteriaQuery.getIndices(), "No index defined for Query");
679717
Assert.notNull(criteriaQuery.getTypes(), "No type define for Query");
680718
Assert.notNull(criteriaQuery.getPageable(), "Query.pageable is required for scan & scroll");
681719

682720
QueryBuilder elasticsearchQuery = new CriteriaQueryProcessor().createQueryFromCriteria(criteriaQuery.getCriteria());
683721
FilterBuilder elasticsearchFilter = new CriteriaFilterProcessor().createFilterFromCriteria(criteriaQuery.getCriteria());
684-
SearchRequestBuilder requestBuilder = prepareScan(criteriaQuery, scrollTimeInMillis, noFields);
685722

686723
if (elasticsearchQuery != null) {
687724
requestBuilder.setQuery(elasticsearchQuery);
@@ -696,37 +733,18 @@ public String scan(CriteriaQuery criteriaQuery, long scrollTimeInMillis, boolean
696733
return getSearchResponse(requestBuilder.execute()).getScrollId();
697734
}
698735

699-
@Override
700-
public String scan(SearchQuery searchQuery, long scrollTimeInMillis, boolean noFields) {
736+
private String doScan(SearchRequestBuilder requestBuilder, SearchQuery searchQuery) {
701737
Assert.notNull(searchQuery.getIndices(), "No index defined for Query");
702738
Assert.notNull(searchQuery.getTypes(), "No type define for Query");
703739
Assert.notNull(searchQuery.getPageable(), "Query.pageable is required for scan & scroll");
704740

705-
SearchRequestBuilder requestBuilder = prepareScan(searchQuery, scrollTimeInMillis, noFields);
706-
707741
if (searchQuery.getFilter() != null) {
708742
requestBuilder.setPostFilter(searchQuery.getFilter());
709743
}
710744

711745
return getSearchResponse(requestBuilder.setQuery(searchQuery.getQuery()).execute()).getScrollId();
712746
}
713747

714-
private SearchRequestBuilder prepareScan(Query query, long scrollTimeInMillis, boolean noFields) {
715-
SearchRequestBuilder requestBuilder = client.prepareSearch(toArray(query.getIndices())).setSearchType(SCAN)
716-
.setTypes(toArray(query.getTypes()))
717-
.setScroll(TimeValue.timeValueMillis(scrollTimeInMillis)).setFrom(0)
718-
.setSize(query.getPageable().getPageSize());
719-
720-
if (isNotEmpty(query.getFields())) {
721-
requestBuilder.addFields(toArray(query.getFields()));
722-
}
723-
724-
if (noFields) {
725-
requestBuilder.setNoFields();
726-
}
727-
return requestBuilder;
728-
}
729-
730748
@Override
731749
public <T> Page<T> scroll(String scrollId, long scrollTimeInMillis, Class<T> clazz) {
732750
SearchResponse response = getSearchResponse(client.prepareSearchScroll(scrollId)

‎src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplateTests.java

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -900,6 +900,64 @@ public <T> FacetedPage<T> mapResults(SearchResponse response, Class<T> clazz, Pa
900900
assertThat(sampleEntities.size(), is(equalTo(30)));
901901
}
902902

903+
/*
904+
DATAES-217
905+
*/
906+
@Test
907+
public void shouldReturnResultsWithScanAndScrollForGivenCriteriaQueryAndClass() {
908+
//given
909+
List<IndexQuery> entities = createSampleEntitiesWithMessage("Test message", 30);
910+
// when
911+
elasticsearchTemplate.bulkIndex(entities);
912+
elasticsearchTemplate.refresh(SampleEntity.class, true);
913+
// then
914+
915+
CriteriaQuery criteriaQuery = new CriteriaQuery(new Criteria());
916+
criteriaQuery.setPageable(new PageRequest(0, 10));
917+
918+
String scrollId = elasticsearchTemplate.scan(criteriaQuery, 1000, false, SampleEntity.class);
919+
List<SampleEntity> sampleEntities = new ArrayList<SampleEntity>();
920+
boolean hasRecords = true;
921+
while (hasRecords) {
922+
Page<SampleEntity> page = elasticsearchTemplate.scroll(scrollId, 5000L, SampleEntity.class);
923+
if (page.hasContent()) {
924+
sampleEntities.addAll(page.getContent());
925+
} else {
926+
hasRecords = false;
927+
}
928+
}
929+
assertThat(sampleEntities.size(), is(equalTo(30)));
930+
}
931+
932+
/*
933+
DATAES-217
934+
*/
935+
@Test
936+
public void shouldReturnResultsWithScanAndScrollForGivenSearchQueryAndClass() {
937+
//given
938+
List<IndexQuery> entities = createSampleEntitiesWithMessage("Test message", 30);
939+
// when
940+
elasticsearchTemplate.bulkIndex(entities);
941+
elasticsearchTemplate.refresh(SampleEntity.class, true);
942+
// then
943+
944+
SearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery())
945+
.withPageable(new PageRequest(0, 10)).build();
946+
947+
String scrollId = elasticsearchTemplate.scan(searchQuery, 1000, false, SampleEntity.class);
948+
List<SampleEntity> sampleEntities = new ArrayList<SampleEntity>();
949+
boolean hasRecords = true;
950+
while (hasRecords) {
951+
Page<SampleEntity> page = elasticsearchTemplate.scroll(scrollId, 5000L, SampleEntity.class);
952+
if (page.hasContent()) {
953+
sampleEntities.addAll(page.getContent());
954+
} else {
955+
hasRecords = false;
956+
}
957+
}
958+
assertThat(sampleEntities.size(), is(equalTo(30)));
959+
}
960+
903961
/*
904962
DATAES-167
905963
*/

0 commit comments

Comments
 (0)