From 6a201b4e34e04d2630e89b5bd645029c6af26ecb Mon Sep 17 00:00:00 2001 From: Peter-Josef Meisch Date: Wed, 13 Apr 2022 22:12:02 +0200 Subject: [PATCH 001/516] Add more implementations using the new client. Original Pull Request #2136 See #1973 (cherry picked from commit 8cef50347ea5a67e945b17312b9dac466fa40527) --- .../elasticsearch/NoSuchIndexException.java | 8 + .../elasticsearch/client/elc/Aggregation.java | 44 +++ .../client/elc/DocumentAdapters.java | 29 +- .../client/elc/ElasticsearchAggregation.java | 37 ++ .../client/elc/ElasticsearchAggregations.java | 28 +- .../elc/ElasticsearchExceptionTranslator.java | 39 ++- .../client/elc/ElasticsearchTemplate.java | 70 +++- .../elasticsearch/client/elc/NativeQuery.java | 46 ++- .../client/elc/NativeQueryBuilder.java | 71 +++- .../elc/ReactiveElasticsearchClient.java | 54 +++ .../ReactiveElasticsearchIndicesClient.java | 53 --- .../elc/ReactiveElasticsearchTemplate.java | 154 ++++++--- .../client/elc/ReactiveIndicesTemplate.java | 11 +- .../client/elc/RequestConverter.java | 321 ++++++++++++++++-- .../client/elc/ResponseConverter.java | 53 ++- .../elasticsearch/client/elc/TypeUtils.java | 65 ++++ ...AbstractReactiveElasticsearchTemplate.java | 2 +- .../core/ReactiveSearchOperations.java | 6 +- .../elasticsearch/core/query/BaseQuery.java | 14 +- .../core/query/BaseQueryBuilder.java | 22 ++ .../core/query/NativeSearchQueryBuilder.java | 10 - .../elasticsearch/core/query/ScriptData.java | 70 ++++ .../core/query/ScriptedField.java | 45 +++ .../elasticsearch/core/query/UpdateQuery.java | 37 +- .../core/query/UpdateResponse.java | 7 + .../core/query/highlight/Highlight.java | 11 + .../core/query/highlight/HighlightField.java | 11 + .../data/elasticsearch/ELCQueries.java | 57 ++++ .../ElasticsearchELCIntegrationTests.java | 114 ++++++- .../ElasticsearchERHLCIntegrationTests.java | 86 +++++ .../core/ElasticsearchIntegrationTests.java | 227 +++++-------- ...ctiveElasticsearchELCIntegrationTests.java | 80 ++++- ...iveElasticsearchERHLCIntegrationTests.java | 63 ++++ ...ReactiveElasticsearchIntegrationTests.java | 124 +++---- .../core/RequestFactoryTests.java | 24 ++ .../AggregationELCIntegrationTests.java | 29 +- .../AggregationIntegrationTests.java | 2 +- .../suggest/CompletionIntegrationTests.java | 6 +- .../ReactiveSuggestIntegrationTests.java | 2 +- ...icsearchRepositoryELCIntegrationTests.java | 44 +++ ...searchRepositoryERHLCIntegrationTests.java | 44 +++ ...sticsearchRepositoryIntegrationTests.java} | 50 +-- ...eReactiveElasticsearchRepositoryTests.java | 1 + 43 files changed, 1760 insertions(+), 511 deletions(-) create mode 100644 src/main/java/org/springframework/data/elasticsearch/client/elc/Aggregation.java create mode 100644 src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchAggregation.java create mode 100644 src/main/java/org/springframework/data/elasticsearch/core/query/ScriptData.java create mode 100644 src/main/java/org/springframework/data/elasticsearch/core/query/ScriptedField.java create mode 100644 src/test/java/org/springframework/data/elasticsearch/ELCQueries.java create mode 100644 src/test/java/org/springframework/data/elasticsearch/repository/support/ElasticsearchRepositoryELCIntegrationTests.java create mode 100644 src/test/java/org/springframework/data/elasticsearch/repository/support/ElasticsearchRepositoryERHLCIntegrationTests.java rename src/test/java/org/springframework/data/elasticsearch/repository/support/{SimpleElasticsearchRepositoryIntegrationTests.java => ElasticsearchRepositoryIntegrationTests.java} (92%) diff --git a/src/main/java/org/springframework/data/elasticsearch/NoSuchIndexException.java b/src/main/java/org/springframework/data/elasticsearch/NoSuchIndexException.java index 3352526b2..0a122c4b0 100644 --- a/src/main/java/org/springframework/data/elasticsearch/NoSuchIndexException.java +++ b/src/main/java/org/springframework/data/elasticsearch/NoSuchIndexException.java @@ -25,6 +25,14 @@ public class NoSuchIndexException extends NonTransientDataAccessResourceExceptio private final String index; + /** + * @since 4.4 + */ + public NoSuchIndexException(String index) { + super(String.format("Index %s not found.", index)); + this.index = index; + } + public NoSuchIndexException(String index, Throwable cause) { super(String.format("Index %s not found.", index), cause); this.index = index; diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/Aggregation.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/Aggregation.java new file mode 100644 index 000000000..3be50a227 --- /dev/null +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/Aggregation.java @@ -0,0 +1,44 @@ +/* + * Copyright 2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch.client.elc; + +import co.elastic.clients.elasticsearch._types.aggregations.Aggregate; + +/** + * Class to combine an Elasticsearch {@link co.elastic.clients.elasticsearch._types.aggregations.Aggregate} with its + * name. Necessary as the Elasticsearch Aggregate does not know i"s name. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public class Aggregation { + + private final String name; + private final Aggregate aggregate; + + public Aggregation(String name, Aggregate aggregate) { + this.name = name; + this.aggregate = aggregate; + } + + public String getName() { + return name; + } + + public Aggregate getAggregate() { + return aggregate; + } +} diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/DocumentAdapters.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/DocumentAdapters.java index 4ed89ee52..fbd2e4f73 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/DocumentAdapters.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/DocumentAdapters.java @@ -28,6 +28,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; import org.apache.commons.logging.Log; @@ -77,18 +78,12 @@ public static SearchDocument from(Hit hit, JsonpMapper jsonpMapper) { NestedMetaData nestedMetaData = from(hit.nested()); - // todo #1973 explanation Explanation explanation = from(hit.explanation()); // todo #1973 matchedQueries List matchedQueries = null; - // todo #1973 documentFields - Map> documentFields = Collections.emptyMap(); - Document document; - Object source = hit.source(); - if (source == null) { - // Elasticsearch provides raw JsonData, so we build the fields into a JSON string + Function, EntityAsMap> fromFields = fields -> { StringBuilder sb = new StringBuilder("{"); final boolean[] firstField = { true }; hit.fields().forEach((key, jsonData) -> { @@ -100,7 +95,25 @@ public static SearchDocument from(Hit hit, JsonpMapper jsonpMapper) { firstField[0] = false; }); sb.append('}'); - document = Document.parse(sb.toString()); + return new EntityAsMap().fromJson(sb.toString()); + }; + + EntityAsMap hitFieldsAsMap = fromFields.apply(hit.fields()); + + Map> documentFields = new LinkedHashMap<>(); + hitFieldsAsMap.entrySet().forEach(entry -> { + if (entry.getValue() instanceof List) { + // noinspection unchecked + documentFields.put(entry.getKey(), (List) entry.getValue()); + } else { + documentFields.put(entry.getKey(), Collections.singletonList(entry.getValue())); + } + }); + + Document document; + Object source = hit.source(); + if (source == null) { + document = Document.from(hitFieldsAsMap); } else { if (source instanceof EntityAsMap) { document = Document.from((EntityAsMap) source); diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchAggregation.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchAggregation.java new file mode 100644 index 000000000..d8a823503 --- /dev/null +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchAggregation.java @@ -0,0 +1,37 @@ +/* + * Copyright 2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch.client.elc; + +import org.springframework.data.elasticsearch.core.AggregationContainer; + +/** + * {@link AggregationContainer} for a {@link Aggregation} that holds Elasticsearch data. + * @author Peter-Josef Meisch + * @since 4.4 + */ +public class ElasticsearchAggregation implements AggregationContainer { + + private final Aggregation aggregation; + + public ElasticsearchAggregation(Aggregation aggregation) { + this.aggregation = aggregation; + } + + @Override + public Aggregation aggregation() { + return aggregation; + } +} diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchAggregations.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchAggregations.java index 13fb295d3..a67da95bc 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchAggregations.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchAggregations.java @@ -17,9 +17,12 @@ import co.elastic.clients.elasticsearch._types.aggregations.Aggregate; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import org.springframework.data.elasticsearch.core.AggregationsContainer; +import org.springframework.util.Assert; /** * AggregationsContainer implementation for the Elasticsearch aggregations. @@ -27,16 +30,33 @@ * @author Peter-Josef Meisch * @since 4.4 */ -public class ElasticsearchAggregations implements AggregationsContainer> { +public class ElasticsearchAggregations implements AggregationsContainer> { - private final Map aggregations; + private final List aggregations; + + public ElasticsearchAggregations(List aggregations) { + + Assert.notNull(aggregations, "aggregations must not be null"); - public ElasticsearchAggregations(Map aggregations) { this.aggregations = aggregations; } + /** + * convenience constructor taking a map as it is returned from the new Elasticsearch client. + * + * @param aggregationsMap aggregate map + */ + public ElasticsearchAggregations(Map aggregationsMap) { + + Assert.notNull(aggregationsMap, "aggregationsMap must not be null"); + + aggregations = new ArrayList<>(aggregationsMap.size()); + aggregationsMap + .forEach((name, aggregate) -> aggregations.add(new ElasticsearchAggregation(new Aggregation(name, aggregate)))); + } + @Override - public Map aggregations() { + public List aggregations() { return aggregations; } } diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchExceptionTranslator.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchExceptionTranslator.java index 1d1ebad2f..d6179d926 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchExceptionTranslator.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchExceptionTranslator.java @@ -20,13 +20,18 @@ import co.elastic.clients.json.JsonpMapper; import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.elasticsearch.client.ResponseException; import org.springframework.dao.DataAccessException; import org.springframework.dao.DataAccessResourceFailureException; +import org.springframework.dao.DataIntegrityViolationException; import org.springframework.dao.OptimisticLockingFailureException; import org.springframework.dao.support.PersistenceExceptionTranslator; -import org.springframework.data.elasticsearch.RestStatusException; +import org.springframework.data.elasticsearch.NoSuchIndexException; import org.springframework.data.elasticsearch.UncategorizedElasticsearchException; +import org.springframework.http.HttpStatus; /** * Simple {@link PersistenceExceptionTranslator} for Elasticsearch. Convert the given runtime exception to an @@ -67,14 +72,28 @@ public DataAccessException translateExceptionIfPossible(RuntimeException ex) { return new OptimisticLockingFailureException("Cannot index a document due to seq_no+primary_term conflict", ex); } - // todo #1973 index unavailable? - if (ex instanceof ElasticsearchException) { ElasticsearchException elasticsearchException = (ElasticsearchException) ex; ErrorResponse response = elasticsearchException.response(); + + if (response.status() == HttpStatus.NOT_FOUND.value() + && "index_not_found_exception".equals(response.error().type())) { + + Pattern pattern = Pattern.compile(".*no such index \\[(.*)\\]"); + String index = ""; + Matcher matcher = pattern.matcher(response.error().reason()); + if (matcher.matches()) { + index = matcher.group(1); + } + return new NoSuchIndexException(index); + } String body = JsonUtils.toJson(response, jsonpMapper); + if (response.error().type().contains("validation_exception")) { + return new DataIntegrityViolationException(response.error().reason()); + } + return new UncategorizedElasticsearchException(ex.getMessage(), response.status(), body, ex); } @@ -86,20 +105,22 @@ public DataAccessException translateExceptionIfPossible(RuntimeException ex) { return null; } - private boolean isSeqNoConflict(Exception exception) { + private boolean isSeqNoConflict(Throwable exception) { // todo #1973 check if this works Integer status = null; String message = null; - if (exception instanceof RestStatusException) { - RestStatusException statusException = (RestStatusException) exception; - status = statusException.getStatus(); - message = statusException.getMessage(); + if (exception instanceof ResponseException) { + ResponseException responseException = (ResponseException) exception; + status = responseException.getResponse().getStatusLine().getStatusCode(); + message = responseException.getMessage(); + } else if (exception.getCause() != null) { + return isSeqNoConflict(exception.getCause()); } if (status != null && message != null) { - return status == 409 && message.contains("type=version_conflict_engine_exception") + return status == 409 && message.contains("type\":\"version_conflict_engine_exception") && message.contains("version conflict, required seqNo"); } diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java index f40abc42c..ca0f319aa 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java @@ -15,10 +15,13 @@ */ package org.springframework.data.elasticsearch.client.elc; +import static org.springframework.data.elasticsearch.client.elc.TypeUtils.*; + import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.Time; import co.elastic.clients.elasticsearch.core.*; import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; +import co.elastic.clients.elasticsearch.core.msearch.MultiSearchResponseItem; import co.elastic.clients.json.JsonpMapper; import co.elastic.clients.transport.Version; @@ -40,6 +43,7 @@ import org.springframework.data.elasticsearch.core.SearchScrollHits; import org.springframework.data.elasticsearch.core.cluster.ClusterOperations; import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; +import org.springframework.data.elasticsearch.core.document.Document; import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; import org.springframework.data.elasticsearch.core.query.BulkOptions; @@ -137,7 +141,12 @@ public List> multiGet(Query query, Class clazz, IndexCoor @Override public void bulkUpdate(List queries, BulkOptions bulkOptions, IndexCoordinates index) { - throw new UnsupportedOperationException("not implemented"); + + Assert.notNull(queries, "queries must not be null"); + Assert.notNull(bulkOptions, "bulkOptions must not be null"); + Assert.notNull(index, "index must not be null"); + + doBulkOperation(queries, bulkOptions, index); } @Override @@ -155,12 +164,25 @@ public ByQueryResponse delete(Query query, Class clazz, IndexCoordinates inde @Override public UpdateResponse update(UpdateQuery updateQuery, IndexCoordinates index) { - throw new UnsupportedOperationException("not implemented"); + + UpdateRequest request = requestConverter.documentUpdateRequest(updateQuery, index, getRefreshPolicy(), + routingResolver.getRouting()); + co.elastic.clients.elasticsearch.core.UpdateResponse response = execute( + client -> client.update(request, Document.class)); + return UpdateResponse.of(result(response.result())); } @Override public ByQueryResponse updateByQuery(UpdateQuery updateQuery, IndexCoordinates index) { - throw new UnsupportedOperationException("not implemented"); + + Assert.notNull(updateQuery, "updateQuery must not be null"); + Assert.notNull(index, "index must not be null"); + + UpdateByQueryRequest request = requestConverter.documentUpdateByQueryRequest(updateQuery, index, + getRefreshPolicy()); + + UpdateByQueryResponse byQueryResponse = execute(client -> client.updateByQuery(request)); + return responseConverter.byQueryResponse(byQueryResponse); } @Override @@ -404,14 +426,52 @@ public List> multiSearch(List queries, List> doMultiSearch(List multiSearchQueryParameters) { - throw new UnsupportedOperationException("not implemented"); + + MsearchRequest request = requestConverter.searchMsearchRequest(multiSearchQueryParameters); + + MsearchResponse msearchResponse = execute(client -> client.msearch(request, EntityAsMap.class)); + List> responseItems = msearchResponse.responses(); + + Assert.isTrue(multiSearchQueryParameters.size() == responseItems.size(), + "number of response items does not match number of requests"); + + List> searchHitsList = new ArrayList<>(multiSearchQueryParameters.size()); + + Iterator queryIterator = multiSearchQueryParameters.iterator(); + Iterator> responseIterator = responseItems.iterator(); + + while (queryIterator.hasNext()) { + MultiSearchQueryParameter queryParameter = queryIterator.next(); + MultiSearchResponseItem responseItem = responseIterator.next(); + + // if responseItem kind is Result then responsItem.value is a MultiSearchItem which is derived from SearchResponse + + if (responseItem.isResult()) { + + Class clazz = queryParameter.clazz; + ReadDocumentCallback documentCallback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, + queryParameter.index); + SearchDocumentResponseCallback> callback = new ReadSearchDocumentResponseCallback<>(clazz, + queryParameter.index); + + SearchHits searchHits = callback.doWith( + SearchDocumentResponseBuilder.from(responseItem.result(), getEntityCreator(documentCallback), jsonpMapper)); + + searchHitsList.add(searchHits); + } else { + // todo #1973 add failure + } + } + + return searchHitsList; } /** * value class combining the information needed for a single query in a multisearch request. */ - private static class MultiSearchQueryParameter { + static class MultiSearchQueryParameter { final Query query; final Class clazz; final IndexCoordinates index; diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/NativeQuery.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/NativeQuery.java index 6f97ab7c6..e5d4a3296 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/NativeQuery.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/NativeQuery.java @@ -17,14 +17,18 @@ import co.elastic.clients.elasticsearch._types.aggregations.Aggregation; import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import co.elastic.clients.elasticsearch.core.search.FieldCollapse; import co.elastic.clients.elasticsearch.core.search.Suggester; +import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import org.springframework.data.elasticsearch.core.query.BaseQuery; +import org.springframework.data.elasticsearch.core.query.RescorerQuery; +import org.springframework.data.elasticsearch.core.query.ScriptedField; import org.springframework.lang.Nullable; -import org.springframework.util.Assert; /** * A {@link org.springframework.data.elasticsearch.core.query.Query} implementation using query builders from the new @@ -36,13 +40,23 @@ public class NativeQuery extends BaseQuery { @Nullable private final Query query; + @Nullable private Query filter; // note: the new client does not have pipeline aggs, these are just set up as normal aggs private final Map aggregations = new LinkedHashMap<>(); @Nullable private Suggester suggester; + @Nullable private FieldCollapse fieldCollapse; + private List scriptedFields = Collections.emptyList(); + private List rescorerQueries = Collections.emptyList(); public NativeQuery(NativeQueryBuilder builder) { super(builder); this.query = builder.getQuery(); + this.filter = builder.getFilter(); + this.aggregations.putAll(builder.getAggregations()); + this.suggester = builder.getSuggester(); + this.fieldCollapse = builder.getFieldCollapse(); + this.scriptedFields = builder.getScriptedFields(); + this.rescorerQueries = builder.getRescorerQueries(); } public NativeQuery(@Nullable Query query) { @@ -58,20 +72,9 @@ public Query getQuery() { return query; } - public void addAggregation(String name, Aggregation aggregation) { - - Assert.notNull(name, "name must not be null"); - Assert.notNull(aggregation, "aggregation must not be null"); - - aggregations.put(name, aggregation); - } - - public void setAggregations(Map aggregations) { - - Assert.notNull(aggregations, "aggregations must not be null"); - - this.aggregations.clear(); - this.aggregations.putAll(aggregations); + @Nullable + public Query getFilter() { + return filter; } public Map getAggregations() { @@ -83,8 +86,17 @@ public Suggester getSuggester() { return suggester; } - public void setSuggester(@Nullable Suggester suggester) { - this.suggester = suggester; + @Nullable + public FieldCollapse getFieldCollapse() { + return fieldCollapse; } + public List getScriptedFields() { + return scriptedFields; + } + + @Override + public List getRescorerQueries() { + return rescorerQueries; + } } diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/NativeQueryBuilder.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/NativeQueryBuilder.java index 2152d1ff7..ff0e79dc0 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/NativeQueryBuilder.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/NativeQueryBuilder.java @@ -17,14 +17,19 @@ import co.elastic.clients.elasticsearch._types.aggregations.Aggregation; import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import co.elastic.clients.elasticsearch.core.search.FieldCollapse; import co.elastic.clients.elasticsearch.core.search.Suggester; import co.elastic.clients.util.ObjectBuilder; +import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.function.Function; import org.springframework.data.elasticsearch.core.query.BaseQueryBuilder; +import org.springframework.data.elasticsearch.core.query.RescorerQuery; +import org.springframework.data.elasticsearch.core.query.ScriptedField; import org.springframework.lang.Nullable; import org.springframework.util.Assert; @@ -35,17 +40,47 @@ public class NativeQueryBuilder extends BaseQueryBuilder { @Nullable private Query query; + @Nullable private Query filter; private final Map aggregations = new LinkedHashMap<>(); @Nullable private Suggester suggester; + @Nullable private FieldCollapse fieldCollapse; + private final List scriptedFields = new ArrayList<>(); + private List rescorerQueries = new ArrayList<>(); - public NativeQueryBuilder() { - } + public NativeQueryBuilder() {} @Nullable public Query getQuery() { return query; } + @Nullable + public Query getFilter() { + return this.filter; + } + + public Map getAggregations() { + return aggregations; + } + + @Nullable + public Suggester getSuggester() { + return suggester; + } + + @Nullable + public FieldCollapse getFieldCollapse() { + return fieldCollapse; + } + + public List getScriptedFields() { + return scriptedFields; + } + + public List getRescorerQueries() { + return rescorerQueries; + } + public NativeQueryBuilder withQuery(Query query) { Assert.notNull(query, "query must not be null"); @@ -54,6 +89,11 @@ public NativeQueryBuilder withQuery(Query query) { return this; } + public NativeQueryBuilder withFilter(@Nullable Query filter) { + this.filter = filter; + return this; + } + public NativeQueryBuilder withQuery(Function> fn) { Assert.notNull(fn, "fn must not be null"); @@ -75,11 +115,28 @@ public NativeQueryBuilder withSuggester(@Nullable Suggester suggester) { return this; } - public NativeQuery build() { - NativeQuery nativeQuery = new NativeQuery(this); - nativeQuery.setAggregations(aggregations); - nativeQuery.setSuggester(suggester); + public NativeQueryBuilder withFieldCollapse(@Nullable FieldCollapse fieldCollapse) { + this.fieldCollapse = fieldCollapse; + return this; + } + + public NativeQueryBuilder withScriptedField(ScriptedField scriptedField) { + + Assert.notNull(scriptedField, "scriptedField must not be null"); + + this.scriptedFields.add(scriptedField); + return this; + } + + public NativeQueryBuilder withResorerQuery(RescorerQuery resorerQuery) { + + Assert.notNull(resorerQuery, "resorerQuery must not be null"); - return nativeQuery; + this.rescorerQueries.add(resorerQuery); + return this; + } + + public NativeQuery build() { + return new NativeQuery(this); } } diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchClient.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchClient.java index 242ed3e7a..2be10341f 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchClient.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchClient.java @@ -127,12 +127,51 @@ public Mono> get(GetRequest request, Class tClass) { return Mono.fromFuture(transport.performRequestAsync(request, endpoint, transportOptions)); } + public Mono> update(UpdateRequest request, Class clazz) { + + Assert.notNull(request, "request must not be null"); + + // noinspection unchecked + JsonEndpoint, UpdateResponse, ErrorResponse> endpoint = new EndpointWithResponseMapperAttr( + UpdateRequest._ENDPOINT, "co.elastic.clients:Deserializer:_global.update.TDocument", + this.getDeserializer(clazz)); + return Mono.fromFuture(transport.performRequestAsync(request, endpoint, this.transportOptions)); + } + + public Mono> update( + Function, ObjectBuilder>> fn, Class clazz) { + + Assert.notNull(fn, "fn must not be null"); + + return update(fn.apply(new UpdateRequest.Builder<>()).build(), clazz); + } + public Mono> get(Function> fn, Class tClass) { Assert.notNull(fn, "fn must not be null"); return get(fn.apply(new GetRequest.Builder()).build(), tClass); } + public Mono> mget(MgetRequest request, Class clazz) { + + Assert.notNull(request, "request must not be null"); + Assert.notNull(clazz, "clazz must not be null"); + + // noinspection unchecked + JsonEndpoint, ErrorResponse> endpoint = (JsonEndpoint, ErrorResponse>) MgetRequest._ENDPOINT; + endpoint = new EndpointWithResponseMapperAttr<>(endpoint, "co.elastic.clients:Deserializer:_global.mget.TDocument", + this.getDeserializer(clazz)); + + return Mono.fromFuture(transport.performRequestAsync(request, endpoint, transportOptions)); + } + + public Mono> mget(Function> fn, Class clazz) { + + Assert.notNull(fn, "fn must not be null"); + + return mget(fn.apply(new MgetRequest.Builder()).build(), clazz); + } + public Mono reindex(ReindexRequest request) { Assert.notNull(request, "request must not be null"); @@ -161,6 +200,21 @@ public Mono delete(Function deleteByQuery(DeleteByQueryRequest request) { + + Assert.notNull(request, "request must not be null"); + + return Mono.fromFuture(transport.performRequestAsync(request, DeleteByQueryRequest._ENDPOINT, transportOptions)); + } + + public Mono deleteByQuery( + Function> fn) { + + Assert.notNull(fn, "fn must not be null"); + + return deleteByQuery(fn.apply(new DeleteByQueryRequest.Builder()).build()); + } + // endregion // region search diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchIndicesClient.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchIndicesClient.java index a84c44fa0..589fb82dc 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchIndicesClient.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchIndicesClient.java @@ -212,14 +212,6 @@ public Mono existsTemplate( return existsTemplate(fn.apply(new ExistsTemplateRequest.Builder()).build()); } - public Mono existsType(ExistsTypeRequest request) { - return Mono.fromFuture(transport.performRequestAsync(request, ExistsTypeRequest._ENDPOINT, transportOptions)); - } - - public Mono existsType(Function> fn) { - return existsType(fn.apply(new ExistsTypeRequest.Builder()).build()); - } - public Mono flush(FlushRequest request) { return Mono.fromFuture(transport.performRequestAsync(request, FlushRequest._ENDPOINT, transportOptions)); } @@ -232,19 +224,6 @@ public Mono flush() { return flush(builder -> builder); } - public Mono flushSynced(FlushSyncedRequest request) { - return Mono.fromFuture(transport.performRequestAsync(request, FlushSyncedRequest._ENDPOINT, transportOptions)); - } - - public Mono flushSynced( - Function> fn) { - return flushSynced(fn.apply(new FlushSyncedRequest.Builder()).build()); - } - - public Mono flushSynced() { - return flushSynced(builder -> builder); - } - @SuppressWarnings("SpellCheckingInspection") public Mono forcemerge(ForcemergeRequest request) { return Mono.fromFuture(transport.performRequestAsync(request, ForcemergeRequest._ENDPOINT, transportOptions)); @@ -260,14 +239,6 @@ public Mono forcemerge() { return forcemerge(builder -> builder); } - public Mono freeze(FreezeRequest request) { - return Mono.fromFuture(transport.performRequestAsync(request, FreezeRequest._ENDPOINT, transportOptions)); - } - - public Mono freeze(Function> fn) { - return freeze(fn.apply(new FreezeRequest.Builder()).build()); - } - public Mono get(GetIndexRequest request) { return Mono.fromFuture(transport.performRequestAsync(request, GetIndexRequest._ENDPOINT, transportOptions)); } @@ -363,18 +334,6 @@ public Mono getTemplate() { return getTemplate(builder -> builder); } - public Mono getUpgrade(GetUpgradeRequest request) { - return Mono.fromFuture(transport.performRequestAsync(request, GetUpgradeRequest._ENDPOINT, transportOptions)); - } - - public Mono getUpgrade(Function> fn) { - return getUpgrade(fn.apply(new GetUpgradeRequest.Builder()).build()); - } - - public Mono getUpgrade() { - return getUpgrade(builder -> builder); - } - public Mono migrateToDataStream(MigrateToDataStreamRequest request) { return Mono .fromFuture(transport.performRequestAsync(request, MigrateToDataStreamRequest._ENDPOINT, transportOptions)); @@ -601,18 +560,6 @@ public Mono updateAliases() { return updateAliases(builder -> builder); } - public Mono upgrade(UpgradeRequest request) { - return Mono.fromFuture(transport.performRequestAsync(request, UpgradeRequest._ENDPOINT, transportOptions)); - } - - public Mono upgrade(Function> fn) { - return upgrade(fn.apply(new UpgradeRequest.Builder()).build()); - } - - public Mono upgrade() { - return upgrade(builder -> builder); - } - public Mono validateQuery(ValidateQueryRequest request) { return Mono.fromFuture(transport.performRequestAsync(request, ValidateQueryRequest._ENDPOINT, transportOptions)); } diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java index 0c182f39c..4ddc4dfa3 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java @@ -15,10 +15,13 @@ */ package org.springframework.data.elasticsearch.client.elc; +import static org.springframework.data.elasticsearch.client.elc.TypeUtils.*; + import co.elastic.clients.elasticsearch._types.Result; import co.elastic.clients.elasticsearch._types.Time; import co.elastic.clients.elasticsearch.core.*; import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; +import co.elastic.clients.elasticsearch.core.get.GetResult; import co.elastic.clients.json.JsonpMapper; import co.elastic.clients.transport.Version; import reactor.core.publisher.Flux; @@ -46,6 +49,7 @@ import org.springframework.data.elasticsearch.core.ReactiveIndexOperations; import org.springframework.data.elasticsearch.core.cluster.ReactiveClusterOperations; import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; +import org.springframework.data.elasticsearch.core.document.Document; import org.springframework.data.elasticsearch.core.document.SearchDocument; import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; @@ -103,6 +107,34 @@ protected Mono> doIndex(T entity, IndexCoor ))); } + @Override + public Flux saveAll(Mono> entitiesPublisher, IndexCoordinates index) { + + Assert.notNull(entitiesPublisher, "entitiesPublisher must not be null!"); + + return entitiesPublisher // + .flatMapMany(entities -> Flux.fromIterable(entities) // + .concatMap(entity -> maybeCallBeforeConvert(entity, index)) // + ).collectList() // + .map(Entities::new) // + .flatMapMany(entities -> { + + if (entities.isEmpty()) { + return Flux.empty(); + } + + return doBulkOperation(entities.indexQueries(), BulkOptions.defaultOptions(), index)// + .index() // + .flatMap(indexAndResponse -> { + T savedEntity = entities.entityAt(indexAndResponse.getT1()); + BulkResponseItem response = indexAndResponse.getT2(); + updateIndexedObject(savedEntity, IndexedObjectInformation.of(response.id(), response.seqNo(), + response.primaryTerm(), response.version())); + return maybeCallAfterSave(savedEntity, index); + }); + }); + } + @Override public Mono get(String id, Class entityType, IndexCoordinates index) { @@ -144,9 +176,9 @@ public Mono submitReindex(ReindexRequest reindexRequest) { return Mono.from(execute( // (ClientCallback>) client -> client .reindex(reindexRequestES))) - .flatMap(response -> (response.task() == null) - ? Mono.error( // todo #1973 check behaviour and create issue in ES if necessary - new UnsupportedBackendOperation("ElasticsearchClient did not return a task id on submit request")) + .flatMap(response -> (response.task() == null) ? Mono.error( // todo #1973 check behaviour and create issue in + // ES if necessary + new UnsupportedBackendOperation("ElasticsearchClient did not return a task id on submit request")) : Mono.just(response.task())); } @@ -170,7 +202,7 @@ private Flux doBulkOperation(List queries, BulkOptions bulk } - private Mono checkForBulkOperationFailure(BulkResponse bulkResponse) { + private Mono checkForBulkOperationFailure(BulkResponse bulkResponse) { if (bulkResponse.errors()) { Map failedDocuments = new HashMap<>(); @@ -214,6 +246,31 @@ private Mono doDelete(DeleteRequest request) { }).onErrorResume(NoSuchIndexException.class, it -> Mono.empty()); } + @Override + public Flux> multiGet(Query query, Class clazz, IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(clazz, "clazz must not be null"); + + MgetRequest request = requestConverter.documentMgetRequest(query, clazz, index); + + ReadDocumentCallback callback = new ReadDocumentCallback<>(converter, clazz, index); + + Publisher> response = execute( + (ClientCallback>>) client -> client.mget(request, EntityAsMap.class)); + + return Mono.from(response)// + .flatMapMany(it -> Flux.fromIterable(DocumentAdapters.from(it))) // + .flatMap(multiGetItem -> { + if (multiGetItem.isFailed()) { + return Mono.just(MultiGetItem.of(null, multiGetItem.getFailure())); + } else { + return callback.toEntity(multiGetItem.getItem()) // + .map(t -> MultiGetItem.of(t, multiGetItem.getFailure())); + } + }); + } + // endregion @Override @@ -223,8 +280,30 @@ protected ReactiveElasticsearchTemplate doCopy() { @Override protected Mono doExists(String id, IndexCoordinates index) { - throw new UnsupportedOperationException("not implemented"); + + Assert.notNull(id, "id must not be null"); + Assert.notNull(index, "index must not be null"); + + GetRequest getRequest = requestConverter.documentGetRequest(id, routingResolver.getRouting(), index, true); + + return Mono.from(execute( + ((ClientCallback>>) client -> client.get(getRequest, EntityAsMap.class)))) + .map(GetResult::found) // + .onErrorReturn(NoSuchIndexException.class, false); + } + + @Override + public Mono delete(Query query, Class entityType, IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + + DeleteByQueryRequest request = requestConverter.documentDeleteByQueryRequest(query, entityType, index, + getRefreshPolicy()); + return Mono + .from(execute((ClientCallback>) client -> client.deleteByQuery(request))) + .map(responseConverter::byQueryResponse); } + // region search operations @Override @@ -307,7 +386,7 @@ protected Mono doFindForResponse(Query query, Class< SearchRequest searchRequest = requestConverter.searchRequest(query, clazz, index, false, false); // noinspection unchecked - SearchDocumentCallback callback = new ReadSearchDocumentCallback((Class) clazz, index); + SearchDocumentCallback callback = new ReadSearchDocumentCallback<>((Class) clazz, index); SearchDocumentResponse.EntityCreator entityCreator = searchDocument -> callback.toEntity(searchDocument) .toFuture(); @@ -317,6 +396,15 @@ protected Mono doFindForResponse(Query query, Class< .map(searchResponse -> SearchDocumentResponseBuilder.from(searchResponse, entityCreator, jsonpMapper)); } + @Override + public Flux> aggregate(Query query, Class entityType, IndexCoordinates index) { + + return doFindForResponse(query, entityType, index).flatMapMany(searchDocumentResponse -> { + ElasticsearchAggregations aggregations = (ElasticsearchAggregations) searchDocumentResponse.getAggregations(); + return aggregations == null ? Flux.empty() : Flux.fromIterable(aggregations.aggregations()); + }); + } + // endregion @Override @@ -326,7 +414,7 @@ protected Mono getVendor() { @Override protected Mono getRuntimeLibraryVersion() { - return Mono.just(Version.VERSION.toString()); + return Mono.just(Version.VERSION != null ? Version.VERSION.toString() : "null"); } @Override @@ -335,48 +423,23 @@ protected Mono getClusterVersion() { } @Override - public Flux saveAll(Mono> entitiesPublisher, IndexCoordinates index) { - - Assert.notNull(entitiesPublisher, "entitiesPublisher must not be null!"); + public Mono update(UpdateQuery updateQuery, IndexCoordinates index) { - return entitiesPublisher // - .flatMapMany(entities -> Flux.fromIterable(entities) // - .concatMap(entity -> maybeCallBeforeConvert(entity, index)) // - ).collectList() // - .map(Entities::new) // - .flatMapMany(entities -> { + Assert.notNull(updateQuery, "UpdateQuery must not be null"); + Assert.notNull(index, "Index must not be null"); - if (entities.isEmpty()) { - return Flux.empty(); - } + UpdateRequest request = requestConverter.documentUpdateRequest(updateQuery, index, getRefreshPolicy(), + routingResolver.getRouting()); - return doBulkOperation(entities.indexQueries(), BulkOptions.defaultOptions(), index)// - .index() // - .flatMap(indexAndResponse -> { - T savedEntity = entities.entityAt(indexAndResponse.getT1()); - BulkResponseItem response = indexAndResponse.getT2(); - updateIndexedObject(savedEntity, IndexedObjectInformation.of(response.id(), response.seqNo(), - response.primaryTerm(), response.version())); - return maybeCallAfterSave(savedEntity, index); - }); + return Mono.from(execute( + (ClientCallback>>) client -> client + .update(request, Document.class))) + .flatMap(response -> { + UpdateResponse.Result result = result(response.result()); + return result == null ? Mono.empty() : Mono.just(UpdateResponse.of(result)); }); } - @Override - public Flux> multiGet(Query query, Class clazz, IndexCoordinates index) { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public Mono delete(Query query, Class entityType, IndexCoordinates index) { - throw new UnsupportedOperationException("not implemented"); - } - - @Override - public Mono update(UpdateQuery updateQuery, IndexCoordinates index) { - throw new UnsupportedOperationException("not implemented"); - } - @Override public Mono updateByQuery(UpdateQuery updateQuery, IndexCoordinates index) { throw new UnsupportedOperationException("not implemented"); @@ -413,11 +476,6 @@ public ReactiveClusterOperations cluster() { return new ReactiveClusterTemplate(client.cluster(), converter); } - @Override - public Flux> aggregate(Query query, Class entityType, IndexCoordinates index) { - throw new UnsupportedOperationException("not implemented"); - } - @Override public Flux suggest(SuggestBuilder suggestion, Class entityType) { throw new UnsupportedOperationException("not implemented"); diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveIndicesTemplate.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveIndicesTemplate.java index ac8d91373..08f9644cc 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveIndicesTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveIndicesTemplate.java @@ -297,7 +297,7 @@ public Mono existsTemplate(ExistsTemplateRequest existsTemplateRequest) co.elastic.clients.elasticsearch.indices.ExistsTemplateRequest existsTemplateRequestES = requestConverter .indicesExistsTemplateRequest(existsTemplateRequest); - return Mono.from(execute(client1 -> client1.existsTemplate(existsTemplateRequestES))).map(BooleanResponse::value); + return Mono.from(execute(client -> client.existsTemplate(existsTemplateRequestES))).map(BooleanResponse::value); } @Override @@ -307,13 +307,18 @@ public Mono deleteTemplate(DeleteTemplateRequest deleteTemplateRequest) co.elastic.clients.elasticsearch.indices.DeleteTemplateRequest deleteTemplateRequestES = requestConverter .indicesDeleteTemplateRequest(deleteTemplateRequest); - return Mono.from(execute(client1 -> client1.deleteTemplate(deleteTemplateRequestES))) + return Mono.from(execute(client -> client.deleteTemplate(deleteTemplateRequestES))) .map(DeleteTemplateResponse::acknowledged); } @Override public Flux getInformation(IndexCoordinates index) { - throw new UnsupportedOperationException("not implemented"); + + GetIndexRequest request = requestConverter.indicesGetIndexRequest(index); + + return Mono.from(execute(client -> client.get(request))) // + .map(responseConverter::indicesGetIndexInformations) // + .flatMapMany(Flux::fromIterable); } @Override diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/RequestConverter.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/RequestConverter.java index 1f65c217e..92cd14692 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/RequestConverter.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/RequestConverter.java @@ -16,14 +16,15 @@ package org.springframework.data.elasticsearch.client.elc; import static org.springframework.data.elasticsearch.client.elc.TypeUtils.*; -import static org.springframework.util.ObjectUtils.*; +import static org.springframework.util.CollectionUtils.*; +import co.elastic.clients.elasticsearch._types.Conflicts; import co.elastic.clients.elasticsearch._types.InlineScript; import co.elastic.clients.elasticsearch._types.OpType; import co.elastic.clients.elasticsearch._types.SortOptions; import co.elastic.clients.elasticsearch._types.SortOrder; -import co.elastic.clients.elasticsearch._types.Time; import co.elastic.clients.elasticsearch._types.VersionType; +import co.elastic.clients.elasticsearch._types.WaitForActiveShardOptions; import co.elastic.clients.elasticsearch._types.mapping.FieldType; import co.elastic.clients.elasticsearch._types.mapping.Property; import co.elastic.clients.elasticsearch._types.mapping.RuntimeField; @@ -31,20 +32,17 @@ import co.elastic.clients.elasticsearch._types.mapping.TypeMapping; import co.elastic.clients.elasticsearch._types.query_dsl.Like; import co.elastic.clients.elasticsearch.cluster.HealthRequest; -import co.elastic.clients.elasticsearch.core.BulkRequest; -import co.elastic.clients.elasticsearch.core.DeleteByQueryRequest; -import co.elastic.clients.elasticsearch.core.DeleteRequest; -import co.elastic.clients.elasticsearch.core.GetRequest; -import co.elastic.clients.elasticsearch.core.IndexRequest; -import co.elastic.clients.elasticsearch.core.MgetRequest; -import co.elastic.clients.elasticsearch.core.SearchRequest; +import co.elastic.clients.elasticsearch.core.*; import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; import co.elastic.clients.elasticsearch.core.bulk.CreateOperation; import co.elastic.clients.elasticsearch.core.bulk.IndexOperation; +import co.elastic.clients.elasticsearch.core.bulk.UpdateOperation; import co.elastic.clients.elasticsearch.core.mget.MultiGetOperation; import co.elastic.clients.elasticsearch.core.search.Highlight; +import co.elastic.clients.elasticsearch.core.search.Rescore; import co.elastic.clients.elasticsearch.core.search.SourceConfig; import co.elastic.clients.elasticsearch.indices.*; +import co.elastic.clients.elasticsearch.indices.ExistsRequest; import co.elastic.clients.elasticsearch.indices.update_aliases.Action; import co.elastic.clients.json.JsonData; import co.elastic.clients.json.JsonpDeserializer; @@ -58,6 +56,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -67,6 +66,7 @@ import org.springframework.dao.InvalidDataAccessApiUsageException; import org.springframework.data.domain.Sort; import org.springframework.data.elasticsearch.core.RefreshPolicy; +import org.springframework.data.elasticsearch.core.ScriptType; import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; import org.springframework.data.elasticsearch.core.document.Document; import org.springframework.data.elasticsearch.core.index.AliasAction; @@ -562,6 +562,79 @@ private CreateOperation bulkCreateOperation(IndexQuery query, IndexCoordinate return builder.build(); } + private UpdateOperation bulkUpdateOperation(UpdateQuery query, IndexCoordinates index, + @Nullable RefreshPolicy refreshPolicy) { + + UpdateOperation.Builder uob = new UpdateOperation.Builder<>(); + String indexName = query.getIndexName() != null ? query.getIndexName() : index.getIndexName(); + + uob.index(indexName).id(query.getId()); + uob.action(a -> { + a // + .script(getScript(query.getScriptData())) // + .doc(query.getDocument()) // + .upsert(query.getUpsert()) // + .scriptedUpsert(query.getScriptedUpsert()) // + .docAsUpsert(query.getDocAsUpsert()) // + ; + + if (query.getFetchSource() != null) { + a.source(sc -> sc.fetch(query.getFetchSource())); + } + + if (query.getFetchSourceIncludes() != null || query.getFetchSourceExcludes() != null) { + List includes = query.getFetchSourceIncludes() != null ? query.getFetchSourceIncludes() + : Collections.emptyList(); + List excludes = query.getFetchSourceExcludes() != null ? query.getFetchSourceExcludes() + : Collections.emptyList(); + a.source(sc -> sc.filter(sf -> sf.includes(includes).excludes(excludes))); + } + + return a; + }); + + uob // + .routing(query.getRouting()) // + .ifSeqNo(query.getIfSeqNo() != null ? Long.valueOf(query.getIfSeqNo()) : null) // + .ifPrimaryTerm(query.getIfPrimaryTerm() != null ? Long.valueOf(query.getIfPrimaryTerm()) : null) // + .retryOnConflict(query.getRetryOnConflict()) // + ; + + // no refresh, timeout, waitForActiveShards on UpdateOperation or UpdateAction + + return uob.build(); + } + + @Nullable + private co.elastic.clients.elasticsearch._types.Script getScript(@Nullable ScriptData scriptData) { + + if (scriptData == null) { + return null; + } + + Map params = new HashMap<>(); + + if (scriptData.getParams() != null) { + scriptData.getParams().forEach((key, value) -> { + params.put(key, JsonData.of(value, jsonpMapper)); + }); + } + return co.elastic.clients.elasticsearch._types.Script.of(sb -> { + if (scriptData.getType() == ScriptType.INLINE) { + sb.inline(is -> is // + .lang(scriptData.getLanguage()) // + .source(scriptData.getScript()) // + .params(params)); // + } else if (scriptData.getType() == ScriptType.STORED) { + sb.stored(ss -> ss // + .id(scriptData.getScript()) // + .params(params) // + ); + } + return sb; + }); + } + public BulkRequest documentBulkRequest(List queries, BulkOptions bulkOptions, IndexCoordinates indexCoordinates, @Nullable RefreshPolicy refreshPolicy) { @@ -599,7 +672,8 @@ public BulkRequest documentBulkRequest(List queries, BulkOptions bulkOptions, ob.index(bulkIndexOperation(indexQuery, indexCoordinates, refreshPolicy)); } } else if (query instanceof UpdateQuery) { - // todo #1973 + UpdateQuery updateQuery = (UpdateQuery) query; + ob.update(bulkUpdateOperation(updateQuery, indexCoordinates, refreshPolicy)); } return ob.build(); }).collect(Collectors.toList()); @@ -674,7 +748,7 @@ public co.elastic.clients.elasticsearch.core.ReindexRequest reindex(ReindexReque } if (source.getQuery() != null) { - s.query(getQuery(source.getQuery())); + s.query(getQuery(source.getQuery(), null)); } if (source.getRemote() != null) { @@ -731,13 +805,8 @@ public co.elastic.clients.elasticsearch.core.ReindexRequest reindex(ReindexReque builder.script(s -> s.inline(InlineScript.of(i -> i.lang(script.getLang()).source(script.getSource())))); } - if (reindexRequest.getTimeout() != null) { - builder.timeout(tv -> tv.time(reindexRequest.getTimeout().toMillis() + "ms")); - } - - if (reindexRequest.getScroll() != null) { - builder.scroll(tv -> tv.time(reindexRequest.getScroll().toMillis() + "ms")); - } + builder.timeout(time(reindexRequest.getTimeout())) // + .scroll(time(reindexRequest.getScroll())); if (reindexRequest.getWaitForActiveShards() != null) { builder.waitForActiveShards(wfas -> wfas // @@ -779,7 +848,7 @@ public DeleteByQueryRequest documentDeleteByQueryRequest(Query query, Class c return DeleteByQueryRequest.of(b -> { b.index(Arrays.asList(index.getIndexNames())) // - .query(getQuery(query))// + .query(getQuery(query, clazz))// .refresh(deleteByQueryRefresh(refreshPolicy)); if (query.isLimiting()) { @@ -787,10 +856,7 @@ public DeleteByQueryRequest documentDeleteByQueryRequest(Query query, Class c b.maxDocs(Long.valueOf(query.getMaxResults())); } - if (query.hasScrollTime()) { - // noinspection ConstantConditions - b.scroll(Time.of(t -> t.time(query.getScrollTime().toMillis() + "ms"))); - } + b.scroll(time(query.getScrollTime())); if (query.getRoute() != null) { b.routing(query.getRoute()); @@ -800,6 +866,140 @@ public DeleteByQueryRequest documentDeleteByQueryRequest(Query query, Class c }); } + public UpdateRequest documentUpdateRequest(UpdateQuery query, IndexCoordinates index, + @Nullable RefreshPolicy refreshPolicy, @Nullable String routing) { + + String indexName = query.getIndexName() != null ? query.getIndexName() : index.getIndexName(); + return UpdateRequest.of(uqb -> { + uqb.index(indexName).id(query.getId()); + + if (query.getScript() != null) { + Map params = new HashMap<>(); + + if (query.getParams() != null) { + query.getParams().forEach((key, value) -> { + params.put(key, JsonData.of(value, jsonpMapper)); + }); + } + + uqb.script(sb -> { + if (query.getScriptType() == ScriptType.INLINE) { + sb.inline(is -> is // + .lang(query.getLang()) // + .source(query.getScript()) // + .params(params)); // + } else if (query.getScriptType() == ScriptType.STORED) { + sb.stored(ss -> ss // + .id(query.getScript()) // + .params(params) // + ); + } + return sb; + } + + ); + } + + uqb // + .doc(query.getDocument()) // + .upsert(query.getUpsert()) // + .routing(query.getRouting() != null ? query.getRouting() : routing) // + .scriptedUpsert(query.getScriptedUpsert()) // + .docAsUpsert(query.getDocAsUpsert()) // + .ifSeqNo(query.getIfSeqNo() != null ? Long.valueOf(query.getIfSeqNo()) : null) // + .ifPrimaryTerm(query.getIfPrimaryTerm() != null ? Long.valueOf(query.getIfPrimaryTerm()) : null) // + .refresh(refresh(refreshPolicy)) // + .retryOnConflict(query.getRetryOnConflict()) // + ; + + if (query.getFetchSource() != null) { + uqb.source(sc -> sc.fetch(query.getFetchSource())); + } + + if (query.getFetchSourceIncludes() != null || query.getFetchSourceExcludes() != null) { + List includes = query.getFetchSourceIncludes() != null ? query.getFetchSourceIncludes() + : Collections.emptyList(); + List excludes = query.getFetchSourceExcludes() != null ? query.getFetchSourceExcludes() + : Collections.emptyList(); + uqb.source(sc -> sc.filter(sf -> sf.includes(includes).excludes(excludes))); + } + + if (query.getTimeout() != null) { + uqb.timeout(tv -> tv.time(query.getTimeout())); + } + + String waitForActiveShards = query.getWaitForActiveShards(); + if (waitForActiveShards != null) { + if ("all".equalsIgnoreCase(waitForActiveShards)) { + uqb.waitForActiveShards(wfa -> wfa.option(WaitForActiveShardOptions.All)); + } else { + int val; + try { + val = Integer.parseInt(waitForActiveShards); + } catch (NumberFormatException var3) { + throw new IllegalArgumentException("cannot parse ActiveShardCount[" + waitForActiveShards + "]", var3); + } + uqb.waitForActiveShards(wfa -> wfa.count(val)); + } + } + + return uqb; + } // + ); + } + + public UpdateByQueryRequest documentUpdateByQueryRequest(UpdateQuery updateQuery, IndexCoordinates index, + @Nullable RefreshPolicy refreshPolicy) { + + return UpdateByQueryRequest.of(ub -> { + ub // + .index(Arrays.asList(index.getIndexNames())) // + .refresh(refreshPolicy == RefreshPolicy.IMMEDIATE) // + .routing(updateQuery.getRouting()) // + .script(getScript(updateQuery.getScriptData())) // + .maxDocs(updateQuery.getMaxDocs() != null ? Long.valueOf(updateQuery.getMaxDocs()) : null) // + .pipeline(updateQuery.getPipeline()) // + .requestsPerSecond( + updateQuery.getRequestsPerSecond() != null ? updateQuery.getRequestsPerSecond().longValue() : null) // + .slices(updateQuery.getSlices() != null ? Long.valueOf(updateQuery.getSlices()) : null) // + ; + + if (updateQuery.getAbortOnVersionConflict() != null) { + ub.conflicts(updateQuery.getAbortOnVersionConflict() ? Conflicts.Abort : Conflicts.Proceed); + } + + if (updateQuery.getBatchSize() != null) { + ub.size(Long.valueOf(updateQuery.getBatchSize())); + } + + if (updateQuery.getQuery() != null) { + Query queryQuery = updateQuery.getQuery(); + ub.query(getQuery(queryQuery, null)); + + // no indicesOptions available like in old client + + ub.scroll(time(queryQuery.getScrollTime())); + } + + // no maxRetries available like in old client + // no shouldStoreResult + + if (updateQuery.getRefreshPolicy() != null) { + ub.refresh(updateQuery.getRefreshPolicy() == RefreshPolicy.IMMEDIATE); + } + + if (updateQuery.getTimeout() != null) { + ub.timeout(tb -> tb.time(updateQuery.getTimeout())); + } + + if (updateQuery.getWaitForActiveShards() != null) { + ub.waitForActiveShards(w -> w.count(waitForActiveShardsCount(updateQuery.getWaitForActiveShards()))); + } + + return ub; + }); + } + // endregion // region search @@ -831,13 +1031,36 @@ public SearchRequest searchRequest(Query query, @Nullable Class clazz, In builder.scroll(t -> t.time(scrollTimeInMillis + "ms")); } - builder.query(getQuery(query)); + builder.query(getQuery(query, clazz)); addFilter(query, builder); return builder.build(); } + public MsearchRequest searchMsearchRequest( + List multiSearchQueryParameters) { + + return MsearchRequest.of(mrb -> { + multiSearchQueryParameters.forEach(param -> { + ElasticsearchPersistentEntity persistentEntity = getPersistentEntity(param.clazz); + + mrb.searches(sb -> sb // + .header(h -> h // + .index(param.index.getIndexName()) // + // todo #1973 add remaining flags for header + ) // + .body(bb -> bb // + .query(getQuery(param.query, param.clazz))// + // #1973 seq_no_primary_term and version not available in client ES issue 161 + // todo #1973 add remaining flags for body + ) // + ); + }); + return mrb; + }); + } + private void prepareSearchRequest(Query query, @Nullable Class clazz, IndexCoordinates indexCoordinates, SearchRequest.Builder builder, boolean forCount, boolean useScroll) { @@ -930,7 +1153,11 @@ private void prepareSearchRequest(Query query, @Nullable Class clazz, Ind if (!isEmpty(query.getSearchAfter())) { builder.searchAfter(query.getSearchAfter().stream().map(Object::toString).collect(Collectors.toList())); } - // todo #1973 rescorer queries + + query.getRescorerQueries().forEach(rescorerQuery -> { + builder.rescore(getRescore(rescorerQuery)); + }); + // todo #1973 request cache if (!query.getRuntimeFields().isEmpty()) { @@ -952,12 +1179,27 @@ private void prepareSearchRequest(Query query, @Nullable Class clazz, Ind // request_cache is not allowed on scroll requests. builder.requestCache(null); Duration scrollTimeout = query.getScrollTime() != null ? query.getScrollTime() : Duration.ofMinutes(1); - builder.scroll(tv -> tv.time(scrollTimeout.toMillis() + "ms")); + builder.scroll(time(scrollTimeout)); // limit the number of documents in a batch builder.size(500); } } + private Rescore getRescore(RescorerQuery rescorerQuery) { + + return Rescore.of(r -> r // + .query(rq -> rq // + .query(getQuery(rescorerQuery.getQuery(), null)) // + .scoreMode(scoreMode(rescorerQuery.getScoreMode())) // + .queryWeight(rescorerQuery.getQueryWeight() != null ? Double.valueOf(rescorerQuery.getQueryWeight()) : 1.0) // + .rescoreQueryWeight( + rescorerQuery.getRescoreQueryWeight() != null ? Double.valueOf(rescorerQuery.getRescoreQueryWeight()) + : 1.0) // + + ) // + .windowSize(rescorerQuery.getWindowSize())); + } + private void addHighlight(Query query, SearchRequest.Builder builder) { Highlight highlight = query.getHighlightQuery() @@ -1032,23 +1274,35 @@ private SortOptions getSortOptions(Sort.Order order, @Nullable ElasticsearchPers } private void prepareNativeSearch(NativeQuery query, SearchRequest.Builder builder) { - // todo #1973 script fields - // todo #1973 collapse builder + + query.getScriptedFields().forEach(scriptedField -> { + builder.scriptFields(scriptedField.getFieldName(), sf -> sf.script(getScript(scriptedField.getScriptData()))); + }); + + builder // + .suggest(query.getSuggester()) // + .collapse(query.getFieldCollapse()) // + ; + // todo #1973 indices boost if (!isEmpty(query.getAggregations())) { builder.aggregations(query.getAggregations()); } - builder.suggest(query.getSuggester()); - // todo #1973 searchExt } @Nullable - private co.elastic.clients.elasticsearch._types.query_dsl.Query getQuery(Query query) { + private co.elastic.clients.elasticsearch._types.query_dsl.Query getQuery(@Nullable Query query, + @Nullable Class clazz) { + + if (query == null) { + return null; + } + + elasticsearchConverter.updateQuery(query, clazz); - // todo #1973 some native stuff co.elastic.clients.elasticsearch._types.query_dsl.Query esQuery = null; if (query instanceof CriteriaQuery) { @@ -1074,7 +1328,7 @@ private void addFilter(Query query, SearchRequest.Builder builder) { } else if (query instanceof StringQuery) { // no filter for StringQuery } else if (query instanceof NativeQuery) { - // todo #1973 NativeQuery filter + builder.postFilter(((NativeQuery) query).getFilter()); } else { throw new IllegalArgumentException("unhandled Query implementation " + query.getClass().getName()); } @@ -1128,6 +1382,7 @@ public co.elastic.clients.elasticsearch._types.query_dsl.MoreLikeThisQuery moreL return moreLikeThisQuery; } + // endregion // region helper functions diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ResponseConverter.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ResponseConverter.java index c2d4bca69..ab43811d4 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ResponseConverter.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ResponseConverter.java @@ -23,6 +23,7 @@ import co.elastic.clients.elasticsearch._types.query_dsl.Query; import co.elastic.clients.elasticsearch.cluster.HealthResponse; import co.elastic.clients.elasticsearch.core.DeleteByQueryResponse; +import co.elastic.clients.elasticsearch.core.UpdateByQueryResponse; import co.elastic.clients.elasticsearch.core.mget.MultiGetError; import co.elastic.clients.elasticsearch.core.mget.MultiGetResponseItem; import co.elastic.clients.elasticsearch.indices.*; @@ -314,6 +315,9 @@ public static MultiGetItem.Failure getFailure(MultiGetResponseItem } public ByQueryResponse byQueryResponse(DeleteByQueryResponse response) { + // the code for the methods taking a DeleteByQueryResponse or a UpdateByQueryResponse is duplicated because the + // Elasticsearch responses do not share a common class + // noinspection DuplicatedCode List failures = response.failures().stream().map(this::byQueryResponseFailureOf) .collect(Collectors.toList()); @@ -357,9 +361,56 @@ public ByQueryResponse byQueryResponse(DeleteByQueryResponse response) { return builder.build(); } - // endregion + public ByQueryResponse byQueryResponse(UpdateByQueryResponse response) { + // the code for the methods taking a DeleteByQueryResponse or a UpdateByQueryResponse is duplicated because the + // Elasticsearch responses do not share a common class + // noinspection DuplicatedCode + List failures = response.failures().stream().map(this::byQueryResponseFailureOf) + .collect(Collectors.toList()); + + ByQueryResponse.ByQueryResponseBuilder builder = ByQueryResponse.builder(); + + if (response.took() != null) { + builder.withTook(response.took()); + } + if (response.timedOut() != null) { + builder.withTimedOut(response.timedOut()); + } + + if (response.total() != null) { + builder.withTotal(response.total()); + } + + if (response.deleted() != null) { + builder.withDeleted(response.deleted()); + } + + if (response.batches() != null) { + builder.withBatches(Math.toIntExact(response.batches())); + } + + if (response.versionConflicts() != null) { + builder.withVersionConflicts(response.versionConflicts()); + } + + if (response.noops() != null) { + builder.withNoops(response.noops()); + } + + if (response.retries() != null) { + builder.withBulkRetries(response.retries().bulk()); + builder.withSearchRetries(response.retries().search()); + } + + builder.withFailures(failures); + + return builder.build(); + } + + // endregion // region helper functions + private long timeToLong(Time time) { if (time.isTime()) { diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/TypeUtils.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/TypeUtils.java index 5a3c59541..07968a4de 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/TypeUtils.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/TypeUtils.java @@ -20,7 +20,9 @@ import co.elastic.clients.elasticsearch._types.GeoDistanceType; import co.elastic.clients.elasticsearch._types.OpType; import co.elastic.clients.elasticsearch._types.Refresh; +import co.elastic.clients.elasticsearch._types.Result; import co.elastic.clients.elasticsearch._types.SortMode; +import co.elastic.clients.elasticsearch._types.Time; import co.elastic.clients.elasticsearch._types.VersionType; import co.elastic.clients.elasticsearch._types.mapping.FieldType; import co.elastic.clients.elasticsearch.core.search.BoundaryScanner; @@ -30,11 +32,16 @@ import co.elastic.clients.elasticsearch.core.search.HighlighterOrder; import co.elastic.clients.elasticsearch.core.search.HighlighterTagsSchema; import co.elastic.clients.elasticsearch.core.search.HighlighterType; +import co.elastic.clients.elasticsearch.core.search.ScoreMode; + +import java.time.Duration; import org.springframework.data.elasticsearch.core.RefreshPolicy; import org.springframework.data.elasticsearch.core.query.GeoDistanceOrder; import org.springframework.data.elasticsearch.core.query.IndexQuery; import org.springframework.data.elasticsearch.core.query.Order; +import org.springframework.data.elasticsearch.core.query.RescorerQuery; +import org.springframework.data.elasticsearch.core.query.UpdateResponse; import org.springframework.data.elasticsearch.core.reindex.ReindexRequest; import org.springframework.lang.Nullable; @@ -243,6 +250,54 @@ static Refresh refresh(@Nullable RefreshPolicy refreshPolicy) { } } + @Nullable + static UpdateResponse.Result result(@Nullable Result result) { + + if (result == null) { + return null; + } + + switch (result) { + case Created: + return UpdateResponse.Result.CREATED; + case Updated: + return UpdateResponse.Result.UPDATED; + case Deleted: + return UpdateResponse.Result.DELETED; + case NotFound: + return UpdateResponse.Result.NOT_FOUND; + case NoOp: + return UpdateResponse.Result.NOOP; + } + + return null; + } + + @Nullable + static ScoreMode scoreMode(@Nullable RescorerQuery.ScoreMode scoreMode) { + + if (scoreMode == null) { + return null; + } + + switch (scoreMode) { + case Default: + return null; + case Avg: + return ScoreMode.Avg; + case Max: + return ScoreMode.Max; + case Min: + return ScoreMode.Min; + case Total: + return ScoreMode.Total; + case Multiply: + return ScoreMode.Multiply; + } + + return null; + } + @Nullable static SortMode sortMode(Order.Mode mode) { @@ -260,6 +315,16 @@ static SortMode sortMode(Order.Mode mode) { return null; } + @Nullable + static Time time(@Nullable Duration duration) { + + if (duration == null) { + return null; + } + + return Time.of(t -> t.time(duration.toMillis() + "ms")); + } + @Nullable static VersionType versionType( @Nullable org.springframework.data.elasticsearch.annotations.Document.VersionType versionType) { diff --git a/src/main/java/org/springframework/data/elasticsearch/core/AbstractReactiveElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/core/AbstractReactiveElasticsearchTemplate.java index b7652a467..bd2974ffd 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/AbstractReactiveElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/AbstractReactiveElasticsearchTemplate.java @@ -443,7 +443,7 @@ abstract protected Mono doFindForResponse(Query quer IndexCoordinates index); @Override - public Flux> aggregate(Query query, Class entityType) { + public Flux> aggregate(Query query, Class entityType) { return aggregate(query, entityType, getIndexCoordinatesFor(entityType)); } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/ReactiveSearchOperations.java b/src/main/java/org/springframework/data/elasticsearch/core/ReactiveSearchOperations.java index 0bf8d3834..26a2687dd 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/ReactiveSearchOperations.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/ReactiveSearchOperations.java @@ -22,8 +22,8 @@ import org.elasticsearch.search.suggest.SuggestBuilder; import org.springframework.data.domain.Pageable; -import org.springframework.data.elasticsearch.core.query.NativeSearchQuery; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.data.elasticsearch.core.query.NativeSearchQuery; import org.springframework.data.elasticsearch.core.query.Query; import org.springframework.data.elasticsearch.core.suggest.response.Suggest; @@ -234,7 +234,7 @@ Mono> searchForHits(Query query, Class entityType, * @return a {@link Flux} emitting matching aggregations one by one. * @since 4.0 */ - Flux> aggregate(Query query, Class entityType); + Flux> aggregate(Query query, Class entityType); /** * Perform an aggregation specified by the given {@link Query query}.
@@ -245,7 +245,7 @@ Mono> searchForHits(Query query, Class entityType, * @return a {@link Flux} emitting matching aggregations one by one. * @since 4.0 */ - Flux> aggregate(Query query, Class entityType, IndexCoordinates index); + Flux> aggregate(Query query, Class entityType, IndexCoordinates index); /** * Does a suggest query diff --git a/src/main/java/org/springframework/data/elasticsearch/core/query/BaseQuery.java b/src/main/java/org/springframework/data/elasticsearch/core/query/BaseQuery.java index e3dfca5cb..39b6d3ad7 100755 --- a/src/main/java/org/springframework/data/elasticsearch/core/query/BaseQuery.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/query/BaseQuery.java @@ -62,15 +62,15 @@ public class BaseQuery implements Query { @Nullable protected Integer maxResults; @Nullable protected HighlightQuery highlightQuery; @Nullable private Boolean trackTotalHits; - @Nullable private Integer trackTotalHitsUpTo; - @Nullable private Duration scrollTime; - @Nullable private Duration timeout; + @Nullable protected Integer trackTotalHitsUpTo; + @Nullable protected Duration scrollTime; + @Nullable protected Duration timeout; private boolean explain = false; - @Nullable private List searchAfter; + @Nullable protected List searchAfter; protected List rescorerQueries = new ArrayList<>(); @Nullable protected Boolean requestCache; - private List idsWithRouting = Collections.emptyList(); - private final List runtimeFields = new ArrayList<>(); + protected List idsWithRouting = Collections.emptyList(); + protected final List runtimeFields = new ArrayList<>(); public BaseQuery() {} @@ -86,6 +86,8 @@ public > BaseQuery(BaseQue this.preference = builder.getPreference(); this.sourceFilter = builder.getSourceFilter(); this.fields = builder.getFields(); + this.highlightQuery = builder.highlightQuery; + this.route = builder.getRoute(); // #1973 add the other fields to the builder } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/query/BaseQueryBuilder.java b/src/main/java/org/springframework/data/elasticsearch/core/query/BaseQueryBuilder.java index 49598038d..fcb5968e1 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/query/BaseQueryBuilder.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/query/BaseQueryBuilder.java @@ -44,6 +44,8 @@ public abstract class BaseQueryBuilder fields = new ArrayList<>(); + @Nullable protected HighlightQuery highlightQuery; + @Nullable private String route; @Nullable public Pageable getPageable() { @@ -92,6 +94,16 @@ public List getFields() { return fields; } + @Nullable + public HighlightQuery getHighlightQuery() { + return highlightQuery; + } + + @Nullable + public String getRoute() { + return route; + } + public SELF withPageable(Pageable pageable) { this.pageable = pageable; return self(); @@ -156,6 +168,16 @@ public SELF withFields(Collection fields) { return self(); } + public SELF withHighlightQuery(HighlightQuery highlightQuery) { + this.highlightQuery = highlightQuery; + return self(); + } + + public SELF withRoute(String route) { + this.route = route; + return self(); + } + public abstract Q build(); private SELF self() { diff --git a/src/main/java/org/springframework/data/elasticsearch/core/query/NativeSearchQueryBuilder.java b/src/main/java/org/springframework/data/elasticsearch/core/query/NativeSearchQueryBuilder.java index ec696c3c9..698489523 100755 --- a/src/main/java/org/springframework/data/elasticsearch/core/query/NativeSearchQueryBuilder.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/query/NativeSearchQueryBuilder.java @@ -65,7 +65,6 @@ public class NativeSearchQueryBuilder extends BaseQueryBuilder indicesBoost = new ArrayList<>(); @Nullable private SearchTemplateRequestBuilder searchTemplateBuilder; - @Nullable private String route; @Nullable private SearchType searchType; @Nullable private Boolean trackTotalHits; @Nullable private Duration timeout; @@ -232,11 +231,6 @@ public NativeSearchQueryBuilder withStoredFields(String... storedFields) { return this; } - public NativeSearchQueryBuilder withRoute(String route) { - this.route = route; - return this; - } - public NativeSearchQueryBuilder withSearchType(SearchType searchType) { this.searchType = searchType; return this; @@ -320,10 +314,6 @@ public NativeSearchQuery build() { nativeSearchQuery.setPipelineAggregations(pipelineAggregationBuilders); } - if (route != null) { - nativeSearchQuery.setRoute(route); - } - if (searchType != null) { nativeSearchQuery.setSearchType(Query.SearchType.valueOf(searchType.name())); } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/query/ScriptData.java b/src/main/java/org/springframework/data/elasticsearch/core/query/ScriptData.java new file mode 100644 index 000000000..f97cb265f --- /dev/null +++ b/src/main/java/org/springframework/data/elasticsearch/core/query/ScriptData.java @@ -0,0 +1,70 @@ +/* + * Copyright 2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch.core.query; + +import java.util.Map; + +import org.springframework.data.elasticsearch.core.ScriptType; +import org.springframework.lang.Nullable; + +/** + * value class combining script information. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public final class ScriptData { + @Nullable private final ScriptType type; + @Nullable private final String language; + @Nullable private final String script; + @Nullable private final String scriptName; + @Nullable private final Map params; + + public ScriptData(@Nullable ScriptType type, @Nullable String language, @Nullable String script, + @Nullable String scriptName, @Nullable Map params) { + + this.type = type; + this.language = language; + this.script = script; + this.scriptName = scriptName; + this.params = params; + } + + @Nullable + public ScriptType getType() { + return type; + } + + @Nullable + public String getLanguage() { + return language; + } + + @Nullable + public String getScript() { + return script; + } + + @Nullable + public String getScriptName() { + return scriptName; + } + + @Nullable + public Map getParams() { + return params; + } +} diff --git a/src/main/java/org/springframework/data/elasticsearch/core/query/ScriptedField.java b/src/main/java/org/springframework/data/elasticsearch/core/query/ScriptedField.java new file mode 100644 index 000000000..5fbd6504a --- /dev/null +++ b/src/main/java/org/springframework/data/elasticsearch/core/query/ScriptedField.java @@ -0,0 +1,45 @@ +/* + * Copyright 2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch.core.query; + +import org.springframework.util.Assert; + +/** + * @author Peter-Josef Meisch + * @since 4.4 + */ +public class ScriptedField { + + private final String fieldName; + private final ScriptData scriptData; + + public ScriptedField(String fieldName, ScriptData scriptData) { + + Assert.notNull(fieldName, "fieldName must not be null"); + Assert.notNull(scriptData, "scriptData must not be null"); + + this.fieldName = fieldName; + this.scriptData = scriptData; + } + + public String getFieldName() { + return fieldName; + } + + public ScriptData getScriptData() { + return scriptData; + } +} diff --git a/src/main/java/org/springframework/data/elasticsearch/core/query/UpdateQuery.java b/src/main/java/org/springframework/data/elasticsearch/core/query/UpdateQuery.java index f695571a9..9b81e211f 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/query/UpdateQuery.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/query/UpdateQuery.java @@ -35,11 +35,8 @@ public class UpdateQuery { private final String id; - @Nullable private final String script; - @Nullable private final Map params; @Nullable private final Document document; @Nullable private final Document upsert; - @Nullable private final String lang; @Nullable private final String routing; @Nullable private final Boolean scriptedUpsert; @Nullable private final Boolean docAsUpsert; @@ -61,9 +58,8 @@ public class UpdateQuery { @Nullable private final Float requestsPerSecond; @Nullable private final Boolean shouldStoreResult; @Nullable private final Integer slices; - @Nullable private final ScriptType scriptType; - @Nullable private final String scriptName; @Nullable private final String indexName; + @Nullable private final ScriptData scriptData; public static Builder builder(String id) { return new Builder(id); @@ -85,11 +81,8 @@ private UpdateQuery(String id, @Nullable String script, @Nullable Map getParams() { - return params; + return scriptData != null ? scriptData.getParams() : null; } @Nullable @@ -142,7 +139,7 @@ public Document getUpsert() { @Nullable public String getLang() { - return lang; + return scriptData != null ? scriptData.getLanguage() : null; } @Nullable @@ -252,12 +249,12 @@ public Integer getSlices() { @Nullable public ScriptType getScriptType() { - return scriptType; + return scriptData != null ? scriptData.getType() : null; } @Nullable public String getScriptName() { - return scriptName; + return scriptData != null ? scriptData.getScriptName() : null; } /** @@ -268,13 +265,21 @@ public String getIndexName() { return indexName; } + /** + * @since 4.4 + */ + @Nullable + public ScriptData getScriptData() { + return scriptData; + } + public static final class Builder { private String id; @Nullable private String script = null; @Nullable private Map params; @Nullable private Document document = null; @Nullable private Document upsert = null; - @Nullable private String lang = "painless"; + @Nullable private String lang = null; @Nullable private String routing = null; @Nullable private Boolean scriptedUpsert; @Nullable private Boolean docAsUpsert; diff --git a/src/main/java/org/springframework/data/elasticsearch/core/query/UpdateResponse.java b/src/main/java/org/springframework/data/elasticsearch/core/query/UpdateResponse.java index 16c271890..f865644de 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/query/UpdateResponse.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/query/UpdateResponse.java @@ -35,6 +35,13 @@ public UpdateResponse(Result result) { this.result = result; } + /** + * @since 4.4 + */ + public static UpdateResponse of(Result result) { + return new UpdateResponse(result); + } + public Result getResult() { return result; } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/query/highlight/Highlight.java b/src/main/java/org/springframework/data/elasticsearch/core/query/highlight/Highlight.java index c2ca6507d..fcae1903a 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/query/highlight/Highlight.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/query/highlight/Highlight.java @@ -30,6 +30,17 @@ public class Highlight { private final HighlightParameters parameters; private final List fields; + /** + * @since 4.4 + */ + public Highlight(List fields) { + + Assert.notNull(fields, "fields must not be null"); + + this.parameters = HighlightParameters.builder().build(); + this.fields = fields; + } + public Highlight(HighlightParameters parameters, List fields) { Assert.notNull(parameters, "parameters must not be null"); diff --git a/src/main/java/org/springframework/data/elasticsearch/core/query/highlight/HighlightField.java b/src/main/java/org/springframework/data/elasticsearch/core/query/highlight/HighlightField.java index 3d9c6d197..73c7de230 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/query/highlight/HighlightField.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/query/highlight/HighlightField.java @@ -25,6 +25,17 @@ public class HighlightField { private final String name; private final HighlightFieldParameters parameters; + /** + * @since 4.4 + */ + public HighlightField(String name) { + + Assert.notNull(name, "name must not be null"); + + this.name = name; + this.parameters = HighlightFieldParameters.builder().build(); + } + public HighlightField(String name, HighlightFieldParameters parameters) { Assert.notNull(name, "name must not be null"); diff --git a/src/test/java/org/springframework/data/elasticsearch/ELCQueries.java b/src/test/java/org/springframework/data/elasticsearch/ELCQueries.java new file mode 100644 index 000000000..c8f0a979f --- /dev/null +++ b/src/test/java/org/springframework/data/elasticsearch/ELCQueries.java @@ -0,0 +1,57 @@ +/* +// * Copyright 2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch; + +import static org.springframework.data.elasticsearch.client.elc.QueryBuilders.*; + +import co.elastic.clients.elasticsearch._types.aggregations.Aggregation; + +import org.springframework.data.elasticsearch.client.elc.NativeQuery; +import org.springframework.data.elasticsearch.client.elc.QueryBuilders; +import org.springframework.data.elasticsearch.core.query.BaseQueryBuilder; +import org.springframework.data.elasticsearch.core.query.Query; + +/** + * Class providing some queries for the new Elasticsearch client needed in different tests. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public final class ELCQueries { + + private ELCQueries() {} + + public static Query getTermsAggsQuery(String aggsName, String aggsField){ + return NativeQuery.builder() // + .withQuery(QueryBuilders.matchAllQueryAsQuery()) // + .withAggregation(aggsName, Aggregation.of(a -> a // + .terms(ta -> ta.field(aggsField)))) // + .withMaxResults(0) // + .build(); + } + + public static Query queryWithIds(String... ids) { + return NativeQuery.builder().withIds(ids).build(); + } + + public static BaseQueryBuilder getBuilderWithMatchAllQuery() { + return NativeQuery.builder().withQuery(matchAllQueryAsQuery()); + } + + public static BaseQueryBuilder getBuilderWithTermQuery(String field, String value) { + return NativeQuery.builder().withQuery(termQueryAsQuery(field, value)); + } +} diff --git a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchELCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchELCIntegrationTests.java index 4c99f5122..dd7cd018d 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchELCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchELCIntegrationTests.java @@ -18,16 +18,29 @@ import static org.springframework.data.elasticsearch.client.elc.QueryBuilders.*; import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.FunctionBoostMode; +import co.elastic.clients.elasticsearch._types.query_dsl.FunctionScoreMode; +import co.elastic.clients.elasticsearch.core.search.FieldCollapse; +import co.elastic.clients.json.JsonData; + +import java.util.Map; import org.junit.jupiter.api.DisplayName; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.ELCQueries; import org.springframework.data.elasticsearch.client.elc.NativeQuery; +import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder; import org.springframework.data.elasticsearch.core.query.BaseQueryBuilder; +import org.springframework.data.elasticsearch.core.query.FetchSourceFilterBuilder; import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.core.query.RescorerQuery; +import org.springframework.data.elasticsearch.core.query.ScriptData; +import org.springframework.data.elasticsearch.core.query.ScriptedField; import org.springframework.data.elasticsearch.junit.jupiter.ElasticsearchTemplateConfiguration; import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.lang.Nullable; import org.springframework.test.context.ContextConfiguration; /** @@ -54,7 +67,7 @@ public boolean usesNewElasticsearchClient() { @Override protected Query queryWithIds(String... ids) { - return NativeQuery.builder().withIds(ids).build(); + return ELCQueries.queryWithIds(ids); } @Override @@ -64,7 +77,7 @@ protected Query getTermQuery(String field, String value) { @Override protected BaseQueryBuilder getBuilderWithMatchAllQuery() { - return NativeQuery.builder().withQuery(matchAllQueryAsQuery()); + return ELCQueries.getBuilderWithMatchAllQuery(); } @Override @@ -93,4 +106,101 @@ protected Query getBoolQueryWithWildcardsFirstMustSecondShouldAndMinScore(String .withMinScore(minScore) // .build(); } + + @Override + protected Query getQueryWithCollapse(String collapseField, @Nullable String innerHits, @Nullable Integer size) { + return NativeQuery.builder() // + .withQuery(matchAllQueryAsQuery()) // + .withFieldCollapse(FieldCollapse.of(fc -> { + fc.field(collapseField); + + if (innerHits != null) { + fc.innerHits(ih -> ih.name(innerHits).size(size)); + } + return fc; + })).build(); + } + + @Override + protected Query getMatchAllQueryWithFilterForId(String id) { + return NativeQuery.builder() // + .withQuery(matchAllQueryAsQuery()) // + .withFilter(termQueryAsQuery("id", id)) // + .build(); + } + + @Override + protected Query getQueryForParentId(String type, String id, @Nullable String route) { + + NativeQueryBuilder queryBuilder = NativeQuery.builder() // + .withQuery(qb -> qb // + .parentId(p -> p.type(type).id(id)) // + ); + + if (route != null) { + queryBuilder.withRoute(route); + } + + return queryBuilder.build(); + } + + @Override + protected Query getMatchAllQueryWithIncludesAndInlineExpressionScript(@Nullable String includes, String fieldName, + String script, Map params) { + + NativeQueryBuilder nativeQueryBuilder = NativeQuery.builder().withQuery(matchAllQueryAsQuery()); + + if (includes != null) { + nativeQueryBuilder.withSourceFilter(new FetchSourceFilterBuilder().withIncludes(includes).build()); + } + + return nativeQueryBuilder.withScriptedField(new ScriptedField( // + fieldName, // + new ScriptData(ScriptType.INLINE, "expression", script, null, params))) // + .build(); + } + + @Override + protected Query getQueryWithRescorer() { + + return NativeQuery.builder() // + .withQuery(q -> q // + .bool(b -> b // + .filter(f -> f.exists(e -> e.field("rate"))) // + .should(s -> s.term(t -> t.field("message").value("message"))) // + )) // + .withResorerQuery( // + new RescorerQuery(NativeQuery.builder() // + .withQuery(q -> q // + .functionScore(fs -> fs // + .functions(f1 -> f1 // + .filter(matchAllQueryAsQuery()) // + .weight(1.0) // + .gauss(d -> d // + .field("rate") // + .placement(dp -> dp // + .origin(JsonData.of(0)) // + .scale(JsonData.of(10)) // + .decay(0.5)) // + )) // + .functions(f2 -> f2 // + .filter(matchAllQueryAsQuery()).weight(100.0) // + .gauss(d -> d // + .field("rate") // + .placement(dp -> dp // + .origin(JsonData.of(0)) // + .scale(JsonData.of(10)) // + .decay(0.5)) // + + )) // + .scoreMode(FunctionScoreMode.Sum) // + .maxBoost(80.0) // + .boostMode(FunctionBoostMode.Replace)) // + ) // + .build() // + ) // + .withScoreMode(RescorerQuery.ScoreMode.Max) // + .withWindowSize(100)) // + .build(); + } } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchERHLCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchERHLCIntegrationTests.java index 9d3a66cf6..ce4332816 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchERHLCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchERHLCIntegrationTests.java @@ -27,8 +27,17 @@ import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.common.lucene.search.function.CombineFunction; +import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.InnerHitBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; +import org.elasticsearch.index.query.functionscore.GaussDecayFunctionBuilder; import org.elasticsearch.index.reindex.UpdateByQueryRequest; +import org.elasticsearch.join.query.ParentIdQueryBuilder; +import org.elasticsearch.script.Script; +import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.json.JSONException; import org.junit.jupiter.api.DisplayName; @@ -38,12 +47,16 @@ import org.springframework.context.annotation.Import; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; import org.springframework.data.elasticsearch.core.query.BaseQueryBuilder; +import org.springframework.data.elasticsearch.core.query.FetchSourceFilterBuilder; import org.springframework.data.elasticsearch.core.query.IndicesOptions; import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder; import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.core.query.RescorerQuery; +import org.springframework.data.elasticsearch.core.query.ScriptField; import org.springframework.data.elasticsearch.core.query.UpdateQuery; import org.springframework.data.elasticsearch.junit.jupiter.ElasticsearchRestTemplateConfiguration; import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.lang.Nullable; import org.springframework.test.context.ContextConfiguration; /** @@ -115,6 +128,79 @@ protected Query getBoolQueryWithWildcardsFirstMustSecondShouldAndMinScore(String .withMinScore(minScore).build(); } + @Override + protected Query getQueryWithCollapse(String collapseField, @Nullable String innerHits, @Nullable Integer size) { + CollapseBuilder collapseBuilder = new CollapseBuilder(collapseField); + + if (innerHits != null) { + InnerHitBuilder innerHitBuilder = new InnerHitBuilder(innerHits); + + if (size != null) { + innerHitBuilder.setSize(size); + } + + collapseBuilder.setInnerHits(innerHitBuilder); + } + + return new NativeSearchQueryBuilder().withQuery(matchAllQuery()).withCollapseBuilder(collapseBuilder).build(); + } + + @Override + protected Query getMatchAllQueryWithFilterForId(String id) { + return new NativeSearchQueryBuilder().withQuery(matchAllQuery()).withFilter(boolQuery().filter(termQuery("id", id))) + .build(); + } + + @Override + protected Query getQueryForParentId(String type, String id, @Nullable String route) { + + NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder() + .withQuery(new ParentIdQueryBuilder(type, id)); + + if (route != null) { + queryBuilder.withRoute(route); + } + return queryBuilder.build(); + } + + @Override + protected Query getMatchAllQueryWithIncludesAndInlineExpressionScript(@Nullable String includes, String fieldName, + String script, Map params) { + + NativeSearchQueryBuilder nativeSearchQueryBuilder = new NativeSearchQueryBuilder().withQuery(matchAllQuery()); + + if (includes != null) { + nativeSearchQueryBuilder.withSourceFilter(new FetchSourceFilterBuilder().withIncludes(includes).build()); + } + return nativeSearchQueryBuilder.withScriptField(new ScriptField(fieldName, + new Script(org.elasticsearch.script.ScriptType.INLINE, "expression", script, params))).build(); + } + + @Override + protected Query getQueryWithRescorer() { + return new NativeSearchQueryBuilder() // + .withQuery( // + boolQuery() // + .filter(existsQuery("rate")) // + .should(termQuery("message", "message"))) // + .withRescorerQuery( // + new RescorerQuery( // + new NativeSearchQueryBuilder() // + .withQuery(QueryBuilders + .functionScoreQuery(new FunctionScoreQueryBuilder.FilterFunctionBuilder[] { + new FunctionScoreQueryBuilder.FilterFunctionBuilder( + new GaussDecayFunctionBuilder("rate", 0, 10, null, 0.5).setWeight(1f)), + new FunctionScoreQueryBuilder.FilterFunctionBuilder( + new GaussDecayFunctionBuilder("rate", 0, 10, null, 0.5).setWeight(100f)) }) // + .scoreMode(FunctionScoreQuery.ScoreMode.SUM) // + .maxBoost(80f) // + .boostMode(CombineFunction.REPLACE)) // + .build())// + .withScoreMode(RescorerQuery.ScoreMode.Max) // + .withWindowSize(100)) // + .build(); + } + @Test // DATAES-768 void shouldUseAllOptionsFromUpdateQuery() { Map doc = new HashMap<>(); diff --git a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchIntegrationTests.java index a0d3c417a..f69ab3535 100755 --- a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchIntegrationTests.java @@ -36,19 +36,6 @@ import org.assertj.core.api.SoftAssertions; import org.assertj.core.util.Lists; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.common.lucene.search.function.CombineFunction; -import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery; -import org.elasticsearch.index.query.InnerHitBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; -import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder.FilterFunctionBuilder; -import org.elasticsearch.index.query.functionscore.GaussDecayFunctionBuilder; -import org.elasticsearch.join.query.ParentIdQueryBuilder; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptType; -import org.elasticsearch.search.collapse.CollapseBuilder; -import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Order; @@ -82,7 +69,8 @@ import org.springframework.data.elasticsearch.core.join.JoinField; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; import org.springframework.data.elasticsearch.core.query.*; -import org.springframework.data.elasticsearch.core.query.RescorerQuery.ScoreMode; +import org.springframework.data.elasticsearch.core.query.highlight.Highlight; +import org.springframework.data.elasticsearch.core.query.highlight.HighlightField; import org.springframework.data.elasticsearch.junit.jupiter.SpringIntegrationTest; import org.springframework.data.elasticsearch.utils.IndexNameProvider; import org.springframework.data.util.StreamUtils; @@ -170,7 +158,23 @@ private Query queryWithIds(Collection ids) { protected abstract BaseQueryBuilder getBuilderWithWildcardQuery(String field, String value); - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation + protected abstract Query getQueryWithCollapse(String collapseField, @Nullable String innerHits, + @Nullable Integer size); + + protected abstract Query getMatchAllQueryWithFilterForId(String id); + + protected abstract Query getQueryForParentId(String type, String id, @Nullable String route); + + protected Query getMatchAllQueryWithInlineExpressionScript(String fieldName, String script, + Map params) { + return getMatchAllQueryWithIncludesAndInlineExpressionScript(null, fieldName, script, params); + } + + protected abstract Query getMatchAllQueryWithIncludesAndInlineExpressionScript(@Nullable String includes, + String fieldName, String script, Map params); + + protected abstract Query getQueryWithRescorer(); + @Test public void shouldThrowDataAccessExceptionIfDocumentDoesNotExistWhileDoingPartialUpdate() { @@ -451,7 +455,6 @@ public void shouldDoBulkIndex() { assertThat(searchHits.getTotalHits()).isEqualTo(2); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test public void shouldDoBulkUpdate() { @@ -618,11 +621,9 @@ public void shouldDeleteAcrossIndexWhenNoMatchingDataPresent() { assertThat(operations.count(searchQuery, IndexCoordinates.of("test-index-*"))).isEqualTo(2); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test public void shouldFilterSearchResultsForGivenFilter() { - // given String documentId = nextIdAsString(); SampleEntity sampleEntity = SampleEntity.builder().id(documentId).message("some message") .version(System.currentTimeMillis()).build(); @@ -630,14 +631,11 @@ public void shouldFilterSearchResultsForGivenFilter() { IndexQuery indexQuery = getIndexQuery(sampleEntity); operations.index(indexQuery, IndexCoordinates.of(indexNameProvider.indexName())); - NativeSearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery()) - .withFilter(boolQuery().filter(termQuery("id", documentId))).build(); + Query query = getMatchAllQueryWithFilterForId(documentId); - // when - SearchHits searchHits = operations.search(searchQuery, SampleEntity.class, + SearchHits searchHits = operations.search(query, SampleEntity.class, IndexCoordinates.of(indexNameProvider.indexName())); - // then assertThat(searchHits.getTotalHits()).isEqualTo(1); } @@ -825,7 +823,6 @@ public void shouldExecuteStringQuery() { assertThat(searchHits.getTotalHits()).isEqualTo(1); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test public void shouldUseScriptedFields() { @@ -847,10 +844,8 @@ public void shouldUseScriptedFields() { params.put("factor", 2); // when - NativeSearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery()).withScriptField( - new ScriptField("scriptedRate", new Script(ScriptType.INLINE, "expression", "doc['rate'] * factor", params))) - .build(); - SearchHits searchHits = operations.search(searchQuery, SampleEntity.class, + Query query = getMatchAllQueryWithInlineExpressionScript("scriptedRate", "doc['rate'] * factor", params); + SearchHits searchHits = operations.search(query, SampleEntity.class, IndexCoordinates.of(indexNameProvider.indexName())); // then @@ -1502,7 +1497,6 @@ public void shouldDeleteIndexForGivenEntity() { assertThat(indexOperations.exists()).isFalse(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test public void shouldDoPartialUpdateForExistingDocument() { @@ -1534,10 +1528,9 @@ public void shouldDoPartialUpdateForExistingDocument() { assertThat(indexedEntity.getMessage()).isEqualTo(messageAfterUpdate); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test void shouldDoUpdateByQueryForExistingDocument() { - // given + final String documentId = nextIdAsString(); final String messageBeforeUpdate = "some test message"; final String messageAfterUpdate = "test message"; @@ -1549,7 +1542,7 @@ void shouldDoUpdateByQueryForExistingDocument() { operations.index(indexQuery, IndexCoordinates.of(indexNameProvider.indexName())); - final NativeSearchQuery query = new NativeSearchQueryBuilder().withQuery(matchAllQuery()).build(); + final Query query = operations.matchAllQuery(); final UpdateQuery updateQuery = UpdateQuery.builder(query) .withScriptType(org.springframework.data.elasticsearch.core.ScriptType.INLINE) @@ -1557,42 +1550,15 @@ void shouldDoUpdateByQueryForExistingDocument() { .withParams(Collections.singletonMap("newMessage", messageAfterUpdate)).withAbortOnVersionConflict(true) .build(); - // when operations.updateByQuery(updateQuery, IndexCoordinates.of(indexNameProvider.indexName())); - // then SampleEntity indexedEntity = operations.get(documentId, SampleEntity.class, IndexCoordinates.of(indexNameProvider.indexName())); assertThat(indexedEntity.getMessage()).isEqualTo(messageAfterUpdate); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation - @Test // DATAES-227 - public void shouldUseUpsertOnUpdate() { - - // given - Map doc = new HashMap<>(); - doc.put("id", "1"); - doc.put("message", "test"); - - org.springframework.data.elasticsearch.core.document.Document document = org.springframework.data.elasticsearch.core.document.Document - .from(doc); - - UpdateQuery updateQuery = UpdateQuery.builder("1") // - .withDocument(document) // - .withUpsert(document) // - .build(); - - // when - UpdateRequest request = getRequestFactory().updateRequest(updateQuery, IndexCoordinates.of("index")); - // then - assertThat(request).isNotNull(); - assertThat(request.upsertRequest()).isNotNull(); - } - - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test public void shouldDoUpsertIfDocumentDoesNotExist() { @@ -1650,63 +1616,57 @@ public void shouldPassIndicesOptionsForGivenSearchScrollQuery() { assertThat(entities.size()).isGreaterThanOrEqualTo(1); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation + @DisabledIf(value = "newElasticsearchClient", + disabledReason = "todo #1973 can't check response, open ES issue 161 that does not allow seqno") + // and version to be set in the request @Test // DATAES-487 public void shouldReturnSameEntityForMultiSearch() { - // given List indexQueries = new ArrayList<>(); - indexQueries.add(buildIndex(SampleEntity.builder().id("1").message("ab").build())); indexQueries.add(buildIndex(SampleEntity.builder().id("2").message("bc").build())); indexQueries.add(buildIndex(SampleEntity.builder().id("3").message("ac").build())); - operations.bulkIndex(indexQueries, IndexCoordinates.of(indexNameProvider.indexName())); + List queries = new ArrayList<>(); + queries.add(getTermQuery("message", "ab")); + queries.add(getTermQuery("message", "bc")); + queries.add(getTermQuery("message", "ac")); - // when - List queries = new ArrayList<>(); - - queries.add(new NativeSearchQueryBuilder().withQuery(termQuery("message", "ab")).build()); - queries.add(new NativeSearchQueryBuilder().withQuery(termQuery("message", "bc")).build()); - queries.add(new NativeSearchQueryBuilder().withQuery(termQuery("message", "ac")).build()); - // then List> searchHits = operations.multiSearch(queries, SampleEntity.class, IndexCoordinates.of(indexNameProvider.indexName())); + for (SearchHits sampleEntity : searchHits) { assertThat(sampleEntity.getTotalHits()).isEqualTo(1); } } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation + @DisabledIf(value = "newElasticsearchClient", + disabledReason = "todo #1973 can't check response, open ES issue 161 that does not allow seqno") + // and version to be set in the request @Test // DATAES-487 public void shouldReturnDifferentEntityForMultiSearch() { - // given Class clazz = Book.class; IndexOperations bookIndexOperations = operations.indexOps(Book.class); bookIndexOperations.delete(); - bookIndexOperations.create(); - indexOperations.putMapping(clazz); + bookIndexOperations.createWithMapping(); bookIndexOperations.refresh(); - - IndexCoordinates bookIndex = IndexCoordinates.of("test-index-book-core-template"); - + IndexCoordinates bookIndex = IndexCoordinates.of("i-need-my-own-index"); operations.index(buildIndex(SampleEntity.builder().id("1").message("ab").build()), IndexCoordinates.of(indexNameProvider.indexName())); operations.index(buildIndex(Book.builder().id("2").description("bc").build()), bookIndex); - bookIndexOperations.refresh(); - // when - List queries = new ArrayList<>(); - queries.add(new NativeSearchQueryBuilder().withQuery(termQuery("message", "ab")).build()); - queries.add(new NativeSearchQueryBuilder().withQuery(termQuery("description", "bc")).build()); + List queries = new ArrayList<>(); + queries.add(getTermQuery("message", "ab")); + queries.add(getTermQuery("description", "bc")); List> searchHitsList = operations.multiSearch(queries, Lists.newArrayList(SampleEntity.class, clazz), IndexCoordinates.of(indexNameProvider.indexName(), bookIndex.getIndexName())); - // then + bookIndexOperations.delete(); + SearchHits searchHits0 = searchHitsList.get(0); assertThat(searchHits0.getTotalHits()).isEqualTo(1L); SearchHit searchHit0 = (SearchHit) searchHits0.getSearchHit(0); @@ -1950,7 +1910,7 @@ public void shouldIndexGteEntityWithVersionType() { @Test public void shouldIndexSampleEntityWithIndexAtRuntime() { - String indexName = "custom-" + indexNameProvider.indexName(); + String indexName = indexNameProvider.indexName() + "-custom"; // given String documentId = nextIdAsString(); @@ -2699,7 +2659,6 @@ public void shouldSortResultsGivenSortCriteriaFromPageableWithScanAndScroll() { assertThat(sampleEntities.get(2).getContent().getMessage()).isEqualTo(sampleEntity1.getMessage()); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-593 public void shouldReturnDocumentWithCollapsedField() { @@ -2715,11 +2674,9 @@ public void shouldReturnDocumentWithCollapsedField() { operations.bulkIndex(indexQueries, IndexCoordinates.of(indexNameProvider.indexName())); - NativeSearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery()).withCollapseField("rate") - .build(); + Query query = getQueryWithCollapse("rate", null, null); - // when - SearchHits searchHits = operations.search(searchQuery, SampleEntity.class, + SearchHits searchHits = operations.search(query, SampleEntity.class, IndexCoordinates.of(indexNameProvider.indexName())); // then @@ -2730,7 +2687,6 @@ public void shouldReturnDocumentWithCollapsedField() { assertThat(searchHits.getSearchHit(1).getContent().getMessage()).isEqualTo("message 2"); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // #1493 @DisplayName("should return document with collapse field and inner hits") public void shouldReturnDocumentWithCollapsedFieldAndInnerHits() { @@ -2747,11 +2703,10 @@ public void shouldReturnDocumentWithCollapsedFieldAndInnerHits() { operations.bulkIndex(indexQueries, IndexCoordinates.of(indexNameProvider.indexName())); - NativeSearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery()) - .withCollapseBuilder(new CollapseBuilder("rate").setInnerHits(new InnerHitBuilder("innerHits"))).build(); + Query query = getQueryWithCollapse("rate", "innerHits", null); // when - SearchHits searchHits = operations.search(searchQuery, SampleEntity.class, + SearchHits searchHits = operations.search(query, SampleEntity.class, IndexCoordinates.of(indexNameProvider.indexName())); // then @@ -2764,7 +2719,7 @@ public void shouldReturnDocumentWithCollapsedFieldAndInnerHits() { assertThat(searchHits.getSearchHit(1).getInnerHits("innerHits").getTotalHits()).isEqualTo(1); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation + @Test // #1997 @DisplayName("should return document with inner hits size zero") void shouldReturnDocumentWithInnerHitsSizeZero() { @@ -2777,12 +2732,10 @@ void shouldReturnDocumentWithInnerHitsSizeZero() { operations.bulkIndex(indexQueries, IndexCoordinates.of(indexNameProvider.indexName())); - NativeSearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery()) - .withCollapseBuilder(new CollapseBuilder("rate").setInnerHits(new InnerHitBuilder("innerHits").setSize(0))) - .build(); + Query query = getQueryWithCollapse("rate", "innerHits", 0); // when - SearchHits searchHits = operations.search(searchQuery, SampleEntity.class, + SearchHits searchHits = operations.search(query, SampleEntity.class, IndexCoordinates.of(indexNameProvider.indexName())); // then @@ -2867,7 +2820,7 @@ void shouldReturnSortFieldsInSearchHits() { List sortValues = searchHit.getSortValues(); assertThat(sortValues).hasSize(2); assertThat(sortValues.get(0)).isInstanceOf(String.class).isEqualTo("thousands"); - // transport client returns Long, RestHghlevelClient Integer, new ElasticsearchClient String + // transport client returns Long, RestHighlevelClient Integer, new ElasticsearchClient String java.lang.Object o = sortValues.get(1); if (o instanceof Integer) { Integer i = (Integer) o; @@ -2882,7 +2835,6 @@ void shouldReturnSortFieldsInSearchHits() { } } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-715 void shouldReturnHighlightFieldsInSearchHit() { IndexCoordinates index = IndexCoordinates.of("test-index-highlight-entity-template"); @@ -2893,11 +2845,10 @@ void shouldReturnHighlightFieldsInSearchHit() { operations.index(indexQuery, index); operations.indexOps(index).refresh(); - NativeSearchQuery query = new NativeSearchQueryBuilder() // - .withQuery(termQuery("message", "message")) // - .withHighlightFields(new HighlightBuilder.Field("message")) // + Query query = getBuilderWithTermQuery("message", "message") // + .withHighlightQuery( + new HighlightQuery(new Highlight(singletonList(new HighlightField("message"))), HighlightEntity.class)) .build(); - SearchHits searchHits = operations.search(query, HighlightEntity.class, index); assertThat(searchHits).isNotNull(); @@ -2910,10 +2861,9 @@ void shouldReturnHighlightFieldsInSearchHit() { assertThat(highlightField.get(1)).contains("message"); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // #1686 void shouldRunRescoreQueryInSearchQuery() { - IndexCoordinates index = IndexCoordinates.of("test-index-rescore-entity-template"); + IndexCoordinates index = IndexCoordinates.of(indexNameProvider.getPrefix() + "rescore-entity"); // matches main query better SampleEntity entity = SampleEntity.builder() // @@ -2933,17 +2883,7 @@ void shouldRunRescoreQueryInSearchQuery() { operations.bulkIndex(indexQueries, index); - NativeSearchQuery query = new NativeSearchQueryBuilder() // - .withQuery(boolQuery().filter(existsQuery("rate")).should(termQuery("message", "message"))) // - .withRescorerQuery( - new RescorerQuery(new NativeSearchQueryBuilder().withQuery(QueryBuilders - .functionScoreQuery(new FunctionScoreQueryBuilder.FilterFunctionBuilder[] { - new FilterFunctionBuilder(new GaussDecayFunctionBuilder("rate", 0, 10, null, 0.5).setWeight(1f)), - new FilterFunctionBuilder( - new GaussDecayFunctionBuilder("rate", 0, 10, null, 0.5).setWeight(100f)) }) - .scoreMode(FunctionScoreQuery.ScoreMode.SUM).maxBoost(80f).boostMode(CombineFunction.REPLACE)).build()) - .withScoreMode(ScoreMode.Max).withWindowSize(100)) - .build(); + Query query = getQueryWithRescorer(); SearchHits searchHits = operations.search(query, SampleEntity.class, index); @@ -2956,6 +2896,7 @@ void shouldRunRescoreQueryInSearchQuery() { assertThat(searchHit.getScore()).isEqualTo(80f); } + @Test // DATAES-738 void shouldSaveEntityWithIndexCoordinates() { @@ -3128,7 +3069,9 @@ void searchShouldReturnSeqNoPrimaryTerm() { assertThatSeqNoPrimaryTermIsFilled(retrieved); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation + @DisabledIf(value = "newElasticsearchClient", + disabledReason = "todo #1973 can't check response, open ES issue 161 that does not allow seqno") + // and version to be set in the request @Test // DATAES-799 void multiSearchShouldReturnSeqNoPrimaryTerm() { OptimisticEntity original = new OptimisticEntity(); @@ -3158,7 +3101,6 @@ void searchForStreamShouldReturnSeqNoPrimaryTerm() { assertThatSeqNoPrimaryTermIsFilled(retrieved); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-799 void shouldThrowOptimisticLockingFailureExceptionWhenConcurrentUpdateOccursOnEntityWithSeqNoPrimaryTermProperty() { OptimisticEntity original = new OptimisticEntity(); @@ -3175,7 +3117,6 @@ void shouldThrowOptimisticLockingFailureExceptionWhenConcurrentUpdateOccursOnEnt assertThatThrownBy(() -> operations.save(forEdit2)).isInstanceOf(OptimisticLockingFailureException.class); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-799 void shouldThrowOptimisticLockingFailureExceptionWhenConcurrentUpdateOccursOnVersionedEntityWithSeqNoPrimaryTermProperty() { OptimisticAndVersionedEntity original = new OptimisticAndVersionedEntity(); @@ -3204,7 +3145,6 @@ void shouldAllowFullReplaceOfEntityWithBothSeqNoPrimaryTermAndVersion() { operations.save(forEdit); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test void shouldSupportCRUDOpsForEntityWithJoinFields() throws Exception { String qId1 = java.util.UUID.randomUUID().toString(); @@ -3258,9 +3198,8 @@ private void shouldSaveEntityWithJoinFields(String qId1, String qId2, String aId operations.save( Arrays.asList(sampleQuestionEntity1, sampleQuestionEntity2, sampleAnswerEntity1, sampleAnswerEntity2), index); - SearchHits hits = operations.search( - new NativeSearchQueryBuilder().withQuery(new ParentIdQueryBuilder("answer", qId1)).build(), - SampleJoinEntity.class); + Query query = getQueryForParentId("answer", qId1, null); + SearchHits hits = operations.search(query, SampleJoinEntity.class); List hitIds = hits.getSearchHits().stream() .map(sampleJoinEntitySearchHit -> sampleJoinEntitySearchHit.getId()).collect(Collectors.toList()); @@ -3287,8 +3226,7 @@ private void shouldUpdateEntityWithJoinFields(String qId1, String qId2, String a // when operations.bulkUpdate(queries, IndexCoordinates.of(indexNameProvider.indexName())); - SearchHits updatedHits = operations.search( - new NativeSearchQueryBuilder().withQuery(new ParentIdQueryBuilder("answer", qId2)).build(), + SearchHits updatedHits = operations.search(getQueryForParentId("answer", qId2, null), SampleJoinEntity.class); List hitIds = updatedHits.getSearchHits().stream().map(new Function, String>() { @@ -3300,8 +3238,7 @@ public String apply(SearchHit sampleJoinEntitySearchHit) { assertThat(hitIds.size()).isEqualTo(1); assertThat(hitIds.get(0)).isEqualTo(aId2); - updatedHits = operations.search( - new NativeSearchQueryBuilder().withQuery(new ParentIdQueryBuilder("answer", qId1)).build(), + updatedHits = operations.search(getQueryForParentId("answer", qId1, null), SampleJoinEntity.class); hitIds = updatedHits.getSearchHits().stream().map(new Function, String>() { @@ -3315,20 +3252,15 @@ public String apply(SearchHit sampleJoinEntitySearchHit) { } private void shouldDeleteEntityWithJoinFields(String qId2, String aId2) throws Exception { - Query query = new NativeSearchQueryBuilder().withQuery(new ParentIdQueryBuilder("answer", qId2)).withRoute(qId2) - .build(); - operations.delete(query, SampleJoinEntity.class, IndexCoordinates.of(indexNameProvider.indexName())); - SearchHits deletedHits = operations.search( - new NativeSearchQueryBuilder().withQuery(new ParentIdQueryBuilder("answer", qId2)).build(), + operations.delete(getQueryForParentId("answer", qId2, qId2), SampleJoinEntity.class, + IndexCoordinates.of(indexNameProvider.indexName())); + + SearchHits deletedHits = operations.search(getQueryForParentId("answer", qId2, null), SampleJoinEntity.class); - List hitIds = deletedHits.getSearchHits().stream().map(new Function, String>() { - @Override - public String apply(SearchHit sampleJoinEntitySearchHit) { - return sampleJoinEntitySearchHit.getId(); - } - }).collect(Collectors.toList()); + List hitIds = deletedHits.getSearchHits().stream() + .map(sampleJoinEntitySearchHit -> sampleJoinEntitySearchHit.getId()).collect(Collectors.toList()); assertThat(hitIds.size()).isEqualTo(0); } @@ -3539,7 +3471,6 @@ void shouldWorkWithImmutableClasses() { assertThat(retrieved).isEqualTo(saved); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // #1488 @DisplayName("should set scripted fields on immutable objects") void shouldSetScriptedFieldsOnImmutableObjects() { @@ -3549,13 +3480,10 @@ void shouldSetScriptedFieldsOnImmutableObjects() { Map params = new HashMap<>(); params.put("factor", 2); - NativeSearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery()) - .withSourceFilter(new FetchSourceFilterBuilder().withIncludes("*").build()) - .withScriptField(new ScriptField("scriptedRate", - new Script(ScriptType.INLINE, "expression", "doc['rate'] * factor", params))) - .build(); + Query query = getMatchAllQueryWithIncludesAndInlineExpressionScript("*", "scriptedRate", "doc['rate'] * factor", + params); - SearchHits searchHits = operations.search(searchQuery, + SearchHits searchHits = operations.search(query, ImmutableWithScriptedEntity.class); assertThat(searchHits.getTotalHits()).isEqualTo(1); @@ -3565,6 +3493,7 @@ void shouldSetScriptedFieldsOnImmutableObjects() { assertThat(foundEntity.getScriptedRate()).isEqualTo(84.0); } + @Test // #1893 @DisplayName("should index document from source with version") void shouldIndexDocumentFromSourceWithVersion() { @@ -3875,7 +3804,7 @@ public void setVersion(@Nullable java.lang.Long version) { } } - @Document(indexName = "test-index-book-core-template") + @Document(indexName = "i-need-my-own-index") static class Book { @Nullable @Id private String id; @@ -4419,7 +4348,7 @@ public void setText(@Nullable String text) { } } - @Document(indexName = "immutable-class") + @Document(indexName = "#{@indexNameProvider.indexName()}-immutable") private static final class ImmutableEntity { @Id @Nullable private final String id; @@ -4477,7 +4406,7 @@ public String toString() { } } - @Document(indexName = "immutable-scripted") + @Document(indexName = "#{@indexNameProvider.indexName()}-immutable-scripted") public static final class ImmutableWithScriptedEntity { @Id private final String id; @Field(type = Integer) diff --git a/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchELCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchELCIntegrationTests.java index 3dd324c82..80ed21033 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchELCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchELCIntegrationTests.java @@ -15,11 +15,30 @@ */ package org.springframework.data.elasticsearch.core; +import static org.assertj.core.api.Assertions.*; +import static org.springframework.data.elasticsearch.client.elc.QueryBuilders.*; + +import co.elastic.clients.elasticsearch._types.aggregations.Aggregate; +import co.elastic.clients.elasticsearch._types.aggregations.Buckets; +import co.elastic.clients.elasticsearch._types.aggregations.StringTermsAggregate; +import co.elastic.clients.elasticsearch._types.aggregations.StringTermsBucket; +import co.elastic.clients.elasticsearch.core.search.FieldCollapse; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.ELCQueries; +import org.springframework.data.elasticsearch.client.elc.Aggregation; +import org.springframework.data.elasticsearch.client.elc.ElasticsearchAggregation; +import org.springframework.data.elasticsearch.client.elc.NativeQuery; +import org.springframework.data.elasticsearch.core.query.BaseQueryBuilder; +import org.springframework.data.elasticsearch.core.query.Query; import org.springframework.data.elasticsearch.junit.jupiter.ReactiveElasticsearchTemplateConfiguration; import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.lang.Nullable; import org.springframework.test.context.ContextConfiguration; /** @@ -37,9 +56,66 @@ IndexNameProvider indexNameProvider() { return new IndexNameProvider("reactive-template"); } } + @Override + protected Query getTermsAggsQuery(String aggsName, String aggsField) { + return ELCQueries.getTermsAggsQuery(aggsName, aggsField); + } + + @Override + protected BaseQueryBuilder getBuilderWithMatchAllQuery() { + return ELCQueries.getBuilderWithMatchAllQuery(); + } + + @Override + protected BaseQueryBuilder getBuilderWithTermQuery(String field, String value) { + return ELCQueries.getBuilderWithTermQuery(field, value); + } + + @Override + protected Query getQueryWithCollapse(String collapseField, @Nullable String innerHits, @Nullable Integer size) { + return NativeQuery.builder() // + .withQuery(matchAllQueryAsQuery()) // + .withFieldCollapse(FieldCollapse.of(fc -> { + fc.field(collapseField); + + if (innerHits != null) { + fc.innerHits(ih -> ih.name(innerHits).size(size)); + } + return fc; + })).build(); + } + + @Override + protected Query queryWithIds(String... ids) { + return ELCQueries.queryWithIds(ids); + } @Override - public boolean usesNewElasticsearchClient() { - return true; + protected > void assertThatAggregationsAreCorrect(A aggregationContainer) { + Aggregation aggregation = ((ElasticsearchAggregation) aggregationContainer).aggregation(); + assertThat(aggregation.getName()).isEqualTo("messages"); + Aggregate aggregate = aggregation.getAggregate(); + assertThat(aggregate.isSterms()).isTrue(); + StringTermsAggregate parsedStringTerms = (StringTermsAggregate) aggregate.sterms(); + Buckets buckets = parsedStringTerms.buckets(); + assertThat(buckets.isArray()).isTrue(); + List bucketList = buckets.array(); + assertThat(bucketList.size()).isEqualTo(3); + AtomicInteger count = new AtomicInteger(); + bucketList.forEach(stringTermsBucket -> { + if ("message".equals(stringTermsBucket.key())) { + count.getAndIncrement(); + assertThat(stringTermsBucket.docCount()).isEqualTo(3); + } + if ("some".equals(stringTermsBucket.key())) { + count.getAndIncrement(); + assertThat(stringTermsBucket.docCount()).isEqualTo(2); + } + if ("other".equals(stringTermsBucket.key())) { + count.getAndIncrement(); + assertThat(stringTermsBucket.docCount()).isEqualTo(1); + } + }); + assertThat(count.get()).isEqualTo(3); } } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchERHLCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchERHLCIntegrationTests.java index a6294b413..71052ddf2 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchERHLCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchERHLCIntegrationTests.java @@ -15,11 +15,23 @@ */ package org.springframework.data.elasticsearch.core; +import static org.assertj.core.api.Assertions.*; +import static org.elasticsearch.index.query.QueryBuilders.*; + +import org.elasticsearch.index.query.InnerHitBuilder; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; +import org.elasticsearch.search.collapse.CollapseBuilder; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.core.query.BaseQueryBuilder; +import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder; +import org.springframework.data.elasticsearch.core.query.Query; import org.springframework.data.elasticsearch.junit.jupiter.ReactiveElasticsearchRestTemplateConfiguration; import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.lang.Nullable; import org.springframework.test.context.ContextConfiguration; /** @@ -37,4 +49,55 @@ IndexNameProvider indexNameProvider() { return new IndexNameProvider("reactive-template-es7"); } } + + @Override + protected Query getTermsAggsQuery(String aggsName, String aggsField) { + return new NativeSearchQueryBuilder().withQuery(matchAllQuery()) + .addAggregation(AggregationBuilders.terms("messages").field("message")).build(); + } + @Override + protected BaseQueryBuilder getBuilderWithMatchAllQuery() { + return new NativeSearchQueryBuilder().withQuery(matchAllQuery()); + + } + + @Override + protected BaseQueryBuilder getBuilderWithTermQuery(String field, String value) { + return new NativeSearchQueryBuilder().withQuery(termQuery(field, value)); + } + + @Override + protected Query getQueryWithCollapse(String collapseField, @Nullable String innerHits, @Nullable Integer size) { + CollapseBuilder collapseBuilder = new CollapseBuilder(collapseField); + + if (innerHits != null) { + InnerHitBuilder innerHitBuilder = new InnerHitBuilder(innerHits); + + if (size != null) { + innerHitBuilder.setSize(size); + } + + collapseBuilder.setInnerHits(innerHitBuilder); + } + + return new NativeSearchQueryBuilder().withQuery(matchAllQuery()).withCollapseBuilder(collapseBuilder).build(); + } + + @Override + protected Query queryWithIds(String... ids) { + return new NativeSearchQueryBuilder().withIds(ids).build(); + } + + @Override + protected > void assertThatAggregationsAreCorrect(A aggregationContainer) { + Aggregation aggregation = (Aggregation) aggregationContainer.aggregation(); + assertThat(aggregation.getName()).isEqualTo("messages"); + assertThat(aggregation instanceof ParsedStringTerms); + ParsedStringTerms parsedStringTerms = (ParsedStringTerms) aggregation; + assertThat(parsedStringTerms.getBuckets().size()).isEqualTo(3); + assertThat(parsedStringTerms.getBucketByKey("message").getDocCount()).isEqualTo(3); + assertThat(parsedStringTerms.getBucketByKey("some").getDocCount()).isEqualTo(2); + assertThat(parsedStringTerms.getBucketByKey("other").getDocCount()).isEqualTo(1); + } + } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchIntegrationTests.java index a55ebbad7..fae7ab12d 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchIntegrationTests.java @@ -24,9 +24,9 @@ import reactor.test.StepVerifier; import java.lang.Boolean; +import java.lang.Integer; import java.lang.Long; import java.lang.Object; -import java.net.ConnectException; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -40,36 +40,26 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; -import org.elasticsearch.index.query.IdsQueryBuilder; -import org.elasticsearch.search.aggregations.Aggregation; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.SortOrder; import org.json.JSONException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.DisabledIf; import org.skyscreamer.jsonassert.JSONAssert; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DataAccessException; -import org.springframework.dao.DataAccessResourceFailureException; import org.springframework.dao.OptimisticLockingFailureException; import org.springframework.data.annotation.Id; import org.springframework.data.annotation.Version; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; -import org.springframework.data.elasticsearch.NewElasticsearchClientDevelopment; import org.springframework.data.elasticsearch.RestStatusException; import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.FieldType; import org.springframework.data.elasticsearch.annotations.Mapping; import org.springframework.data.elasticsearch.annotations.Setting; -import org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient; import org.springframework.data.elasticsearch.core.document.Explanation; import org.springframework.data.elasticsearch.core.index.AliasAction; import org.springframework.data.elasticsearch.core.index.AliasActionParameters; @@ -98,7 +88,7 @@ */ @SuppressWarnings("SpringJavaAutowiredMembersInspection") @SpringIntegrationTest -public abstract class ReactiveElasticsearchIntegrationTests implements NewElasticsearchClientDevelopment { +public abstract class ReactiveElasticsearchIntegrationTests { @Autowired private ReactiveElasticsearchOperations operations; @Autowired private IndexNameProvider indexNameProvider; @@ -118,28 +108,18 @@ void cleanup() { } // endregion - // region Tests - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation - @Test // DATAES-504 - public void executeShouldProvideResource() { + protected abstract Query getTermsAggsQuery(String aggsName, String aggsField); - Mono.from(operations.execute(ReactiveElasticsearchClient::ping)) // - .as(StepVerifier::create) // - .expectNext(true) // - .verifyComplete(); - } + protected abstract BaseQueryBuilder getBuilderWithMatchAllQuery(); - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation - @Test // DATAES-504 - public void executeShouldConvertExceptions() { + protected abstract BaseQueryBuilder getBuilderWithTermQuery(String field, String value); - Mono.from(operations.execute(client -> { - throw new RuntimeException(new ConnectException("we're doomed")); - })) // - .as(StepVerifier::create) // - .expectError(DataAccessResourceFailureException.class) // - .verify(); - } + protected abstract Query getQueryWithCollapse(String collapseField, @Nullable String innerHits, + @Nullable Integer size); + + protected abstract Query queryWithIds(String... ids); + + // region Tests @Test // DATAES-504 public void insertWithIdShouldWork() { @@ -157,7 +137,6 @@ public void insertWithIdShouldWork() { .verifyComplete(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-504 public void insertWithAutogeneratedIdShouldUpdateEntityId() { @@ -175,7 +154,6 @@ private Mono documentWithIdExistsInIndex(String id, String index) { return operations.exists(id, IndexCoordinates.of(index)); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-504 public void insertWithExplicitIndexNameShouldOverwriteMetadata() { @@ -290,7 +268,6 @@ public void getByIdWithExplicitIndexNameShouldOverwriteMetadata() { .verifyComplete(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-519 public void existsShouldReturnFalseWhenIndexDoesNotExist() { @@ -300,7 +277,6 @@ public void existsShouldReturnFalseWhenIndexDoesNotExist() { .verifyComplete(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-504 public void existsShouldReturnTrueWhenFound() { @@ -313,7 +289,6 @@ public void existsShouldReturnTrueWhenFound() { .verifyComplete(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-504 public void existsShouldReturnFalseWhenNotFound() { @@ -432,7 +407,6 @@ public void shouldReturnListUsingLocalPreferenceForGivenCriteria() { .verifyComplete(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-595, DATAES-767 public void shouldThrowDataAccessExceptionWhenInvalidPreferenceForGivenCriteria() { @@ -445,6 +419,8 @@ public void shouldThrowDataAccessExceptionWhenInvalidPreferenceForGivenCriteria( CriteriaQuery queryWithInvalidPreference = new CriteriaQuery( new Criteria("message").contains("some").and("message").contains("message")); queryWithInvalidPreference.setPreference("_only_nodes:oops"); + // add a pageable to not use scrolling,otherwise the exception class does not match + queryWithInvalidPreference.setPageable(PageRequest.of(0, 10)); operations.search(queryWithInvalidPreference, SampleEntity.class) // .as(StepVerifier::create) // @@ -498,7 +474,6 @@ public void findWithoutPagingShouldReadAll() { .verifyComplete(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-567 public void aggregateShouldReturnAggregations() { @@ -508,24 +483,17 @@ public void aggregateShouldReturnAggregations() { index(sampleEntity1, sampleEntity2, sampleEntity3); - NativeSearchQuery query = new NativeSearchQueryBuilder().withQuery(matchAllQuery()) - .addAggregation(AggregationBuilders.terms("messages").field("message")).build(); + Query query = getTermsAggsQuery("messages", "message"); operations.aggregate(query, SampleEntity.class) // .as(StepVerifier::create) // .consumeNextWith(aggregationContainer -> { - Aggregation aggregation = ((ElasticsearchAggregation) aggregationContainer).aggregation(); - assertThat(aggregation.getName()).isEqualTo("messages"); - assertThat(aggregation instanceof ParsedStringTerms); - ParsedStringTerms parsedStringTerms = (ParsedStringTerms) aggregation; - assertThat(parsedStringTerms.getBuckets().size()).isEqualTo(3); - assertThat(parsedStringTerms.getBucketByKey("message").getDocCount()).isEqualTo(3); - assertThat(parsedStringTerms.getBucketByKey("some").getDocCount()).isEqualTo(2); - assertThat(parsedStringTerms.getBucketByKey("other").getDocCount()).isEqualTo(1); + assertThatAggregationsAreCorrect(aggregationContainer); }).verifyComplete(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation + protected abstract > void assertThatAggregationsAreCorrect(A aggregationContainer); + @Test // DATAES-567, DATAES-767 public void aggregateShouldErrorWhenIndexDoesNotExist() { operations @@ -621,7 +589,6 @@ public void deleteShouldCompleteWhenNothingDeleted() { .verifyComplete(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-519 public void deleteByQueryShouldReturnZeroWhenIndexDoesNotExist() { @@ -634,7 +601,6 @@ public void deleteByQueryShouldReturnZeroWhenIndexDoesNotExist() { }).verifyComplete(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-547 public void shouldDeleteAcrossIndex() { @@ -650,11 +616,9 @@ public void shouldDeleteAcrossIndex() { operations.indexOps(thisIndex).refresh().then(operations.indexOps(thatIndex).refresh()).block(); - NativeSearchQuery searchQuery = new NativeSearchQueryBuilder() // - .withQuery(termQuery("message", "test")) // - .build(); + Query query = getBuilderWithTermQuery("message", "test").build(); - operations.delete(searchQuery, SampleEntity.class, IndexCoordinates.of(indexPrefix + '*')) // + operations.delete(query, SampleEntity.class, IndexCoordinates.of(indexPrefix + '*')) // .map(ByQueryResponse::getDeleted) // .as(StepVerifier::create) // .expectNext(2L) // @@ -663,7 +627,6 @@ public void shouldDeleteAcrossIndex() { operations.indexOps(thisIndex).delete().then(operations.indexOps(thatIndex).delete()).block(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-547 public void shouldDeleteAcrossIndexWhenNoMatchingDataPresent() { @@ -679,11 +642,9 @@ public void shouldDeleteAcrossIndexWhenNoMatchingDataPresent() { operations.indexOps(thisIndex).refresh().then(operations.indexOps(thatIndex).refresh()).block(); - NativeSearchQuery searchQuery = new NativeSearchQueryBuilder() // - .withQuery(termQuery("message", "negative")) // - .build(); + Query query = getBuilderWithTermQuery("message", "negative").build(); - operations.delete(searchQuery, SampleEntity.class, IndexCoordinates.of(indexPrefix + '*')) // + operations.delete(query, SampleEntity.class, IndexCoordinates.of(indexPrefix + '*')) // .map(ByQueryResponse::getDeleted) // .as(StepVerifier::create) // .expectNext(0L) // @@ -692,7 +653,6 @@ public void shouldDeleteAcrossIndexWhenNoMatchingDataPresent() { operations.indexOps(thisIndex).delete().then(operations.indexOps(thatIndex).delete()).block(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-504 public void deleteByQueryShouldReturnNumberOfDeletedDocuments() { @@ -707,7 +667,6 @@ public void deleteByQueryShouldReturnNumberOfDeletedDocuments() { .verifyComplete(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-504 public void deleteByQueryShouldReturnZeroIfNothingDeleted() { @@ -722,7 +681,6 @@ public void deleteByQueryShouldReturnZeroIfNothingDeleted() { .verifyComplete(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-593 public void shouldReturnDocumentWithCollapsedField() { @@ -734,28 +692,20 @@ public void shouldReturnDocumentWithCollapsedField() { entity3.setRate(1); index(entity1, entity2, entity3); - NativeSearchQuery query = new NativeSearchQueryBuilder() // - .withQuery(matchAllQuery()) // - .withCollapseField("rate") // - .withPageable(PageRequest.of(0, 25)) // - .build(); - + Query query = getQueryWithCollapse("rate", null, null); operations.search(query, SampleEntity.class, IndexCoordinates.of(indexNameProvider.indexName())) // .as(StepVerifier::create) // .expectNextCount(2) // .verifyComplete(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test void shouldReturnSortFields() { SampleEntity entity = randomEntity("test message"); entity.rate = 42; index(entity); - NativeSearchQuery query = new NativeSearchQueryBuilder() // - .withQuery(matchAllQuery()) // - .withSort(new FieldSortBuilder("rate").order(SortOrder.DESC)) // + Query query = getBuilderWithMatchAllQuery().withSort(Sort.by(Sort.Direction.DESC, "rate")) .build(); operations.search(query, SampleEntity.class) // @@ -763,12 +713,23 @@ void shouldReturnSortFields() { .consumeNextWith(it -> { List sortValues = it.getSortValues(); assertThat(sortValues).hasSize(1); - assertThat(sortValues.get(0)).isEqualTo(42); + // old client returns Integer, new ElasticsearchClient String + java.lang.Object o = sortValues.get(0); + if (o instanceof Integer) { + Integer i = (Integer) o; + assertThat(o).isInstanceOf(Integer.class).isEqualTo(42); + } else if (o instanceof Long) { + Long l = (Long) o; + assertThat(o).isInstanceOf(Long.class).isEqualTo(42L); + } else if (o instanceof String) { + assertThat(o).isInstanceOf(String.class).isEqualTo("42"); + } else { + fail("unexpected object type " + o); + } }) // .verifyComplete(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-623, #1678 public void shouldReturnObjectsForGivenIdsUsingMultiGet() { SampleEntity entity1 = randomEntity("test message 1"); @@ -788,7 +749,6 @@ public void shouldReturnObjectsForGivenIdsUsingMultiGet() { .verifyComplete(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-623 public void shouldReturnObjectsForGivenIdsUsingMultiGetWithFields() { SampleEntity entity1 = randomEntity("test message 1"); @@ -809,7 +769,6 @@ public void shouldReturnObjectsForGivenIdsUsingMultiGetWithFields() { .verifyComplete(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-623. #1678 public void shouldDoBulkUpdate() { SampleEntity entity1 = randomEntity("test message 1"); @@ -847,7 +806,6 @@ public void shouldDoBulkUpdate() { .verifyComplete(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-623 void shouldSaveAll() { SampleEntity entity1 = randomEntity("test message 1"); @@ -858,7 +816,7 @@ void shouldSaveAll() { operations.saveAll(Mono.just(Arrays.asList(entity1, entity2)), IndexCoordinates.of(indexNameProvider.indexName())) // .then().block(); - NativeSearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery()).build(); + Query searchQuery = operations.matchAllQuery(); operations.search(searchQuery, SampleEntity.class, IndexCoordinates.of(indexNameProvider.indexName())) // .as(StepVerifier::create) // .expectNextMatches(hit -> entity1.equals(hit.getContent()) || entity2.equals(hit.getContent())) // @@ -891,7 +849,6 @@ private void assertThatSeqNoPrimaryTermIsFilled(OptimisticEntity retrieved) { assertThat(retrieved.seqNoPrimaryTerm.getPrimaryTerm()).isPositive(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-799, #1678 void multiGetShouldReturnSeqNoPrimaryTerm() { OptimisticEntity original = new OptimisticEntity(); @@ -910,7 +867,6 @@ private Query multiGetQueryForOne(String id) { return new NativeSearchQueryBuilder().withIds(singletonList(id)).build(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-799 void searchShouldReturnSeqNoPrimaryTerm() { OptimisticEntity original = new OptimisticEntity(); @@ -927,10 +883,9 @@ void searchShouldReturnSeqNoPrimaryTerm() { } private Query searchQueryForOne(String id) { - return new NativeSearchQueryBuilder().withFilter(new IdsQueryBuilder().addIds(id)).build(); + return queryWithIds(id); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-799 void shouldThrowOptimisticLockingFailureExceptionWhenConcurrentUpdateOccursOnEntityWithSeqNoPrimaryTermProperty() { OptimisticEntity original = new OptimisticEntity(); @@ -950,7 +905,6 @@ void shouldThrowOptimisticLockingFailureExceptionWhenConcurrentUpdateOccursOnEnt .verify(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-799 void shouldThrowOptimisticLockingFailureExceptionWhenConcurrentUpdateOccursOnVersionedEntityWithSeqNoPrimaryTermProperty() { OptimisticAndVersionedEntity original = new OptimisticAndVersionedEntity(); @@ -979,7 +933,6 @@ void shouldAllowFullReplaceOfEntityWithBothSeqNoPrimaryTermAndVersion() { operations.save(forEdit).as(StepVerifier::create).expectNextCount(1).verifyComplete(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // DATAES-909 void shouldDoUpdate() { SampleEntity entity = randomEntity("test message"); @@ -1119,7 +1072,6 @@ void shouldReturnExplanationWhenRequested() { }).verifyComplete(); } - @DisabledIf("newElasticsearchClient") // todo #1973 still needs implementation @Test // #1646, #1718 @DisplayName("should return a list of info for specific index") void shouldReturnInformationListOfAllIndices() { diff --git a/src/test/java/org/springframework/data/elasticsearch/core/RequestFactoryTests.java b/src/test/java/org/springframework/data/elasticsearch/core/RequestFactoryTests.java index 714a9aeac..9fca88cc1 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/RequestFactoryTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/RequestFactoryTests.java @@ -116,6 +116,30 @@ public long getOffset() { assertThat(searchRequest.source().from()).isEqualTo(30); } + @Test // DATAES-227 + public void shouldUseUpsertOnUpdate() { + + // given + Map doc = new HashMap<>(); + doc.put("id", "1"); + doc.put("message", "test"); + + org.springframework.data.elasticsearch.core.document.Document document = org.springframework.data.elasticsearch.core.document.Document + .from(doc); + + UpdateQuery updateQuery = UpdateQuery.builder("1") // + .withDocument(document) // + .withUpsert(document) // + .build(); + + // when + UpdateRequest request = requestFactory.updateRequest(updateQuery, IndexCoordinates.of("index")); + + // then + assertThat(request).isNotNull(); + assertThat(request.upsertRequest()).isNotNull(); + } + @Test // DATAES-693 public void shouldReturnSourceWhenRequested() { // given diff --git a/src/test/java/org/springframework/data/elasticsearch/core/aggregation/AggregationELCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/aggregation/AggregationELCIntegrationTests.java index 5b52b9a62..7bfed99ad 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/aggregation/AggregationELCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/aggregation/AggregationELCIntegrationTests.java @@ -21,11 +21,15 @@ import co.elastic.clients.elasticsearch._types.aggregations.Aggregation; import co.elastic.clients.elasticsearch._types.aggregations.StatsBucketAggregate; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.ELCQueries; +import org.springframework.data.elasticsearch.client.elc.ElasticsearchAggregation; import org.springframework.data.elasticsearch.client.elc.ElasticsearchAggregations; import org.springframework.data.elasticsearch.client.elc.NativeQuery; import org.springframework.data.elasticsearch.client.elc.QueryBuilders; @@ -55,18 +59,17 @@ IndexNameProvider indexNameProvider() { @Override protected Query getTermsAggsQuery(String aggsName, String aggsField) { - return NativeQuery.builder() // - .withQuery(QueryBuilders.matchAllQueryAsQuery()) // - .withAggregation(aggsName, Aggregation.of(a -> a // - .terms(ta -> ta.field(aggsField)))) // - .withMaxResults(0) // - .build(); + return ELCQueries.getTermsAggsQuery(aggsName, aggsField); } @Override protected void assertThatAggsHasResult(AggregationsContainer aggregationsContainer, String aggsName) { - Map aggregations = ((ElasticsearchAggregations) aggregationsContainer).aggregations(); - assertThat(aggregations).containsKey(aggsName); + List aggregations = ((ElasticsearchAggregations) aggregationsContainer).aggregations(); + List aggNames = aggregations.stream() // + .map(ElasticsearchAggregation::aggregation) // + .map(org.springframework.data.elasticsearch.client.elc.Aggregation::getName) // + .collect(Collectors.toList()); + assertThat(aggNames).contains(aggsName); } @Override @@ -84,9 +87,13 @@ protected Query getPipelineAggsQuery(String aggsName, String aggsField, String a @Override protected void assertThatPipelineAggsAreCorrect(AggregationsContainer aggregationsContainer, String aggsName, String pipelineAggsName) { - Map aggregations = ((ElasticsearchAggregations) aggregationsContainer).aggregations(); - assertThat(aggregations).containsKey(aggsName); - Aggregate aggregate = aggregations.get(pipelineAggsName); + Map aggregates = ((ElasticsearchAggregations) aggregationsContainer).aggregations().stream() // + .map(ElasticsearchAggregation::aggregation) // + .collect(Collectors.toMap(org.springframework.data.elasticsearch.client.elc.Aggregation::getName, + org.springframework.data.elasticsearch.client.elc.Aggregation::getAggregate)); + + assertThat(aggregates).containsKey(aggsName); + Aggregate aggregate = aggregates.get(pipelineAggsName); assertThat(aggregate.isStatsBucket()).isTrue(); StatsBucketAggregate statsBucketAggregate = aggregate.statsBucket(); assertThat(statsBucketAggregate.min()).isEqualTo(1.0); diff --git a/src/test/java/org/springframework/data/elasticsearch/core/aggregation/AggregationIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/aggregation/AggregationIntegrationTests.java index 41524b3cd..c941ca380 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/aggregation/AggregationIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/aggregation/AggregationIntegrationTests.java @@ -234,7 +234,7 @@ public void setScore(int score) { */ static class ArticleEntityBuilder { - private ArticleEntity result; + private final ArticleEntity result; public ArticleEntityBuilder(String id) { result = new ArticleEntity(id); diff --git a/src/test/java/org/springframework/data/elasticsearch/core/suggest/CompletionIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/suggest/CompletionIntegrationTests.java index 9f9f96c39..4fe48d4f1 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/suggest/CompletionIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/suggest/CompletionIntegrationTests.java @@ -116,7 +116,7 @@ private void loadAnnotatedCompletionObjectEntitiesWithWeights() { operations.bulkIndex(indexQueries, AnnotatedCompletionEntity.class); } - @DisabledIf("newElasticsearchClient") // todo #1973, ES issue 150 + @DisabledIf(value = "newElasticsearchClient", disabledReason="todo #1973, ES issue 150") @Test public void shouldFindSuggestionsForGivenCriteriaQueryUsingCompletionEntity() { @@ -148,7 +148,7 @@ void shouldRetrieveEntityWithCompletion() { operations.get("1", CompletionEntity.class); } - @DisabledIf("newElasticsearchClient") // todo #1973, ES issue 150 + @DisabledIf(value = "newElasticsearchClient", disabledReason="todo #1973, ES issue 150") @Test public void shouldFindSuggestionsForGivenCriteriaQueryUsingAnnotatedCompletionEntity() { @@ -172,7 +172,7 @@ public void shouldFindSuggestionsForGivenCriteriaQueryUsingAnnotatedCompletionEn assertThat(options.get(1).getText()).isIn("Marchand", "Mohsin"); } - @DisabledIf("newElasticsearchClient") // todo #1973, ES issue 150 + @DisabledIf(value = "newElasticsearchClient", disabledReason="todo #1973, ES issue 150") @Test public void shouldFindSuggestionsWithWeightsForGivenCriteriaQueryUsingAnnotatedCompletionEntity() { diff --git a/src/test/java/org/springframework/data/elasticsearch/core/suggest/ReactiveSuggestIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/suggest/ReactiveSuggestIntegrationTests.java index 3c4e03046..a6d57feeb 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/suggest/ReactiveSuggestIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/suggest/ReactiveSuggestIntegrationTests.java @@ -67,7 +67,7 @@ void cleanup() { operations.indexOps(IndexCoordinates.of(indexNameProvider.getPrefix() + "*")).delete().block(); } - @DisabledIf("newElasticsearchClient") // todo #1973, ES issue 150 + @DisabledIf(value = "newElasticsearchClient", disabledReason="todo #1973, ES issue 150") @Test // #1302 @DisplayName("should find suggestions for given prefix completion") void shouldFindSuggestionsForGivenPrefixCompletion() { diff --git a/src/test/java/org/springframework/data/elasticsearch/repository/support/ElasticsearchRepositoryELCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/repository/support/ElasticsearchRepositoryELCIntegrationTests.java new file mode 100644 index 000000000..551c5da14 --- /dev/null +++ b/src/test/java/org/springframework/data/elasticsearch/repository/support/ElasticsearchRepositoryELCIntegrationTests.java @@ -0,0 +1,44 @@ +/* + * Copyright 2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch.repository.support; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.junit.jupiter.ElasticsearchTemplateConfiguration; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +/** + * @author Peter-Josef Meisch + * @since 4.4 + */ +@ContextConfiguration(classes = {ElasticsearchRepositoryELCIntegrationTests.Config.class }) +public class ElasticsearchRepositoryELCIntegrationTests extends ElasticsearchRepositoryIntegrationTests { + + @Configuration + @Import({ElasticsearchTemplateConfiguration.class }) + @EnableElasticsearchRepositories(basePackages = {"org.springframework.data.elasticsearch.repository.support" }, + considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("repository"); + } + + } +} diff --git a/src/test/java/org/springframework/data/elasticsearch/repository/support/ElasticsearchRepositoryERHLCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/repository/support/ElasticsearchRepositoryERHLCIntegrationTests.java new file mode 100644 index 000000000..120a237b6 --- /dev/null +++ b/src/test/java/org/springframework/data/elasticsearch/repository/support/ElasticsearchRepositoryERHLCIntegrationTests.java @@ -0,0 +1,44 @@ +/* + * Copyright 2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch.repository.support; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.junit.jupiter.ElasticsearchRestTemplateConfiguration; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +/** + * @author Peter-Josef Meisch + * @since 4.4 + */ +@ContextConfiguration(classes = {ElasticsearchRepositoryERHLCIntegrationTests.Config.class }) +public class ElasticsearchRepositoryERHLCIntegrationTests extends ElasticsearchRepositoryIntegrationTests { + + @Configuration + @Import({ElasticsearchRestTemplateConfiguration.class }) + @EnableElasticsearchRepositories(basePackages = {"org.springframework.data.elasticsearch.repository.support" }, + considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("repository-es7"); + } + + } +} diff --git a/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleElasticsearchRepositoryIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/repository/support/ElasticsearchRepositoryIntegrationTests.java similarity index 92% rename from src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleElasticsearchRepositoryIntegrationTests.java rename to src/test/java/org/springframework/data/elasticsearch/repository/support/ElasticsearchRepositoryIntegrationTests.java index 596a07356..4ffcddcd5 100644 --- a/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleElasticsearchRepositoryIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/repository/support/ElasticsearchRepositoryIntegrationTests.java @@ -17,12 +17,9 @@ import static org.assertj.core.api.Assertions.*; import static org.elasticsearch.index.query.QueryBuilders.*; -import static org.springframework.data.elasticsearch.annotations.FieldType.*; import static org.springframework.data.elasticsearch.utils.IdGenerator.*; import java.io.IOException; -import java.lang.Long; -import java.lang.Object; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -30,12 +27,10 @@ import java.util.Optional; import java.util.stream.Collectors; -import org.junit.jupiter.api.AfterEach; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; import org.springframework.dao.DataIntegrityViolationException; import org.springframework.data.annotation.Id; import org.springframework.data.annotation.Version; @@ -45,18 +40,16 @@ import org.springframework.data.domain.Sort.Order; import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Field; +import org.springframework.data.elasticsearch.annotations.FieldType; import org.springframework.data.elasticsearch.core.ElasticsearchOperations; -import org.springframework.data.elasticsearch.core.IndexOperations; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; import org.springframework.data.elasticsearch.core.query.NativeSearchQuery; import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder; -import org.springframework.data.elasticsearch.junit.jupiter.ElasticsearchRestTemplateConfiguration; import org.springframework.data.elasticsearch.junit.jupiter.SpringIntegrationTest; import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; -import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; -import org.springframework.data.elasticsearch.utils.IndexInitializer; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; import org.springframework.data.util.StreamUtils; import org.springframework.lang.Nullable; -import org.springframework.test.context.ContextConfiguration; /** * @author Rizwan Idrees @@ -68,29 +61,22 @@ * @author Murali Chevuri */ @SpringIntegrationTest -@ContextConfiguration(classes = { SimpleElasticsearchRepositoryIntegrationTests.Config.class }) -class SimpleElasticsearchRepositoryIntegrationTests { - - @Configuration - @Import({ ElasticsearchRestTemplateConfiguration.class }) - @EnableElasticsearchRepositories(basePackages = { "org.springframework.data.elasticsearch.repository.support" }, - considerNestedRepositories = true) - static class Config {} +abstract class ElasticsearchRepositoryIntegrationTests { @Autowired private SampleElasticsearchRepository repository; - @Autowired private ElasticsearchOperations operations; - private IndexOperations indexOperations; + @Autowired private IndexNameProvider indexNameProvider; @BeforeEach void before() { - indexOperations = operations.indexOps(SampleEntity.class); - IndexInitializer.init(indexOperations); + indexNameProvider.increment(); + operations.indexOps(SampleEntity.class).createWithMapping(); } - @AfterEach - void after() { - indexOperations.delete(); + @Test + @org.junit.jupiter.api.Order(Integer.MAX_VALUE) + public void cleanup() { + operations.indexOps(IndexCoordinates.of(indexNameProvider.getPrefix() + "*")).delete(); } @Test @@ -369,7 +355,7 @@ void shouldDeleteAllById() { repository.deleteAllById(Arrays.asList(id1, id3)); // then - assertThat(repository.findAll()).extracting(SampleEntity::getId).containsExactly(id2); + Assertions.assertThat(repository.findAll()).extracting(SampleEntity::getId).containsExactly(id2); } @Test @@ -539,8 +525,8 @@ void shouldDeleteIterableEntities() { repository.deleteAll(sampleEntities); // then - assertThat(repository.findById(documentId1)).isNotPresent(); - assertThat(repository.findById(documentId2)).isNotPresent(); + Assertions.assertThat(repository.findById(documentId1)).isNotPresent(); + Assertions.assertThat(repository.findById(documentId2)).isNotPresent(); } @Test @@ -677,14 +663,14 @@ private static List createSampleEntitiesWithMessage(String message return sampleEntities; } - @Document(indexName = "test-index-sample-simple-repository") + @Document(indexName = "#{@indexNameProvider.indexName()}") static class SampleEntity { @Nullable @Id private String id; @Nullable - @Field(type = Text, store = true, fielddata = true) private String type; + @Field(type = FieldType.Text, store = true, fielddata = true) private String type; @Nullable - @Field(type = Text, store = true, fielddata = true) private String message; + @Field(type = FieldType.Text, store = true, fielddata = true) private String message; @Nullable private int rate; @Nullable private boolean available; @Nullable diff --git a/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryTests.java b/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryTests.java index 605a210d5..61d7ee87e 100644 --- a/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryTests.java @@ -67,6 +67,7 @@ * @author Peter-Josef Meisch * @author Jens Schauder */ +// todo #1973 test for both clients @SpringIntegrationTest @ContextConfiguration(classes = { SimpleReactiveElasticsearchRepositoryTests.Config.class }) class SimpleReactiveElasticsearchRepositoryTests { From e1c926e1341435fbf751880f80a9c32782d18f5c Mon Sep 17 00:00:00 2001 From: Peter-Josef Meisch Date: Fri, 15 Apr 2022 08:42:26 +0200 Subject: [PATCH 002/516] Use Range class from spring-data-commons. Original Pull Request #2137 Closes #2098 --- .../springframework/data/elasticsearch/core/Range.java | 8 +++++--- .../core/convert/AbstractRangePropertyValueConverter.java | 8 +++++--- .../mapping/SimpleElasticsearchPersistentProperty.java | 2 +- .../data/elasticsearch/core/RangeUnitTests.java | 1 + .../convert/MappingElasticsearchConverterUnitTests.java | 2 +- .../elasticsearch/core/index/MappingBuilderUnitTests.java | 4 ++-- 6 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/springframework/data/elasticsearch/core/Range.java b/src/main/java/org/springframework/data/elasticsearch/core/Range.java index cdd6d0697..0c336627c 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/Range.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/Range.java @@ -25,7 +25,9 @@ * * @author Sascha Woo * @since 4.3 + * @deprecated use {org.springframework.data.domain.Range} instead. */ +@Deprecated(since = "5.0", forRemoval = true) public class Range { private final static Range UNBOUNDED = Range.of(Bound.unbounded(), Bound.UNBOUNDED); @@ -58,7 +60,7 @@ public static Range closed(T from, T to) { * @param * @param value must not be {@literal null}. * @return - * @see Range#closed(Object, Object) + * @see Range#closed(Object, Object) */ public static Range just(T value) { return Range.closed(value, value); @@ -231,8 +233,8 @@ public String toString() { } /** - * Value object representing a boundary. A boundary can either be {@link #unbounded() unbounded}, {@link #inclusive(Object)} - * including its value} or {@link #exclusive(Object)} its value}. + * Value object representing a boundary. A boundary can either be {@link #unbounded() unbounded}, + * {@link #inclusive(Object)} including its value} or {@link #exclusive(Object)} its value}. */ public static final class Bound { diff --git a/src/main/java/org/springframework/data/elasticsearch/core/convert/AbstractRangePropertyValueConverter.java b/src/main/java/org/springframework/data/elasticsearch/core/convert/AbstractRangePropertyValueConverter.java index 6c07b4bc6..1bce2a99d 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/convert/AbstractRangePropertyValueConverter.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/convert/AbstractRangePropertyValueConverter.java @@ -18,7 +18,7 @@ import java.util.LinkedHashMap; import java.util.Map; -import org.springframework.data.elasticsearch.core.Range; +import org.springframework.data.domain.Range; import org.springframework.data.mapping.PersistentProperty; import org.springframework.util.Assert; @@ -44,6 +44,7 @@ public Object read(Object value) { Assert.isInstanceOf(Map.class, value, "value must be instance of Map."); try { + // noinspection unchecked Map source = (Map) value; Range.Bound lowerBound; Range.Bound upperBound; @@ -82,12 +83,13 @@ public Object write(Object value) { } try { + // noinspection unchecked Range range = (Range) value; Range.Bound lowerBound = range.getLowerBound(); Range.Bound upperBound = range.getUpperBound(); Map target = new LinkedHashMap<>(); - if (lowerBound.isBounded()) { + if (lowerBound.getValue().isPresent()) { String lowerBoundValue = format(lowerBound.getValue().get()); if (lowerBound.isInclusive()) { target.put(GTE_FIELD, lowerBoundValue); @@ -96,7 +98,7 @@ public Object write(Object value) { } } - if (upperBound.isBounded()) { + if (upperBound.getValue().isPresent()) { String upperBoundValue = format(upperBound.getValue().get()); if (upperBound.isInclusive()) { target.put(LTE_FIELD, upperBoundValue); diff --git a/src/main/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentProperty.java b/src/main/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentProperty.java index 6c6d695f0..02fea106f 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentProperty.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentProperty.java @@ -24,6 +24,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.BeanUtils; +import org.springframework.data.domain.Range; import org.springframework.data.elasticsearch.annotations.DateFormat; import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.FieldType; @@ -31,7 +32,6 @@ import org.springframework.data.elasticsearch.annotations.GeoShapeField; import org.springframework.data.elasticsearch.annotations.MultiField; import org.springframework.data.elasticsearch.annotations.ValueConverter; -import org.springframework.data.elasticsearch.core.Range; import org.springframework.data.elasticsearch.core.convert.DatePropertyValueConverter; import org.springframework.data.elasticsearch.core.convert.DateRangePropertyValueConverter; import org.springframework.data.elasticsearch.core.convert.ElasticsearchDateConverter; diff --git a/src/test/java/org/springframework/data/elasticsearch/core/RangeUnitTests.java b/src/test/java/org/springframework/data/elasticsearch/core/RangeUnitTests.java index e95be4121..1b69f7d07 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/RangeUnitTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/RangeUnitTests.java @@ -26,6 +26,7 @@ * @author Sascha Woo * @since 4.3 */ +@Deprecated(since = "5.0", forRemoval = true) public class RangeUnitTests { @Test diff --git a/src/test/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverterUnitTests.java b/src/test/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverterUnitTests.java index d669d6443..60590cc6b 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverterUnitTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverterUnitTests.java @@ -48,12 +48,12 @@ import org.springframework.data.annotation.TypeAlias; import org.springframework.data.convert.ReadingConverter; import org.springframework.data.convert.WritingConverter; +import org.springframework.data.domain.Range; import org.springframework.data.elasticsearch.annotations.DateFormat; import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.FieldType; import org.springframework.data.elasticsearch.annotations.GeoPointField; import org.springframework.data.elasticsearch.annotations.ValueConverter; -import org.springframework.data.elasticsearch.core.Range; import org.springframework.data.elasticsearch.core.document.Document; import org.springframework.data.elasticsearch.core.geo.GeoJsonEntity; import org.springframework.data.elasticsearch.core.geo.GeoJsonGeometryCollection; diff --git a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderUnitTests.java b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderUnitTests.java index 63aed56c5..4d2773070 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderUnitTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderUnitTests.java @@ -40,9 +40,9 @@ import org.junit.jupiter.api.Test; import org.springframework.data.annotation.Id; import org.springframework.data.annotation.Transient; +import org.springframework.data.domain.Range; import org.springframework.data.elasticsearch.annotations.*; import org.springframework.data.elasticsearch.core.MappingContextBaseTests; -import org.springframework.data.elasticsearch.core.Range; import org.springframework.data.elasticsearch.core.geo.GeoPoint; import org.springframework.data.elasticsearch.core.mapping.SimpleElasticsearchMappingContext; import org.springframework.data.elasticsearch.core.query.SeqNoPrimaryTerm; @@ -167,7 +167,7 @@ void shouldBuildMappingsForGeoTypes() throws JSONException { " \"coerce\": true\n" + // " }\n" + // " }\n" + // - "}\n}"; // + "}\n"; // // when String mapping; From 0950dd6c7a1fc2f3f94516e8720e83e1fb577deb Mon Sep 17 00:00:00 2001 From: Peter-Josef Meisch Date: Sun, 24 Apr 2022 11:53:19 +0200 Subject: [PATCH 003/516] Update to Elasticsearch 7.17.3. Original Pull Request #2145 Closes #2144 --- pom.xml | 10 +++-- src/main/asciidoc/preface.adoc | 2 +- ...elasticsearch-migration-guide-4.3-4.4.adoc | 4 +- .../asciidoc/reference/elasticsearch-new.adoc | 2 +- .../client/elc/ElasticsearchTemplate.java | 8 ++-- .../elc/ReactiveElasticsearchClient.java | 5 ++- .../elc/ReactiveElasticsearchTemplate.java | 44 +++++++++++-------- .../elc/SearchDocumentResponseBuilder.java | 16 ++++--- .../elasticsearch/client/elc/DevTests.java | 8 ++-- .../testcontainers-elasticsearch.properties | 2 +- 10 files changed, 57 insertions(+), 44 deletions(-) diff --git a/pom.xml b/pom.xml index ae19fa3cf..1eec82e2c 100644 --- a/pom.xml +++ b/pom.xml @@ -18,8 +18,10 @@ https://github.com/spring-projects/spring-data-elasticsearch - 7.17.2 - 7.17.2 + + 7.17.3 + + 7.17.3 2.17.1 4.1.65.Final 3.0.0-SNAPSHOT @@ -139,7 +141,7 @@ org.elasticsearch.client elasticsearch-rest-high-level-client - ${elasticsearch} + ${elasticsearch-rhlc} commons-logging @@ -163,7 +165,7 @@ org.elasticsearch.client elasticsearch-rest-client - ${elasticsearch} + ${elasticsearch-java} commons-logging diff --git a/src/main/asciidoc/preface.adoc b/src/main/asciidoc/preface.adoc index e713f544e..9fe6f8beb 100644 --- a/src/main/asciidoc/preface.adoc +++ b/src/main/asciidoc/preface.adoc @@ -34,7 +34,7 @@ The following table shows the Elasticsearch versions that are used by Spring Dat [cols="^,^,^,^,^",options="header"] |=== | Spring Data Release Train | Spring Data Elasticsearch | Elasticsearch | Spring Framework | Spring Boot -| 2022.0 (Raj) | 4.4.x | 7.17.2 | 5.3.x | 2.7.x +| 2022.0 (Raj) | 4.4.x | 7.17.3 | 5.3.x | 2.7.x | 2021.1 (Q) | 4.3.x | 7.15.2 | 5.3.x | 2.6.x | 2021.0 (Pascal) | 4.2.x | 7.12.0 | 5.3.x | 2.5.x | 2020.0 (Ockham)footnote:oom[Out of maintenance] | 4.1.xfootnote:oom[] | 7.9.3 | 5.3.2 | 2.4.x diff --git a/src/main/asciidoc/reference/elasticsearch-migration-guide-4.3-4.4.adoc b/src/main/asciidoc/reference/elasticsearch-migration-guide-4.3-4.4.adoc index bffc1c4fc..929961ac6 100644 --- a/src/main/asciidoc/reference/elasticsearch-migration-guide-4.3-4.4.adoc +++ b/src/main/asciidoc/reference/elasticsearch-migration-guide-4.3-4.4.adoc @@ -82,7 +82,7 @@ The dependencies for the new Elasticsearch client are still optional in Spring D co.elastic.clients elasticsearch-java - 7.17.2 + 7.17.3 commons-logging @@ -93,7 +93,7 @@ The dependencies for the new Elasticsearch client are still optional in Spring D org.elasticsearch.client elasticsearch-rest-client - 7.17.2 + 7.17.3 commons-logging diff --git a/src/main/asciidoc/reference/elasticsearch-new.adoc b/src/main/asciidoc/reference/elasticsearch-new.adoc index e03d693c7..11feabdcc 100644 --- a/src/main/asciidoc/reference/elasticsearch-new.adoc +++ b/src/main/asciidoc/reference/elasticsearch-new.adoc @@ -5,7 +5,7 @@ == New in Spring Data Elasticsearch 4.4 * Introduction of new imperative and reactive clients using the classes from the new Elasticsearch Java client -* Upgrade to Elasticsearch 7.17.2. +* Upgrade to Elasticsearch 7.17.3. [[new-features.4-3-0]] == New in Spring Data Elasticsearch 4.3 diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java index ca0f319aa..545c23aff 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java @@ -22,6 +22,7 @@ import co.elastic.clients.elasticsearch.core.*; import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; import co.elastic.clients.elasticsearch.core.msearch.MultiSearchResponseItem; +import co.elastic.clients.elasticsearch.core.search.ResponseBody; import co.elastic.clients.json.JsonpMapper; import co.elastic.clients.transport.Version; @@ -353,8 +354,8 @@ protected SearchScrollHits searchScrollContinue(String scrollId, long scr return getSearchScrollHits(clazz, index, response); } - private > SearchScrollHits getSearchScrollHits(Class clazz, - IndexCoordinates index, R response) { + private SearchScrollHits getSearchScrollHits(Class clazz, IndexCoordinates index, + ResponseBody response) { ReadDocumentCallback documentCallback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, index); SearchDocumentResponseCallback> callback = new ReadSearchScrollDocumentResponseCallback<>(clazz, index); @@ -446,7 +447,8 @@ private List> doMultiSearch(List multiS MultiSearchQueryParameter queryParameter = queryIterator.next(); MultiSearchResponseItem responseItem = responseIterator.next(); - // if responseItem kind is Result then responsItem.value is a MultiSearchItem which is derived from SearchResponse + // if responseItem kind is Result then responseItem.value is a MultiSearchItem which is derived from + // SearchResponse if (responseItem.isResult()) { diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchClient.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchClient.java index 2be10341f..6b0782c0e 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchClient.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchClient.java @@ -18,6 +18,7 @@ import co.elastic.clients.ApiClient; import co.elastic.clients.elasticsearch._types.ErrorResponse; import co.elastic.clients.elasticsearch.core.*; +import co.elastic.clients.elasticsearch.core.search.ResponseBody; import co.elastic.clients.transport.ElasticsearchTransport; import co.elastic.clients.transport.JsonEndpoint; import co.elastic.clients.transport.TransportOptions; @@ -218,7 +219,7 @@ public Mono deleteByQuery( // endregion // region search - public Mono> search(SearchRequest request, Class tDocumentClass) { + public Mono> search(SearchRequest request, Class tDocumentClass) { Assert.notNull(request, "request must not be null"); Assert.notNull(tDocumentClass, "tDocumentClass must not be null"); @@ -227,7 +228,7 @@ public Mono> search(SearchRequest request, Class tDocum SearchRequest.createSearchEndpoint(this.getDeserializer(tDocumentClass)), transportOptions)); } - public Mono> search(Function> fn, + public Mono> search(Function> fn, Class tDocumentClass) { Assert.notNull(fn, "fn must not be null"); diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java index 4ddc4dfa3..f2587b023 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java @@ -22,6 +22,7 @@ import co.elastic.clients.elasticsearch.core.*; import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; import co.elastic.clients.elasticsearch.core.get.GetResult; +import co.elastic.clients.elasticsearch.core.search.ResponseBody; import co.elastic.clients.json.JsonpMapper; import co.elastic.clients.transport.Version; import reactor.core.publisher.Flux; @@ -326,22 +327,27 @@ private Flux doScroll(SearchRequest searchRequest) { Time scrollTimeout = searchRequest.scroll() != null ? searchRequest.scroll() : Time.of(t -> t.time("1m")); - Flux> searchResponses = Flux.usingWhen(Mono.fromSupplier(ScrollState::new), // - state -> Mono - .from(execute((ClientCallback>>) client -> client - .search(searchRequest, EntityAsMap.class))) // - .expand(entityAsMapSearchResponse -> { - - state.updateScrollId(entityAsMapSearchResponse.scrollId()); - - if (entityAsMapSearchResponse.hits() == null - || CollectionUtils.isEmpty(entityAsMapSearchResponse.hits().hits())) { - return Mono.empty(); - } - - return Mono.from(execute((ClientCallback>>) client -> client.scroll( - ScrollRequest.of(sr -> sr.scrollId(state.getScrollId()).scroll(scrollTimeout)), EntityAsMap.class))); - }), + Flux> searchResponses = Flux.usingWhen(Mono.fromSupplier(ScrollState::new), // + state -> { + return Mono + .from(execute((ClientCallback>>) client1 -> client1 + .search(searchRequest, EntityAsMap.class))) // + .expand(entityAsMapSearchResponse -> { + + state.updateScrollId(entityAsMapSearchResponse.scrollId()); + + if (entityAsMapSearchResponse.hits() == null + || CollectionUtils.isEmpty(entityAsMapSearchResponse.hits().hits())) { + return Mono.empty(); + } + + return Mono.from(execute((ClientCallback>>) client1 -> { + ScrollRequest scrollRequest = ScrollRequest + .of(sr -> sr.scrollId(state.getScrollId()).scroll(scrollTimeout)); + return client1.scroll(scrollRequest, EntityAsMap.class); + })); + }); + }, this::cleanupScroll, (state, ex) -> cleanupScroll(state), this::cleanupScroll); return searchResponses.flatMapIterable(entityAsMapSearchResponse -> entityAsMapSearchResponse.hits().hits()) @@ -363,7 +369,7 @@ protected Mono doCount(Query query, Class entityType, IndexCoordinates SearchRequest searchRequest = requestConverter.searchRequest(query, entityType, index, true, false); return Mono - .from(execute((ClientCallback>>) client -> client.search(searchRequest, + .from(execute((ClientCallback>>) client -> client.search(searchRequest, EntityAsMap.class))) .map(searchResponse -> searchResponse.hits().total() != null ? searchResponse.hits().total().value() : 0L); } @@ -371,7 +377,7 @@ protected Mono doCount(Query query, Class entityType, IndexCoordinates private Flux doFind(SearchRequest searchRequest) { return Mono - .from(execute((ClientCallback>>) client -> client.search(searchRequest, + .from(execute((ClientCallback>>) client -> client.search(searchRequest, EntityAsMap.class))) // .flatMapIterable(entityAsMapSearchResponse -> entityAsMapSearchResponse.hits().hits()) // .map(entityAsMapHit -> DocumentAdapters.from(entityAsMapHit, jsonpMapper)); @@ -391,7 +397,7 @@ protected Mono doFindForResponse(Query query, Class< .toFuture(); return Mono - .from(execute((ClientCallback>>) client -> client.search(searchRequest, + .from(execute((ClientCallback>>) client -> client.search(searchRequest, EntityAsMap.class))) .map(searchResponse -> SearchDocumentResponseBuilder.from(searchResponse, entityCreator, jsonpMapper)); } diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/SearchDocumentResponseBuilder.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/SearchDocumentResponseBuilder.java index ab2b63739..0b0aa014f 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/SearchDocumentResponseBuilder.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/SearchDocumentResponseBuilder.java @@ -19,6 +19,7 @@ import co.elastic.clients.elasticsearch.core.SearchResponse; import co.elastic.clients.elasticsearch.core.search.Hit; import co.elastic.clients.elasticsearch.core.search.HitsMetadata; +import co.elastic.clients.elasticsearch.core.search.ResponseBody; import co.elastic.clients.elasticsearch.core.search.Suggestion; import co.elastic.clients.elasticsearch.core.search.TotalHits; import co.elastic.clients.json.JsonpMapper; @@ -45,21 +46,22 @@ class SearchDocumentResponseBuilder { /** * creates a SearchDocumentResponse from the {@link SearchResponse} * - * @param searchResponse the Elasticsearch search response + * @param responseBody the Elasticsearch response body * @param entityCreator function to create an entity from a {@link SearchDocument} * @param jsonpMapper to map JsonData objects * @return the SearchDocumentResponse */ - public static SearchDocumentResponse from(SearchResponse searchResponse, + @SuppressWarnings("DuplicatedCode") + public static SearchDocumentResponse from(ResponseBody responseBody, SearchDocumentResponse.EntityCreator entityCreator, JsonpMapper jsonpMapper) { - Assert.notNull(searchResponse, "searchResponse must not be null"); + Assert.notNull(responseBody, "responseBody must not be null"); Assert.notNull(entityCreator, "entityCreator must not be null"); - HitsMetadata hitsMetadata = searchResponse.hits(); - String scrollId = searchResponse.scrollId(); - Map aggregations = searchResponse.aggregations(); - Map>> suggest = searchResponse.suggest(); + HitsMetadata hitsMetadata = responseBody.hits(); + String scrollId = responseBody.scrollId(); + Map aggregations = responseBody.aggregations(); + Map>> suggest = responseBody.suggest(); return from(hitsMetadata, scrollId, aggregations, suggest, entityCreator, jsonpMapper); } diff --git a/src/test/java/org/springframework/data/elasticsearch/client/elc/DevTests.java b/src/test/java/org/springframework/data/elasticsearch/client/elc/DevTests.java index b28275186..b0c3e9819 100644 --- a/src/test/java/org/springframework/data/elasticsearch/client/elc/DevTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/client/elc/DevTests.java @@ -25,7 +25,7 @@ import co.elastic.clients.elasticsearch.core.IndexRequest; import co.elastic.clients.elasticsearch.core.IndexResponse; import co.elastic.clients.elasticsearch.core.SearchRequest; -import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.search.ResponseBody; import co.elastic.clients.elasticsearch.indices.ElasticsearchIndicesClient; import co.elastic.clients.elasticsearch.indices.IndexSettings; import co.elastic.clients.transport.ElasticsearchTransport; @@ -300,7 +300,7 @@ void search() { .query(query -> query.match(matchQuery -> matchQuery.field("content").query(FieldValue.of("content1")))) .build(); - SearchResponse searchResponse = null; + ResponseBody searchResponse = null; try { searchResponse = searchImperative(searchRequest); assertThat(searchResponse).isNotNull(); @@ -316,11 +316,11 @@ void search() { } } - private SearchResponse searchImperative(SearchRequest searchRequest) throws IOException { + private ResponseBody searchImperative(SearchRequest searchRequest) throws IOException { return imperativeElasticsearchClient.search(searchRequest, EntityAsMap.class); } - private SearchResponse searchReactive(SearchRequest searchRequest) { + private ResponseBody searchReactive(SearchRequest searchRequest) { return Objects.requireNonNull(reactiveElasticsearchClient.search(searchRequest, EntityAsMap.class).block()); } // endregion diff --git a/src/test/resources/testcontainers-elasticsearch.properties b/src/test/resources/testcontainers-elasticsearch.properties index fc55f6131..94557e34b 100644 --- a/src/test/resources/testcontainers-elasticsearch.properties +++ b/src/test/resources/testcontainers-elasticsearch.properties @@ -15,7 +15,7 @@ # # sde.testcontainers.image-name=docker.elastic.co/elasticsearch/elasticsearch -sde.testcontainers.image-version=7.17.2 +sde.testcontainers.image-version=7.17.3 # # # needed as we do a DELETE /* at the end of the tests, will be required from 8.0 on, produces a warning since 7.13 From a86658c3970f72f8780a8c2d2fd45bb5cb2faafe Mon Sep 17 00:00:00 2001 From: Peter-Josef Meisch Date: Thu, 12 May 2022 07:32:39 +0200 Subject: [PATCH 004/516] Add new Elasticsearch client as an alternative to the existing REST client. Original Pull Request #2160 Closes #1973 (cherry picked from commit c0b26a51f1299fdc20998cc54c82ffbc7c005dae) --- .../client/elc/CriteriaFilterProcessor.java | 28 +++++++- .../client/elc/DocumentAdapters.java | 3 +- .../elc/ElasticsearchExceptionTranslator.java | 1 - .../client/elc/ElasticsearchTemplate.java | 13 ++-- .../client/elc/HighlightQueryBuilder.java | 8 --- .../client/elc/QueryBuilders.java | 17 +++++ .../elc/ReactiveElasticsearchTemplate.java | 5 +- .../client/elc/RequestConverter.java | 72 ++++++++++--------- .../elc/SearchDocumentResponseBuilder.java | 2 +- .../elasticsearch/client/elc/TypeUtils.java | 38 +++++++--- .../core/ReactiveElasticsearchTemplate.java | 6 +- .../core/document/Explanation.java | 3 +- .../elasticsearch/core/query/BaseQuery.java | 9 ++- .../core/query/BaseQueryBuilder.java | 16 +++++ .../core/query/NativeSearchQuery.java | 6 -- .../core/query/NativeSearchQueryBuilder.java | 18 ----- .../data/elasticsearch/core/query/Query.java | 6 ++ src/main/resources/versions.properties | 2 +- .../NewElasticsearchClientDevelopment.java | 2 +- .../client/elc/DocumentAdaptersUnitTests.java | 52 ++++++++++++++ .../core/ElasticsearchIntegrationTests.java | 6 +- .../suggest/CompletionIntegrationTests.java | 6 +- .../ReactiveSuggestIntegrationTests.java | 2 +- ...icsearchRepositoryELCIntegrationTests.java | 44 ++++++++++++ ...searchRepositoryERHLCIntegrationTests.java | 43 +++++++++++ ...sticsearchRepositoryIntegrationTests.java} | 59 +++++++-------- 26 files changed, 329 insertions(+), 138 deletions(-) create mode 100644 src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryELCIntegrationTests.java create mode 100644 src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryERHLCIntegrationTests.java rename src/test/java/org/springframework/data/elasticsearch/repository/support/{SimpleReactiveElasticsearchRepositoryTests.java => SimpleReactiveElasticsearchRepositoryIntegrationTests.java} (92%) diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/CriteriaFilterProcessor.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/CriteriaFilterProcessor.java index 11c2ee5c6..523e83efe 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/CriteriaFilterProcessor.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/CriteriaFilterProcessor.java @@ -31,6 +31,7 @@ import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import org.springframework.data.elasticsearch.core.convert.GeoConverters; @@ -67,9 +68,13 @@ public static Optional createQuery(Criteria criteria) { for (Criteria chainedCriteria : criteria.getCriteriaChain()) { if (chainedCriteria.isOr()) { - // todo #1973 + BoolQuery.Builder boolQueryBuilder = QueryBuilders.bool(); + queriesForEntries(chainedCriteria).forEach(boolQueryBuilder::should); + filterQueries.add(new Query(boolQueryBuilder.build())); } else if (chainedCriteria.isNegating()) { - // todo #1973 + Collection negatingFilters = buildNegatingFilter(criteria.getField().getName(), + criteria.getFilterCriteriaEntries()); + filterQueries.addAll(negatingFilters); } else { filterQueries.addAll(queriesForEntries(chainedCriteria)); } @@ -85,11 +90,28 @@ public static Optional createQuery(Criteria criteria) { BoolQuery.Builder boolQueryBuilder = QueryBuilders.bool(); filterQueries.forEach(boolQueryBuilder::must); BoolQuery boolQuery = boolQueryBuilder.build(); - return Optional.of(boolQuery._toQuery()); + return Optional.of(new Query(boolQuery)); } } } + private static Collection buildNegatingFilter(String fieldName, + Set filterCriteriaEntries) { + + List negationFilters = new ArrayList<>(); + + filterCriteriaEntries.forEach(criteriaEntry -> { + Optional query = queryFor(criteriaEntry.getKey(), criteriaEntry.getValue(), fieldName); + + if (query.isPresent()) { + BoolQuery negatingFilter = QueryBuilders.bool().mustNot(query.get()).build(); + negationFilters.add(new Query(negatingFilter)); + } + }); + + return negationFilters; + } + private static Collection queriesForEntries(Criteria criteria) { Assert.notNull(criteria.getField(), "criteria must have a field"); diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/DocumentAdapters.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/DocumentAdapters.java index fbd2e4f73..b90ae49e7 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/DocumentAdapters.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/DocumentAdapters.java @@ -80,8 +80,7 @@ public static SearchDocument from(Hit hit, JsonpMapper jsonpMapper) { Explanation explanation = from(hit.explanation()); - // todo #1973 matchedQueries - List matchedQueries = null; + List matchedQueries = hit.matchedQueries(); Function, EntityAsMap> fromFields = fields -> { StringBuilder sb = new StringBuilder("{"); diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchExceptionTranslator.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchExceptionTranslator.java index d6179d926..ba8fadfcd 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchExceptionTranslator.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchExceptionTranslator.java @@ -106,7 +106,6 @@ public DataAccessException translateExceptionIfPossible(RuntimeException ex) { } private boolean isSeqNoConflict(Throwable exception) { - // todo #1973 check if this works Integer status = null; String message = null; diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java index 545c23aff..8f1111354 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java @@ -34,6 +34,8 @@ import java.util.Map; import java.util.stream.Collectors; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.springframework.data.elasticsearch.BulkFailureException; import org.springframework.data.elasticsearch.client.UnsupportedBackendOperation; import org.springframework.data.elasticsearch.core.AbstractElasticsearchTemplate; @@ -68,6 +70,8 @@ */ public class ElasticsearchTemplate extends AbstractElasticsearchTemplate { + private static final Log LOGGER = LogFactory.getLog(ElasticsearchTemplate.class); + private final ElasticsearchClient client; private final RequestConverter requestConverter; private final ResponseConverter responseConverter; @@ -249,7 +253,6 @@ public String submitReindex(ReindexRequest reindexRequest) { client -> client.reindex(reindexRequestES)); if (reindexResponse.task() == null) { - // todo #1973 check behaviour and create issue in ES if necessary throw new UnsupportedBackendOperation("ElasticsearchClient did not return a task id on submit request"); } @@ -447,9 +450,6 @@ private List> doMultiSearch(List multiS MultiSearchQueryParameter queryParameter = queryIterator.next(); MultiSearchResponseItem responseItem = responseIterator.next(); - // if responseItem kind is Result then responseItem.value is a MultiSearchItem which is derived from - // SearchResponse - if (responseItem.isResult()) { Class clazz = queryParameter.clazz; @@ -463,7 +463,10 @@ private List> doMultiSearch(List multiS searchHitsList.add(searchHits); } else { - // todo #1973 add failure + if (LOGGER.isWarnEnabled()) { + LOGGER + .warn(String.format("multisearch responsecontains failure: {}", responseItem.failure().error().reason())); + } } } diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/HighlightQueryBuilder.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/HighlightQueryBuilder.java index 3e8c72b5b..5cceb0cd1 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/HighlightQueryBuilder.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/HighlightQueryBuilder.java @@ -87,10 +87,6 @@ private void addParameters(HighlightParameters parameters, builder.boundaryScannerLocale(parameters.getBoundaryScannerLocale()); } - if (parameters.getForceSource()) { // default is false - // todo #1973 parameter missing in new client - } - if (StringUtils.hasLength(parameters.getFragmenter())) { builder.fragmenter(highlighterFragmenter(parameters.getFragmenter())); } @@ -111,10 +107,6 @@ private void addParameters(HighlightParameters parameters, builder.order(highlighterOrder(parameters.getOrder())); } - if (parameters.getPhraseLimit() > -1) { - // todo #1973 parameter missing in new client - } - if (parameters.getPreTags().length > 0) { builder.preTags(Arrays.asList(parameters.getPreTags())); } diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/QueryBuilders.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/QueryBuilders.java index 2bc28a116..81fc6d387 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/QueryBuilders.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/QueryBuilders.java @@ -17,6 +17,7 @@ import co.elastic.clients.elasticsearch._types.FieldValue; import co.elastic.clients.elasticsearch._types.LatLonGeoLocation; +import co.elastic.clients.elasticsearch._types.query_dsl.IdsQuery; import co.elastic.clients.elasticsearch._types.query_dsl.MatchAllQuery; import co.elastic.clients.elasticsearch._types.query_dsl.MatchQuery; import co.elastic.clients.elasticsearch._types.query_dsl.Operator; @@ -29,6 +30,7 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; +import java.util.List; import java.util.function.Function; import org.springframework.data.elasticsearch.core.geo.GeoPoint; @@ -45,6 +47,21 @@ public final class QueryBuilders { private QueryBuilders() {} + public static IdsQuery idsQuery(List ids) { + + Assert.notNull(ids, "ids must not be null"); + + return IdsQuery.of(i -> i.values(ids)); + } + + public static Query idsQueryAsQuery(List ids) { + + Assert.notNull(ids, "ids must not be null"); + + Function> builder = b -> b.ids(idsQuery(ids)); + + return builder.apply(new Query.Builder()).build(); + } public static MatchQuery matchQuery(String fieldName, String query, @Nullable Operator operator, @Nullable Float boost) { diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java index f2587b023..0c9d41520 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java @@ -177,8 +177,7 @@ public Mono submitReindex(ReindexRequest reindexRequest) { return Mono.from(execute( // (ClientCallback>) client -> client .reindex(reindexRequestES))) - .flatMap(response -> (response.task() == null) ? Mono.error( // todo #1973 check behaviour and create issue in - // ES if necessary + .flatMap(response -> (response.task() == null) ? Mono.error( new UnsupportedBackendOperation("ElasticsearchClient did not return a task id on submit request")) : Mono.just(response.task())); } @@ -499,7 +498,7 @@ public Query matchAllQuery() { @Override public Query idsQuery(List ids) { - throw new UnsupportedOperationException("not implemented"); + return NativeQuery.builder().withQuery(QueryBuilders.idsQueryAsQuery(ids)).build(); } /** diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/RequestConverter.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/RequestConverter.java index 92cd14692..fec2d1ac4 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/RequestConverter.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/RequestConverter.java @@ -58,6 +58,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -192,7 +193,7 @@ public UpdateAliasesRequest indicesUpdateAliasesRequest(AliasActions aliasAction if (filterQuery != null) { elasticsearchConverter.updateQuery(filterQuery, parameters.getFilterQueryClass()); - co.elastic.clients.elasticsearch._types.query_dsl.Query esQuery = getFilter(filterQuery); + co.elastic.clients.elasticsearch._types.query_dsl.Query esQuery = getQuery(filterQuery, null); if (esQuery != null) { addActionBuilder.filter(esQuery); @@ -239,7 +240,8 @@ public PutMappingRequest indicesPutMappingRequest(IndexCoordinates indexCoordina PutMappingRequest.Builder builder = new PutMappingRequest.Builder(); builder.index(Arrays.asList(indexCoordinates.getIndexNames())); addPropertiesToMapping(builder, mapping); - // TODO #1973 what else to add + + // TODO #2155 what else to add return builder.build(); } @@ -374,7 +376,7 @@ public co.elastic.clients.elasticsearch.indices.PutTemplateRequest indicesPutTem Query filterQuery = parameters.getFilterQuery(); if (filterQuery != null) { - co.elastic.clients.elasticsearch._types.query_dsl.Query esQuery = getFilter(filterQuery); + co.elastic.clients.elasticsearch._types.query_dsl.Query esQuery = getQuery(filterQuery, null); if (esQuery != null) { aliasBuilder.filter(esQuery); @@ -475,7 +477,7 @@ public IndexRequest documentIndexRequest(IndexQuery query, IndexCoordinates i } } - builder.refresh(refresh(refreshPolicy)); + builder.refresh(TypeUtils.refresh(refreshPolicy)); return builder.build(); } @@ -644,9 +646,9 @@ public BulkRequest documentBulkRequest(List queries, BulkOptions bulkOptions, builder.timeout(tb -> tb.time(Long.valueOf(bulkOptions.getTimeout().toMillis()).toString() + "ms")); } - builder.refresh(refresh(refreshPolicy)); + builder.refresh(TypeUtils.refresh(refreshPolicy)); if (bulkOptions.getRefreshPolicy() != null) { - builder.refresh(refresh(bulkOptions.getRefreshPolicy())); + builder.refresh(TypeUtils.refresh(bulkOptions.getRefreshPolicy())); } if (bulkOptions.getWaitForActiveShards() != null) { @@ -791,13 +793,13 @@ public co.elastic.clients.elasticsearch.core.ReindexRequest reindex(ReindexReque ReindexRequest.Dest dest = reindexRequest.getDest(); return d // .index(dest.getIndex().getIndexName()) // - .versionType(versionType(dest.getVersionType())) // - .opType(opType(dest.getOpType())); + .versionType(TypeUtils.versionType(dest.getVersionType())) // + .opType(TypeUtils.opType(dest.getOpType())); } // ); if (reindexRequest.getConflicts() != null) { - builder.conflicts(conflicts(reindexRequest.getConflicts())); + builder.conflicts(TypeUtils.conflicts(reindexRequest.getConflicts())); } ReindexRequest.Script script = reindexRequest.getScript(); @@ -810,7 +812,7 @@ public co.elastic.clients.elasticsearch.core.ReindexRequest reindex(ReindexReque if (reindexRequest.getWaitForActiveShards() != null) { builder.waitForActiveShards(wfas -> wfas // - .count(waitForActiveShardsCount(reindexRequest.getWaitForActiveShards()))); + .count(TypeUtils.waitForActiveShardsCount(reindexRequest.getWaitForActiveShards()))); } builder // @@ -835,7 +837,7 @@ public DeleteRequest documentDeleteRequest(String id, @Nullable String routing, if (routing != null) { r.routing(routing); } - r.refresh(refresh(refreshPolicy)); + r.refresh(TypeUtils.refresh(refreshPolicy)); return r; }); } @@ -908,7 +910,7 @@ public DeleteByQueryRequest documentDeleteByQueryRequest(Query query, Class c .docAsUpsert(query.getDocAsUpsert()) // .ifSeqNo(query.getIfSeqNo() != null ? Long.valueOf(query.getIfSeqNo()) : null) // .ifPrimaryTerm(query.getIfPrimaryTerm() != null ? Long.valueOf(query.getIfPrimaryTerm()) : null) // - .refresh(refresh(refreshPolicy)) // + .refresh(TypeUtils.refresh(refreshPolicy)) // .retryOnConflict(query.getRetryOnConflict()) // ; @@ -993,7 +995,7 @@ public UpdateByQueryRequest documentUpdateByQueryRequest(UpdateQuery updateQuery } if (updateQuery.getWaitForActiveShards() != null) { - ub.waitForActiveShards(w -> w.count(waitForActiveShardsCount(updateQuery.getWaitForActiveShards()))); + ub.waitForActiveShards(w -> w.count(TypeUtils.waitForActiveShardsCount(updateQuery.getWaitForActiveShards()))); } return ub; @@ -1048,12 +1050,12 @@ public MsearchRequest searchMsearchRequest( mrb.searches(sb -> sb // .header(h -> h // .index(param.index.getIndexName()) // - // todo #1973 add remaining flags for header + // todo #2156 add remaining flags for header ) // .body(bb -> bb // .query(getQuery(param.query, param.clazz))// - // #1973 seq_no_primary_term and version not available in client ES issue 161 - // todo #1973 add remaining flags for body + // todo #2138 seq_no_primary_term and version not available in client ES issue 161 + // todo #2156 add remaining flags for body ) // ); }); @@ -1101,7 +1103,7 @@ private void prepareSearchRequest(Query query, @Nullable Class clazz, Ind } if (query.getIndicesOptions() != null) { - // todo #1973 indices options + // new Elasticsearch client does not support the old Indices options, need to be adapted } if (query.isLimiting()) { @@ -1116,7 +1118,7 @@ private void prepareSearchRequest(Query query, @Nullable Class clazz, Ind builder.preference(query.getPreference()); } - // todo #1973 searchType + builder.searchType(searchType(query.getSearchType())); if (query.getSort() != null) { List sortOptions = getSortOptions(query.getSort(), persistentEntity); @@ -1144,7 +1146,7 @@ private void prepareSearchRequest(Query query, @Nullable Class clazz, Ind builder.routing(query.getRoute()); } - // todo #1973 timeout + builder.timeout(timeStringMs(query.getTimeout())); if (query.getExplain()) { builder.explain(true); @@ -1158,7 +1160,7 @@ private void prepareSearchRequest(Query query, @Nullable Class clazz, Ind builder.rescore(getRescore(rescorerQuery)); }); - // todo #1973 request cache + builder.requestCache(query.getRequestCache()); if (!query.getRuntimeFields().isEmpty()) { @@ -1183,6 +1185,15 @@ private void prepareSearchRequest(Query query, @Nullable Class clazz, Ind // limit the number of documents in a batch builder.size(500); } + + if (!isEmpty(query.getIndicesBoost())) { + Map boosts = new LinkedHashMap<>(); + query.getIndicesBoost().forEach(indexBoost -> { + boosts.put(indexBoost.getIndexName(), (double) indexBoost.getBoost()); + }); + // noinspection unchecked + builder.indicesBoost(boosts); + } } private Rescore getRescore(RescorerQuery rescorerQuery) { @@ -1190,7 +1201,7 @@ private Rescore getRescore(RescorerQuery rescorerQuery) { return Rescore.of(r -> r // .query(rq -> rq // .query(getQuery(rescorerQuery.getQuery(), null)) // - .scoreMode(scoreMode(rescorerQuery.getScoreMode())) // + .scoreMode(TypeUtils.scoreMode(rescorerQuery.getScoreMode())) // .queryWeight(rescorerQuery.getQueryWeight() != null ? Double.valueOf(rescorerQuery.getQueryWeight()) : 1.0) // .rescoreQueryWeight( rescorerQuery.getRescoreQueryWeight() != null ? Double.valueOf(rescorerQuery.getRescoreQueryWeight()) @@ -1242,8 +1253,9 @@ private SortOptions getSortOptions(Sort.Order order, @Nullable ElasticsearchPers .geoDistance(gd -> gd // .field(fieldName) // .location(loc -> loc.latlon(QueryBuilders.latLon(geoDistanceOrder.getGeoPoint())))// - .distanceType(geoDistanceType(geoDistanceOrder.getDistanceType())).mode(sortMode(finalMode)) // - .unit(distanceUnit(geoDistanceOrder.getUnit())) // + .distanceType(TypeUtils.geoDistanceType(geoDistanceOrder.getDistanceType())) + .mode(TypeUtils.sortMode(finalMode)) // + .unit(TypeUtils.distanceUnit(geoDistanceOrder.getUnit())) // .ignoreUnmapped(geoDistanceOrder.getIgnoreUnmapped()))); } else { String missing = (order.getNullHandling() == Sort.NullHandling.NULLS_FIRST) ? "_first" @@ -1253,10 +1265,10 @@ private SortOptions getSortOptions(Sort.Order order, @Nullable ElasticsearchPers .field(f -> { f.field(fieldName) // .order(sortOrder) // - .mode(sortMode(finalMode)); + .mode(TypeUtils.sortMode(finalMode)); if (finalUnmappedType != null) { - FieldType fieldType = fieldType(finalUnmappedType); + FieldType fieldType = TypeUtils.fieldType(finalUnmappedType); if (fieldType != null) { f.unmappedType(fieldType); @@ -1284,13 +1296,11 @@ private void prepareNativeSearch(NativeQuery query, SearchRequest.Builder builde .collapse(query.getFieldCollapse()) // ; - // todo #1973 indices boost - if (!isEmpty(query.getAggregations())) { builder.aggregations(query.getAggregations()); } - // todo #1973 searchExt + // todo #2150 searchExt, currently not supported by the new client } @Nullable @@ -1463,12 +1473,6 @@ private VersionType retrieveVersionTypeFromPersistentEntity(@Nullable Class c return versionType != null ? versionType : VersionType.External; } - private co.elastic.clients.elasticsearch._types.query_dsl.Query getFilter(Query filterQuery) { - // TODO #1973 add filter query - - throw new UnsupportedOperationException("not implemented"); - } - @Nullable private SourceConfig getSourceConfig(Query query) { diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/SearchDocumentResponseBuilder.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/SearchDocumentResponseBuilder.java index 0b0aa014f..bdf3a69ca 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/SearchDocumentResponseBuilder.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/SearchDocumentResponseBuilder.java @@ -116,7 +116,7 @@ public static SearchDocumentResponse from(HitsMetadata hitsMetadata, @Nul ElasticsearchAggregations aggregationsContainer = aggregations != null ? new ElasticsearchAggregations(aggregations) : null; - // todo #1973 + // todo #2154 Suggest suggest = null; return new SearchDocumentResponse(totalHits, totalHitsRelation, maxScore, scrollId, searchDocuments, diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/TypeUtils.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/TypeUtils.java index 07968a4de..f68533a82 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/TypeUtils.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/TypeUtils.java @@ -15,15 +15,7 @@ */ package org.springframework.data.elasticsearch.client.elc; -import co.elastic.clients.elasticsearch._types.Conflicts; -import co.elastic.clients.elasticsearch._types.DistanceUnit; -import co.elastic.clients.elasticsearch._types.GeoDistanceType; -import co.elastic.clients.elasticsearch._types.OpType; -import co.elastic.clients.elasticsearch._types.Refresh; -import co.elastic.clients.elasticsearch._types.Result; -import co.elastic.clients.elasticsearch._types.SortMode; -import co.elastic.clients.elasticsearch._types.Time; -import co.elastic.clients.elasticsearch._types.VersionType; +import co.elastic.clients.elasticsearch._types.*; import co.elastic.clients.elasticsearch._types.mapping.FieldType; import co.elastic.clients.elasticsearch.core.search.BoundaryScanner; import co.elastic.clients.elasticsearch.core.search.BuiltinHighlighterType; @@ -40,6 +32,7 @@ import org.springframework.data.elasticsearch.core.query.GeoDistanceOrder; import org.springframework.data.elasticsearch.core.query.IndexQuery; import org.springframework.data.elasticsearch.core.query.Order; +import org.springframework.data.elasticsearch.core.query.Query; import org.springframework.data.elasticsearch.core.query.RescorerQuery; import org.springframework.data.elasticsearch.core.query.UpdateResponse; import org.springframework.data.elasticsearch.core.reindex.ReindexRequest; @@ -298,6 +291,23 @@ static ScoreMode scoreMode(@Nullable RescorerQuery.ScoreMode scoreMode) { return null; } + @Nullable + static SearchType searchType(@Nullable Query.SearchType searchType) { + + if (searchType == null) { + return null; + } + + switch (searchType) { + case QUERY_THEN_FETCH: + return SearchType.QueryThenFetch; + case DFS_QUERY_THEN_FETCH: + return SearchType.DfsQueryThenFetch; + } + + return null; + } + @Nullable static SortMode sortMode(Order.Mode mode) { @@ -325,6 +335,16 @@ static Time time(@Nullable Duration duration) { return Time.of(t -> t.time(duration.toMillis() + "ms")); } + @Nullable + static String timeStringMs(@Nullable Duration duration) { + + if (duration == null) { + return null; + } + + return duration.toMillis() + "ms"; + } + @Nullable static VersionType versionType( @Nullable org.springframework.data.elasticsearch.annotations.Document.VersionType versionType) { diff --git a/src/main/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchTemplate.java index 974f79baa..ae3a86c1d 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchTemplate.java @@ -277,8 +277,7 @@ public Mono get(String id, Class entityType, IndexCoordinates index) { */ protected Mono doGet(GetRequest request) { - return Mono.from(execute(client -> client.get(request))) // - .onErrorResume(NoSuchIndexException.class, it -> Mono.empty()); + return Mono.from(execute(client -> client.get(request))); } protected Mono doDeleteById(String id, @Nullable String routing, IndexCoordinates index) { @@ -633,8 +632,7 @@ protected Mono doCount(SearchRequest request) { QUERY_LOGGER.debug(String.format("Executing doCount: %s", request)); } - return Mono.from(execute(client -> client.count(request))) // - .onErrorResume(NoSuchIndexException.class, it -> Mono.just(0L)); + return Mono.from(execute(client -> client.count(request))); } /** diff --git a/src/main/java/org/springframework/data/elasticsearch/core/document/Explanation.java b/src/main/java/org/springframework/data/elasticsearch/core/document/Explanation.java index 3d1c41277..891c64708 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/document/Explanation.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/document/Explanation.java @@ -22,7 +22,8 @@ import org.springframework.util.Assert; /** - * class that holds explanations returned from an Elasticsearch search. + * class that holds explanations returned from an Elasticsearch search. Note: the new Elasticsearch client does not + * return the match property in search hits anymore, probably because a returned hit always is a match. * * @author Peter-Josef Meisch */ diff --git a/src/main/java/org/springframework/data/elasticsearch/core/query/BaseQuery.java b/src/main/java/org/springframework/data/elasticsearch/core/query/BaseQuery.java index 39b6d3ad7..0a34a3197 100755 --- a/src/main/java/org/springframework/data/elasticsearch/core/query/BaseQuery.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/query/BaseQuery.java @@ -71,6 +71,7 @@ public class BaseQuery implements Query { @Nullable protected Boolean requestCache; protected List idsWithRouting = Collections.emptyList(); protected final List runtimeFields = new ArrayList<>(); + @Nullable protected List indicesBoost; public BaseQuery() {} @@ -88,7 +89,7 @@ public > BaseQuery(BaseQue this.fields = builder.getFields(); this.highlightQuery = builder.highlightQuery; this.route = builder.getRoute(); - // #1973 add the other fields to the builder + this.indicesBoost = builder.getIndicesBoost(); } @Override @@ -433,4 +434,10 @@ public void addRuntimeField(RuntimeField runtimeField) { public List getRuntimeFields() { return runtimeFields; } + + @Override + @Nullable + public List getIndicesBoost() { + return indicesBoost; + } } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/query/BaseQueryBuilder.java b/src/main/java/org/springframework/data/elasticsearch/core/query/BaseQueryBuilder.java index fcb5968e1..29799064e 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/query/BaseQueryBuilder.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/query/BaseQueryBuilder.java @@ -46,6 +46,7 @@ public abstract class BaseQueryBuilder fields = new ArrayList<>(); @Nullable protected HighlightQuery highlightQuery; @Nullable private String route; + @Nullable private List indicesBoost; @Nullable public Pageable getPageable() { @@ -104,6 +105,11 @@ public String getRoute() { return route; } + @Nullable + public List getIndicesBoost() { + return indicesBoost; + } + public SELF withPageable(Pageable pageable) { this.pageable = pageable; return self(); @@ -178,6 +184,16 @@ public SELF withRoute(String route) { return self(); } + public SELF withIndicesBoost(List indicesBoost) { + this.indicesBoost = indicesBoost; + return self(); + } + + public SELF withIndicesBoost(IndexBoost... indicesBoost) { + this.indicesBoost = Arrays.asList(indicesBoost); + return self(); + } + public abstract Q build(); private SELF self() { diff --git a/src/main/java/org/springframework/data/elasticsearch/core/query/NativeSearchQuery.java b/src/main/java/org/springframework/data/elasticsearch/core/query/NativeSearchQuery.java index fd76d6dcd..893aae972 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/query/NativeSearchQuery.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/query/NativeSearchQuery.java @@ -55,7 +55,6 @@ public class NativeSearchQuery extends BaseQuery { @Nullable private List pipelineAggregations; @Nullable private HighlightBuilder highlightBuilder; @Nullable private HighlightBuilder.Field[] highlightFields; - @Nullable private List indicesBoost; @Nullable private SearchTemplateRequestBuilder searchTemplate; @Nullable private SuggestBuilder suggestBuilder; @Nullable private List searchExtBuilders; @@ -182,11 +181,6 @@ public void setPipelineAggregations(List pipelineAgg this.pipelineAggregations = pipelineAggregationBuilders; } - @Nullable - public List getIndicesBoost() { - return indicesBoost; - } - public void setIndicesBoost(List indicesBoost) { this.indicesBoost = indicesBoost; } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/query/NativeSearchQueryBuilder.java b/src/main/java/org/springframework/data/elasticsearch/core/query/NativeSearchQueryBuilder.java index 698489523..30549372f 100755 --- a/src/main/java/org/springframework/data/elasticsearch/core/query/NativeSearchQueryBuilder.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/query/NativeSearchQueryBuilder.java @@ -63,7 +63,6 @@ public class NativeSearchQueryBuilder extends BaseQueryBuilder highlightFields = new ArrayList<>(); @Nullable protected List storedFields; @Nullable private CollapseBuilder collapseBuilder; - @Nullable private List indicesBoost = new ArrayList<>(); @Nullable private SearchTemplateRequestBuilder searchTemplateBuilder; @Nullable private SearchType searchType; @Nullable private Boolean trackTotalHits; @@ -194,19 +193,6 @@ public NativeSearchQueryBuilder withHighlightFields(Collection indicesBoost) { - this.indicesBoost.addAll(indicesBoost); - return this; - } - - /** - * @since 4.3 - */ - public NativeSearchQueryBuilder withIndicesBoost(IndexBoost... indicesBoost) { - Collections.addAll(this.indicesBoost, indicesBoost); - return this; - } - public NativeSearchQueryBuilder withSearchTemplate(SearchTemplateRequestBuilder searchTemplateBuilder) { this.searchTemplateBuilder = searchTemplateBuilder; return this; @@ -290,10 +276,6 @@ public NativeSearchQuery build() { nativeSearchQuery.setStoredFields(storedFields); } - if (indicesBoost != null) { - nativeSearchQuery.setIndicesBoost(indicesBoost); - } - if (searchTemplateBuilder != null) { nativeSearchQuery.setSearchTemplate(searchTemplateBuilder); } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/query/Query.java b/src/main/java/org/springframework/data/elasticsearch/core/query/Query.java index 2b714c55f..c2080e4a3 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/query/Query.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/query/Query.java @@ -433,6 +433,12 @@ default List getRescorerQueries() { */ List getRuntimeFields(); + /** + * @since 4.4 + */ + @Nullable + List getIndicesBoost(); + /** * @since 4.3 */ diff --git a/src/main/resources/versions.properties b/src/main/resources/versions.properties index 58f49b58d..782d47e5b 100644 --- a/src/main/resources/versions.properties +++ b/src/main/resources/versions.properties @@ -1,2 +1,2 @@ version.spring-data-elasticsearch=${project.version} -version.elasticsearch-client=${elasticsearch} +version.elasticsearch-client=${elasticsearch-rhlc} diff --git a/src/test/java/org/springframework/data/elasticsearch/NewElasticsearchClientDevelopment.java b/src/test/java/org/springframework/data/elasticsearch/NewElasticsearchClientDevelopment.java index 81b13ea90..5c098ccd9 100644 --- a/src/test/java/org/springframework/data/elasticsearch/NewElasticsearchClientDevelopment.java +++ b/src/test/java/org/springframework/data/elasticsearch/NewElasticsearchClientDevelopment.java @@ -16,7 +16,7 @@ package org.springframework.data.elasticsearch; /** - * TODO #1973 remove when the new Elasticsearch client is fully working + * TODO remove when the new Elasticsearch client is fully working * * @author Peter-Josef Meisch */ diff --git a/src/test/java/org/springframework/data/elasticsearch/client/elc/DocumentAdaptersUnitTests.java b/src/test/java/org/springframework/data/elasticsearch/client/elc/DocumentAdaptersUnitTests.java index 5133d39a4..ce0d9c58e 100644 --- a/src/test/java/org/springframework/data/elasticsearch/client/elc/DocumentAdaptersUnitTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/client/elc/DocumentAdaptersUnitTests.java @@ -20,12 +20,15 @@ import co.elastic.clients.json.JsonpMapper; import co.elastic.clients.json.jackson.JacksonJsonpMapper; +import java.util.Arrays; import java.util.Collections; import java.util.List; import org.assertj.core.api.SoftAssertions; +import org.assertj.core.data.Offset; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.springframework.data.elasticsearch.core.document.Explanation; import org.springframework.data.elasticsearch.core.document.SearchDocument; /** @@ -106,4 +109,53 @@ void shouldAdaptSearchHitFromSource() { softly.assertAll(); } + + @Test // #725 #1973 + @DisplayName("should adapt returned explanations") + void shouldAdaptReturnedExplanations() { + + Hit searchHit = new Hit.Builder() // + .index("index") // + .id("42") // + .explanation(eb -> eb // + .value(3.14f) // + .description("explanation 3.14") // + .details(edb -> edb.description("explanation noMatch").value(0f))) + .build(); + + SearchDocument searchDocument = DocumentAdapters.from(searchHit, jsonpMapper); + + SoftAssertions softly = new SoftAssertions(); + + Explanation explanation = searchDocument.getExplanation(); + softly.assertThat(explanation).isNotNull(); + softly.assertThat(explanation.isMatch()).isTrue(); + softly.assertThat(explanation.getValue()).isCloseTo(3.14, Offset.offset(0.001)); + softly.assertThat(explanation.getDescription()).isEqualTo("explanation 3.14"); + List details = explanation.getDetails(); + softly.assertThat(details) + .containsExactly(new Explanation(null, 0.0, "explanation noMatch", Collections.emptyList())); + softly.assertAll(); + } + + @Test // DATAES-979 #1973 + @DisplayName("should adapt returned matched queries") + void shouldAdaptReturnedMatchedQueries() { + + Hit searchHit = new Hit.Builder() // + .index("index") // + .id("42") // + .matchedQueries("query1", "query2") // + .build(); + + SearchDocument searchDocument = DocumentAdapters.from(searchHit, jsonpMapper); + + SoftAssertions softly = new SoftAssertions(); + + List matchedQueries = searchDocument.getMatchedQueries(); + softly.assertThat(matchedQueries).isNotNull(); + softly.assertThat(matchedQueries).hasSize(2); + softly.assertThat(matchedQueries).isEqualTo(Arrays.asList("query1", "query2")); + softly.assertAll(); + } } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchIntegrationTests.java index f69ab3535..20bcb1c79 100755 --- a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchIntegrationTests.java @@ -1617,7 +1617,7 @@ public void shouldPassIndicesOptionsForGivenSearchScrollQuery() { } @DisabledIf(value = "newElasticsearchClient", - disabledReason = "todo #1973 can't check response, open ES issue 161 that does not allow seqno") + disabledReason = "todo #2138 can't check response, open ES issue 161 that does not allow seqno") // and version to be set in the request @Test // DATAES-487 public void shouldReturnSameEntityForMultiSearch() { @@ -1642,7 +1642,7 @@ public void shouldReturnSameEntityForMultiSearch() { } @DisabledIf(value = "newElasticsearchClient", - disabledReason = "todo #1973 can't check response, open ES issue 161 that does not allow seqno") + disabledReason = "todo #2138 can't check response, open ES issue 161 that does not allow seqno") // and version to be set in the request @Test // DATAES-487 public void shouldReturnDifferentEntityForMultiSearch() { @@ -3070,7 +3070,7 @@ void searchShouldReturnSeqNoPrimaryTerm() { } @DisabledIf(value = "newElasticsearchClient", - disabledReason = "todo #1973 can't check response, open ES issue 161 that does not allow seqno") + disabledReason = "todo #2138 can't check response, open ES issue 161 that does not allow seqno") // and version to be set in the request @Test // DATAES-799 void multiSearchShouldReturnSeqNoPrimaryTerm() { diff --git a/src/test/java/org/springframework/data/elasticsearch/core/suggest/CompletionIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/suggest/CompletionIntegrationTests.java index 4fe48d4f1..e0b51c4fb 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/suggest/CompletionIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/suggest/CompletionIntegrationTests.java @@ -116,7 +116,7 @@ private void loadAnnotatedCompletionObjectEntitiesWithWeights() { operations.bulkIndex(indexQueries, AnnotatedCompletionEntity.class); } - @DisabledIf(value = "newElasticsearchClient", disabledReason="todo #1973, ES issue 150") + @DisabledIf(value = "newElasticsearchClient", disabledReason = "todo #2139, ES issue 150") @Test public void shouldFindSuggestionsForGivenCriteriaQueryUsingCompletionEntity() { @@ -148,7 +148,7 @@ void shouldRetrieveEntityWithCompletion() { operations.get("1", CompletionEntity.class); } - @DisabledIf(value = "newElasticsearchClient", disabledReason="todo #1973, ES issue 150") + @DisabledIf(value = "newElasticsearchClient", disabledReason = "todo #2139, ES issue 150") @Test public void shouldFindSuggestionsForGivenCriteriaQueryUsingAnnotatedCompletionEntity() { @@ -172,7 +172,7 @@ public void shouldFindSuggestionsForGivenCriteriaQueryUsingAnnotatedCompletionEn assertThat(options.get(1).getText()).isIn("Marchand", "Mohsin"); } - @DisabledIf(value = "newElasticsearchClient", disabledReason="todo #1973, ES issue 150") + @DisabledIf(value = "newElasticsearchClient", disabledReason = "todo #2139, ES 1issue 150") @Test public void shouldFindSuggestionsWithWeightsForGivenCriteriaQueryUsingAnnotatedCompletionEntity() { diff --git a/src/test/java/org/springframework/data/elasticsearch/core/suggest/ReactiveSuggestIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/suggest/ReactiveSuggestIntegrationTests.java index a6d57feeb..026ff84c6 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/suggest/ReactiveSuggestIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/suggest/ReactiveSuggestIntegrationTests.java @@ -67,7 +67,7 @@ void cleanup() { operations.indexOps(IndexCoordinates.of(indexNameProvider.getPrefix() + "*")).delete().block(); } - @DisabledIf(value = "newElasticsearchClient", disabledReason="todo #1973, ES issue 150") + @DisabledIf(value = "newElasticsearchClient", disabledReason = "todo #2139, ES issue 150") @Test // #1302 @DisplayName("should find suggestions for given prefix completion") void shouldFindSuggestionsForGivenPrefixCompletion() { diff --git a/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryELCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryELCIntegrationTests.java new file mode 100644 index 000000000..8781ad271 --- /dev/null +++ b/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryELCIntegrationTests.java @@ -0,0 +1,44 @@ +/* + * Copyright 2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch.repository.support; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.junit.jupiter.ReactiveElasticsearchTemplateConfiguration; +import org.springframework.data.elasticsearch.repository.config.EnableReactiveElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +/** + * @author Peter-Josef Meisch + * @since 4.4 + */ +@ContextConfiguration(classes = { SimpleReactiveElasticsearchRepositoryELCIntegrationTests.Config.class }) +public class SimpleReactiveElasticsearchRepositoryELCIntegrationTests + extends SimpleReactiveElasticsearchRepositoryIntegrationTests { + + @Configuration + @Import({ ReactiveElasticsearchTemplateConfiguration.class }) + @EnableReactiveElasticsearchRepositories(considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("simple-reactive-repository"); + } + } + +} diff --git a/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryERHLCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryERHLCIntegrationTests.java new file mode 100644 index 000000000..361cb4be1 --- /dev/null +++ b/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryERHLCIntegrationTests.java @@ -0,0 +1,43 @@ +/* + * Copyright 2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch.repository.support; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.junit.jupiter.ReactiveElasticsearchRestTemplateConfiguration; +import org.springframework.data.elasticsearch.repository.config.EnableReactiveElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +/** + * @author Peter-Josef Meisch + */ +@ContextConfiguration(classes = { SimpleReactiveElasticsearchRepositoryERHLCIntegrationTests.Config.class }) +public class SimpleReactiveElasticsearchRepositoryERHLCIntegrationTests + extends SimpleReactiveElasticsearchRepositoryIntegrationTests { + + @Configuration + @Import({ ReactiveElasticsearchRestTemplateConfiguration.class }) + @EnableReactiveElasticsearchRepositories(considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("simple-reactive-repository-es7"); + } + } + +} diff --git a/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryTests.java b/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryIntegrationTests.java similarity index 92% rename from src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryTests.java rename to src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryIntegrationTests.java index 61d7ee87e..2f50033b8 100644 --- a/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryIntegrationTests.java @@ -16,81 +16,68 @@ package org.springframework.data.elasticsearch.repository.support; import static org.assertj.core.api.Assertions.*; -import static org.springframework.data.elasticsearch.annotations.FieldType.*; import static org.springframework.data.elasticsearch.core.query.Query.*; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.test.StepVerifier; -import java.lang.Boolean; -import java.lang.Long; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.reactivestreams.Publisher; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; import org.springframework.data.annotation.Id; import org.springframework.data.annotation.Version; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; import org.springframework.data.domain.Sort.Order; -import org.springframework.data.elasticsearch.RestStatusException; +import org.springframework.data.elasticsearch.NoSuchIndexException; import org.springframework.data.elasticsearch.annotations.CountQuery; import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Field; +import org.springframework.data.elasticsearch.annotations.FieldType; import org.springframework.data.elasticsearch.annotations.Highlight; import org.springframework.data.elasticsearch.annotations.HighlightField; import org.springframework.data.elasticsearch.annotations.Query; import org.springframework.data.elasticsearch.core.ReactiveElasticsearchOperations; import org.springframework.data.elasticsearch.core.SearchHit; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; -import org.springframework.data.elasticsearch.junit.jupiter.ReactiveElasticsearchRestTemplateConfiguration; import org.springframework.data.elasticsearch.junit.jupiter.SpringIntegrationTest; -import org.springframework.data.elasticsearch.repository.config.EnableReactiveElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; import org.springframework.data.repository.reactive.ReactiveCrudRepository; import org.springframework.lang.Nullable; -import org.springframework.test.context.ContextConfiguration; /** * @author Christoph Strobl * @author Peter-Josef Meisch * @author Jens Schauder */ -// todo #1973 test for both clients @SpringIntegrationTest -@ContextConfiguration(classes = { SimpleReactiveElasticsearchRepositoryTests.Config.class }) -class SimpleReactiveElasticsearchRepositoryTests { - - @Configuration - @Import({ ReactiveElasticsearchRestTemplateConfiguration.class }) - @EnableReactiveElasticsearchRepositories(considerNestedRepositories = true) - static class Config {} - - static final String INDEX = "test-index-sample-simple-reactive"; +abstract class SimpleReactiveElasticsearchRepositoryIntegrationTests { @Autowired ReactiveElasticsearchOperations operations; - @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") @Autowired ReactiveSampleEntityRepository repository; + @Autowired private IndexNameProvider indexNameProvider; + @BeforeEach - void setUp() { - operations.indexOps(IndexCoordinates.of(INDEX)).delete().block(); + void before() { + indexNameProvider.increment(); + operations.indexOps(SampleEntity.class).createWithMapping().block(); } - @AfterEach - void after() { - operations.indexOps(IndexCoordinates.of(INDEX)).delete().block(); + @Test + @org.junit.jupiter.api.Order(Integer.MAX_VALUE) + public void cleanup() { + operations.indexOps(IndexCoordinates.of(indexNameProvider.getPrefix() + "*")).delete().block(); } @Test // DATAES-519 @@ -104,7 +91,7 @@ void saveShouldSaveSingleEntity() { } private Mono documentWithIdExistsInIndex(String id) { - return operations.exists(id, IndexCoordinates.of(INDEX)); + return operations.exists(id, IndexCoordinates.of(indexNameProvider.indexName())); } @Test // DATAES-519 @@ -122,9 +109,12 @@ void saveShouldComputeMultipleEntities() { @Test // DATAES-519, DATAES-767, DATAES-822 void findByIdShouldErrorIfIndexDoesNotExist() { + + operations.indexOps(SampleEntity.class).delete().block(); repository.findById("id-two") // .as(StepVerifier::create) // - .expectError(RestStatusException.class); + .expectError(NoSuchIndexException.class) // + .verify(); } @Test // DATAES-519 @@ -268,9 +258,12 @@ void shouldReturnHighlightsOnAnnotatedStringQueryMethod() { @Test // DATAES-519, DATAES-767, DATAES-822 void countShouldErrorWhenIndexDoesNotExist() { + + operations.indexOps(SampleEntity.class).delete().block(); repository.count() // .as(StepVerifier::create) // - .expectError(RestStatusException.class); + .expectError(NoSuchIndexException.class) // + .verify(); } @Test // DATAES-519 @@ -596,7 +589,7 @@ void derivedDeleteMethodShouldBeExecutedCorrectly() { } Mono bulkIndex(SampleEntity... entities) { - return operations.saveAll(Arrays.asList(entities), IndexCoordinates.of(INDEX)).then(); + return operations.saveAll(Arrays.asList(entities), IndexCoordinates.of(indexNameProvider.indexName())).then(); } interface ReactiveSampleEntityRepository extends ReactiveCrudRepository { @@ -636,14 +629,14 @@ interface ReactiveSampleEntityRepository extends ReactiveCrudRepository retrieveCountByText(String message); } - @Document(indexName = INDEX) + @Document(indexName = "#{@indexNameProvider.indexName()}") static class SampleEntity { @Nullable @Id private String id; @Nullable - @Field(type = Text, store = true, fielddata = true) private String type; + @Field(type = FieldType.Text, store = true, fielddata = true) private String type; @Nullable - @Field(type = Text, store = true, fielddata = true) private String message; + @Field(type = FieldType.Text, store = true, fielddata = true) private String message; @Nullable private int rate; @Nullable private boolean available; @Nullable From 7cb46b6ab0f0fdaea323709d0a512599ccf743ad Mon Sep 17 00:00:00 2001 From: Christoph Strobl Date: Fri, 13 May 2022 10:43:21 +0200 Subject: [PATCH 005/516] Prepare 5.0 M4 (2022.0.0). See #2122 --- pom.xml | 1463 +++++++++++++++++++++------------ src/main/resources/notice.txt | 3 +- 2 files changed, 943 insertions(+), 523 deletions(-) diff --git a/pom.xml b/pom.xml index 1eec82e2c..aaad295c8 100644 --- a/pom.xml +++ b/pom.xml @@ -1,285 +1,496 @@ - - 4.0.0 - - org.springframework.data - spring-data-elasticsearch - 5.0.0-SNAPSHOT - - - org.springframework.data.build - spring-data-parent - 3.0.0-SNAPSHOT - - - Spring Data Elasticsearch - Spring Data Implementation for Elasticsearch - https://github.com/spring-projects/spring-data-elasticsearch - - - - 7.17.3 - - 7.17.3 - 2.17.1 - 4.1.65.Final - 3.0.0-SNAPSHOT - 1.16.2 - 1.0.6.RELEASE - spring.data.elasticsearch - - + + 7.17.3 + + + + 7.17.3 + + 2.17.1 + + 4.1.65.Final + + 3.0.0-M4 + + 1.16.2 + + 1.0.6.RELEASE + + spring.data.elasticsearch + + - test - integration-test - none - - - - - biomedcentral - BioMed Central Development Team - +0 - - - cstrobl - Christoph Strobl - cstrobl at pivotal.io - Pivotal - https://www.pivotal.io - - Developer - - +1 - - - mpaluch - Mark Paluch - mpaluch at pivotal.io - Pivotal - https://www.pivotal.io - - Developer - - +1 - - - - - https://github.com/spring-projects/spring-data-elasticsearch - scm:git:git://github.com/spring-projects/spring-data-elasticsearch.git - scm:git:ssh://git@github.com/spring-projects/spring-data-elasticsearch.git + + test + + integration-test + + none + + + + + + + + biomedcentral + + BioMed Central Development Team + + +0 + + + + + + cstrobl + + Christoph Strobl + + cstrobl at pivotal.io + + Pivotal + + https://www.pivotal.io + + + + Developer + + + + +1 + + + + + + mpaluch + + Mark Paluch + + mpaluch at pivotal.io + + Pivotal + + https://www.pivotal.io + + + + Developer + + + + +1 + + + + + + + + https://github.com/spring-projects/spring-data-elasticsearch + + scm:git:git://github.com/spring-projects/spring-data-elasticsearch.git + + scm:git:ssh://git@github.com/spring-projects/spring-data-elasticsearch.git - - - - Bamboo - https://build.spring.io/browse/SPRINGDATAES - - - - GitHub - https://github.com/spring-projects/spring-data-elasticsearch/issues - - - - - - io.netty - netty-bom - ${netty} - pom - import - - - - - - - - - org.springframework - spring-context - - - - org.springframework - spring-tx - - - - - org.springframework.data - spring-data-commons - ${springdata.commons} - - - - - org.springframework - spring-webflux - true - - - - io.projectreactor.netty - reactor-netty-http - true - - - - io.projectreactor - reactor-test - test - - - - - org.elasticsearch.client - elasticsearch-rest-high-level-client - ${elasticsearch-rhlc} - - - commons-logging - commons-logging - - - - - - - co.elastic.clients - elasticsearch-java - ${elasticsearch-java} - - - commons-logging - commons-logging - - - - - org.elasticsearch.client - elasticsearch-rest-client - ${elasticsearch-java} - - - commons-logging - commons-logging - - - - - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - - - - org.apache.geronimo.specs - geronimo-jcdi_2.0_spec - 1.3 - test - - - - javax.interceptor - javax.interceptor-api - 1.2.1 - test - - - - jakarta.enterprise - jakarta.enterprise.cdi-api - ${cdi} - provided - true - - - - jakarta.annotation - jakarta.annotation-api - ${jakarta-annotation-api} - test - - - - org.apache.openwebbeans - openwebbeans-se - jakarta - ${webbeans} - test - - - - org.apache.openwebbeans - openwebbeans-spi - jakarta - ${webbeans} - test - - - - org.apache.openwebbeans - openwebbeans-impl - jakarta - ${webbeans} - test - - - - - org.springframework - spring-test - test - - - ch.qos.logback - logback-classic - - - - - - org.slf4j - log4j-over-slf4j - ${slf4j} - test - - - org.apache.logging.log4j - log4j-core - ${log4j} - test - - - org.apache.logging.log4j - log4j-to-slf4j - ${log4j} - test - - - - io.projectreactor.tools - blockhound-junit-platform - ${blockhound-junit} - test - - - - + + + + org.springframework + + spring-context + + + + + + org.springframework + + spring-tx + + + + + + + + org.springframework.data + + spring-data-commons + + ${springdata.commons} + + + + + + + + org.springframework + + spring-webflux + + true + + + + + + io.projectreactor.netty + + reactor-netty-http + + true + + + + + + io.projectreactor + + reactor-test + + test + + + + + + + + org.elasticsearch.client + + elasticsearch-rest-high-level-client + + ${elasticsearch-rhlc} + + + + + + commons-logging + + commons-logging + + + + + + + + + + + + co.elastic.clients + + elasticsearch-java + + ${elasticsearch-java} + + + + + + commons-logging + + commons-logging + + + + + + + + + + org.elasticsearch.client + + elasticsearch-rest-client + + + + ${elasticsearch-java} + + + + + + commons-logging + + commons-logging + + + + + + + + + + + + com.fasterxml.jackson.core + + jackson-core + + + + + + com.fasterxml.jackson.core + + jackson-databind + + + + + + + + + + org.apache.geronimo.specs + + geronimo-jcdi_2.0_spec + + 1.3 + + test + + + + + + javax.interceptor + + javax.interceptor-api + + 1.2.1 + + test + + + + + + jakarta.enterprise + + jakarta.enterprise.cdi-api + + ${cdi} + + provided + + true + + + + + + jakarta.annotation + + jakarta.annotation-api + + ${jakarta-annotation-api} + + test + + + + + + org.apache.openwebbeans + + openwebbeans-se + + jakarta + + ${webbeans} + + test + + + + + + org.apache.openwebbeans + + openwebbeans-spi + + jakarta + + ${webbeans} + + test + + + + + + org.apache.openwebbeans + + openwebbeans-impl + + jakarta + + ${webbeans} + + test + + + + + + + + org.springframework + + spring-test + + test + + + + + + ch.qos.logback + + logback-classic + + + + + + + + + + org.slf4j + + log4j-over-slf4j + + ${slf4j} + + test + + + + + + org.apache.logging.log4j + + log4j-core + + ${log4j} + + test + + + + + + org.apache.logging.log4j + + log4j-to-slf4j + + ${log4j} + + test + + + + + + io.projectreactor.tools + + blockhound-junit-platform + + ${blockhound-junit} + + test + + + + - - org.projectlombok - lombok - - 999999 - test - - - - org.skyscreamer - jsonassert - 1.5.0 - test - - - - com.github.tomakehurst - wiremock-jre8 - 2.32.0 - test - - - - commons-logging - commons-logging - - - org.ow2.asm - asm - - - - - - io.specto - hoverfly-java-junit5 - 0.14.1 - test - - - - - org.apache.xbean - xbean-asm5-shaded - 4.5 - test - - - - javax.servlet - javax.servlet-api - 3.1.0 - test - - - - org.mockito - mockito-junit-jupiter - ${mockito} - test - - - - org.testcontainers - elasticsearch - ${testcontainers} - test - - - - - - - - src/main/resources - true - - **/versions.properties - - - - src/main/resources - false - - **/versions.properties - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - true - false - - **/*Tests.java - **/*Test.java - - - false - - - - - - default-test - ${mvn.unit-test.goal} - - test - - - integration-test - - - - - integration-test-elasticsearch - ${mvn.integration-test-elasticsearch.goal} - - test - - - integration-test - - elasticsearch - - - - - - integration-test-opensearch - ${mvn.integration-test-opensearch.goal} - - test - - - integration-test - - opensearch - - - - - - - org.pitest - pitest-maven - 1.7.5 - - - org.pitest - pitest-junit5-plugin - 0.12 - - - - - -XX:+AllowRedefinitionToAddDeleteMethods - - integration-test - - org.springframework.data.elasticsearch.core.geo.* - - toString - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - org.asciidoctor - asciidoctor-maven-plugin - - - - - - - ci - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - - - - - - - - - **/* - + + + + org.projectlombok + + lombok + + + + 999999 + + test + + + + + + org.skyscreamer + + jsonassert + + 1.5.0 + + test + + + + + + com.github.tomakehurst + + wiremock-jre8 + + 2.32.0 + + test + + + + + + + + commons-logging + + commons-logging + + + + + + org.ow2.asm + + asm + + + + + + + + + + io.specto + + hoverfly-java-junit5 + + 0.14.1 + + test + + + + + + + + org.apache.xbean + + xbean-asm5-shaded + + 4.5 + + test + + + + + + javax.servlet + + javax.servlet-api + + 3.1.0 + + test + + + + + + org.mockito + + mockito-junit-jupiter + + ${mockito} + + test + + + + + + org.testcontainers + + elasticsearch + + ${testcontainers} + + test + + + + + + + + + + + + src/main/resources + + true + + + + **/versions.properties + + + + + + + + src/main/resources + + false + + + + **/versions.properties + + + + + + + + + + + + org.apache.maven.plugins + + maven-surefire-plugin + + + + true + + false + + + + **/*Tests.java + + **/*Test.java + + + + + + false + + + + + + + + + + + + default-test + + ${mvn.unit-test.goal} + + + + test + + + + + + integration-test + + + + + + + + + + integration-test-elasticsearch + + ${mvn.integration-test-elasticsearch.goal} + + + + test + + + + + + integration-test + + + + elasticsearch + + + + + + + + + + + + integration-test-opensearch + + ${mvn.integration-test-opensearch.goal} + + + + test + + + + + + integration-test + + + + opensearch + + + + + + + + + + + + + + org.pitest + + pitest-maven + + 1.7.5 + + + + + + org.pitest + + pitest-junit5-plugin + + 0.12 + + + + + + + + + + -XX:+AllowRedefinitionToAddDeleteMethods + + + + integration-test + + + + org.springframework.data.elasticsearch.core.geo.* + + + + toString + + + + + + + + org.apache.maven.plugins + + maven-assembly-plugin + + + + + + org.asciidoctor + + asciidoctor-maven-plugin + + + + + + + + + + + + ci + + + + + + + + org.apache.maven.plugins + + maven-checkstyle-plugin + + + + + + + + + + + + + + + + + + **/* + + .git/**/*,target/**/*,**/target/**/*,.idea/**/*,**/spring.schemas,**/*.svg,mvnw,mvnw.cmd,**/*.policy - ./ - - - - - - - - jdk13+ - - - [13,) - - - - - org.apache.maven.plugins - maven-surefire-plugin - - -XX:+AllowRedefinitionToAddDeleteMethods - - - - - - - - - - spring-libs-snapshot - https://repo.spring.io/libs-snapshot - - - - local-maven-repo - file:///${project.basedir}/src/test/resources/local-maven-repo - - - - - - - spring-plugins-release - https://repo.spring.io/plugins-release - - - + + ./ + + + + + + + + + + + + + + jdk13+ + + + + + + [13,) + + + + + + + + + + org.apache.maven.plugins + + maven-surefire-plugin + + + + -XX:+AllowRedefinitionToAddDeleteMethods + + + + + + + + + + + + + + + + + + spring-libs-milestone + + https://repo.spring.io/libs-milestone + + + + + + local-maven-repo + + file:///${project.basedir}/src/test/resources/local-maven-repo + + + + + + + + + + spring-plugins-release + + https://repo.spring.io/plugins-release + + + + + diff --git a/src/main/resources/notice.txt b/src/main/resources/notice.txt index 648b319ac..94475c30d 100644 --- a/src/main/resources/notice.txt +++ b/src/main/resources/notice.txt @@ -1,4 +1,4 @@ -Spring Data Elasticsearch 5.0 M3 (2022.0.0) +Spring Data Elasticsearch 5.0 M4 (2022.0.0) Copyright (c) [2013-2021] Pivotal Software, Inc. This product is licensed to you under the Apache License, Version 2.0 (the "License"). @@ -10,3 +10,4 @@ code for the these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. + From 1bff5dd7495d5d38e9e1ad9c2e7d12b30644ab46 Mon Sep 17 00:00:00 2001 From: Christoph Strobl Date: Fri, 13 May 2022 10:44:00 +0200 Subject: [PATCH 006/516] Release version 5.0 M4 (2022.0.0). See #2122 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index aaad295c8..504328d84 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ spring-data-elasticsearch - 5.0.0-SNAPSHOT + 5.0.0-M4 From d676ffe0660d8f2ea57d836be2b1ea95653707ad Mon Sep 17 00:00:00 2001 From: Christoph Strobl Date: Fri, 13 May 2022 10:53:24 +0200 Subject: [PATCH 007/516] Prepare next development iteration. See #2122 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 504328d84..aaad295c8 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ spring-data-elasticsearch - 5.0.0-M4 + 5.0.0-SNAPSHOT From d37ca4d533b04c1bb1a8236ba1cc90b7ef6458b2 Mon Sep 17 00:00:00 2001 From: Christoph Strobl Date: Fri, 13 May 2022 10:53:27 +0200 Subject: [PATCH 008/516] After release cleanups. See #2122 --- pom.xml | 1421 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 946 insertions(+), 475 deletions(-) diff --git a/pom.xml b/pom.xml index aaad295c8..f8d78075e 100644 --- a/pom.xml +++ b/pom.xml @@ -1,495 +1,738 @@ - + + 4.0.0 - + + org.springframework.data - + + spring-data-elasticsearch - + + 5.0.0-SNAPSHOT - + + - + + org.springframework.data.build - + + spring-data-parent - - 3.0.0-M4 - + + + 3.0.0-SNAPSHOT + + - + + Spring Data Elasticsearch - + + Spring Data Implementation for Elasticsearch - + + https://github.com/spring-projects/spring-data-elasticsearch - + + - + + - + + 7.17.3 - + + - + + 7.17.3 - + + 2.17.1 - + + 4.1.65.Final - - 3.0.0-M4 - + + + 3.0.0-SNAPSHOT + + 1.16.2 - + + 1.0.6.RELEASE - + + spring.data.elasticsearch - + + - + + test - + + integration-test - + + none - + + - + + - + + - + + biomedcentral - + + BioMed Central Development Team - + + +0 - + + - + + - + + cstrobl - + + Christoph Strobl - + + cstrobl at pivotal.io - + + Pivotal - + + https://www.pivotal.io - + + - + + Developer - + + - + + +1 - + + - + + - + + mpaluch - + + Mark Paluch - + + mpaluch at pivotal.io - + + Pivotal - + + https://www.pivotal.io - + + - + + Developer - + + - + + +1 - + + - + + - + + - + + https://github.com/spring-projects/spring-data-elasticsearch - + + scm:git:git://github.com/spring-projects/spring-data-elasticsearch.git - + + scm:git:ssh://git@github.com/spring-projects/spring-data-elasticsearch.git - + + - + + - + + Bamboo - + + https://build.spring.io/browse/SPRINGDATAES - + + - + + - + + GitHub - + + https://github.com/spring-projects/spring-data-elasticsearch/issues - + + - + + - + + - + + - + + io.netty - + + netty-bom - + + ${netty} - + + pom - + + import - + + - + + - + + - + + - + + - + + - + + org.springframework - + + spring-context - + + - + + - + + org.springframework - + + spring-tx - + + - + + - + + - + + org.springframework.data - + + spring-data-commons - + + ${springdata.commons} - + + - + + - + + - + + org.springframework - + + spring-webflux - + + true - + + - + + - + + io.projectreactor.netty - + + reactor-netty-http - + + true - + + - + + - + + io.projectreactor - + + reactor-test - + + test - + + - + + - + + - + + org.elasticsearch.client - + + elasticsearch-rest-high-level-client - + + ${elasticsearch-rhlc} - + + - + + - + + commons-logging - + + commons-logging - + + - + + - + + - + + - + + - + + co.elastic.clients - + + elasticsearch-java - + + ${elasticsearch-java} - + + - + + - + + commons-logging - + + commons-logging - + + - + + - + + - + + - + + org.elasticsearch.client - + + elasticsearch-rest-client - + + - + + ${elasticsearch-java} - + + - + + - + + commons-logging - + + commons-logging - + + - + + - + + - + + - + + - + + com.fasterxml.jackson.core - + + jackson-core - + + - + + - + + com.fasterxml.jackson.core - + + jackson-databind - + + - + + - + + - + + - + + org.apache.geronimo.specs - + + geronimo-jcdi_2.0_spec - + + 1.3 - + + test - + + - + + - + + javax.interceptor - + + javax.interceptor-api - + + 1.2.1 - + + test - + + - + + - + + jakarta.enterprise - + + jakarta.enterprise.cdi-api - + + ${cdi} - + + provided - + + true - + + - + + - + + jakarta.annotation - + + jakarta.annotation-api - + + ${jakarta-annotation-api} - + + test - + + - + + - + + org.apache.openwebbeans - + + openwebbeans-se - + + jakarta - + + ${webbeans} - + + test - + + - + + - + + org.apache.openwebbeans - + + openwebbeans-spi - + + jakarta - + + ${webbeans} - + + test - + + - + + - + + org.apache.openwebbeans - + + openwebbeans-impl - + + jakarta - + + ${webbeans} - + + test - + + - + + - + + - + + org.springframework - + + spring-test - + + test - + + - + + - + + ch.qos.logback - + + logback-classic - + + - + + - + + - + + - + + org.slf4j - + + log4j-over-slf4j - + + ${slf4j} - + + test - + + - + + - + + org.apache.logging.log4j - + + log4j-core - + + ${log4j} - + + test - + + - + + - + + org.apache.logging.log4j - + + log4j-to-slf4j - + + ${log4j} - + + test - + + - + + - + + io.projectreactor.tools - + + blockhound-junit-platform - + + ${blockhound-junit} - + + test - + + - + + - + + - + + org.projectlombok - + + lombok - + + - + + 999999 - + + test - + + - + + - + + org.skyscreamer - + + jsonassert - + + 1.5.0 - + + test - + + - + + - + + com.github.tomakehurst - + + wiremock-jre8 - + + 2.32.0 - + + test - + + - + + - + + - + + commons-logging - + + commons-logging - + + - + + - + + org.ow2.asm - + + asm - + + - + + - + + - + + - + + io.specto - + + hoverfly-java-junit5 - + + 0.14.1 - + + test - + + - + + - + + - + + org.apache.xbean - + + xbean-asm5-shaded - + + 4.5 - + + test - + + - + + - + + javax.servlet - + + javax.servlet-api - + + 3.1.0 - + + test - + + - + + - + + org.mockito - + + mockito-junit-jupiter - + + ${mockito} - + + test - + + - + + - + + org.testcontainers - + + elasticsearch - + + ${testcontainers} - + + test - + + - + + - + + - + + - + + - + + src/main/resources - + + true - + + - + + **/versions.properties - + + - + + - + + - + + src/main/resources - + + false - + + - + + **/versions.properties - + + - + + - + + - + + - + + - + + org.apache.maven.plugins - + + maven-surefire-plugin - + + - + + true - + + false - + + - + + **/*Tests.java - + + **/*Test.java - + + - + + - + + false - + + - + + - + + - + + - + + - + + default-test - + + ${mvn.unit-test.goal} - + + - + + test - + + - + + - + + integration-test - + + - + + - + + - + + - + + integration-test-elasticsearch - + + ${mvn.integration-test-elasticsearch.goal} - + + - + + test - + + - + + - + + integration-test - + + - + + elasticsearch - + + - + + - + + - + + - + + - + + integration-test-opensearch - + + ${mvn.integration-test-opensearch.goal} - + + - + + test - + + - + + - + + integration-test - + + - + + opensearch - + + - + + - + + - + + - + + - + + - + + org.pitest - + + pitest-maven - + + 1.7.5 - + + - + + - + + org.pitest - + + pitest-junit5-plugin - + + 0.12 - + + - + + - + + - + + - + + -XX:+AllowRedefinitionToAddDeleteMethods - + + - + + integration-test - + + - + + org.springframework.data.elasticsearch.core.geo.* - + + - + + toString - + + - + + - + + - + + org.apache.maven.plugins - + + maven-assembly-plugin - + + - + + - + + org.asciidoctor - + + asciidoctor-maven-plugin - + + - + + - + + - + + - + + - + + ci - + + - + + - + + - + + org.apache.maven.plugins - + + maven-checkstyle-plugin - + + - + + - + + - + + - + + - + + - + + - + + - + + **/* - + + .git/**/*,target/**/*,**/target/**/*,.idea/**/*,**/spring.schemas,**/*.svg,mvnw,mvnw.cmd,**/*.policy - + + ./ - + + - + + - + + - + + - + + - + + - + + jdk13+ - + + - + + - + + [13,) - + + - + + - + + - + + - + + org.apache.maven.plugins - + + maven-surefire-plugin - + + - + + -XX:+AllowRedefinitionToAddDeleteMethods - + + - + + - + + - + + - + + - + + - + + - + + - - spring-libs-milestone - - https://repo.spring.io/libs-milestone - + + + spring-libs-snapshot + + + https://repo.spring.io/libs-snapshot + + - + + - + + local-maven-repo - + + file:///${project.basedir}/src/test/resources/local-maven-repo - + + - + + - + + - + + - + + spring-plugins-release - + + https://repo.spring.io/plugins-release - + + - - + + + + From 1eaebdec331762b23819155831cfcb68efb19dcb Mon Sep 17 00:00:00 2001 From: Christoph Strobl Date: Fri, 13 May 2022 13:45:30 +0200 Subject: [PATCH 009/516] Fix pom.xml formatting. This commit reverts formatting changes introduced via 7cb46b6ab0f0fdaea323709d0a512599ccf743ad. See #2122 --- pom.xml | 1932 +++++++++++++++---------------------------------------- 1 file changed, 521 insertions(+), 1411 deletions(-) diff --git a/pom.xml b/pom.xml index f8d78075e..1eec82e2c 100644 --- a/pom.xml +++ b/pom.xml @@ -1,739 +1,285 @@ - - - 4.0.0 - - - org.springframework.data - - - spring-data-elasticsearch - - - 5.0.0-SNAPSHOT - - - - - - org.springframework.data.build - - - spring-data-parent - - - 3.0.0-SNAPSHOT - - - - - - Spring Data Elasticsearch - - - Spring Data Implementation for Elasticsearch - - - https://github.com/spring-projects/spring-data-elasticsearch - - - - - - - - - 7.17.3 - - - - - - 7.17.3 - - - 2.17.1 - - - 4.1.65.Final - - - 3.0.0-SNAPSHOT - - - 1.16.2 - - - 1.0.6.RELEASE - - - spring.data.elasticsearch - - - + 7.17.3 + + 7.17.3 + 2.17.1 + 4.1.65.Final + 3.0.0-SNAPSHOT + 1.16.2 + 1.0.6.RELEASE + spring.data.elasticsearch + + - - - test - - - integration-test - - - none - - - - - - - - - - - - biomedcentral - - - BioMed Central Development Team - - - +0 - - - - - - - - - cstrobl - - - Christoph Strobl - - - cstrobl at pivotal.io - - - Pivotal - - - https://www.pivotal.io - - - - - - Developer - - - - - - +1 - - - - - - - - - mpaluch - - - Mark Paluch - - - mpaluch at pivotal.io - - - Pivotal - - - https://www.pivotal.io - - - - - - Developer - - - - - - +1 - - - - - - - - - - - - https://github.com/spring-projects/spring-data-elasticsearch - - - scm:git:git://github.com/spring-projects/spring-data-elasticsearch.git - - - scm:git:ssh://git@github.com/spring-projects/spring-data-elasticsearch.git + test + integration-test + none + + + + + biomedcentral + BioMed Central Development Team + +0 + + + cstrobl + Christoph Strobl + cstrobl at pivotal.io + Pivotal + https://www.pivotal.io + + Developer + + +1 + + + mpaluch + Mark Paluch + mpaluch at pivotal.io + Pivotal + https://www.pivotal.io + + Developer + + +1 + + + + + https://github.com/spring-projects/spring-data-elasticsearch + scm:git:git://github.com/spring-projects/spring-data-elasticsearch.git + scm:git:ssh://git@github.com/spring-projects/spring-data-elasticsearch.git - - - - - - - - - Bamboo - - - https://build.spring.io/browse/SPRINGDATAES - - - - - - - - - GitHub - - - https://github.com/spring-projects/spring-data-elasticsearch/issues - - - - - - - - - - - - - - - io.netty - - - netty-bom - - - ${netty} - - - pom - - - import - - - - - - - - - - - - - - - - - - - - - org.springframework - - - spring-context - - - - - - - - - org.springframework - - - spring-tx - - - - - - - - - - - - org.springframework.data - - - spring-data-commons - - - ${springdata.commons} - - - - - - - - - - - - org.springframework - - - spring-webflux - - - true - - - - - - - - - io.projectreactor.netty - - - reactor-netty-http - - - true - - - - - - - - - io.projectreactor - - - reactor-test - - - test - - - - - - - - - - - - org.elasticsearch.client - - - elasticsearch-rest-high-level-client - - - ${elasticsearch-rhlc} - - - - - - - - - commons-logging - - - commons-logging - - - - - - - - - - - - - - - - - - co.elastic.clients - - - elasticsearch-java - - - ${elasticsearch-java} - - - - - - - - - commons-logging - - - commons-logging - - - - - - - - - - - - - - - org.elasticsearch.client - - - elasticsearch-rest-client - - - - - - ${elasticsearch-java} - - - - - - - - - commons-logging - - - commons-logging - - - - - - - - - - - - - - - - - - com.fasterxml.jackson.core - - - jackson-core - - - - - - - - - com.fasterxml.jackson.core - - - jackson-databind - - - - - - - - - - - - - - - org.apache.geronimo.specs - - - geronimo-jcdi_2.0_spec - - - 1.3 - - - test - - - - - - - - - javax.interceptor - - - javax.interceptor-api - - - 1.2.1 - - - test - - - - - - - - - jakarta.enterprise - - - jakarta.enterprise.cdi-api - - - ${cdi} - - - provided - - - true - - - - - - - - - jakarta.annotation - - - jakarta.annotation-api - - - ${jakarta-annotation-api} - - - test - - - - - - - - - org.apache.openwebbeans - - - openwebbeans-se - - - jakarta - - - ${webbeans} - - - test - - - - - - - - - org.apache.openwebbeans - - - openwebbeans-spi - - - jakarta - - - ${webbeans} - - - test - - - - - - - - - org.apache.openwebbeans - - - openwebbeans-impl - - - jakarta - - - ${webbeans} - - - test - - - - - - - - - - - - org.springframework - - - spring-test - - - test - - - - - - - - - ch.qos.logback - - - logback-classic - - - - - - - - - - - - - - - org.slf4j - - - log4j-over-slf4j - - - ${slf4j} - - - test - - - - - - - - - org.apache.logging.log4j - - - log4j-core - - - ${log4j} - - - test - - - - - - - - - org.apache.logging.log4j - - - log4j-to-slf4j - - - ${log4j} - - - test - - - - - - - - - io.projectreactor.tools - - - blockhound-junit-platform - - - ${blockhound-junit} - - - test - - - - - - + + org.springframework + spring-context + + + + org.springframework + spring-tx + + + + + org.springframework.data + spring-data-commons + ${springdata.commons} + + + + + org.springframework + spring-webflux + true + + + + io.projectreactor.netty + reactor-netty-http + true + + + + io.projectreactor + reactor-test + test + + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${elasticsearch-rhlc} + + + commons-logging + commons-logging + + + + + + + co.elastic.clients + elasticsearch-java + ${elasticsearch-java} + + + commons-logging + commons-logging + + + + + org.elasticsearch.client + elasticsearch-rest-client + ${elasticsearch-java} + + + commons-logging + commons-logging + + + + + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + + + + org.apache.geronimo.specs + geronimo-jcdi_2.0_spec + 1.3 + test + + + + javax.interceptor + javax.interceptor-api + 1.2.1 + test + + + + jakarta.enterprise + jakarta.enterprise.cdi-api + ${cdi} + provided + true + + + + jakarta.annotation + jakarta.annotation-api + ${jakarta-annotation-api} + test + + + + org.apache.openwebbeans + openwebbeans-se + jakarta + ${webbeans} + test + + + + org.apache.openwebbeans + openwebbeans-spi + jakarta + ${webbeans} + test + + + + org.apache.openwebbeans + openwebbeans-impl + jakarta + ${webbeans} + test + + + + + org.springframework + spring-test + test + + + ch.qos.logback + logback-classic + + + + + + org.slf4j + log4j-over-slf4j + ${slf4j} + test + + + org.apache.logging.log4j + log4j-core + ${log4j} + test + + + org.apache.logging.log4j + log4j-to-slf4j + ${log4j} + test + + + + io.projectreactor.tools + blockhound-junit-platform + ${blockhound-junit} + test + + + + - - - - - - org.projectlombok - - - lombok - - - - - - 999999 - - - test - - - - - - - - - org.skyscreamer - - - jsonassert - - - 1.5.0 - - - test - - - - - - - - - com.github.tomakehurst - - - wiremock-jre8 - - - 2.32.0 - - - test - - - - - - - - - - - - commons-logging - - - commons-logging - - - - - - - - - org.ow2.asm - - - asm - - - - - - - - - - - - - - - io.specto - - - hoverfly-java-junit5 - - - 0.14.1 - - - test - - - - - - - - - - - - org.apache.xbean - - - xbean-asm5-shaded - - - 4.5 - - - test - - - - - - - - - javax.servlet - - - javax.servlet-api - - - 3.1.0 - - - test - - - - - - - - - org.mockito - - - mockito-junit-jupiter - - - ${mockito} - - - test - - - - - - - - - org.testcontainers - - - elasticsearch - - - ${testcontainers} - - - test - - - - - - - - - - - - - - - - - - src/main/resources - - - true - - - - - - **/versions.properties - - - - - - - - - - - - src/main/resources - - - false - - - - - - **/versions.properties - - - - - - - - - - - - - - - - - - org.apache.maven.plugins - - - maven-surefire-plugin - - - - - - true - - - false - - - - - - **/*Tests.java - - - **/*Test.java - - - - - - - - - false - - - - - - - - - - - - - - - - - - default-test - - - ${mvn.unit-test.goal} - - - - - - test - - - - - - - - - integration-test - - - - - - - - - - - - - - - integration-test-elasticsearch - - - ${mvn.integration-test-elasticsearch.goal} - - - - - - test - - - - - - - - - integration-test - - - - - - elasticsearch - - - - - - - - - - - - - - - - - - integration-test-opensearch - - - ${mvn.integration-test-opensearch.goal} - - - - - - test - - - - - - - - - integration-test - - - - - - opensearch - - - - - - - - - - - - - - - - - - - - - org.pitest - - - pitest-maven - - - 1.7.5 - - - - - - - - - org.pitest - - - pitest-junit5-plugin - - - 0.12 - - - - - - - - - - - - - - - -XX:+AllowRedefinitionToAddDeleteMethods - - - - - - integration-test - - - - - - org.springframework.data.elasticsearch.core.geo.* - - - - - - toString - - - - - - - - - - - - org.apache.maven.plugins - - - maven-assembly-plugin - - - - - - - - - org.asciidoctor - - - asciidoctor-maven-plugin - - - - - - - - - - - - - - - - - - ci - - - - - - - - - - - - org.apache.maven.plugins - - - maven-checkstyle-plugin - - - - - - - - - - - - - - - - - - - - - - - - - - - **/* - - - + + org.projectlombok + lombok + + 999999 + test + + + + org.skyscreamer + jsonassert + 1.5.0 + test + + + + com.github.tomakehurst + wiremock-jre8 + 2.32.0 + test + + + + commons-logging + commons-logging + + + org.ow2.asm + asm + + + + + + io.specto + hoverfly-java-junit5 + 0.14.1 + test + + + + + org.apache.xbean + xbean-asm5-shaded + 4.5 + test + + + + javax.servlet + javax.servlet-api + 3.1.0 + test + + + + org.mockito + mockito-junit-jupiter + ${mockito} + test + + + + org.testcontainers + elasticsearch + ${testcontainers} + test + + + + + + + + src/main/resources + true + + **/versions.properties + + + + src/main/resources + false + + **/versions.properties + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + true + false + + **/*Tests.java + **/*Test.java + + + false + + + + + + default-test + ${mvn.unit-test.goal} + + test + + + integration-test + + + + + integration-test-elasticsearch + ${mvn.integration-test-elasticsearch.goal} + + test + + + integration-test + + elasticsearch + + + + + + integration-test-opensearch + ${mvn.integration-test-opensearch.goal} + + test + + + integration-test + + opensearch + + + + + + + org.pitest + pitest-maven + 1.7.5 + + + org.pitest + pitest-junit5-plugin + 0.12 + + + + + -XX:+AllowRedefinitionToAddDeleteMethods + + integration-test + + org.springframework.data.elasticsearch.core.geo.* + + toString + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.asciidoctor + asciidoctor-maven-plugin + + + + + + + ci + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + + + + + + + + + **/* + .git/**/*,target/**/*,**/target/**/*,.idea/**/*,**/spring.schemas,**/*.svg,mvnw,mvnw.cmd,**/*.policy - - - ./ - - - - - - - - - - - - - - - - - - - - - jdk13+ - - - - - - - - - [13,) - - - - - - - - - - - - - - - org.apache.maven.plugins - - - maven-surefire-plugin - - - - - - -XX:+AllowRedefinitionToAddDeleteMethods - - - - - - - - - - - - - - - - - - - - - - - - - - - spring-libs-snapshot - - - https://repo.spring.io/libs-snapshot - - - - - - - - - local-maven-repo - - - file:///${project.basedir}/src/test/resources/local-maven-repo - - - - - - - - - - - - - - - spring-plugins-release - - - https://repo.spring.io/plugins-release - - - - - - - + ./ + + + + + + + + jdk13+ + + + [13,) + + + + + org.apache.maven.plugins + maven-surefire-plugin + + -XX:+AllowRedefinitionToAddDeleteMethods + + + + + + + + + + spring-libs-snapshot + https://repo.spring.io/libs-snapshot + + + + local-maven-repo + file:///${project.basedir}/src/test/resources/local-maven-repo + + + + + + + spring-plugins-release + https://repo.spring.io/plugins-release + + From c826adb152fb1b00b49f9a8b69db8f969b9ba486 Mon Sep 17 00:00:00 2001 From: panzhenchao <52995949+panzhenchao@users.noreply.github.com> Date: Fri, 27 May 2022 22:40:38 +0800 Subject: [PATCH 010/516] Fix incorrect argument check asserts. Original Pull Request #2169 Closes #2170 --- .../elasticsearch/core/convert/ElasticsearchDateConverter.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/springframework/data/elasticsearch/core/convert/ElasticsearchDateConverter.java b/src/main/java/org/springframework/data/elasticsearch/core/convert/ElasticsearchDateConverter.java index f1050dacd..9db656197 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/convert/ElasticsearchDateConverter.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/convert/ElasticsearchDateConverter.java @@ -66,7 +66,6 @@ public static ElasticsearchDateConverter of(DateFormat dateFormat) { */ public static ElasticsearchDateConverter of(String pattern) { - Assert.notNull(pattern, "pattern must not be null"); Assert.hasText(pattern, "pattern must not be empty"); String[] subPatterns = pattern.split("\\|\\|"); @@ -86,7 +85,7 @@ private ElasticsearchDateConverter(DateFormatter dateFormatter) { */ public String format(TemporalAccessor accessor) { - Assert.notNull("accessor", "accessor must not be null"); + Assert.notNull(accessor, "accessor must not be null"); if (accessor instanceof Instant) { Instant instant = (Instant) accessor; From ac64a6a73343d7fa9be0651ad85ef13dcf85941b Mon Sep 17 00:00:00 2001 From: Peter-Josef Meisch Date: Tue, 31 May 2022 22:13:51 +0200 Subject: [PATCH 011/516] Use Elasticsearch 8.2.2. Original Pull Request #2174 Closes #2158 --- pom.xml | 4 +- .../elasticsearch/client/RestClients.java | 3 +- .../client/elc/ElasticsearchClients.java | 15 +++++ .../client/elc/ElasticsearchTemplate.java | 6 +- .../elc/ReactiveElasticsearchTemplate.java | 6 +- .../client/elc/RequestConverter.java | 21 +++--- .../client/elc/ResponseConverter.java | 16 ++--- .../elasticsearch/client/elc/TypeUtils.java | 17 +++-- .../client/reactive/WebClientProvider.java | 10 +++ .../core/AbstractElasticsearchTemplate.java | 10 +-- ...AbstractReactiveElasticsearchTemplate.java | 6 +- .../core/ElasticsearchRestTemplate.java | 6 +- .../core/ReactiveElasticsearchTemplate.java | 6 +- .../SimpleElasticsearchPersistentEntity.java | 23 +++---- .../NewElasticsearchClientDevelopment.java | 10 ++- .../elasticsearch/client/elc/DevTests.java | 65 +++++++++++++++---- .../ElasticsearchELCIntegrationTests.java | 2 +- .../core/ElasticsearchIntegrationTests.java | 1 + ...ctiveElasticsearchELCIntegrationTests.java | 6 ++ ...ReactiveElasticsearchIntegrationTests.java | 5 +- .../RuntimeFieldsELCIntegrationTests.java | 5 ++ .../core/RuntimeFieldsIntegrationTests.java | 5 +- .../core/geo/GeoELCIntegrationTests.java | 5 ++ .../core/geo/GeoIntegrationTests.java | 16 ++++- .../core/geo/GeoJsonELCIntegrationTests.java | 17 ++++- .../geo/GeoJsonERHLCIntegrationTests.java | 18 ++++- .../elasticsearch/core/geo/GeoJsonEntity.java | 2 +- .../core/geo/GeoJsonIntegrationTests.java | 47 ++++++-------- .../IndexTemplateELCIntegrationTests.java | 7 +- .../IndexTemplateERHLCIntegrationTests.java | 3 +- .../index/IndexTemplateIntegrationTests.java | 14 +++- .../MappingBuilderELCIntegrationTests.java | 2 +- .../IndexOperationsELCIntegrationTests.java | 4 +- .../IndexOperationsIntegrationTests.java | 2 + ...iveIndexOperationsELCIntegrationTests.java | 6 +- ...activeIndexOperationsIntegrationTests.java | 44 ++++++------- ...pleElasticsearchPersistentEntityTests.java | 27 ++++++-- .../CompletionELCIntegrationTests.java | 2 +- .../ReactiveSuggestELCIntegrationTests.java | 2 +- ...MethodRepositoryERHLCIntegrationTests.java | 5 ++ ...ustomMethodRepositoryIntegrationTests.java | 12 +++- .../testcontainers-elasticsearch.properties | 6 +- 42 files changed, 337 insertions(+), 152 deletions(-) diff --git a/pom.xml b/pom.xml index 1eec82e2c..c4fe1db23 100644 --- a/pom.xml +++ b/pom.xml @@ -21,8 +21,8 @@ 7.17.3 - 7.17.3 - 2.17.1 + 8.2.2 + 2.17.2 4.1.65.Final 3.0.0-SNAPSHOT 1.16.2 diff --git a/src/main/java/org/springframework/data/elasticsearch/client/RestClients.java b/src/main/java/org/springframework/data/elasticsearch/client/RestClients.java index 866822943..8d059e5e3 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/RestClients.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/RestClients.java @@ -43,6 +43,7 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.RestHighLevelClientBuilder; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.util.Assert; @@ -132,7 +133,7 @@ public static ElasticsearchRestClient create(ClientConfiguration clientConfigura return clientBuilder; }); - RestHighLevelClient client = new RestHighLevelClient(builder); + RestHighLevelClient client = new RestHighLevelClientBuilder(builder.build()).setApiCompatibilityMode(true).build(); return () -> client; } diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchClients.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchClients.java index cc845b111..53d434fb0 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchClients.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchClients.java @@ -19,6 +19,7 @@ import co.elastic.clients.json.jackson.JacksonJsonpMapper; import co.elastic.clients.transport.ElasticsearchTransport; import co.elastic.clients.transport.TransportOptions; +import co.elastic.clients.transport.Version; import co.elastic.clients.transport.rest_client.RestClientOptions; import co.elastic.clients.transport.rest_client.RestClientTransport; @@ -41,8 +42,10 @@ import org.apache.http.HttpResponseInterceptor; import org.apache.http.client.config.RequestConfig; import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.entity.ContentType; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.message.BasicHeader; +import org.apache.http.message.BasicNameValuePair; import org.apache.http.protocol.HttpContext; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; @@ -242,6 +245,18 @@ private static ElasticsearchTransport getElasticsearchTransport(RestClient restC TransportOptions.Builder transportOptionsBuilder = transportOptions != null ? transportOptions.toBuilder() : new RestClientOptions(RequestOptions.DEFAULT).toBuilder(); + + // need to add the compatibility header, this is only done automatically when not passing in custom options. + // code copied from RestClientTransport as it is not available outside the package + ContentType jsonContentType = null; + if (Version.VERSION == null) { + jsonContentType = ContentType.APPLICATION_JSON; + } else { + jsonContentType = ContentType.create("application/vnd.elasticsearch+json", + new BasicNameValuePair("compatible-with", String.valueOf(Version.VERSION.major()))); + } + transportOptionsBuilder.addHeader("Accept", jsonContentType.toString()); + TransportOptions transportOptionsWithHeader = transportOptionsBuilder .addHeader(X_SPRING_DATA_ELASTICSEARCH_CLIENT, clientType).build(); diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java index 8f1111354..99fc7a4ba 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java @@ -273,18 +273,18 @@ public List doBulkOperation(List queries, BulkOptio // endregion @Override - protected String getClusterVersion() { + public String getClusterVersion() { return execute(client -> client.info().version().number()); } @Override - protected String getVendor() { + public String getVendor() { return "Elasticsearch"; } @Override - protected String getRuntimeLibraryVersion() { + public String getRuntimeLibraryVersion() { return Version.VERSION.toString(); } diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java index 0c9d41520..c7a3b0702 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java @@ -413,17 +413,17 @@ public Flux> aggregate(Query query, Class e // endregion @Override - protected Mono getVendor() { + public Mono getVendor() { return Mono.just("Elasticsearch"); } @Override - protected Mono getRuntimeLibraryVersion() { + public Mono getRuntimeLibraryVersion() { return Mono.just(Version.VERSION != null ? Version.VERSION.toString() : "null"); } @Override - protected Mono getClusterVersion() { + public Mono getClusterVersion() { return Mono.from(execute(ReactiveElasticsearchClient::info)).map(infoResponse -> infoResponse.version().number()); } diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/RequestConverter.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/RequestConverter.java index fec2d1ac4..78df81450 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/RequestConverter.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/RequestConverter.java @@ -820,7 +820,7 @@ public co.elastic.clients.elasticsearch.core.ReindexRequest reindex(ReindexReque .refresh(reindexRequest.getRefresh()) // .requireAlias(reindexRequest.getRequireAlias()) // .requestsPerSecond(reindexRequest.getRequestsPerSecond()) // - .slices(reindexRequest.getSlices()); + .slices(slices(reindexRequest.getSlices())); return builder.build(); } @@ -963,24 +963,24 @@ public UpdateByQueryRequest documentUpdateByQueryRequest(UpdateQuery updateQuery .pipeline(updateQuery.getPipeline()) // .requestsPerSecond( updateQuery.getRequestsPerSecond() != null ? updateQuery.getRequestsPerSecond().longValue() : null) // - .slices(updateQuery.getSlices() != null ? Long.valueOf(updateQuery.getSlices()) : null) // - ; + .slices(slices(updateQuery.getSlices() != null ? Long.valueOf(updateQuery.getSlices()) : null)); if (updateQuery.getAbortOnVersionConflict() != null) { ub.conflicts(updateQuery.getAbortOnVersionConflict() ? Conflicts.Abort : Conflicts.Proceed); } - if (updateQuery.getBatchSize() != null) { - ub.size(Long.valueOf(updateQuery.getBatchSize())); - } - if (updateQuery.getQuery() != null) { Query queryQuery = updateQuery.getQuery(); + + if (updateQuery.getBatchSize() != null) { + ((BaseQuery) queryQuery).setMaxResults(updateQuery.getBatchSize()); + } ub.query(getQuery(queryQuery, null)); // no indicesOptions available like in old client ub.scroll(time(queryQuery.getScrollTime())); + } // no maxRetries available like in old client @@ -1164,11 +1164,11 @@ private void prepareSearchRequest(Query query, @Nullable Class clazz, Ind if (!query.getRuntimeFields().isEmpty()) { - Map runtimeMappings = new HashMap<>(); + Map> runtimeMappings = new HashMap<>(); query.getRuntimeFields().forEach(runtimeField -> { - runtimeMappings.put(runtimeField.getName(), RuntimeField.of(rt -> rt // + runtimeMappings.put(runtimeField.getName(), Collections.singletonList(RuntimeField.of(rt -> rt // .type(RuntimeFieldType._DESERIALIZER.parse(runtimeField.getType())) // - .script(s -> s.inline(is -> is.source(runtimeField.getScript()))))); + .script(s -> s.inline(is -> is.source(runtimeField.getScript())))))); }); builder.runtimeMappings(runtimeMappings); } @@ -1328,6 +1328,7 @@ private co.elastic.clients.elasticsearch._types.query_dsl.Query getQuery(@Nullab } else { throw new IllegalArgumentException("unhandled Query implementation " + query.getClass().getName()); } + return esQuery; } diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ResponseConverter.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ResponseConverter.java index ab43811d4..67341b449 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ResponseConverter.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ResponseConverter.java @@ -90,7 +90,7 @@ public ClusterHealth clusterHealth(HealthResponse healthResponse) { .withNumberOfPendingTasks(healthResponse.numberOfPendingTasks()) // .withRelocatingShards(healthResponse.relocatingShards()) // .withStatus(healthResponse.status().toString()) // - .withTaskMaxWaitingTimeMillis(Long.parseLong(healthResponse.taskMaxWaitingInQueueMillis())) // + .withTaskMaxWaitingTimeMillis(healthResponse.taskMaxWaitingInQueueMillis().toEpochMilli()) // .withTimedOut(healthResponse.timedOut()) // .withUnassignedShards(healthResponse.unassignedShards()) // .build(); // @@ -144,11 +144,11 @@ public Document indicesGetMapping(GetMappingResponse getMappingResponse, IndexCo if (indexMappingRecord == null) { if (mappings.size() != 1) { - LOGGER.warn("no mapping returned for index {}", indexCoordinates.getIndexName()); - return Document.create(); - } - String index = mappings.keySet().iterator().next(); - indexMappingRecord = mappings.get(index); + LOGGER.warn("no mapping returned for index {}", indexCoordinates.getIndexName()); + return Document.create(); + } + String index = mappings.keySet().iterator().next(); + indexMappingRecord = mappings.get(index); } return Document.parse(toJson(indexMappingRecord.mappings(), jsonpMapper)); @@ -277,9 +277,9 @@ public ReindexResponse reindexResponse(co.elastic.clients.elasticsearch.core.Rei .withNoops(reindexResponse.noops()) // .withBulkRetries(reindexResponse.retries().bulk()) // .withSearchRetries(reindexResponse.retries().search()) // - .withThrottledMillis(Long.parseLong(reindexResponse.throttledMillis())) // + .withThrottledMillis(reindexResponse.throttledMillis().toEpochMilli()) // .withRequestsPerSecond(reindexResponse.requestsPerSecond()) // - .withThrottledUntilMillis(Long.parseLong(reindexResponse.throttledUntilMillis())).withFailures(failures) // + .withThrottledUntilMillis(reindexResponse.throttledUntilMillis().toEpochMilli()).withFailures(failures) // .build(); } diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/TypeUtils.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/TypeUtils.java index f68533a82..9e96d98be 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/TypeUtils.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/TypeUtils.java @@ -18,7 +18,6 @@ import co.elastic.clients.elasticsearch._types.*; import co.elastic.clients.elasticsearch._types.mapping.FieldType; import co.elastic.clients.elasticsearch.core.search.BoundaryScanner; -import co.elastic.clients.elasticsearch.core.search.BuiltinHighlighterType; import co.elastic.clients.elasticsearch.core.search.HighlighterEncoder; import co.elastic.clients.elasticsearch.core.search.HighlighterFragmenter; import co.elastic.clients.elasticsearch.core.search.HighlighterOrder; @@ -170,11 +169,11 @@ static HighlighterType highlighterType(@Nullable String value) { if (value != null) { switch (value.toLowerCase()) { case "unified": - return HighlighterType.of(b -> b.builtin(BuiltinHighlighterType.Unified)); + return HighlighterType.Unified; case "plain": - return HighlighterType.of(b -> b.builtin(BuiltinHighlighterType.Plain)); + return HighlighterType.Plain; case "fvh": - return HighlighterType.of(b -> b.builtin(BuiltinHighlighterType.FastVector)); + return HighlighterType.FastVector; default: return null; } @@ -308,6 +307,16 @@ static SearchType searchType(@Nullable Query.SearchType searchType) { return null; } + @Nullable + static Slices slices(@Nullable Long count) { + + if (count == null) { + return null; + } + + return Slices.of(s -> s.value(Math.toIntExact(count))); + } + @Nullable static SortMode sortMode(Order.Mode mode) { diff --git a/src/main/java/org/springframework/data/elasticsearch/client/reactive/WebClientProvider.java b/src/main/java/org/springframework/data/elasticsearch/client/reactive/WebClientProvider.java index 44ea51ed5..01da85fd2 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/reactive/WebClientProvider.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/reactive/WebClientProvider.java @@ -257,6 +257,16 @@ static WebClientProvider getWebClientProvider(ClientConfiguration clientConfigur if (suppliedHeaders != null && suppliedHeaders != HttpHeaders.EMPTY) { httpHeaders.addAll(suppliedHeaders); } + + // this WebClientProvider is built with ES 7 and not used on 8 anymore + httpHeaders.add("Accept", "application/vnd.elasticsearch+json;compatible-with=7"); + + var contentTypeHeader = "Content-Type"; + if (httpHeaders.containsKey(contentTypeHeader)) { + httpHeaders.remove(contentTypeHeader); + } + httpHeaders.add(contentTypeHeader, "application/vnd.elasticsearch+json;compatible-with=7"); + })); return provider; diff --git a/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java index 6980e41dd..07ace4c68 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java @@ -145,8 +145,8 @@ public void setApplicationContext(ApplicationContext applicationContext) throws /** * Set the {@link EntityCallbacks} instance to use when invoking {@link EntityCallbacks callbacks} like the - * {@link org.springframework.data.elasticsearch.core.event.BeforeConvertCallback}. - * Overrides potentially existing {@link EntityCallbacks}. + * {@link org.springframework.data.elasticsearch.core.event.BeforeConvertCallback}. Overrides potentially existing + * {@link EntityCallbacks}. * * @param entityCallbacks must not be {@literal null}. * @throws IllegalArgumentException if the given instance is {@literal null}. @@ -588,19 +588,19 @@ protected SearchDocumentResponse.EntityCreator getEntityCreator(ReadDocum * @return the version as string if it can be retrieved */ @Nullable - abstract protected String getClusterVersion(); + public abstract String getClusterVersion(); /** * @return the vendor name of the used cluster and client library * @since 4.3 */ - abstract protected String getVendor(); + public abstract String getVendor(); /** * @return the version of the used client runtime library. * @since 4.3 */ - abstract protected String getRuntimeLibraryVersion(); + public abstract String getRuntimeLibraryVersion(); // endregion diff --git a/src/main/java/org/springframework/data/elasticsearch/core/AbstractReactiveElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/core/AbstractReactiveElasticsearchTemplate.java index bd2974ffd..83b834d98 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/AbstractReactiveElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/AbstractReactiveElasticsearchTemplate.java @@ -611,15 +611,15 @@ public ElasticsearchPersistentEntity getPersistentEntityFor(@Nullable Class getVendor(); + public abstract Mono getVendor(); /** * @return the version of the used client runtime library. * @since 4.3 */ - abstract protected Mono getRuntimeLibraryVersion(); + public abstract Mono getRuntimeLibraryVersion(); - abstract protected Mono getClusterVersion(); + public abstract Mono getClusterVersion(); /** * Value class to capture client independent information from a response to an index request. diff --git a/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchRestTemplate.java b/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchRestTemplate.java index 2eb74cf44..ecce20f0d 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchRestTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchRestTemplate.java @@ -607,7 +607,7 @@ private RuntimeException translateException(Exception exception) { // region helper methods @Override - protected String getClusterVersion() { + public String getClusterVersion() { try { return execute(client -> client.info(RequestOptions.DEFAULT)).getVersion().getNumber(); } catch (Exception ignored) {} @@ -629,12 +629,12 @@ public Query idsQuery(List ids) { } @Override - protected String getVendor() { + public String getVendor() { return "Elasticsearch"; } @Override - protected String getRuntimeLibraryVersion() { + public String getRuntimeLibraryVersion() { return Version.CURRENT.toString(); } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchTemplate.java index ae3a86c1d..e29864a65 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchTemplate.java @@ -676,7 +676,7 @@ protected SearchRequest prepareSearchRequest(SearchRequest request, boolean useS // region Helper methods @Override - protected Mono getClusterVersion() { + public Mono getClusterVersion() { try { return Mono.from(execute(ReactiveElasticsearchClient::info)) .map(mainResponse -> mainResponse.getVersion().toString()); @@ -689,7 +689,7 @@ protected Mono getClusterVersion() { * @since 4.3 */ @Override - protected Mono getVendor() { + public Mono getVendor() { return Mono.just("Elasticsearch"); } @@ -698,7 +698,7 @@ protected Mono getVendor() { * @since 4.3 */ @Override - protected Mono getRuntimeLibraryVersion() { + public Mono getRuntimeLibraryVersion() { return Mono.just(Version.CURRENT.toString()); } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentEntity.java b/src/main/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentEntity.java index e63f24ded..4028efb74 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentEntity.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentEntity.java @@ -510,36 +510,37 @@ Settings toSettings() { return new Settings(); } - Settings settings = new Settings() // - .append("index.number_of_shards", String.valueOf(shards)) - .append("index.number_of_replicas", String.valueOf(replicas)); + var index = new Settings() // + .append("number_of_shards", String.valueOf(shards)) // + .append("number_of_replicas", String.valueOf(replicas)); if (refreshIntervall != null) { - settings.append("index.refresh_interval", refreshIntervall); + index.append("refresh_interval", refreshIntervall); } if (indexStoreType != null) { - settings.append("index.store.type", indexStoreType); + index.append("store", new Settings().append("type", indexStoreType)); } if (sortFields != null && sortFields.length > 0) { - settings.append("index.sort.field", sortFields); + var sort = new Settings().append("field", sortFields); if (sortOrders != null && sortOrders.length > 0) { - settings.append("index.sort.order", sortOrders); + sort.append("order", sortOrders); } if (sortModes != null && sortModes.length > 0) { - settings.append("index.sort.mode", sortModes); + sort.append("mode", sortModes); } if (sortMissingValues != null && sortMissingValues.length > 0) { - settings.append("index.sort.missing", sortMissingValues); + sort.append("missing", sortMissingValues); } - } - return settings; // + index.append("sort", sort); + } + return new Settings().append("index", index); // } } diff --git a/src/test/java/org/springframework/data/elasticsearch/NewElasticsearchClientDevelopment.java b/src/test/java/org/springframework/data/elasticsearch/NewElasticsearchClientDevelopment.java index 5c098ccd9..7e2d229d3 100644 --- a/src/test/java/org/springframework/data/elasticsearch/NewElasticsearchClientDevelopment.java +++ b/src/test/java/org/springframework/data/elasticsearch/NewElasticsearchClientDevelopment.java @@ -16,19 +16,17 @@ package org.springframework.data.elasticsearch; /** - * TODO remove when the new Elasticsearch client is fully working + * TODO remove when the new Elasticsearch client is fully working and the old client is removed * * @author Peter-Josef Meisch */ public interface NewElasticsearchClientDevelopment { - boolean forceEnable = false; - - default boolean usesNewElasticsearchClient() { + default boolean newElasticsearchClient() { return false; } - default boolean newElasticsearchClient() { - return !forceEnable && usesNewElasticsearchClient(); + default boolean oldElasticsearchClient() { + return !newElasticsearchClient(); } } diff --git a/src/test/java/org/springframework/data/elasticsearch/client/elc/DevTests.java b/src/test/java/org/springframework/data/elasticsearch/client/elc/DevTests.java index b0c3e9819..07ba20822 100644 --- a/src/test/java/org/springframework/data/elasticsearch/client/elc/DevTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/client/elc/DevTests.java @@ -19,6 +19,9 @@ import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.FieldValue; +import co.elastic.clients.elasticsearch._types.Script; +import co.elastic.clients.elasticsearch._types.mapping.RuntimeField; +import co.elastic.clients.elasticsearch._types.mapping.RuntimeFieldType; import co.elastic.clients.elasticsearch._types.mapping.TypeMapping; import co.elastic.clients.elasticsearch.cluster.HealthRequest; import co.elastic.clients.elasticsearch.cluster.HealthResponse; @@ -36,6 +39,7 @@ import java.io.IOException; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; +import java.util.Collections; import java.util.Objects; import java.util.function.Function; @@ -83,22 +87,29 @@ void someTest() throws IOException { ElasticsearchClient client = imperativeElasticsearchClient; - String index = "testindex"; +String index = "testindex"; - Person person = new Person("42", new Name("Ford", "Prefect")); +var p = new Product("p1", 42.0); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Person: %s", person)); - } - - IndexRequest indexRequest = IndexRequest.of(b -> b.index(index).id(person.id).document(person)); +client.index(ir -> ir // + .index(index)// + .document(p)); - client.index(indexRequest); - client.search(srb -> srb.index(index).trackTotalHits(thb -> thb.enabled(false)), Person.class); +client.indices().flush(f -> f.index(index)); - ReactiveClient asyncClient = new ReactiveClient(client._transport()); +RuntimeField runtimeField = RuntimeField.of(rf -> rf // + .type(RuntimeFieldType.Double) // + .script(Script.of(s -> s // + .inline(i -> i. // + source("emit(doc['price'].value * 1.19)") // + ) // + )) // +); // - asyncClient.index(indexRequest).block(); +client.search(sr -> sr // + .index(index) // + .runtimeMappings("priceWithTax", Collections.singletonList(runtimeField)), // + Person.class); // } static class ReactiveClient { @@ -113,6 +124,36 @@ public Mono index(IndexRequest request) { } } + static class Product { + @Nullable String id; + @Nullable Double price; + + public Product() {} + + public Product(@Nullable String id, @Nullable Double price) { + this.id = id; + this.price = price; + } + + @Nullable + public String getId() { + return id; + } + + public void setId(@Nullable String id) { + this.id = id; + } + + @Nullable + public Double getPrice() { + return price; + } + + public void setPrice(@Nullable Double price) { + this.price = price; + } + } + static class Person { @Nullable String id; @Nullable Name name; @@ -327,7 +368,7 @@ private ResponseBody searchReactive(SearchRequest searchRequest) { private ClientConfiguration clientConfiguration() { return ClientConfiguration.builder() // - .connectedTo("localhost:9200")// + .connectedTo("thranduil.local.:9200")// .withBasicAuth("elastic", "hcraescitsale").withProxy("localhost:8080") // .withHeaders(() -> { HttpHeaders headers = new HttpHeaders(); diff --git a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchELCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchELCIntegrationTests.java index dd7cd018d..89253b901 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchELCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchELCIntegrationTests.java @@ -61,7 +61,7 @@ IndexNameProvider indexNameProvider() { } @Override - public boolean usesNewElasticsearchClient() { + public boolean newElasticsearchClient() { return true; } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchIntegrationTests.java index 20bcb1c79..a58cc2b01 100755 --- a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchIntegrationTests.java @@ -2783,6 +2783,7 @@ public void shouldNotIncludeDefaultsGetIndexSettings() { assertThat(settings).doesNotContainKey("index.max_result_window"); } + @DisabledIf(value = "newElasticsearchClient", disabledReason = "todo #2165, ES issue 286") @Test // DATAES-709 public void shouldIncludeDefaultsOnGetIndexSettings() { diff --git a/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchELCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchELCIntegrationTests.java index 80ed21033..5af02c1f7 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchELCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchELCIntegrationTests.java @@ -56,6 +56,12 @@ IndexNameProvider indexNameProvider() { return new IndexNameProvider("reactive-template"); } } + + @Override + public boolean newElasticsearchClient() { + return true; + } + @Override protected Query getTermsAggsQuery(String aggsName, String aggsField) { return ELCQueries.getTermsAggsQuery(aggsName, aggsField); diff --git a/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchIntegrationTests.java index fae7ab12d..6aaf1ac41 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchIntegrationTests.java @@ -45,6 +45,7 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIf; import org.skyscreamer.jsonassert.JSONAssert; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DataAccessException; @@ -54,6 +55,7 @@ import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; +import org.springframework.data.elasticsearch.NewElasticsearchClientDevelopment; import org.springframework.data.elasticsearch.RestStatusException; import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Field; @@ -88,7 +90,7 @@ */ @SuppressWarnings("SpringJavaAutowiredMembersInspection") @SpringIntegrationTest -public abstract class ReactiveElasticsearchIntegrationTests { +public abstract class ReactiveElasticsearchIntegrationTests implements NewElasticsearchClientDevelopment { @Autowired private ReactiveElasticsearchOperations operations; @Autowired private IndexNameProvider indexNameProvider; @@ -1072,6 +1074,7 @@ void shouldReturnExplanationWhenRequested() { }).verifyComplete(); } + @DisabledIf(value = "newElasticsearchClient", disabledReason = "todo #2165, ES issue 286") @Test // #1646, #1718 @DisplayName("should return a list of info for specific index") void shouldReturnInformationListOfAllIndices() { diff --git a/src/test/java/org/springframework/data/elasticsearch/core/RuntimeFieldsELCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/RuntimeFieldsELCIntegrationTests.java index 417554f4a..b232fa31e 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/RuntimeFieldsELCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/RuntimeFieldsELCIntegrationTests.java @@ -37,4 +37,9 @@ IndexNameProvider indexNameProvider() { return new IndexNameProvider("runtime-fields-rest-template"); } } + + @Override + public boolean newElasticsearchClient() { + return true; + } } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/RuntimeFieldsIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/RuntimeFieldsIntegrationTests.java index 06dbf157e..55800c2a1 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/RuntimeFieldsIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/RuntimeFieldsIntegrationTests.java @@ -21,8 +21,10 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIf; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.annotation.Id; +import org.springframework.data.elasticsearch.NewElasticsearchClientDevelopment; import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.FieldType; @@ -38,7 +40,7 @@ * @author Peter-Josef Meisch */ @SpringIntegrationTest -public abstract class RuntimeFieldsIntegrationTests { +public abstract class RuntimeFieldsIntegrationTests implements NewElasticsearchClientDevelopment { @Autowired private ElasticsearchOperations operations; @Autowired protected IndexNameProvider indexNameProvider; @@ -58,6 +60,7 @@ void cleanup() { operations.indexOps(IndexCoordinates.of(indexNameProvider.getPrefix() + "*")).delete(); } + @DisabledIf(value = "newElasticsearchClient", disabledReason = "todo #2171, ES issue 298") @Test // #1971 @DisplayName("should use runtime-field from query in search") void shouldUseRuntimeFieldFromQueryInSearch() { diff --git a/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoELCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoELCIntegrationTests.java index 05742ca77..b1dcf322e 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoELCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoELCIntegrationTests.java @@ -45,6 +45,11 @@ IndexNameProvider indexNameProvider() { } } + @Override + public boolean newElasticsearchClient() { + return true; + } + @Override protected Query nativeQueryForBoundingBox(String fieldName, double top, double left, double bottom, double right) { return NativeQuery.builder() // diff --git a/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoIntegrationTests.java index 95a76292a..e56fc67b7 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoIntegrationTests.java @@ -23,10 +23,13 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIf; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.annotation.Id; +import org.springframework.data.elasticsearch.NewElasticsearchClientDevelopment; import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.GeoPointField; +import org.springframework.data.elasticsearch.core.AbstractElasticsearchTemplate; import org.springframework.data.elasticsearch.core.ElasticsearchOperations; import org.springframework.data.elasticsearch.core.SearchHit; import org.springframework.data.elasticsearch.core.SearchHits; @@ -53,11 +56,16 @@ * Latitude , max Longitude , max Latitude */ @SpringIntegrationTest -public abstract class GeoIntegrationTests { +public abstract class GeoIntegrationTests implements NewElasticsearchClientDevelopment { @Autowired private ElasticsearchOperations operations; @Autowired private IndexNameProvider indexNameProvider; + boolean rhlcWithCluster8() { + var clusterVersion = ((AbstractElasticsearchTemplate) operations).getClusterVersion(); + return (oldElasticsearchClient() && clusterVersion != null && clusterVersion.startsWith("8")); + } + @BeforeEach public void before() { indexNameProvider.increment(); @@ -219,6 +227,7 @@ public void shouldFindAnnotatedGeoMarkersInRangeForGivenCriteriaQueryUsingGeohas assertThat(geoAuthorsForGeoCriteria).hasSize(3); } + @DisabledIf(value = "rhlcWithCluster8", disabledReason = "RHLC fails to parse response from ES 8.2") @Test public void shouldFindAllMarkersForNativeSearchQuery() { @@ -236,6 +245,7 @@ public void shouldFindAllMarkersForNativeSearchQuery() { protected abstract Query nativeQueryForBoundingBox(String fieldName, double top, double left, double bottom, double right); + @DisabledIf(value = "rhlcWithCluster8", disabledReason = "RHLC fails to parse response from ES 8.2") @Test public void shouldFindAuthorMarkersInBoxForGivenCriteriaQueryUsingGeoBox() { @@ -254,6 +264,7 @@ public void shouldFindAuthorMarkersInBoxForGivenCriteriaQueryUsingGeoBox() { .containsExactlyInAnyOrder("def", "ghi"); } + @DisabledIf(value = "rhlcWithCluster8", disabledReason = "RHLC fails to parse response from ES 8.2") @Test public void shouldFindAuthorMarkersInBoxForGivenCriteriaQueryUsingGeohash() { @@ -272,6 +283,7 @@ public void shouldFindAuthorMarkersInBoxForGivenCriteriaQueryUsingGeohash() { .containsExactlyInAnyOrder("def", "ghi"); } + @DisabledIf(value = "rhlcWithCluster8", disabledReason = "RHLC fails to parse response from ES 8.2") @Test public void shouldFindAuthorMarkersInBoxForGivenCriteriaQueryUsingGeoPoints() { @@ -290,6 +302,7 @@ public void shouldFindAuthorMarkersInBoxForGivenCriteriaQueryUsingGeoPoints() { .containsExactlyInAnyOrder("def", "ghi"); } + @DisabledIf(value = "rhlcWithCluster8", disabledReason = "RHLC fails to parse response from ES 8.2") @Test public void shouldFindAuthorMarkersInBoxForGivenCriteriaQueryUsingPoints() { @@ -308,6 +321,7 @@ public void shouldFindAuthorMarkersInBoxForGivenCriteriaQueryUsingPoints() { .containsExactlyInAnyOrder("def", "ghi"); } + @DisabledIf(value = "rhlcWithCluster8", disabledReason = "RHLC fails to parse response from ES 8.2") @Test public void shouldFindLocationWithGeoHashPrefix() { diff --git a/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoJsonELCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoJsonELCIntegrationTests.java index 9995237ae..03de9bc21 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoJsonELCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoJsonELCIntegrationTests.java @@ -16,13 +16,26 @@ package org.springframework.data.elasticsearch.core.geo; import org.junit.jupiter.api.DisplayName; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; import org.springframework.data.elasticsearch.junit.jupiter.ElasticsearchTemplateConfiguration; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; import org.springframework.test.context.ContextConfiguration; /** * @author Peter-Josef Meisch * @since 4.4 */ -@ContextConfiguration(classes = { ElasticsearchTemplateConfiguration.class }) +@ContextConfiguration(classes = { GeoJsonELCIntegrationTests.Config.class }) @DisplayName("GeoJson integration test with ElasticsearchClient") -public class GeoJsonELCIntegrationTests extends GeoJsonIntegrationTests {} +public class GeoJsonELCIntegrationTests extends GeoJsonIntegrationTests { + @Configuration + @Import({ ElasticsearchTemplateConfiguration.class }) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("geojson-integration"); + } + } +} diff --git a/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoJsonERHLCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoJsonERHLCIntegrationTests.java index 958d83f17..2a629bac1 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoJsonERHLCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoJsonERHLCIntegrationTests.java @@ -16,12 +16,26 @@ package org.springframework.data.elasticsearch.core.geo; import org.junit.jupiter.api.DisplayName; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; import org.springframework.data.elasticsearch.junit.jupiter.ElasticsearchRestTemplateConfiguration; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; import org.springframework.test.context.ContextConfiguration; /** * @author Peter-Josef Meisch */ -@ContextConfiguration(classes = { ElasticsearchRestTemplateConfiguration.class }) +@ContextConfiguration(classes = { GeoJsonERHLCIntegrationTests.Config.class }) @DisplayName("GeoJson integration test with RestHighLevelClient") -public class GeoJsonERHLCIntegrationTests extends GeoJsonIntegrationTests {} +public class GeoJsonERHLCIntegrationTests extends GeoJsonIntegrationTests { + @Configuration + @Import({ ElasticsearchRestTemplateConfiguration.class }) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("geojson-integration-es7"); + } + } + +} diff --git a/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoJsonEntity.java b/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoJsonEntity.java index 7c93f3bac..5e7c3ac0a 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoJsonEntity.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoJsonEntity.java @@ -23,7 +23,7 @@ /** * this class contains each GeoJson type as explicit type and as GeoJson interface. Used by several test classes */ -@Document(indexName = "geojson-index") +@Document(indexName = "#{@indexNameProvider.indexName()}-geojson") public class GeoJsonEntity { @Nullable @Id private String id; diff --git a/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoJsonIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoJsonIntegrationTests.java index 998ced193..9b9f3bbab 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoJsonIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/geo/GeoJsonIntegrationTests.java @@ -19,20 +19,21 @@ import java.util.Arrays; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.annotation.Id; import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.core.ElasticsearchOperations; -import org.springframework.data.elasticsearch.core.IndexOperations; import org.springframework.data.elasticsearch.core.SearchHits; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; import org.springframework.data.elasticsearch.core.query.Criteria; import org.springframework.data.elasticsearch.core.query.CriteriaQuery; import org.springframework.data.elasticsearch.junit.jupiter.SpringIntegrationTest; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; import org.springframework.data.geo.Point; import org.springframework.lang.Nullable; @@ -44,7 +45,21 @@ abstract class GeoJsonIntegrationTests { @Autowired private ElasticsearchOperations operations; - private IndexOperations indexOps; + @Autowired private IndexNameProvider indexNameProvider; + + @BeforeEach + public void before() { + indexNameProvider.increment(); + operations.indexOps(Area.class).createWithMapping(); + operations.indexOps(GeoJsonEntity.class).createWithMapping(); + operations.save(Arrays.asList(area10To20, area30To40)); + } + + @Test + @Order(java.lang.Integer.MAX_VALUE) + void cleanup() { + operations.indexOps(IndexCoordinates.of("*" + indexNameProvider.getPrefix() + "*")).delete(); + } // region data @@ -89,29 +104,6 @@ abstract class GeoJsonIntegrationTests { private final Area area32To37 = new Area("area32To37", geoShape30To40); // endregion - // region setup - @BeforeEach - void setUp() { - indexOps = operations.indexOps(GeoJsonEntity.class); - indexOps.delete(); - indexOps.create(); - indexOps.putMapping(); - - IndexOperations indexOpsArea = operations.indexOps(Area.class); - indexOpsArea.delete(); - indexOpsArea.create(); - indexOpsArea.putMapping(); - - operations.save(Arrays.asList(area10To20, area30To40)); - indexOpsArea.refresh(); - } - - @AfterEach - void tearDown() { - indexOps.delete(); - } - // endregion - // region tests @Test // DATAES-930 @DisplayName("should write and read an entity with GeoJson properties") @@ -149,7 +141,6 @@ void shouldWriteAndReadAnEntityWithGeoJsonProperties() { entity.setGeometryCollection2(geoJsonGeometryCollection); operations.save(entity); - indexOps.refresh(); GeoJsonEntity result = operations.get("42", GeoJsonEntity.class); @@ -202,7 +193,7 @@ void shouldFindContainsObjectsWithCriteriaQuery() { // endregion // region test classes - @Document(indexName = "areas") + @Document(indexName = "#{@indexNameProvider.indexName()}-area") static class Area { @Nullable @Id private String id; diff --git a/src/test/java/org/springframework/data/elasticsearch/core/index/IndexTemplateELCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/index/IndexTemplateELCIntegrationTests.java index f7d3de0dc..625f5a16e 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/index/IndexTemplateELCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/index/IndexTemplateELCIntegrationTests.java @@ -23,4 +23,9 @@ * @since 4.4 */ @ContextConfiguration(classes = { ElasticsearchTemplateConfiguration.class }) -public class IndexTemplateELCIntegrationTests extends IndexTemplateIntegrationTests {} +public class IndexTemplateELCIntegrationTests extends IndexTemplateIntegrationTests { + @Override + public boolean newElasticsearchClient() { + return true; + } +} diff --git a/src/test/java/org/springframework/data/elasticsearch/core/index/IndexTemplateERHLCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/index/IndexTemplateERHLCIntegrationTests.java index 3bb251cb7..e7828ddd8 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/index/IndexTemplateERHLCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/index/IndexTemplateERHLCIntegrationTests.java @@ -22,4 +22,5 @@ * @author Peter-Josef Meisch */ @ContextConfiguration(classes = { ElasticsearchRestTemplateConfiguration.class }) -public class IndexTemplateERHLCIntegrationTests extends IndexTemplateIntegrationTests {} +public class IndexTemplateERHLCIntegrationTests extends IndexTemplateIntegrationTests { +} diff --git a/src/test/java/org/springframework/data/elasticsearch/core/index/IndexTemplateIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/index/IndexTemplateIntegrationTests.java index a275349e3..6bae8d826 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/index/IndexTemplateIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/index/IndexTemplateIntegrationTests.java @@ -23,12 +23,15 @@ import org.json.JSONException; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIf; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.annotation.Id; +import org.springframework.data.elasticsearch.NewElasticsearchClientDevelopment; import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.FieldType; import org.springframework.data.elasticsearch.annotations.Setting; +import org.springframework.data.elasticsearch.core.AbstractElasticsearchTemplate; import org.springframework.data.elasticsearch.core.ElasticsearchOperations; import org.springframework.data.elasticsearch.core.IndexOperations; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; @@ -39,10 +42,16 @@ * @author Peter-Josef Meisch */ @SpringIntegrationTest -public abstract class IndexTemplateIntegrationTests { +public abstract class IndexTemplateIntegrationTests implements NewElasticsearchClientDevelopment { @Autowired ElasticsearchOperations operations; + boolean rhlcWithCluster8() { + var clusterVersion = ((AbstractElasticsearchTemplate) operations).getClusterVersion(); + return (oldElasticsearchClient() && clusterVersion != null && clusterVersion.startsWith("8")); + } + + @DisabledIf(value = "rhlcWithCluster8", disabledReason = "RHLC fails to parse response from ES 8.2") @Test // DATAES-612 void shouldCreateTemplate() { @@ -76,6 +85,7 @@ void shouldReturnNullOnNonExistingGetTemplate() { assertThat(templateData).isNull(); } + @DisabledIf(value = "rhlcWithCluster8", disabledReason = "RHLC fails to parse response from ES 8.2") @Test // DATAES-612 void shouldGetTemplate() throws JSONException { IndexOperations indexOps = operations.indexOps(IndexCoordinates.of("dont-care")); @@ -101,7 +111,7 @@ void shouldGetTemplate() throws JSONException { assertThat(templateData).isNotNull(); assertThat(templateData.getIndexPatterns()).containsExactlyInAnyOrder(putTemplateRequest.getIndexPatterns()); - assertEquals(settings.toJson(), templateData.getSettings().toJson(), false); + assertEquals(settings.flatten().toJson(), templateData.getSettings().toJson(), false); assertEquals(mapping.toJson(), templateData.getMapping().toJson(), false); Map aliases = templateData.getAliases(); assertThat(aliases).hasSize(2); diff --git a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderELCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderELCIntegrationTests.java index 6fbfd2e24..0fcf4567a 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderELCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderELCIntegrationTests.java @@ -39,7 +39,7 @@ IndexNameProvider indexNameProvider() { } @Override - public boolean usesNewElasticsearchClient() { + public boolean newElasticsearchClient() { return true; } } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/indices/IndexOperationsELCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/indices/IndexOperationsELCIntegrationTests.java index a035bb5d7..7dfd87f6f 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/indices/IndexOperationsELCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/indices/IndexOperationsELCIntegrationTests.java @@ -33,12 +33,12 @@ public class IndexOperationsELCIntegrationTests extends IndexOperationsIntegrati static class Config { @Bean IndexNameProvider indexNameProvider() { - return new IndexNameProvider("indexoperations-es"); + return new IndexNameProvider("indexoperations"); } } @Override - public boolean usesNewElasticsearchClient() { + public boolean newElasticsearchClient() { return true; } } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/indices/IndexOperationsIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/indices/IndexOperationsIntegrationTests.java index e69ee61eb..17b5654a1 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/indices/IndexOperationsIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/indices/IndexOperationsIntegrationTests.java @@ -25,6 +25,7 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIf; import org.skyscreamer.jsonassert.JSONAssert; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.annotation.Id; @@ -70,6 +71,7 @@ void cleanup() { operations.indexOps(IndexCoordinates.of(indexNameProvider.getPrefix() + "*")).delete(); } + @DisabledIf(value = "newElasticsearchClient", disabledReason = "todo #2165, ES issue 286") @Test // #1646, #1718 @DisplayName("should return a list of info for specific index") void shouldReturnInformationList() throws JSONException { diff --git a/src/test/java/org/springframework/data/elasticsearch/core/indices/ReactiveIndexOperationsELCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/indices/ReactiveIndexOperationsELCIntegrationTests.java index fa981d2c5..6fc55aead 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/indices/ReactiveIndexOperationsELCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/indices/ReactiveIndexOperationsELCIntegrationTests.java @@ -27,7 +27,6 @@ */ @ContextConfiguration(classes = { ReactiveIndexOperationsELCIntegrationTests.Config.class }) public class ReactiveIndexOperationsELCIntegrationTests extends ReactiveIndexOperationsIntegrationTests { - @Configuration @Import({ ReactiveElasticsearchTemplateConfiguration.class }) static class Config { @@ -36,4 +35,9 @@ IndexNameProvider indexNameProvider() { return new IndexNameProvider("reactive-indexoperations"); } } + + @Override + public boolean newElasticsearchClient() { + return true; + } } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/indices/ReactiveIndexOperationsIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/indices/ReactiveIndexOperationsIntegrationTests.java index d9d308f20..8534928b2 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/indices/ReactiveIndexOperationsIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/indices/ReactiveIndexOperationsIntegrationTests.java @@ -31,26 +31,20 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIf; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.annotation.Id; +import org.springframework.data.elasticsearch.NewElasticsearchClientDevelopment; import org.springframework.data.elasticsearch.annotations.DateFormat; import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.FieldType; import org.springframework.data.elasticsearch.annotations.Mapping; import org.springframework.data.elasticsearch.annotations.Setting; +import org.springframework.data.elasticsearch.core.AbstractReactiveElasticsearchTemplate; import org.springframework.data.elasticsearch.core.ReactiveElasticsearchOperations; import org.springframework.data.elasticsearch.core.ReactiveIndexOperations; -import org.springframework.data.elasticsearch.core.index.AliasAction; -import org.springframework.data.elasticsearch.core.index.AliasActionParameters; -import org.springframework.data.elasticsearch.core.index.AliasActions; -import org.springframework.data.elasticsearch.core.index.AliasData; -import org.springframework.data.elasticsearch.core.index.DeleteTemplateRequest; -import org.springframework.data.elasticsearch.core.index.ExistsTemplateRequest; -import org.springframework.data.elasticsearch.core.index.GetTemplateRequest; -import org.springframework.data.elasticsearch.core.index.PutTemplateRequest; -import org.springframework.data.elasticsearch.core.index.Settings; -import org.springframework.data.elasticsearch.core.index.TemplateData; +import org.springframework.data.elasticsearch.core.index.*; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; import org.springframework.data.elasticsearch.junit.jupiter.SpringIntegrationTest; import org.springframework.data.elasticsearch.utils.IndexNameProvider; @@ -60,12 +54,17 @@ * @author Peter-Josef Meisch */ @SpringIntegrationTest -public abstract class ReactiveIndexOperationsIntegrationTests { +public abstract class ReactiveIndexOperationsIntegrationTests implements NewElasticsearchClientDevelopment { @Autowired private ReactiveElasticsearchOperations operations; @Autowired private IndexNameProvider indexNameProvider; private ReactiveIndexOperations indexOperations; + boolean rhlcWithCluster8() { + var clusterVersion = ((AbstractReactiveElasticsearchTemplate) operations).getClusterVersion().block(); + return (oldElasticsearchClient() && clusterVersion != null && clusterVersion.startsWith("8")); + } + @BeforeEach void setUp() { indexNameProvider.increment(); @@ -124,11 +123,11 @@ void shouldCreateIndexForEntity() { @Test // DATAES-678 void shouldCreateIndexWithGivenSettings() { - org.springframework.data.elasticsearch.core.document.Document requiredSettings = org.springframework.data.elasticsearch.core.document.Document - .create(); - requiredSettings.put("index.number_of_replicas", 3); - requiredSettings.put("index.number_of_shards", 4); - requiredSettings.put("index.refresh_interval", "5s"); + var index = new Settings() // + .append("number_of_replicas", 3) // + .append("number_of_shards", 4)// + .append("refresh_interval", "5s"); + var requiredSettings = new Settings().append("index", index); indexOperations.create(requiredSettings) // .as(StepVerifier::create) // @@ -136,9 +135,10 @@ void shouldCreateIndexWithGivenSettings() { .verifyComplete(); indexOperations.getSettings().as(StepVerifier::create).consumeNextWith(settings -> { - assertThat(settings.get("index.number_of_replicas")).isEqualTo("3"); - assertThat(settings.get("index.number_of_shards")).isEqualTo("4"); - assertThat(settings.get("index.refresh_interval")).isEqualTo("5s"); + var flattened = settings.flatten(); + assertThat(flattened.get("index.number_of_replicas")).isEqualTo("3"); + assertThat(flattened.get("index.number_of_shards")).isEqualTo("4"); + assertThat(flattened.get("index.refresh_interval")).isEqualTo("5s"); }).verifyComplete(); } @@ -359,6 +359,7 @@ void shouldGetAliasData() { .verifyComplete(); } + @DisabledIf(value = "rhlcWithCluster8", disabledReason = "RHLC fails to parse response from ES 8.2") @Test // DATAES-612 void shouldPutTemplate() { @@ -391,6 +392,7 @@ void shouldReturnNullOnNonExistingGetTemplate() { .verifyComplete(); } + @DisabledIf(value = "rhlcWithCluster8", disabledReason = "RHLC fails to parse response from ES 8.2") @Test // DATAES-612 void shouldGetTemplate() throws JSONException { @@ -417,10 +419,8 @@ void shouldGetTemplate() throws JSONException { SoftAssertions softly = new SoftAssertions(); softly.assertThat(templateData).isNotNull(); softly.assertThat(templateData.getIndexPatterns()).containsExactlyInAnyOrder(putTemplateRequest.getIndexPatterns()); - assertEquals(settings.toJson(), templateData.getSettings().toJson(), false); - + assertEquals(settings.flatten().toJson(), templateData.getSettings().toJson(), false); assertEquals(mapping.toJson(), templateData.getMapping().toJson(), false); - Map aliases = templateData.getAliases(); softly.assertThat(aliases).hasSize(2); AliasData alias1 = aliases.get("alias1"); diff --git a/src/test/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentEntityTests.java b/src/test/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentEntityTests.java index 3ba22a3b8..197d0bb03 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentEntityTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentEntityTests.java @@ -177,16 +177,29 @@ void shouldErrorIfIndexSortingParametersDoNotHaveTheSameNumberOfArguments() { .isInstanceOf(IllegalArgumentException.class); } - @Test // #1719 + @Test // #1719, #2158 @DisplayName("should write sort parameters to Settings object") void shouldWriteSortParametersToSettingsObject() throws JSONException { - String expected = "{\n" + // - " \"index.sort.field\": [\"second_field\", \"first_field\"],\n" + // - " \"index.sort.mode\": [\"max\", \"min\"],\n" + // - " \"index.sort.order\": [\"desc\",\"asc\"],\n" + // - " \"index.sort.missing\": [\"_last\",\"_first\"]\n" + // - "}\n"; // + String expected = """ + { + "index": { + "sort": { + "field": [ + "second_field", + "first_field" + ], + "mode": [ + "max", + "min" + ], + "missing": [ + "_last", + "_first" + ] + } + } + } """; ElasticsearchPersistentEntity entity = elasticsearchConverter.get().getMappingContext() .getRequiredPersistentEntity(SettingsValidSortParameterSizes.class); diff --git a/src/test/java/org/springframework/data/elasticsearch/core/suggest/CompletionELCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/suggest/CompletionELCIntegrationTests.java index 00b675833..a020bb39d 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/suggest/CompletionELCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/suggest/CompletionELCIntegrationTests.java @@ -47,7 +47,7 @@ IndexNameProvider indexNameProvider() { } @Override - public boolean usesNewElasticsearchClient() { + public boolean newElasticsearchClient() { return true; } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/suggest/ReactiveSuggestELCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/suggest/ReactiveSuggestELCIntegrationTests.java index b15dac6ea..cb07c6535 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/suggest/ReactiveSuggestELCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/suggest/ReactiveSuggestELCIntegrationTests.java @@ -45,7 +45,7 @@ IndexNameProvider indexNameProvider() { } @Override - public boolean usesNewElasticsearchClient() { + public boolean newElasticsearchClient() { return true; } diff --git a/src/test/java/org/springframework/data/elasticsearch/repositories/custommethod/CustomMethodRepositoryERHLCIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/repositories/custommethod/CustomMethodRepositoryERHLCIntegrationTests.java index 92051ec99..6805329f8 100644 --- a/src/test/java/org/springframework/data/elasticsearch/repositories/custommethod/CustomMethodRepositoryERHLCIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/repositories/custommethod/CustomMethodRepositoryERHLCIntegrationTests.java @@ -42,4 +42,9 @@ IndexNameProvider indexNameProvider() { return new IndexNameProvider("custom-method-repository-es7"); } } + + @Override + public boolean newElasticsearchClient() { + return false; + } } diff --git a/src/test/java/org/springframework/data/elasticsearch/repositories/custommethod/CustomMethodRepositoryIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/repositories/custommethod/CustomMethodRepositoryIntegrationTests.java index 145ce57d8..1d3feef59 100644 --- a/src/test/java/org/springframework/data/elasticsearch/repositories/custommethod/CustomMethodRepositoryIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/repositories/custommethod/CustomMethodRepositoryIntegrationTests.java @@ -32,6 +32,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIf; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.annotation.Id; import org.springframework.data.annotation.Version; @@ -40,12 +41,14 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; import org.springframework.data.domain.Sort.Order; +import org.springframework.data.elasticsearch.NewElasticsearchClientDevelopment; import org.springframework.data.elasticsearch.annotations.CountQuery; import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.Highlight; import org.springframework.data.elasticsearch.annotations.HighlightField; import org.springframework.data.elasticsearch.annotations.Query; +import org.springframework.data.elasticsearch.core.AbstractElasticsearchTemplate; import org.springframework.data.elasticsearch.core.ElasticsearchOperations; import org.springframework.data.elasticsearch.core.SearchHit; import org.springframework.data.elasticsearch.core.SearchHits; @@ -75,7 +78,7 @@ * @author James Mudd */ @SpringIntegrationTest -public abstract class CustomMethodRepositoryIntegrationTests { +public abstract class CustomMethodRepositoryIntegrationTests implements NewElasticsearchClientDevelopment { @Autowired private IndexNameProvider indexNameProvider; @Autowired private SampleCustomMethodRepository repository; @@ -83,6 +86,11 @@ public abstract class CustomMethodRepositoryIntegrationTests { @Autowired ElasticsearchOperations operations; + boolean rhlcWithCluster8() { + var clusterVersion = ((AbstractElasticsearchTemplate) operations).getClusterVersion(); + return (oldElasticsearchClient() && clusterVersion != null && clusterVersion.startsWith("8")); + } + @BeforeEach public void before() { @@ -806,6 +814,7 @@ public void shouldExecuteCustomMethodWithWithinPoint() { assertThat(page.getTotalElements()).isEqualTo(1L); } + @DisabledIf(value = "rhlcWithCluster8", disabledReason = "RHLC fails to parse response from ES 8.2") @Test public void shouldExecuteCustomMethodWithNearBox() { @@ -1365,6 +1374,7 @@ public void shouldCountCustomMethodWithWithinPoint() { assertThat(count).isEqualTo(1L); } + @DisabledIf(value = "rhlcWithCluster8", disabledReason = "RHLC fails to parse response from ES 8.2") @Test // DATAES-106 public void shouldCountCustomMethodWithNearBox() { diff --git a/src/test/resources/testcontainers-elasticsearch.properties b/src/test/resources/testcontainers-elasticsearch.properties index 94557e34b..9529d120e 100644 --- a/src/test/resources/testcontainers-elasticsearch.properties +++ b/src/test/resources/testcontainers-elasticsearch.properties @@ -15,10 +15,14 @@ # # sde.testcontainers.image-name=docker.elastic.co/elasticsearch/elasticsearch -sde.testcontainers.image-version=7.17.3 +sde.testcontainers.image-version=8.2.2 # # # needed as we do a DELETE /* at the end of the tests, will be required from 8.0 on, produces a warning since 7.13 # action.destructive_requires_name=false reindex.remote.whitelist=localhost:9200 +# +# we do not want to complicate the test setup with the security features, makes it harder for debugging, proxyuing +xpack.security.enabled=false +xpack.security.http.ssl.enabled=false From 97d0566e94f31c4a9de277cd53a1b5418b797bc7 Mon Sep 17 00:00:00 2001 From: Peter-Josef Meisch Date: Wed, 1 Jun 2022 14:48:46 +0200 Subject: [PATCH 012/516] Fix msearch setup after ES fixed it's issue. Original Pull Request #2176 Closes #2138 --- .../client/elc/RequestConverter.java | 5 +++-- .../core/ElasticsearchIntegrationTests.java | 15 +++------------ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/RequestConverter.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/RequestConverter.java index 78df81450..3051591e7 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/RequestConverter.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/RequestConverter.java @@ -1049,12 +1049,13 @@ public MsearchRequest searchMsearchRequest( mrb.searches(sb -> sb // .header(h -> h // - .index(param.index.getIndexName()) // + .index(Arrays.asList(param.index.getIndexNames())) // // todo #2156 add remaining flags for header ) // .body(bb -> bb // .query(getQuery(param.query, param.clazz))// - // todo #2138 seq_no_primary_term and version not available in client ES issue 161 + .seqNoPrimaryTerm(persistentEntity.hasSeqNoPrimaryTermProperty()) + .version(true) // todo #2156 add remaining flags for body ) // ); diff --git a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchIntegrationTests.java index a58cc2b01..33e23baf0 100755 --- a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchIntegrationTests.java @@ -1616,9 +1616,6 @@ public void shouldPassIndicesOptionsForGivenSearchScrollQuery() { assertThat(entities.size()).isGreaterThanOrEqualTo(1); } - @DisabledIf(value = "newElasticsearchClient", - disabledReason = "todo #2138 can't check response, open ES issue 161 that does not allow seqno") - // and version to be set in the request @Test // DATAES-487 public void shouldReturnSameEntityForMultiSearch() { @@ -1641,13 +1638,9 @@ public void shouldReturnSameEntityForMultiSearch() { } } - @DisabledIf(value = "newElasticsearchClient", - disabledReason = "todo #2138 can't check response, open ES issue 161 that does not allow seqno") - // and version to be set in the request @Test // DATAES-487 public void shouldReturnDifferentEntityForMultiSearch() { - Class clazz = Book.class; IndexOperations bookIndexOperations = operations.indexOps(Book.class); bookIndexOperations.delete(); bookIndexOperations.createWithMapping(); @@ -1662,7 +1655,8 @@ public void shouldReturnDifferentEntityForMultiSearch() { queries.add(getTermQuery("message", "ab")); queries.add(getTermQuery("description", "bc")); - List> searchHitsList = operations.multiSearch(queries, Lists.newArrayList(SampleEntity.class, clazz), + List> searchHitsList = operations.multiSearch(queries, Lists.newArrayList(SampleEntity.class, + Book.class), IndexCoordinates.of(indexNameProvider.indexName(), bookIndex.getIndexName())); bookIndexOperations.delete(); @@ -1674,7 +1668,7 @@ public void shouldReturnDifferentEntityForMultiSearch() { SearchHits searchHits1 = searchHitsList.get(1); assertThat(searchHits1.getTotalHits()).isEqualTo(1L); SearchHit searchHit1 = (SearchHit) searchHits1.getSearchHit(0); - assertThat(searchHit1.getContent().getClass()).isEqualTo(clazz); + assertThat(searchHit1.getContent().getClass()).isEqualTo(Book.class); } @Test @@ -3070,9 +3064,6 @@ void searchShouldReturnSeqNoPrimaryTerm() { assertThatSeqNoPrimaryTermIsFilled(retrieved); } - @DisabledIf(value = "newElasticsearchClient", - disabledReason = "todo #2138 can't check response, open ES issue 161 that does not allow seqno") - // and version to be set in the request @Test // DATAES-799 void multiSearchShouldReturnSeqNoPrimaryTerm() { OptimisticEntity original = new OptimisticEntity(); From 3835e8e3632fdd73f1b607e3a94f4ff42e5befdc Mon Sep 17 00:00:00 2001 From: Mark Paluch Date: Fri, 3 Jun 2022 09:32:51 +0200 Subject: [PATCH 013/516] Upgrade to Maven Wrapper 3.8.5. See #2177 --- .mvn/wrapper/maven-wrapper.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties index fdfa741df..5f8d5af69 100644 --- a/.mvn/wrapper/maven-wrapper.properties +++ b/.mvn/wrapper/maven-wrapper.properties @@ -1,3 +1,3 @@ -#Tue Feb 22 13:59:12 CET 2022 +#Fri Jun 03 09:32:51 CEST 2022 wrapperUrl=https\://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar -distributionUrl=https\://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.4/apache-maven-3.8.4-bin.zip +distributionUrl=https\://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.5/apache-maven-3.8.5-bin.zip From ef880e6d3b814a447629ebc5f41535f6836c14b0 Mon Sep 17 00:00:00 2001 From: Mark Paluch Date: Fri, 3 Jun 2022 09:34:20 +0200 Subject: [PATCH 014/516] Update CI properties. See #2167 --- ci/pipeline.properties | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/pipeline.properties b/ci/pipeline.properties index 4aeb53c20..57e4868d4 100644 --- a/ci/pipeline.properties +++ b/ci/pipeline.properties @@ -1,11 +1,10 @@ # Java versions -java.main.tag=17.0.2_8-jdk +java.main.tag=17.0.3_7-jdk # Docker container images - standard docker.java.main.image=harbor-repo.vmware.com/dockerhub-proxy-cache/library/eclipse-temurin:${java.main.tag} # Supported versions of MongoDB -docker.mongodb.4.0.version=4.0.28 docker.mongodb.4.4.version=4.4.12 docker.mongodb.5.0.version=5.0.6 From 0a0fc75faa0852740d3ce9150be21b03aa88cd5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gon=C3=A7alo=20Montalv=C3=A3o=20Marques?= <9379664+GonMMarques@users.noreply.github.com> Date: Fri, 3 Jun 2022 18:35:40 +0100 Subject: [PATCH 015/516] fix typos in migration guides documentation. Original Pull Request #2181 Closes #2180 --- .../reference/elasticsearch-migration-guide-4.1-4.2.adoc | 2 +- .../reference/elasticsearch-migration-guide-4.3-4.4.adoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/asciidoc/reference/elasticsearch-migration-guide-4.1-4.2.adoc b/src/main/asciidoc/reference/elasticsearch-migration-guide-4.1-4.2.adoc index 860e09226..6a1977237 100644 --- a/src/main/asciidoc/reference/elasticsearch-migration-guide-4.1-4.2.adoc +++ b/src/main/asciidoc/reference/elasticsearch-migration-guide-4.1-4.2.adoc @@ -14,7 +14,7 @@ The parameters of the `@Document` annotation that are relevant for the index se == Removals The `@Score` annotation that was used to set the score return value in an entity was deprecated in version 4.0 and has been removed. -Scroe values are returned in the `SearchHit` instances that encapsulate the returned entities. +Score values are returned in the `SearchHit` instances that encapsulate the returned entities. The `org.springframework.data.elasticsearch.ElasticsearchException` class has been removed. The remaining usages have been replaced with `org.springframework.data.mapping.MappingException` and `org.springframework.dao.InvalidDataAccessApiUsageException`. diff --git a/src/main/asciidoc/reference/elasticsearch-migration-guide-4.3-4.4.adoc b/src/main/asciidoc/reference/elasticsearch-migration-guide-4.3-4.4.adoc index 929961ac6..7e5308d1e 100644 --- a/src/main/asciidoc/reference/elasticsearch-migration-guide-4.3-4.4.adoc +++ b/src/main/asciidoc/reference/elasticsearch-migration-guide-4.3-4.4.adoc @@ -69,7 +69,7 @@ public class SpringdataElasticTestApplication { ==== Remove Spring Data Elasticsearch related properties from your application configuration. -If Spring Data Elasticsearch was configured using a programmatic configuration (see <>), remove these beans from the Spring application context. +If Spring Data Elasticsearch was configured using a programmatic configuration (see <>), remove these beans from the Spring application context. ==== Add dependencies From 96d0781f24f3378a08ff6d70eafd3ce4dfbc5400 Mon Sep 17 00:00:00 2001 From: Peter-Josef Meisch Date: Wed, 22 Jun 2022 18:34:47 +0200 Subject: [PATCH 016/516] Parse returned suggest data using the new client. Original Pull Request #2187 Closes #2154 --- .../client/elc/DocumentAdapters.java | 16 +++ .../elc/SearchDocumentResponseBuilder.java | 131 ++++++++++++++++-- .../core/AbstractElasticsearchTemplate.java | 6 +- ...AbstractReactiveElasticsearchTemplate.java | 6 +- .../SearchDocumentResponseBuilder.java | 4 +- .../response/CompletionSuggestion.java | 2 +- .../suggest/response/PhraseSuggestion.java | 11 +- .../core/suggest/response/Suggest.java | 11 +- .../core/suggest/response/TermSuggestion.java | 9 +- .../data/elasticsearch/support/ScoreDoc.java | 18 ++- .../CompletionELCIntegrationTests.java | 4 +- .../suggest/CompletionIntegrationTests.java | 10 +- .../ReactiveSuggestELCIntegrationTests.java | 2 +- .../ReactiveSuggestIntegrationTests.java | 2 +- 14 files changed, 187 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/DocumentAdapters.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/DocumentAdapters.java index b90ae49e7..3c00f745e 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/DocumentAdapters.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/DocumentAdapters.java @@ -19,6 +19,7 @@ import co.elastic.clients.elasticsearch.core.MgetResponse; import co.elastic.clients.elasticsearch.core.explain.ExplanationDetail; import co.elastic.clients.elasticsearch.core.get.GetResult; +import co.elastic.clients.elasticsearch.core.search.CompletionSuggestOption; import co.elastic.clients.elasticsearch.core.search.Hit; import co.elastic.clients.elasticsearch.core.search.NestedIdentity; import co.elastic.clients.json.JsonData; @@ -142,6 +143,21 @@ public static SearchDocument from(Hit hit, JsonpMapper jsonpMapper) { highlightFields, innerHits, nestedMetaData, explanation, matchedQueries, hit.routing()); } + public static SearchDocument from(CompletionSuggestOption completionSuggestOption) { + + Document document = completionSuggestOption.source() != null ? Document.from(completionSuggestOption.source()) + : Document.create(); + document.setIndex(completionSuggestOption.index()); + + if (completionSuggestOption.id() != null) { + document.setId(completionSuggestOption.id()); + } + + float score = completionSuggestOption.score() != null ? completionSuggestOption.score().floatValue() : Float.NaN; + return new SearchDocumentAdapter(document, score, new Object[] {}, Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap(), null, null, null, completionSuggestOption.routing()); + } + @Nullable private static Explanation from(@Nullable co.elastic.clients.elasticsearch.core.explain.Explanation explanation) { diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/SearchDocumentResponseBuilder.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/SearchDocumentResponseBuilder.java index bdf3a69ca..4202b7a85 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/SearchDocumentResponseBuilder.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/SearchDocumentResponseBuilder.java @@ -17,24 +17,30 @@ import co.elastic.clients.elasticsearch._types.aggregations.Aggregate; import co.elastic.clients.elasticsearch.core.SearchResponse; -import co.elastic.clients.elasticsearch.core.search.Hit; -import co.elastic.clients.elasticsearch.core.search.HitsMetadata; -import co.elastic.clients.elasticsearch.core.search.ResponseBody; -import co.elastic.clients.elasticsearch.core.search.Suggestion; -import co.elastic.clients.elasticsearch.core.search.TotalHits; +import co.elastic.clients.elasticsearch.core.search.*; import co.elastic.clients.json.JsonpMapper; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.elasticsearch.search.SearchHits; import org.springframework.data.elasticsearch.core.TotalHitsRelation; import org.springframework.data.elasticsearch.core.document.SearchDocument; import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse; +import org.springframework.data.elasticsearch.core.suggest.response.CompletionSuggestion; +import org.springframework.data.elasticsearch.core.suggest.response.PhraseSuggestion; import org.springframework.data.elasticsearch.core.suggest.response.Suggest; +import org.springframework.data.elasticsearch.core.suggest.response.TermSuggestion; +import org.springframework.data.elasticsearch.support.ScoreDoc; import org.springframework.lang.Nullable; import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; /** * Factory class to create {@link SearchDocumentResponse} instances. @@ -43,6 +49,9 @@ * @since 4.4 */ class SearchDocumentResponseBuilder { + + private static final Log LOGGER = LogFactory.getLog(SearchDocumentResponseBuilder.class); + /** * creates a SearchDocumentResponse from the {@link SearchResponse} * @@ -80,7 +89,7 @@ public static SearchDocumentResponse from(ResponseBody response * @return the {@link SearchDocumentResponse} */ public static SearchDocumentResponse from(HitsMetadata hitsMetadata, @Nullable String scrollId, - Map aggregations, Map>> suggestES, + @Nullable Map aggregations, Map>> suggestES, SearchDocumentResponse.EntityCreator entityCreator, JsonpMapper jsonpMapper) { Assert.notNull(hitsMetadata, "hitsMetadata must not be null"); @@ -116,10 +125,116 @@ public static SearchDocumentResponse from(HitsMetadata hitsMetadata, @Nul ElasticsearchAggregations aggregationsContainer = aggregations != null ? new ElasticsearchAggregations(aggregations) : null; - // todo #2154 - Suggest suggest = null; + Suggest suggest = suggestFrom(suggestES, entityCreator); return new SearchDocumentResponse(totalHits, totalHitsRelation, maxScore, scrollId, searchDocuments, aggregationsContainer, suggest); } + + @Nullable + private static Suggest suggestFrom(Map>> suggestES, + SearchDocumentResponse.EntityCreator entityCreator) { + + if (CollectionUtils.isEmpty(suggestES)) { + return null; + } + + List>> suggestions = new ArrayList<>(); + + suggestES.forEach((name, suggestionsES) -> { + + if (!suggestionsES.isEmpty()) { + // take the type from the first entry + switch (suggestionsES.get(0)._kind()) { + case Term: { + suggestions.add(getTermSuggestion(name, suggestionsES)); + break; + } + case Phrase: { + suggestions.add(getPhraseSuggestion(name, suggestionsES)); + break; + } + case Completion: { + suggestions.add(getCompletionSuggestion(name, suggestionsES, entityCreator)); + break; + } + default: + break; + } + } + }); + + // todo: hasScoreDocs checks if any one + boolean hasScoreDocs = false; + + return new Suggest(suggestions, hasScoreDocs); + } + + private static TermSuggestion getTermSuggestion(String name, List> suggestionsES) { + + List entries = new ArrayList<>(); + suggestionsES.forEach(suggestionES -> { + TermSuggest termSuggest = suggestionES.term(); + + TermSuggestOption optionES = termSuggest.options(); + List options = new ArrayList<>(); + options.add(new TermSuggestion.Entry.Option(optionES.text(), null, optionES.score(), null, + Math.toIntExact(optionES.freq()))); + entries.add(new TermSuggestion.Entry(termSuggest.text(), termSuggest.offset(), termSuggest.length(), options)); + }); + return new TermSuggestion(name, suggestionsES.size(), entries, null); + } + + private static PhraseSuggestion getPhraseSuggestion(String name, List> suggestionsES) { + + List entries = new ArrayList<>(); + suggestionsES.forEach(suggestionES -> { + PhraseSuggest phraseSuggest = suggestionES.phrase(); + PhraseSuggestOption optionES = phraseSuggest.options(); + List options = new ArrayList<>(); + options.add(new PhraseSuggestion.Entry.Option(optionES.text(), optionES.highlighted(), null, null)); + entries.add(new PhraseSuggestion.Entry(phraseSuggest.text(), phraseSuggest.offset(), phraseSuggest.length(), + options, null)); + }); + return new PhraseSuggestion(name, suggestionsES.size(), entries); + } + + private static CompletionSuggestion getCompletionSuggestion(String name, + List> suggestionsES, SearchDocumentResponse.EntityCreator entityCreator) { + List> entries = new ArrayList<>(); + suggestionsES.forEach(suggestionES -> { + CompletionSuggest completionSuggest = suggestionES.completion(); + List> options = new ArrayList<>(); + List> optionsES = completionSuggest.options(); + optionsES.forEach(optionES -> { + SearchDocument searchDocument = (optionES.source() != null) ? DocumentAdapters.from(optionES) : null; + T hitEntity = null; + + if (searchDocument != null) { + try { + hitEntity = entityCreator.apply(searchDocument).get(); + } catch (Exception e) { + if (LOGGER.isWarnEnabled()) { + LOGGER.warn("Error creating entity from SearchDocument: " + e.getMessage()); + } + } + } + + Map> contexts = new HashMap<>(); + optionES.contexts().forEach((key, contextList) -> contexts.put(key, + contextList.stream().map(context -> context._get().toString()).collect(Collectors.toSet()))); + + // response from the new client does not have a doc and shardindex as the ScoreDoc from the old client responses + + options.add(new CompletionSuggestion.Entry.Option<>(optionES.text(), null, optionES.score(), + optionES.collateMatch() != null ? optionES.collateMatch() : false, contexts, + new ScoreDoc(optionES.score() != null ? optionES.score() : Double.NaN, null, null), searchDocument, + hitEntity)); + }); + + entries.add(new CompletionSuggestion.Entry<>(completionSuggest.text(), completionSuggest.offset(), + completionSuggest.length(), options)); + }); + return new CompletionSuggestion<>(name, suggestionsES.size(), entries); + } } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java index 07ace4c68..d1dae04b3 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java @@ -745,9 +745,9 @@ public T doWith(@Nullable Document document) { IndexedObjectInformation indexedObjectInformation = IndexedObjectInformation.of( // documentAfterLoad.hasId() ? documentAfterLoad.getId() : null, // - documentAfterLoad.getSeqNo(), // - documentAfterLoad.getPrimaryTerm(), // - documentAfterLoad.getVersion()); // + documentAfterLoad.hasSeqNo() ? documentAfterLoad.getSeqNo() : null, // + documentAfterLoad.hasPrimaryTerm() ? documentAfterLoad.getPrimaryTerm() : null, // + documentAfterLoad.hasVersion() ? documentAfterLoad.getVersion() : null); // entity = updateIndexedObject(entity, indexedObjectInformation); return maybeCallbackAfterConvert(entity, documentAfterLoad, index); diff --git a/src/main/java/org/springframework/data/elasticsearch/core/AbstractReactiveElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/core/AbstractReactiveElasticsearchTemplate.java index 83b834d98..a13d18454 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/AbstractReactiveElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/AbstractReactiveElasticsearchTemplate.java @@ -549,9 +549,9 @@ public Mono toEntity(@Nullable Document document) { IndexedObjectInformation indexedObjectInformation = IndexedObjectInformation.of( // documentAfterLoad.hasId() ? documentAfterLoad.getId() : null, // - documentAfterLoad.getSeqNo(), // - documentAfterLoad.getPrimaryTerm(), // - documentAfterLoad.getVersion()); // + documentAfterLoad.hasSeqNo() ? documentAfterLoad.getSeqNo() : null, // + documentAfterLoad.hasPrimaryTerm() ? documentAfterLoad.getPrimaryTerm() : null, // + documentAfterLoad.hasVersion() ? documentAfterLoad.getVersion() : null); // entity = updateIndexedObject(entity, indexedObjectInformation); return maybeCallAfterConvert(entity, documentAfterLoad, index); diff --git a/src/main/java/org/springframework/data/elasticsearch/core/document/SearchDocumentResponseBuilder.java b/src/main/java/org/springframework/data/elasticsearch/core/document/SearchDocumentResponseBuilder.java index bb8b12849..51e1da753 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/document/SearchDocumentResponseBuilder.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/document/SearchDocumentResponseBuilder.java @@ -156,7 +156,7 @@ private static Suggest suggestFrom(@Nullable org.elasticsearch.search.sugges List options = new ArrayList<>(); for (org.elasticsearch.search.suggest.phrase.PhraseSuggestion.Entry.Option optionES : entryES) { options.add(new PhraseSuggestion.Entry.Option(textToString(optionES.getText()), - textToString(optionES.getHighlighted()), optionES.getScore(), optionES.collateMatch())); + textToString(optionES.getHighlighted()), (double) optionES.getScore(), optionES.collateMatch())); } entries.add(new PhraseSuggestion.Entry(textToString(entryES.getText()), entryES.getOffset(), @@ -188,7 +188,7 @@ private static Suggest suggestFrom(@Nullable org.elasticsearch.search.sugges } options.add(new CompletionSuggestion.Entry.Option<>(textToString(optionES.getText()), - textToString(optionES.getHighlighted()), optionES.getScore(), optionES.collateMatch(), + textToString(optionES.getHighlighted()), (double) optionES.getScore(), optionES.collateMatch(), optionES.getContexts(), scoreDocFrom(optionES.getDoc()), searchDocument, hitEntity)); } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/suggest/response/CompletionSuggestion.java b/src/main/java/org/springframework/data/elasticsearch/core/suggest/response/CompletionSuggestion.java index 1963766bb..e7410c499 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/suggest/response/CompletionSuggestion.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/suggest/response/CompletionSuggestion.java @@ -49,7 +49,7 @@ public static class Option extends Suggest.Suggestion.Entry.Option { @Nullable private final T hitEntity; @Nullable private SearchHit searchHit; - public Option(String text, String highlighted, float score, Boolean collateMatch, + public Option(String text, @Nullable String highlighted, @Nullable Double score, Boolean collateMatch, Map> contexts, ScoreDoc scoreDoc, @Nullable SearchDocument searchDocument, @Nullable T hitEntity) { super(text, highlighted, score, collateMatch); diff --git a/src/main/java/org/springframework/data/elasticsearch/core/suggest/response/PhraseSuggestion.java b/src/main/java/org/springframework/data/elasticsearch/core/suggest/response/PhraseSuggestion.java index 4a2999f16..80fc67662 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/suggest/response/PhraseSuggestion.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/suggest/response/PhraseSuggestion.java @@ -15,6 +15,8 @@ */ package org.springframework.data.elasticsearch.core.suggest.response; +import org.springframework.lang.Nullable; + import java.util.List; /** @@ -29,20 +31,21 @@ public PhraseSuggestion(String name, int size, List entries) { public static class Entry extends Suggest.Suggestion.Entry { - private final double cutoffScore; + @Nullable private final Double cutoffScore; - public Entry(String text, int offset, int length, List