All Field: Automatically detect when field level boosting is used, and optimize when its not, closes elastic#2189.

Author: kimchy

src/main/java/org/elasticsearch/common/lucene/all/AllEntries.java

@@ -79,6 +79,10 @@ public void addText(String name, String text, float boost) {
         entries.add(entry);
     }
 
+    public boolean customBoost() {
+        return customBoost;
+    }
+
     public void clear() {
         this.entries.clear();
         this.current = null;

src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java

@@ -69,6 +69,9 @@ public static class Builder extends AbstractFieldMapper.Builder<Builder, AllFiel
 
         private boolean enabled = Defaults.ENABLED;
 
+        // an internal flag, automatically set if we encounter boosting
+        boolean autoBoost = false;
+
         public Builder() {
             super(Defaults.NAME);
             builder = this;
@@ -103,7 +106,7 @@ protected Builder searchAnalyzer(NamedAnalyzer searchAnalyzer) {
         @Override
         public AllFieldMapper build(BuilderContext context) {
             return new AllFieldMapper(name, store, termVector, omitNorms, indexOptions,
-                    indexAnalyzer, searchAnalyzer, enabled);
+                    indexAnalyzer, searchAnalyzer, enabled, autoBoost);
         }
     }
 
@@ -117,6 +120,8 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
                 Object fieldNode = entry.getValue();
                 if (fieldName.equals("enabled")) {
                     builder.enabled(nodeBooleanValue(fieldNode));
+                } else if (fieldName.equals("auto_boost")) {
+                    builder.autoBoost = nodeBooleanValue(fieldNode);
                 }
             }
             return builder;
@@ -125,16 +130,24 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
 
 
     private boolean enabled;
+    // The autoBoost flag is automatically set based on indexed docs on the mappings
+    // if a doc is indexed with a specific boost value and part of _all, it is automatically
+    // set to true. This allows to optimize (automatically, which we like) for the common case
+    // where fields don't usually have boost associated with them, and we don't need to use the
+    // special SpanTermQuery to look at payloads
+    private volatile boolean autoBoost;
 
     public AllFieldMapper() {
-        this(Defaults.NAME, Defaults.STORE, Defaults.TERM_VECTOR, Defaults.OMIT_NORMS, Defaults.INDEX_OPTIONS, null, null, Defaults.ENABLED);
+        this(Defaults.NAME, Defaults.STORE, Defaults.TERM_VECTOR, Defaults.OMIT_NORMS, Defaults.INDEX_OPTIONS, null, null, Defaults.ENABLED, false);
     }
 
     protected AllFieldMapper(String name, Field.Store store, Field.TermVector termVector, boolean omitNorms, IndexOptions indexOptions,
-                             NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer, boolean enabled) {
+                             NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer, boolean enabled, boolean autoBoost) {
         super(new Names(name, name, name, name), Field.Index.ANALYZED, store, termVector, 1.0f, omitNorms, indexOptions, indexAnalyzer,
                 searchAnalyzer);
         this.enabled = enabled;
+        this.autoBoost = autoBoost;
+
     }
 
     public boolean enabled() {
@@ -143,16 +156,19 @@ public boolean enabled() {
 
     @Override
     public Query queryStringTermQuery(Term term) {
+        if (!autoBoost) {
+            return new TermQuery(term);
+        }
         if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
             return new AllTermQuery(term);
-        } 
+        }
         return new TermQuery(term);
     }
 
     @Override
     public Query fieldQuery(String value, QueryParseContext context) {
         return queryStringTermQuery(names().createIndexNameTerm(value));
-        
+
     }
 
     @Override
@@ -186,6 +202,13 @@ protected Fieldable parseCreateField(ParseContext context) throws IOException {
         // reset the entries
         context.allEntries().reset();
 
+        // if the autoBoost flag is not set, and we indexed a doc with custom boost, make
+        // sure to update the flag, and notify mappings on change
+        if (!autoBoost && context.allEntries().customBoost()) {
+            autoBoost = true;
+            context.setMappingsModified();
+        }
+
         Analyzer analyzer = findAnalyzer(context);
         return new AllField(names.indexName(), store, termVector, context.allEntries(), analyzer);
     }
@@ -240,6 +263,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (enabled != Defaults.ENABLED) {
             builder.field("enabled", enabled);
         }
+        if (autoBoost != false) {
+            builder.field("auto_boost", autoBoost);
+        }
         if (store != Defaults.STORE) {
             builder.field("store", store.name().toLowerCase());
         }

src/test/java/org/elasticsearch/test/unit/index/mapper/all/SimpleAllMapperTests.java

@@ -29,7 +29,6 @@
 import org.elasticsearch.common.lucene.all.AllTokenStream;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.FieldMappers;
 import org.elasticsearch.test.unit.index.mapper.MapperTests;
 import org.hamcrest.Matchers;
 import org.testng.annotations.Test;
@@ -60,7 +59,23 @@ public void testSimpleAllMappers() throws Exception {
         FieldMapper mapper = docMapper.mappers().smartNameFieldMapper("_all");
         assertThat(mapper.queryStringTermQuery(new Term("_all", "foobar")), Matchers.instanceOf(AllTermQuery.class));
     }
-    
+
+    @Test
+    public void testAllMappersNoBoost() throws Exception {
+        String mapping = copyToStringFromClasspath("/org/elasticsearch/test/unit/index/mapper/all/noboost-mapping.json");
+        DocumentMapper docMapper = MapperTests.newParser().parse(mapping);
+        byte[] json = copyToBytesFromClasspath("/org/elasticsearch/test/unit/index/mapper/all/test1.json");
+        Document doc = docMapper.parse(new BytesArray(json)).rootDoc();
+        AllField field = (AllField) doc.getFieldable("_all");
+        AllEntries allEntries = ((AllTokenStream) field.tokenStreamValue()).allEntries();
+        assertThat(allEntries.fields().size(), equalTo(3));
+        assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
+        assertThat(allEntries.fields().contains("name.last"), equalTo(true));
+        assertThat(allEntries.fields().contains("simple1"), equalTo(true));
+        FieldMapper mapper = docMapper.mappers().smartNameFieldMapper("_all");
+        assertThat(mapper.queryStringTermQuery(new Term("_all", "foobar")), Matchers.instanceOf(TermQuery.class));
+    }
+
     @Test
     public void testAllMappersTermQuery() throws Exception {
         String mapping = copyToStringFromClasspath("/org/elasticsearch/test/unit/index/mapper/all/mapping_omit_positions_on_all.json");

src/test/java/org/elasticsearch/test/unit/index/mapper/all/mapping.json

@@ -15,7 +15,8 @@
                     },
                     "last":{
                         "type":"string",
-                        "index":"not_analyzed"
+                        "index":"not_analyzed",
+                        "boost":2.0
                     }
                 }
             },

src/test/java/org/elasticsearch/test/unit/index/mapper/all/noboost-mapping.json

@@ -0,0 +1,55 @@
+{
+    "person":{
+        "_all":{
+            "enabled":true
+        },
+        "properties":{
+            "name":{
+                "type":"object",
+                "dynamic":false,
+                "properties":{
+                    "first":{
+                        "type":"string",
+                        "store":"yes",
+                        "include_in_all":false
+                    },
+                    "last":{
+                        "type":"string",
+                        "index":"not_analyzed"
+                    }
+                }
+            },
+            "address":{
+                "type":"object",
+                "include_in_all":false,
+                "properties":{
+                    "first":{
+                        "properties":{
+                            "location":{
+                                "type":"string",
+                                "store":"yes",
+                                "index_name":"firstLocation"
+                            }
+                        }
+                    },
+                    "last":{
+                        "properties":{
+                            "location":{
+                                "type":"string",
+                                "include_in_all":true
+                            }
+                        }
+                    }
+                }
+            },
+            "simple1":{
+                "type":"long",
+                "include_in_all":true
+            },
+            "simple2":{
+                "type":"long",
+                "include_in_all":false
+            }
+        }
+    }
+}

src/test/java/org/elasticsearch/test/unit/index/mapper/all/store-mapping.json

@@ -16,7 +16,8 @@
                     },
                     "last":{
                         "type":"string",
-                        "index":"not_analyzed"
+                        "index":"not_analyzed",
+                        "boost":2.0
                     }
                 }
             },