add filter_cache_size to node stats

kimchy · kimchy · commit c03b95e9348f · 2010-12-29T12:53:09.000+02:00
diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/AllDocSet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/AllDocSet.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.common.lucene.docset;
 
 import org.apache.lucene.search.DocIdSetIterator;
+import org.elasticsearch.common.RamUsage;
 
 import java.io.IOException;
 
@@ -44,6 +45,10 @@ public AllDocSet(int maxDoc) {
         return doc < maxDoc;
     }
 
+    @Override public long sizeInBytes() {
+        return RamUsage.NUM_BYTES_INT;
+    }
+
     @Override public DocIdSetIterator iterator() throws IOException {
         return new AllDocIdSetIterator(maxDoc);
     }
diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/AndDocSet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/AndDocSet.java
@@ -55,6 +55,14 @@ public AndDocSet(List<DocSet> sets) {
 //        return true;
     }
 
+    @Override public long sizeInBytes() {
+        long sizeInBytes = 0;
+        for (DocSet set : sets) {
+            sizeInBytes += set.sizeInBytes();
+        }
+        return sizeInBytes;
+    }
+
     @Override public DocIdSetIterator iterator() throws IOException {
         return new AndDocIdSetIterator();
     }
diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/DocSet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/DocSet.java
@@ -41,7 +41,13 @@ public abstract class DocSet extends DocIdSet {
         @Override public boolean isCacheable() {
             return true;
         }
+
+        @Override public long sizeInBytes() {
+            return 0;
+        }
     };
 
     public abstract boolean get(int doc) throws IOException;
+
+    public abstract long sizeInBytes();
 }
diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/GetDocSet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/GetDocSet.java
@@ -34,6 +34,10 @@ protected GetDocSet(int maxDoc) {
         this.maxDoc = maxDoc;
     }
 
+    @Override public long sizeInBytes() {
+        return 0;
+    }
+
     @Override public DocIdSetIterator iterator() throws IOException {
         return new DocIdSetIterator() {
             private int doc = -1;
diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/NotDocSet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/NotDocSet.java
@@ -44,6 +44,10 @@ public NotDocSet(DocSet set, int max) {
         return !set.get(doc);
     }
 
+    @Override public long sizeInBytes() {
+        return set.sizeInBytes();
+    }
+
     // This seems like overhead compared to testing with get and iterating over docs
 //    @Override public DocIdSetIterator iterator() throws IOException {
 //        return new NotDocIdSetIterator();
diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/OpenBitDocSet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/OpenBitDocSet.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.util.OpenBitSet;
 import org.apache.lucene.util.OpenBitSetDISI;
+import org.elasticsearch.common.RamUsage;
 
 import java.io.IOException;
 
@@ -59,4 +60,8 @@ public OpenBitSet set() {
     @Override public DocIdSetIterator iterator() throws IOException {
         return set.iterator();
     }
+
+    @Override public long sizeInBytes() {
+        return set.getBits().length * RamUsage.NUM_BYTES_LONG + RamUsage.NUM_BYTES_ARRAY_HEADER + RamUsage.NUM_BYTES_INT /* wlen */;
+    }
 }
diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/OrDocSet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/OrDocSet.java
@@ -37,22 +37,30 @@ public OrDocSet(List<DocSet> sets) {
     }
 
     @Override public boolean get(int doc) throws IOException {
+        for (DocSet s : sets) {
+            if (s.get(doc)) return true;
+        }
+        return false;
+    }
+
+    @Override public boolean isCacheable() {
         // not cacheable, the reason is that by default, when constructing the filter, it is not cacheable,
         // so if someone wants it to be cacheable, we might as well construct a cached version of the result
         return false;
-//        for (DocSet s : sets) {
-//            if (s.get(doc)) return true;
+//        for (DocSet set : sets) {
+//            if (!set.isCacheable()) {
+//                return false;
+//            }
 //        }
-//        return false;
+//        return true;
     }
 
-    @Override public boolean isCacheable() {
+    @Override public long sizeInBytes() {
+        long sizeInBytes = 0;
         for (DocSet set : sets) {
-            if (!set.isCacheable()) {
-                return false;
-            }
+            sizeInBytes += set.sizeInBytes();
         }
-        return true;
+        return sizeInBytes;
     }
 
     @Override public DocIdSetIterator iterator() throws IOException {
diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/FilterCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/FilterCache.java
@@ -45,4 +45,6 @@ public interface FilterCache extends IndexComponent, CloseableComponent {
      * Clears unreferenced filters.
      */
     void clearUnreferenced();
+
+    long sizeInBytes();
 }
diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/none/NoneFilterCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/none/NoneFilterCache.java
@@ -69,4 +69,8 @@ public class NoneFilterCache extends AbstractIndexComponent implements FilterCac
     @Override public void clearUnreferenced() {
         // nothing to do here
     }
+
+    @Override public long sizeInBytes() {
+        return 0;
+    }
 }
diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java
@@ -88,6 +88,16 @@ protected AbstractConcurrentMapFilterCache(Index index, @IndexSettings Settings
 //        }
     }
 
+    @Override public long sizeInBytes() {
+        long sizeInBytes = 0;
+        for (ConcurrentMap<Filter, DocSet> map : cache.values()) {
+            for (DocSet docSet : map.values()) {
+                sizeInBytes += docSet.sizeInBytes();
+            }
+        }
+        return sizeInBytes;
+    }
+
     @Override public Filter cache(Filter filterToCache) {
         if (isCached(filterToCache)) {
             return filterToCache;
diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractDoubleConcurrentMapFilterCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractDoubleConcurrentMapFilterCache.java
@@ -79,6 +79,21 @@ protected AbstractDoubleConcurrentMapFilterCache(Index index, @IndexSettings Set
     @Override public void clearUnreferenced() {
     }
 
+    @Override public long sizeInBytes() {
+        long sizeInBytes = 0;
+        for (ConcurrentMap<Filter, DocSet> map : cache.values()) {
+            for (DocSet docSet : map.values()) {
+                sizeInBytes += docSet.sizeInBytes();
+            }
+        }
+        for (ConcurrentMap<Filter, DocSet> map : weakCache.values()) {
+            for (DocSet docSet : map.values()) {
+                sizeInBytes += docSet.sizeInBytes();
+            }
+        }
+        return sizeInBytes;
+    }
+
     @Override public Filter cache(Filter filterToCache) {
         if (isCached(filterToCache)) {
             return filterToCache;
diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/ScriptFilterParser.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/ScriptFilterParser.java
@@ -164,6 +164,10 @@ public ScriptDocSet(IndexReader reader, SearchScript searchScript) {
                 this.searchScript = searchScript;
             }
 
+            @Override public long sizeInBytes() {
+                return 0;
+            }
+
             @Override public boolean isCacheable() {
                 // not cacheable for several reasons:
                 // 1. The script service is shared and holds the current reader executing against, and it
diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/IndicesStats.java b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/IndicesStats.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentBuilderString;
 
 import java.io.IOException;
 import java.io.Serializable;
@@ -40,12 +41,15 @@ public class IndicesStats implements Streamable, Serializable, ToXContent {
 
     private ByteSizeValue fieldCacheSize;
 
+    private ByteSizeValue filterCacheSize;
+
     IndicesStats() {
     }
 
-    public IndicesStats(ByteSizeValue storeSize, ByteSizeValue fieldCacheSize) {
+    public IndicesStats(ByteSizeValue storeSize, ByteSizeValue fieldCacheSize, ByteSizeValue filterCacheSize) {
         this.storeSize = storeSize;
         this.fieldCacheSize = fieldCacheSize;
+        this.filterCacheSize = filterCacheSize;
     }
 
     /**
@@ -70,6 +74,14 @@ public ByteSizeValue getFieldCacheSize() {
         return this.fieldCacheSize;
     }
 
+    public ByteSizeValue filterCacheSize() {
+        return this.filterCacheSize;
+    }
+
+    public ByteSizeValue getFilterCacheSize() {
+        return this.filterCacheSize;
+    }
+
     public static IndicesStats readIndicesStats(StreamInput in) throws IOException {
         IndicesStats stats = new IndicesStats();
         stats.readFrom(in);
@@ -79,19 +91,33 @@ public static IndicesStats readIndicesStats(StreamInput in) throws IOException {
     @Override public void readFrom(StreamInput in) throws IOException {
         storeSize = ByteSizeValue.readBytesSizeValue(in);
         fieldCacheSize = ByteSizeValue.readBytesSizeValue(in);
+        filterCacheSize = ByteSizeValue.readBytesSizeValue(in);
     }
 
     @Override public void writeTo(StreamOutput out) throws IOException {
         storeSize.writeTo(out);
         fieldCacheSize.writeTo(out);
+        filterCacheSize.writeTo(out);
     }
 
     @Override public void toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject("indices");
-        builder.field("store_size", storeSize.toString());
-        builder.field("store_size_in_bytes", storeSize.bytes());
-        builder.field("field_cache_size", fieldCacheSize.toString());
-        builder.field("field_cache_size_in_bytes", fieldCacheSize.bytes());
+        builder.startObject(Fields.INDICES);
+        builder.field(Fields.STORE_SIZE, storeSize.toString());
+        builder.field(Fields.STORE_SIZE_IN_BYTES, storeSize.bytes());
+        builder.field(Fields.FIELD_CACHE_SIZE, fieldCacheSize.toString());
+        builder.field(Fields.FIELD_CACHE_SIZE_IN_BYTES, fieldCacheSize.bytes());
+        builder.field(Fields.FILTER_CACHE_SIZE, filterCacheSize.toString());
+        builder.field(Fields.FILTER_CACHE_SIZE_IN_BYTES, filterCacheSize.bytes());
         builder.endObject();
     }
+
+    static final class Fields {
+        static final XContentBuilderString INDICES = new XContentBuilderString("indices");
+        static final XContentBuilderString STORE_SIZE = new XContentBuilderString("store_size");
+        static final XContentBuilderString STORE_SIZE_IN_BYTES = new XContentBuilderString("store_size_in_bytes");
+        static final XContentBuilderString FIELD_CACHE_SIZE = new XContentBuilderString("field_cache_size");
+        static final XContentBuilderString FIELD_CACHE_SIZE_IN_BYTES = new XContentBuilderString("field_cache_size_in_bytes");
+        static final XContentBuilderString FILTER_CACHE_SIZE = new XContentBuilderString("filter_cache_size");
+        static final XContentBuilderString FILTER_CACHE_SIZE_IN_BYTES = new XContentBuilderString("filter_cache_size_in_bytes");
+    }
 }
diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/InternalIndicesService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/InternalIndicesService.java
@@ -157,6 +157,7 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
     @Override public IndicesStats stats() {
         long storeTotalSize = 0;
         long fieldCacheTotalSize = 0;
+        long filterCacheTotalSize = 0;
         for (IndexService indexService : indices.values()) {
             for (IndexShard indexShard : indexService) {
                 try {
@@ -166,8 +167,9 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
                 }
             }
             fieldCacheTotalSize += indexService.cache().fieldData().sizeInBytes();
+            filterCacheTotalSize += indexService.cache().filter().sizeInBytes();
         }
-        return new IndicesStats(new ByteSizeValue(storeTotalSize), new ByteSizeValue(fieldCacheTotalSize));
+        return new IndicesStats(new ByteSizeValue(storeTotalSize), new ByteSizeValue(fieldCacheTotalSize), new ByteSizeValue(filterCacheTotalSize));
     }
 
     /**

Original file line number	Diff line number	Diff line change
`@@ -20,6 +20,7 @@`
`20`	`20`	`package org.elasticsearch.common.lucene.docset;`
`21`	`21`
`22`	`22`	`import org.apache.lucene.search.DocIdSetIterator;`
	`23`	`+import org.elasticsearch.common.RamUsage;`
`23`	`24`
`24`	`25`	`import java.io.IOException;`
`25`	`26`
`@@ -44,6 +45,10 @@ public AllDocSet(int maxDoc) {`
`44`	`45`	`return doc < maxDoc;`
`45`	`46`	`}`
`46`	`47`
	`48`	`+ @Override public long sizeInBytes() {`
	`49`	`+ return RamUsage.NUM_BYTES_INT;`
	`50`	`+ }`
	`51`	`+`
`47`	`52`	`@Override public DocIdSetIterator iterator() throws IOException {`
`48`	`53`	`return new AllDocIdSetIterator(maxDoc);`
`49`	`54`	`}`
Original file line number	Diff line number	Diff line change
`@@ -55,6 +55,14 @@ public AndDocSet(List<DocSet> sets) {`
`55`	`55`	`// return true;`
`56`	`56`	`}`
`57`	`57`
	`58`	`+ @Override public long sizeInBytes() {`
	`59`	`+ long sizeInBytes = 0;`
	`60`	`+ for (DocSet set : sets) {`
	`61`	`+ sizeInBytes += set.sizeInBytes();`
	`62`	`+ }`
	`63`	`+ return sizeInBytes;`
	`64`	`+ }`
	`65`	`+`
`58`	`66`	`@Override public DocIdSetIterator iterator() throws IOException {`
`59`	`67`	`return new AndDocIdSetIterator();`
`60`	`68`	`}`
Original file line number	Diff line number	Diff line change
`@@ -41,7 +41,13 @@ public abstract class DocSet extends DocIdSet {`
`41`	`41`	`@Override public boolean isCacheable() {`
`42`	`42`	`return true;`
`43`	`43`	`}`
	`44`	`+`
	`45`	`+ @Override public long sizeInBytes() {`
	`46`	`+ return 0;`
	`47`	`+ }`
`44`	`48`	`};`
`45`	`49`
`46`	`50`	`public abstract boolean get(int doc) throws IOException;`
	`51`	`+`
	`52`	`+ public abstract long sizeInBytes();`
`47`	`53`	`}`
Original file line number	Diff line number	Diff line change
`@@ -22,6 +22,7 @@`
`22`	`22`	`import org.apache.lucene.search.DocIdSetIterator;`
`23`	`23`	`import org.apache.lucene.util.OpenBitSet;`
`24`	`24`	`import org.apache.lucene.util.OpenBitSetDISI;`
	`25`	`+import org.elasticsearch.common.RamUsage;`
`25`	`26`
`26`	`27`	`import java.io.IOException;`
`27`	`28`
`@@ -59,4 +60,8 @@ public OpenBitSet set() {`
`59`	`60`	`@Override public DocIdSetIterator iterator() throws IOException {`
`60`	`61`	`return set.iterator();`
`61`	`62`	`}`
	`63`	`+`
	`64`	`+ @Override public long sizeInBytes() {`
	`65`	`+ return set.getBits().length * RamUsage.NUM_BYTES_LONG + RamUsage.NUM_BYTES_ARRAY_HEADER + RamUsage.NUM_BYTES_INT /* wlen */;`
	`66`	`+ }`
`62`	`67`	`}`
Original file line number	Diff line number	Diff line change
`@@ -45,4 +45,6 @@ public interface FilterCache extends IndexComponent, CloseableComponent {`
`45`	`45`	`* Clears unreferenced filters.`
`46`	`46`	`*/`
`47`	`47`	`void clearUnreferenced();`
	`48`	`+`
	`49`	`+ long sizeInBytes();`
`48`	`50`	`}`
Original file line number	Diff line number	Diff line change
`@@ -69,4 +69,8 @@ public class NoneFilterCache extends AbstractIndexComponent implements FilterCac`
`69`	`69`	`@Override public void clearUnreferenced() {`
`70`	`70`	`// nothing to do here`
`71`	`71`	`}`
	`72`	`+`
	`73`	`+ @Override public long sizeInBytes() {`
	`74`	`+ return 0;`
	`75`	`+ }`
`72`	`76`	`}`
Original file line number	Diff line number	Diff line change
`@@ -157,6 +157,7 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe`
`157`	`157`	`@Override public IndicesStats stats() {`
`158`	`158`	`long storeTotalSize = 0;`
`159`	`159`	`long fieldCacheTotalSize = 0;`
	`160`	`+ long filterCacheTotalSize = 0;`
`160`	`161`	`for (IndexService indexService : indices.values()) {`
`161`	`162`	`for (IndexShard indexShard : indexService) {`
`162`	`163`	`try {`
`@@ -166,8 +167,9 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe`
`166`	`167`	`}`
`167`	`168`	`}`
`168`	`169`	`fieldCacheTotalSize += indexService.cache().fieldData().sizeInBytes();`
	`170`	`+ filterCacheTotalSize += indexService.cache().filter().sizeInBytes();`
`169`	`171`	`}`
`170`		`- return new IndicesStats(new ByteSizeValue(storeTotalSize), new ByteSizeValue(fieldCacheTotalSize));`
	`172`	`+ return new IndicesStats(new ByteSizeValue(storeTotalSize), new ByteSizeValue(fieldCacheTotalSize), new ByteSizeValue(filterCacheTotalSize));`
`171`	`173`	`}`
`172`	`174`
`173`	`175`	`/**`