Skip to content

Commit 0279f41

Browse files
cipriancraciuncipriancraciun
committed
Added compressed serializer.
1 parent 3c31633 commit 0279f41

File tree

4 files changed

+174
-30
lines changed

4 files changed

+174
-30
lines changed

‎logback-common/src/main/java/eu/arkitech/logback/common/CompressedBinarySerializer.java

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
2+
package eu.arkitech.logback.common;
3+
4+
5+
import java.io.IOException;
6+
import java.io.InputStream;
7+
import java.io.OutputStream;
8+
import java.util.zip.Deflater;
9+
import java.util.zip.DeflaterOutputStream;
10+
import java.util.zip.Inflater;
11+
import java.util.zip.InflaterInputStream;
12+
13+
14+
public class CompressedBinarySerializer
15+
extends DefaultBinarySerializer
16+
{
17+
public CompressedBinarySerializer ()
18+
{
19+
this (
20+
CompressedBinarySerializer.compressedContentType, CompressedBinarySerializer.compressedContentEncoding,
21+
CompressedBinarySerializer.compressedBufferSize, CompressedBinarySerializer.compressedLevel);
22+
}
23+
24+
public CompressedBinarySerializer (final int level)
25+
{
26+
this (
27+
CompressedBinarySerializer.compressedContentType, CompressedBinarySerializer.compressedContentEncoding,
28+
CompressedBinarySerializer.compressedBufferSize, level);
29+
}
30+
31+
public CompressedBinarySerializer (
32+
final String contentType, final String contentEncoding, final int bufferSize, final int level)
33+
{
34+
super (contentType, contentEncoding, bufferSize);
35+
this.level = level;
36+
}
37+
38+
protected InputStream decorate (final InputStream stream)
39+
{
40+
return (new CompressedInputStream (stream));
41+
}
42+
43+
protected OutputStream decorate (final OutputStream stream)
44+
{
45+
return (new CompressedOutputStream (stream, this.level));
46+
}
47+
48+
protected int level;
49+
50+
public static final int compressedBufferSize = 1024;
51+
public static final String compressedContentEncoding = "binary";
52+
public static final String compressedContentType = "application/x-java-serialized-object+zlib";
53+
public static final int compressedLevel = 5;
54+
55+
public static class CompressedInputStream
56+
extends InflaterInputStream
57+
{
58+
public CompressedInputStream (final InputStream stream)
59+
{
60+
super (stream, new Inflater ());
61+
}
62+
63+
public void close () throws IOException
64+
{
65+
super.close ();
66+
this.inf.end ();
67+
}
68+
}
69+
70+
public static class CompressedOutputStream
71+
extends DeflaterOutputStream
72+
{
73+
public CompressedOutputStream (final OutputStream stream, final int level)
74+
{
75+
super (stream, new Deflater (level));
76+
}
77+
78+
public void close () throws IOException
79+
{
80+
super.close ();
81+
this.def.end ();
82+
}
83+
}
84+
}

‎logback-common/src/main/java/eu/arkitech/logback/common/DefaultBinarySerializer.java

Lines changed: 51 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,31 @@
44

55
import java.io.ByteArrayInputStream;
66
import java.io.ByteArrayOutputStream;
7+
import java.io.InputStream;
78
import java.io.ObjectInputStream;
89
import java.io.ObjectOutputStream;
10+
import java.io.OutputStream;
911

1012

1113
public class DefaultBinarySerializer
1214
implements
1315
Serializer
1416
{
17+
public DefaultBinarySerializer ()
18+
{
19+
this (
20+
DefaultBinarySerializer.defaultContentType, DefaultBinarySerializer.defaultContentEncoding,
21+
DefaultBinarySerializer.defaultBufferSize);
22+
}
23+
24+
public DefaultBinarySerializer (final String contentType, final String contentEncoding, final int bufferSize)
25+
{
26+
super ();
27+
this.contentType = contentType;
28+
this.contentEncoding = contentEncoding;
29+
this.bufferSize = bufferSize;
30+
}
31+
1532
public Object deserialize (final byte[] data)
1633
throws Throwable
1734
{
@@ -22,11 +39,18 @@ public Object deserialize (final byte[] data, final int offset, final int size)
2239
throws Throwable
2340
{
2441
final ByteArrayInputStream stream = new ByteArrayInputStream (data, offset, size);
25-
final ObjectInputStream decoder = new ObjectInputStream (stream);
42+
final InputStream decoratedStream = this.decorate (stream);
43+
final ObjectInputStream decoder = new ObjectInputStream (decoratedStream);
2644
final Object object = decoder.readObject ();
45+
decoder.close ();
2746
return (object);
2847
}
2948

49+
public int getBufferSize ()
50+
{
51+
return (this.bufferSize);
52+
}
53+
3054
public String getContentEncoding ()
3155
{
3256
return (this.contentEncoding);
@@ -37,21 +61,22 @@ public String getContentType ()
3761
return (this.contentType);
3862
}
3963

40-
public int getDefaultBufferSize ()
41-
{
42-
return (this.defaultBufferSize);
43-
}
44-
4564
public byte[] serialize (final Object object)
4665
throws Throwable
4766
{
48-
final ByteArrayOutputStream stream = new ByteArrayOutputStream (this.defaultBufferSize);
49-
final ObjectOutputStream encoder = new ObjectOutputStream (stream);
67+
final ByteArrayOutputStream stream = new ByteArrayOutputStream (this.bufferSize);
68+
final OutputStream decoratedStream = this.decorate (stream);
69+
final ObjectOutputStream encoder = new ObjectOutputStream (decoratedStream);
5070
encoder.writeObject (object);
5171
encoder.close ();
5272
return (stream.toByteArray ());
5373
}
5474

75+
public void setBufferSize (final int bufferSize)
76+
{
77+
this.bufferSize = bufferSize;
78+
}
79+
5580
public void setContentEncoding (final String contentEncoding)
5681
{
5782
this.contentEncoding = contentEncoding;
@@ -62,12 +87,25 @@ public void setContentType (final String contentType)
6287
this.contentType = contentType;
6388
}
6489

65-
public void setDefaultBufferSize (final int defaultBufferSize)
90+
@SuppressWarnings ("unused")
91+
protected InputStream decorate (final InputStream stream)
92+
throws Throwable
6693
{
67-
this.defaultBufferSize = defaultBufferSize;
94+
return (stream);
6895
}
6996

70-
protected String contentEncoding = "binary";
71-
protected String contentType = "application/x-java-serialized-object";
72-
protected int defaultBufferSize = 2048;
97+
@SuppressWarnings ("unused")
98+
protected OutputStream decorate (final OutputStream stream)
99+
throws Throwable
100+
{
101+
return (stream);
102+
}
103+
104+
protected int bufferSize;
105+
protected String contentEncoding;
106+
protected String contentType;
107+
108+
public static final int defaultBufferSize = 2048;
109+
public static final String defaultContentEncoding = "binary";
110+
public static final String defaultContentType = "application/x-java-serialized-object";
73111
}

‎logging-datastore-lucene/src/main/java/eu/arkitech/logging/datastore/lucene/LuceneDatastore.java

Lines changed: 30 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@
55
import java.io.File;
66
import java.util.List;
77

8+
import eu.arkitech.logback.common.DefaultBinarySerializer;
9+
810
import ch.qos.logback.classic.spi.ILoggingEvent;
911
import ch.qos.logback.core.filter.Filter;
10-
import eu.arkitech.logback.common.DefaultBinarySerializer;
12+
import eu.arkitech.logback.common.CompressedBinarySerializer;
1113
import eu.arkitech.logback.common.Serializer;
1214
import eu.arkitech.logging.datastore.common.Datastore;
1315
import org.apache.lucene.queryParser.ParseException;
@@ -18,19 +20,28 @@ public final class LuceneDatastore
1820
implements
1921
Datastore
2022
{
21-
public LuceneDatastore (final File path)
23+
public LuceneDatastore (final File path, final int compressed, final boolean indexed)
2224
{
2325
super ();
2426
this.path = path;
25-
this.serializer = new DefaultBinarySerializer ();
27+
this.compressed = compressed;
28+
this.indexed = indexed;
29+
if (this.compressed != -1)
30+
this.serializer = new CompressedBinarySerializer (this.compressed);
31+
else
32+
this.serializer = new DefaultBinarySerializer ();
2633
this.bdb = new BdbDatastore (this.path, true, this.serializer);
27-
this.index = new LuceneIndex (this.bdb);
34+
if (this.indexed)
35+
this.index = new LuceneIndex (this.bdb);
36+
else
37+
this.index = null;
2838
}
2939

3040
public final boolean close ()
3141
{
3242
boolean succeeded = true;
33-
succeeded |= this.index.close ();
43+
if (this.index != null)
44+
succeeded |= this.index.close ();
3445
succeeded |= this.bdb.close ();
3546
return (succeeded);
3647
}
@@ -39,10 +50,12 @@ public final boolean open ()
3950
{
4051
boolean succeeded = this.bdb.open ();
4152
if (succeeded) {
42-
succeeded = this.index.open ();
43-
if (!succeeded) {
44-
this.index.close ();
45-
this.bdb.close ();
53+
if (this.index != null) {
54+
succeeded = this.index.open ();
55+
if (!succeeded) {
56+
this.index.close ();
57+
this.bdb.close ();
58+
}
4659
}
4760
}
4861
return (succeeded);
@@ -51,11 +64,15 @@ public final boolean open ()
5164
public final Query parseQuery (final String query)
5265
throws ParseException
5366
{
67+
if (this.index == null)
68+
throw (new IllegalStateException ());
5469
return (this.index.parseQuery (query));
5570
}
5671

5772
public final List<LuceneQueryResult> query (final Query query, final int maxCount)
5873
{
74+
if (this.index == null)
75+
throw (new IllegalStateException ());
5976
return (this.index.query (query, maxCount));
6077
}
6178

@@ -79,10 +96,13 @@ public final ILoggingEvent select (final String key)
7996
public final String store (final ILoggingEvent event)
8097
{
8198
final String key = this.bdb.store (event);
82-
this.index.store (key, event);
99+
if (key != null && this.index != null)
100+
this.index.store (key, event);
83101
return (key);
84102
}
85103

104+
private final boolean indexed;
105+
private final int compressed;
86106
private final BdbDatastore bdb;
87107
private final LuceneIndex index;
88108
private final File path;

‎logging-datastore-lucene/src/main/java/eu/arkitech/logging/datastore/lucene/LuceneDatastoreMain.java

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,21 +28,23 @@ public static final void main (final String[] arguments)
2828
if (arguments.length != 0)
2929
throw (new IllegalArgumentException ());
3030

31-
final int storeCount = 0;
32-
final int selectCount = 0;
31+
final int compressed = 0;
32+
final boolean indexed = true;
33+
final int storeCount = 10 * 1000;
34+
final int selectCount = 10;
3335
final int queryCount = 10;
3436

3537
final Logger logger = LoggerFactory.getLogger (LuceneDatastoreMain.class);
3638

3739
logger.info ("opening");
3840
final File path = new File ("/tmp/logging");
39-
final LuceneDatastore datastore = new LuceneDatastore (path);
41+
final LuceneDatastore datastore = new LuceneDatastore (path, compressed, indexed);
4042
datastore.open ();
4143

4244
final LinkedList<String> keys;
4345
if (storeCount > 0) {
4446
logger.info ("storing");
45-
keys = new LinkedList<String> ();
47+
keys = new LinkedList<String> ();
4648
final RandomEventGenerator generator = new RandomEventGenerator ();
4749
for (int i = 0; i < storeCount; i++) {
4850
final ILoggingEvent event = generator.generate ();
@@ -56,7 +58,7 @@ public static final void main (final String[] arguments)
5658
} else
5759
keys = null;
5860

59-
if (keys != null && selectCount > 0) {
61+
if ((keys != null) && (selectCount > 0)) {
6062
logger.info ("selecting");
6163
int i = 0;
6264
for (final String key : keys) {
@@ -69,7 +71,7 @@ public static final void main (final String[] arguments)
6971
}
7072
}
7173

72-
if (queryCount > 0) {
74+
if ((queryCount > 0) && indexed) {
7375
final String queryString = "(level:INFO OR level:ERROR) AND message:a";
7476
logger.info ("querying `{}`", queryString);
7577
Query query = null;
@@ -92,7 +94,7 @@ public static final void main (final String[] arguments)
9294
}
9395
}
9496

95-
logger.info ("cloning");
97+
logger.info ("closing");
9698
datastore.close ();
9799
}
98100
}

0 commit comments

Comments
 (0)