Use C* as a blob storage for small emails. Major refactoring of blob

storage. fixed #32

Author: Rustam Aliyev

config/elasticinbox.cml

@@ -26,6 +26,12 @@ CREATE COLUMN FAMILY MessageMetadata WITH
 	caching = keys_only AND
 	comment='Message metadata including headers, labels, markers, physical location, etc.';
 
+CREATE COLUMN FAMILY MessageBlob WITH
+    key_validation_class = 'CompositeType(TimeUUIDType, Int32Type)' AND
+    comparator = Int32Type AND
+    caching = keys_only AND
+    comment='Chunked message blobs';
+
 CREATE COLUMN FAMILY IndexLabels WITH
 	key_validation_class = UTF8Type AND
 	comparator = TimeUUIDType AND 

config/elasticinbox.yaml

@@ -26,9 +26,11 @@ lmtp_enable_pop3: true
 pop3_port: 2110
 pop3_max_connections: 20
 
-### Metadata storage driver
-# currently only "cassandra" supported
-metadata_storage_driver: cassandra
+### Database settings
+# Database used for storing metadata of accounts, labels and messages.
+# Optionally, it is possible to store blobs in the database (see database_blob_max_threshold).  
+# Currently only "cassandra" is supported.
+database_driver: cassandra
 
 # If you want to store parsed HTML and/or PLAIN text of the message body
 # in the metadata storage, enable below.
@@ -38,6 +40,11 @@ metadata_storage_driver: cassandra
 store_html_message: true
 store_plain_message: false
 
+# Maximum blob size in bytes which can be stored in the database.
+# Blobs larger than this value will be stored with the deafult blob profile (blobstore_write_profile).
+# Set to 0 to disable using database for blobs. Currently value can't be more than 128K.
+database_blob_max_size: 65536
+
 ### Cassandra settings
 # Specify Cassandra hosts (multiple for LB), cluster name, keyspace
 # and auto discovery
@@ -85,10 +92,10 @@ blobstore_profiles:
 # only one profile can be used for writing at the same time 
 blobstore_write_profile: fs-local
 
-# compress objects written to the blob store
+# compress objects written to the blob store (including database blobs)
 blobstore_enable_compression: true
 
-# encrypt objects written to the blob store
+# encrypt objects written to the blob store (excluding database blobs)
 blobstore_enable_encryption: false
 #blobstore_default_encryption_key: mykey1
 

itests/src/test/java/com/elasticinbox/itests/AbstractIntegrationTest.java

@@ -4,7 +4,6 @@
 import static org.ops4j.pax.exam.CoreOptions.felix;
 import static org.ops4j.pax.exam.CoreOptions.mavenBundle;
 import static org.ops4j.pax.exam.CoreOptions.options;
-import static org.ops4j.pax.exam.CoreOptions.repository;
 import static org.ops4j.pax.exam.CoreOptions.scanDir;
 import static org.ops4j.pax.exam.CoreOptions.systemProperty;
 import static org.ops4j.pax.exam.CoreOptions.workingDirectory;
@@ -46,7 +45,7 @@ public Option[] config()
 				//junitBundles(),
 				felix().version("3.2.2"),
 				workingDirectory("target/paxrunner/"),
-				repository("https://repository.apache.org/snapshots/").allowSnapshots(),
+				//repository("https://repository.apache.org/snapshots/").allowSnapshots(),
 
 				// Configs
 				systemProperty("elasticinbox.config").value("../test-classes/elasticinbox.yaml"),

itests/src/test/java/com/elasticinbox/itests/Pop3IT.java

@@ -94,7 +94,7 @@ public Option[] config()
 	public void testListUidl() throws IOException
 	{
 		initAccount();
-
+		
 		// load messages with POP3 label
 		long mailSizeRegular = getResourceSize(EMAIL_REGULAR);
 		long mailSizeAttach = getResourceSize(EMAIL_LARGE_ATT);

itests/src/test/resources/elasticinbox.yaml

@@ -26,9 +26,11 @@ lmtp_enable_pop3: true
 pop3_port: 2110
 pop3_max_connections: 20
 
-### Metadata storage driver
-# currently only "cassandra" supported
-metadata_storage_driver: cassandra
+### Database settings
+# Database used for storing metadata of accounts, labels and messages.
+# Optionally, it is possible to store blobs in the database (see database_blob_max_threshold).  
+# Currently only "cassandra" is supported.
+database_driver: cassandra
 
 # If you want to store parsed HTML and/or PLAIN text of the message body
 # in the metadata storage, enable below.
@@ -38,6 +40,11 @@ metadata_storage_driver: cassandra
 store_html_message: true
 store_plain_message: false
 
+# Maximum blob size in bytes which can be stored in the database.
+# Blobs larger than this value will be stored with the deafult blob profile (blobstore_write_profile).
+# Set to 0 to disable using database for blobs. Currently value can't be more than 128K.
+database_blob_max_size: 65536
+
 ### Cassandra settings
 # Specify Cassandra hosts (multiple for LB), cluster name, keyspace
 # and auto discovery

modules/config/src/main/java/com/elasticinbox/config/Config.java

@@ -53,10 +53,11 @@ public class Config
 	public Integer pop3_port;
 	public Integer pop3_max_connections;
 
-	// metadata storage settings
-	public String metadata_storage_driver;
+	// Database settings
+	public String database_driver;
 	public Boolean store_html_message;
 	public Boolean store_plain_message;
+	public Long database_blob_max_size;
 
 	// Cassandra settings
 	public List<String> cassandra_hosts;

modules/config/src/main/java/com/elasticinbox/config/Configurator.java

@@ -112,6 +112,18 @@ static URI getStorageConfigURL() throws ConfigurationException
 			// TODO: add config verification here
 			// ...
 
+			// verify max database blob size
+			if(conf.database_blob_max_size > DatabaseConstants.MAX_BLOB_SIZE) {
+				throw new ConfigurationException("Blobs larger than "
+						+ DatabaseConstants.MAX_BLOB_SIZE + " bytes cannot be stored in the database");
+			}
+
+			// verify that blobstore profile name is not conflicting with internal name
+			if (conf.blobstore_profiles.containsKey(DatabaseConstants.DATABASE_PROFILE)) {
+				throw new ConfigurationException("BlobStore profile name cannot be '"
+						+ DatabaseConstants.DATABASE_PROFILE + "'");
+			}
+
 			// verify that default blobstore profile exists
 			if (!conf.blobstore_profiles.containsKey(conf.blobstore_write_profile)) {
 				throw new ConfigurationException("Default BlobStore Profile '"
@@ -167,8 +179,12 @@ public static Integer getPop3MaxConnections() {
 		return conf.pop3_max_connections;
 	}
 
-	public static String getMetadataStorageDriver() {
-		return (conf.metadata_storage_driver.equalsIgnoreCase("cassandra")) ? "CASSANDRA" : "UNKNOWN";
+	public static String getDatabaseDriver() {
+		return (conf.database_driver.equalsIgnoreCase("cassandra")) ? "CASSANDRA" : "UNKNOWN";
+	}
+	
+	public static Long getDatabaseBlobMaxSize() {
+		return conf.database_blob_max_size;
 	}
 
 	public static List<String> getCassandraHosts() {

modules/config/src/main/java/com/elasticinbox/config/DatabaseConstants.java

@@ -0,0 +1,41 @@
+/**
+ * Copyright (c) 2011-2013 Optimax Software Ltd.
+ * All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Optimax Software, ElasticInbox, nor the names
+ *    of its contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.elasticinbox.config;
+
+public class DatabaseConstants
+{
+	/** Maximum blob size allowed in database. 128KB */
+	public static long MAX_BLOB_SIZE = 0x20000;
+
+	/** Maximum blob block size. 128KB */
+	public static long BLOB_BLOCK_SIZE = 0x20000;
+
+	/** Reserved profile name for internal database storage (e.g. Cassandra) */
+	public static final String DATABASE_PROFILE = "db";
+}

modules/core/osgi.bnd

@@ -13,8 +13,7 @@ Export-Package: \
 	com.elasticinbox.core.log
 
 Private-Package: \
-	com.elasticinbox.core.blob.store,\
-	com.elasticinbox.core.blob.naming,\
+	com.elasticinbox.core.blob.*,\
 	com.elasticinbox.core.cassandra,\
 	com.elasticinbox.core.cassandra.persistence,\
 	com.elasticinbox.core.cassandra.utils

modules/core/src/main/java/com/elasticinbox/core/MessageDAO.java

@@ -224,8 +224,9 @@ public void removeLabel(Mailbox mailbox, Set<Integer> labelIds, UUID messageId)
 	 * 
 	 * @param mailbox
 	 * @param age
+	 * @throws IOException 
 	 */
-	public void purge(Mailbox mailbox, Date age);
+	public void purge(Mailbox mailbox, Date age) throws IOException;
 
 	/**
 	 * Calculates counters for all labels bu scanning through all messages. Used