Implementing keep-alive connections in HTTP 1.1, but no chunked transfer encoding support yet.

Author: deanberris

boost/network/protocol/http/client.hpp

@@ -71,6 +71,10 @@ namespace boost { namespace network { namespace http {
         //
         // =================================================================
 
+        ~basic_client() {
+            connection_base::cleanup();
+        }
+
         void clear_resolved_cache() {
             connection_base::endpoint_cache_.clear();
         }

boost/network/protocol/http/detail/connection_helper.hpp

@@ -8,6 +8,7 @@
 
 #include <boost/network/version.hpp>
 #include <boost/foreach.hpp>
+#include <boost/network/protocol/http/traits/connection_keepalive.hpp>
 
 namespace boost { namespace network { namespace http { namespace detail {
 
@@ -47,6 +48,10 @@ namespace boost { namespace network { namespace http { namespace detail {
                 << "Host: " << request_.host() << "\r\n"
                 << "Accept: */*\r\n";
 
+            if (version_major == 1 && version_minor == 1)
+                request_stream 
+                    << "Accept-Encoding: identity;q=1.0, *;q=0\r\n"; // only accept identity encoding
+
             typename headers_range<http::basic_request<Tag> >::type range = headers(request_);
             BOOST_FOREACH(typename headers_range<http::basic_request<Tag> >::type::value_type const & header, range) {
                 request_stream << header.first << ": " << header.second << "\r\n";
@@ -55,8 +60,12 @@ namespace boost { namespace network { namespace http { namespace detail {
             range = headers(request_)["user-agent"];
             if (empty(range)) request_stream << "User-Agent: cpp-netlib/" << BOOST_NETLIB_VERSION << "\r\n";
 
+            if (!connection_keepalive<Tag>::value) {
+                request_stream
+                    << "Connection: close\r\n";
+            }
             request_stream
-                << "Connection: close\r\n\r\n";
+                << "\r\n";
 
             string_type body_ = body(request_);
             if (!body_.empty())

boost/network/protocol/http/impl/sync_connection_base.hpp

@@ -30,6 +30,8 @@ namespace boost { namespace network { namespace http { namespace impl {
         void read_status(basic_response<Tag> & response_, boost::asio::streambuf & response_buffer) {}
         void read_headers(basic_response<Tag> & response_, boost::asio::streambuf & response_buffer) {}
         void read_body(basic_response<Tag> & response_, boost::asio::streambuf & response_buffer) {}
+        bool is_open() { return socket_.lowest_layer().is_open(); }
+        void close_socket() { if (is_open()) { socket_.lowest_layer().close(); } }
         ~https_sync_connection() {}
         private:
         resolver_type & resolver_;
@@ -129,15 +131,38 @@ namespace boost { namespace network { namespace http { namespace impl {
                 body_stream << &response_buffer;
 
             boost::system::error_code error;
-            while (boost::asio::read(socket_, response_buffer, boost::asio::transfer_at_least(1), error))
-                body_stream << &response_buffer;
+            if (!connection_keepalive<Tag>::value) {
+                while (boost::asio::read(socket_, response_buffer, boost::asio::transfer_at_least(1), error)) {
+                    body_stream << &response_buffer;
+                }
+            } else {
+                // look for the content-length header
+                typename headers_range<basic_response<Tag> >::type content_length_range =
+                    headers(response_)["Content-Length"];
+                if (empty(content_length_range))
+                    throw std::runtime_error("Unsupported response, missing 'Content-Length' header.");
+                size_t length = lexical_cast<size_t>(begin(content_length_range)->second);
+                std::cerr << "Before reading body of size " << length << "...\n";
+                size_t bytes_read = 0;
+                while ((bytes_read = boost::asio::read(socket_, response_buffer, boost::asio::transfer_at_least(1), error))) {
+                    body_stream << &response_buffer;
+                    length -= bytes_read;
+                    if ((length <= 0) or error)
+                        break;
+                }
+                std::cerr << "After reading body of size " << length << "...\n";
+            }
 
             if (error != boost::asio::error::eof)
                 throw boost::system::system_error(error);
 
             response_ << body(body_stream.str());
         }
 
+        bool is_open() { return socket_.is_open(); }
+
+        void close_socket() { if (is_open()) { socket_.close(); } }
+
         private:
 
         resolver_type & resolver_;
@@ -165,6 +190,8 @@ namespace boost { namespace network { namespace http { namespace impl {
         virtual void read_status(basic_response<Tag> & response_, boost::asio::streambuf & response_buffer) = 0;
         virtual void read_headers(basic_response<Tag> & response_, boost::asio::streambuf & response_buffer) = 0;
         virtual void read_body(basic_response<Tag> & response_, boost::asio::streambuf & response_buffer) = 0;
+        virtual bool is_open() = 0;
+        virtual void close_socket() = 0;
         virtual ~sync_connection_base() {}
         protected:
         sync_connection_base() {}

boost/network/protocol/http/policies/pooled_connection.hpp

@@ -10,8 +10,12 @@
 
 #include <boost/unordered_map.hpp>
 #include <boost/shared_ptr.hpp>
+#include <utility>
 #include <boost/network/protocol/http/detail/connection_helper.hpp>
 #include <boost/network/protocol/http/impl/sync_connection_base.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+
+#include <iostream>
 
 #ifndef BOOST_NETWORK_HTTP_MAXIMUM_REDIRECT_COUNT
 #define BOOST_NETWORK_HTTP_MAXIMUM_REDIRECT_COUNT 5
@@ -29,14 +33,22 @@ namespace boost { namespace network { namespace http {
         typedef function<typename resolver_base::resolver_iterator_pair(resolver_type &, string_type const &, string_type const &)> resolver_function_type;
 
         struct connection_impl {
+            typedef function<shared_ptr<connection_impl>(resolver_type &,basic_request<Tag> const &)> get_connection_function;
 
-            connection_impl(resolver_type & resolver, bool follow_redirect, string_type const & host, string_type const & port, resolver_function_type resolve, pooled_connection_policy & source)
-            : pimpl(impl::sync_connection_base<Tag,version_major,version_minor>::new_connection(resolver, resolve)), source_(source) {}
+            connection_impl(resolver_type & resolver, bool follow_redirect, string_type const & host, string_type const & port, resolver_function_type resolve, get_connection_function get_connection, bool https)
+            : pimpl(impl::sync_connection_base<Tag,version_major,version_minor>::new_connection(resolver, resolve, https))
+            , resolver_(resolver)
+            , connection_follow_redirect_(follow_redirect)
+            , get_connection_(get_connection) {}
 
             basic_response<Tag> send_request(string_type const & method, basic_request<Tag> request_, bool get_body) {
                 return send_request_recursive(method, request_, get_body, 0);
             }
 
+            ~connection_impl () {
+                pimpl.reset();
+            }
+
             private:
 
             basic_response<Tag> send_request_recursive(string_type const & method, basic_request<Tag> request_, bool get_body, int count) {
@@ -45,49 +57,64 @@ namespace boost { namespace network { namespace http {
 
                 basic_response<Tag> response_;
                 // check if the socket is open first
-                if (!pimpl->socket_.is_open()) {
+                if (!pimpl->is_open()) {
                     pimpl->init_socket(request_.host(), lexical_cast<string_type>(request_.port()));
                 }
+                std::cerr << "Sending request to " << request_.host() << "...\n";
                 pimpl->send_request_impl(method, request_);
                 response_ = basic_response<Tag>();
                 response_ << source(request_.host());
 
                 boost::asio::streambuf response_buffer;
+                std::cerr << "Reading status from " << request_.host() << "...\n";
                 pimpl->read_status(response_, response_buffer);
+                std::cerr << "Reading headers from " << request_.host() << "...\n";
                 pimpl->read_headers(response_, response_buffer);
-                if (get_body) pimpl->read_body(response_, response_buffer);
+                if (
+                    get_body && response_.status() != 304 
+                    && (response_.status() != 204)
+                    && not (response_.status() >= 100 && response_.status() <= 199)
+                   ) {
+                    std::cerr << "Reading body from " << request_.host() << "...\n";
+                    pimpl->read_body(response_, response_buffer);
+                }
 
-                if (follow_redirect_) {
+                if (connection_follow_redirect_) {
                     boost::uint16_t status = response_.status();
                     if (status >= 300 && status <= 307) {
                         typename headers_range<basic_response<Tag> >::type location_range = headers(response_)["Location"];
                         typename range_iterator<typename headers_range<basic_request<Tag> >::type>::type location_header = begin(location_range);
                         if (location_header != end(location_range)) {
                             request_.uri(location_header->second);
                             connection_ptr connection_;
-                            connection_ = source_.get_connection(pimpl->resolver_, request_);
+                            connection_ = get_connection_(resolver_, request_);
                             return connection_->send_request_recursive(method, request_, get_body, ++count);
                         } else throw std::runtime_error("Location header not defined in redirect response.");
                     }
                 }
 
+                typename headers_range<basic_response<Tag> >::type connection_range = headers(response_)["Connection"];
+                if (!empty(connection_range) && begin(connection_range)->second == string_type("close")) {
+                    std::cerr << "Before closing socket for " << request_.host() << ":" << request_.port() << "...\n";
+                    pimpl->close_socket();
+                    std::cerr << "After closing socket for " << request_.host() << ":" << request_.port() << "...\n";
+                }
+
                 return response_;
             }
+
             shared_ptr<http::impl::sync_connection_base<Tag,version_major,version_minor> > pimpl;
-            pooled_connection_policy & source_;
-            template <class T, unsigned vma, unsigned vmi> friend struct pooled_connection;
+            resolver_type & resolver_;
+            bool connection_follow_redirect_;
+            get_connection_function get_connection_;
         };
 
         typedef shared_ptr<connection_impl> connection_ptr;
 
-        private:
-
         typedef unordered_map<string_type, connection_ptr> host_connection_map;
         host_connection_map host_connections;
         bool follow_redirect_;
 
-        protected:
-
         connection_ptr get_connection(resolver_type & resolver, basic_request<Tag> const & request_) {
             string_type index = (request_.host() + ':') + lexical_cast<string_type>(request_.port());
             connection_ptr connection_;
@@ -104,15 +131,24 @@ namespace boost { namespace network { namespace http {
                         this,
                         _1, _2, _3
                         )
-                    , *this
+                    , bind(
+                        &pooled_connection_policy<Tag,version_major,version_minor>::get_connection,
+                        this,
+                        _1, _2
+                        )
+                    , boost::iequals(request_.protocol(), string_type("https"))
                     )
                 );
-                host_connections.insert(index, connection_);
+                host_connections.insert(std::make_pair(index, connection_));
                 return connection_;
             }
             return it->second;
         }
 
+        void cleanup() {
+            host_connection_map().swap(host_connections);
+        }
+
         pooled_connection_policy(bool cache_resolved, bool follow_redirect)
         : resolver_base(cache_resolved), host_connections(), follow_redirect_(follow_redirect) {}
 

boost/network/protocol/http/policies/simple_connection.hpp

@@ -92,6 +92,8 @@ namespace boost { namespace network { namespace http {
             return connection_;
         }
 
+        void cleanup() { }
+
         simple_connection_policy(bool cache_resolved, bool follow_redirect) 
         : resolver_base(cache_resolved), follow_redirect_(follow_redirect) {}
 

boost/network/protocol/http/traits.hpp

@@ -13,5 +13,6 @@
 // Convenience header for including different traits implementations.
 #include <boost/network/protocol/http/traits/message_traits.hpp>
 #include <boost/network/protocol/http/traits/parser_traits.hpp>
+#include <boost/network/protocol/http/traits/connection_keepalive.hpp>
 
 #endif // BOOST_NETWORK_PROTOCOL_HTTP_TRAITS_HPP

boost/network/protocol/http/traits/connection_keepalive.hpp

@@ -0,0 +1,30 @@
+#ifndef BOOST_NETWORK_PROTOCOL_HTTP_TRAITS_CONECTION_KEEPALIVE_20091218
+#define BOOST_NETWORK_PROTOCOL_HTTP_TRAITS_CONECTION_KEEPALIVE_20091218
+
+#include <boost/network/tags.hpp>
+
+namespace boost { namespace network { namespace http {
+
+    template <class Tag>
+    struct connection_keepalive {
+        static bool const value = false;
+    };
+
+    template <>
+    struct connection_keepalive<tags::http_keepalive_8bit_udp_resolve> {
+        static bool const value = true;
+    };
+
+    template <>
+    struct connection_keepalive<tags::http_keepalive_8bit_tcp_resolve> {
+        static bool const value = true;
+    };
+    
+} /* http */
+    
+} /* network */
+    
+} /* boost */
+
+#endif // BOOST_NETWORK_PROTOCOL_HTTP_TRAITS_CONECTION_KEEPALIVE_20091218
+

boost/network/protocol/http/traits/resolver.hpp

@@ -28,6 +28,16 @@ namespace boost { namespace network { namespace http {
         typedef boost::asio::ip::udp::resolver type;
     };
 
+    template <>
+    struct resolver<tags::http_keepalive_8bit_tcp_resolve> {
+        typedef boost::asio::ip::tcp::resolver type;
+    };
+
+    template <>
+    struct resolver<tags::http_keepalive_8bit_udp_resolve> {
+        typedef boost::asio::ip::udp::resolver type;
+    };
+
 } // namespace http
 
 } // namespace network

boost/network/protocol/http/traits/resolver_policy.hpp

@@ -29,6 +29,16 @@ namespace boost { namespace network { namespace http {
         typedef policies::sync_resolver<tags::http_default_8bit_udp_resolve> type;
     };
 
+    template <>
+    struct resolver_policy<tags::http_keepalive_8bit_udp_resolve> {
+        typedef policies::sync_resolver<tags::http_keepalive_8bit_udp_resolve> type;
+    };
+
+    template <>
+    struct resolver_policy<tags::http_keepalive_8bit_tcp_resolve> {
+        typedef policies::sync_resolver<tags::http_keepalive_8bit_tcp_resolve> type;
+    };
+
 } // namespace http
 
 } // namespace network

boost/network/traits/ostringstream.hpp

@@ -1,4 +1,5 @@
 //            Copyright (c) Glyn Matthews 2009.
+//         Copyright (c) Dean Michael Berris 2009.
 // Distributed under the Boost Software License, Version 1.0.
 //    (See accompanying file LICENSE_1_0.txt or copy at
 //          http://www.boost.org/LICENSE_1_0.txt)
@@ -11,14 +12,14 @@
 
 namespace boost { namespace network {
 
-    template <
-        class Tag
-        >
+    template <class Tag>
+    struct unsupported_tag;
+
+    template <class Tag>
     struct ostringstream {
-        typedef void type;
+        typedef unsupported_tag<Tag> type;
     };
 
-
     template <>
     struct ostringstream<tags::default_string> {
         typedef std::ostringstream type;
@@ -40,6 +41,16 @@ namespace boost { namespace network {
         typedef std::ostringstream type;
     };
 
+    template <>
+    struct ostringstream<tags::http_keepalive_8bit_tcp_resolve> {
+        typedef std::ostringstream type;
+    };
+
+    template <>
+    struct ostringstream<tags::http_keepalive_8bit_udp_resolve> {
+        typedef std::ostringstream type;
+    };
+
 } // namespace network
 
 } // namespace boost

boost/network/traits/string.hpp

@@ -46,6 +46,16 @@ namespace boost { namespace network {
         typedef std::string type;
     };
 
+    template <>
+    struct string<tags::http_keepalive_8bit_tcp_resolve> {
+        typedef std::string type;
+    };
+
+    template <>
+    struct string<tags::http_keepalive_8bit_udp_resolve> {
+        typedef std::string type;
+    };
+
 } // namespace network
 
 } // namespace boost