Implementing support for HTTP 1.1 chunked encoding, and keepalive connections.

Author: deanberris

boost/network/protocol/http/impl/sync_connection_base.hpp

@@ -8,6 +8,8 @@
 
 #include <boost/network/protocol/http/traits/resolver_policy.hpp>
 #include <boost/network/protocol/http/detail/connection_helper.hpp>
+#include <boost/network/traits/ostringstream.hpp>
+#include <boost/network/traits/istringstream.hpp>
 #include <boost/asio/ssl.hpp>
 #include <boost/tuple/tuple.hpp>
 
@@ -97,35 +99,75 @@ namespace boost { namespace network { namespace http { namespace impl {
         void read_body(Socket & socket_, basic_response<Tag> & response_, boost::asio::streambuf & response_buffer) {
             typename ostringstream<Tag>::type body_stream;
 
-            if (response_buffer.size() > 0)
-                body_stream << &response_buffer;
-
             boost::system::error_code error;
-            if (!connection_keepalive<Tag>::value) {
+            // TODO tag dispatch based on whether it's HTTP 1.0 or HTTP 1.1
+            if (version_major == 1 && version_minor == 0) {
+                if (response_buffer.size() > 0)
+                    body_stream << &response_buffer;
+
                 while (boost::asio::read(socket_, response_buffer, boost::asio::transfer_at_least(1), error)) {
                     body_stream << &response_buffer;
                 }
-            } else {
+            } else if (version_major == 1 && version_minor == 1) {
                 // look for the content-length header
                 typename headers_range<basic_response<Tag> >::type content_length_range =
                     headers(response_)["Content-Length"];
-                if (empty(content_length_range))
-                    throw std::runtime_error("Unsupported response, missing 'Content-Length' header.");
-                size_t length = lexical_cast<size_t>(begin(content_length_range)->second);
-                std::cerr << "Before reading body of size " << length << "...\n";
-                size_t bytes_read = 0;
-                while ((bytes_read = boost::asio::read(socket_, response_buffer, boost::asio::transfer_at_least(1), error))) {
-                    body_stream << &response_buffer;
-                    length -= bytes_read;
-                    if ((length <= 0) or error)
-                        break;
+                if (empty(content_length_range)) {
+                    typename headers_range<basic_response<Tag> >::type transfer_encoding_range =
+                        headers(response_)["Transfer-Encoding"];
+                    if (empty(transfer_encoding_range)) throw std::runtime_error("Missing Transfer-Encoding Header from response.");
+                    if (boost::iequals(begin(transfer_encoding_range)->second, "chunked")) {
+                        bool stopping = false;
+                        do { 
+                            std::size_t chunk_size_line = read_until(socket_, response_buffer, "\r\n", error);
+                            if ((chunk_size_line == 0) && (error != boost::asio::error::eof)) throw boost::system::system_error(error);
+                            std::size_t chunk_size = 0;
+                            string_type data;
+                            {
+                                std::istream chunk_stream(&response_buffer);
+                                std::getline(chunk_stream, data);
+                                typename istringstream<Tag>::type chunk_size_stream(data);
+                                chunk_size_stream >> std::hex >> chunk_size;
+                            }
+                            if (chunk_size == 0) {
+                                stopping = true;
+                                if (!read_until(socket_, response_buffer, "\r\n", error) && (error != boost::asio::error::eof))
+                                    throw boost::system::system_error(error);
+                            } else {
+                                bool stopping_inner = false;
+                                do {
+                                    std::size_t chunk_bytes_read = read(socket_, response_buffer, boost::asio::transfer_at_least(chunk_size + 2), error);
+                                    if (chunk_bytes_read == 0) {
+                                        if (error != boost::asio::error::eof) throw boost::system::system_error(error);
+                                        stopping_inner = true;
+                                    }
+
+                                    std::istreambuf_iterator<char> eos;
+                                    std::istreambuf_iterator<char> stream_iterator(&response_buffer);
+                                    for (; chunk_size > 0 && stream_iterator != eos; --chunk_size)
+                                        body_stream << *stream_iterator++;
+                                    response_buffer.consume(2);
+                                } while (!stopping_inner && chunk_size != 0);
+
+                                if (chunk_size != 0)
+                                    throw std::runtime_error("Size mismatch between tranfer encoding chunk data size and declared chunk size.");
+                            } 
+                        } while (!stopping);
+                    } else throw std::runtime_error("Unsupported Transfer-Encoding.");
+                } else {
+                    size_t length = lexical_cast<size_t>(begin(content_length_range)->second);
+                    size_t bytes_read = 0;
+                    while ((bytes_read = boost::asio::read(socket_, response_buffer, boost::asio::transfer_at_least(1), error))) {
+                        body_stream << &response_buffer;
+                        length -= bytes_read;
+                        if ((length <= 0) or error)
+                            break;
+                    }
                 }
-                std::cerr << "After reading body of size " << length << "...\n";
+            } else {
+                throw std::runtime_error("Unsupported HTTP version number.");
             }
 
-            if (error != boost::asio::error::eof)
-                throw boost::system::system_error(error);
-
             response_ << body(body_stream.str());
         }
 

boost/network/protocol/http/policies/pooled_connection.hpp

@@ -10,12 +10,10 @@
 
 #include <boost/unordered_map.hpp>
 #include <boost/shared_ptr.hpp>
-#include <utility>
 #include <boost/network/protocol/http/detail/connection_helper.hpp>
 #include <boost/network/protocol/http/impl/sync_connection_base.hpp>
 #include <boost/algorithm/string/predicate.hpp>
-
-#include <iostream>
+#include <utility>
 
 #ifndef BOOST_NETWORK_HTTP_MAXIMUM_REDIRECT_COUNT
 #define BOOST_NETWORK_HTTP_MAXIMUM_REDIRECT_COUNT 5
@@ -60,22 +58,18 @@ namespace boost { namespace network { namespace http {
                 if (!pimpl->is_open()) {
                     pimpl->init_socket(request_.host(), lexical_cast<string_type>(request_.port()));
                 }
-                std::cerr << "Sending request to " << request_.host() << "...\n";
                 pimpl->send_request_impl(method, request_);
                 response_ = basic_response<Tag>();
                 response_ << source(request_.host());
 
                 boost::asio::streambuf response_buffer;
-                std::cerr << "Reading status from " << request_.host() << "...\n";
                 pimpl->read_status(response_, response_buffer);
-                std::cerr << "Reading headers from " << request_.host() << "...\n";
                 pimpl->read_headers(response_, response_buffer);
                 if (
                     get_body && response_.status() != 304 
                     && (response_.status() != 204)
                     && not (response_.status() >= 100 && response_.status() <= 199)
                    ) {
-                    std::cerr << "Reading body from " << request_.host() << "...\n";
                     pimpl->read_body(response_, response_buffer);
                 }
 
@@ -95,9 +89,7 @@ namespace boost { namespace network { namespace http {
 
                 typename headers_range<basic_response<Tag> >::type connection_range = headers(response_)["Connection"];
                 if (!empty(connection_range) && begin(connection_range)->second == string_type("close")) {
-                    std::cerr << "Before closing socket for " << request_.host() << ":" << request_.port() << "...\n";
                     pimpl->close_socket();
-                    std::cerr << "After closing socket for " << request_.host() << ":" << request_.port() << "...\n";
                 }
 
                 return response_;

boost/network/traits/istringstream.hpp

@@ -0,0 +1,60 @@
+//            Copyright (c) Glyn Matthews 2009.
+//         Copyright (c) Dean Michael Berris 2009.
+// Distributed under the Boost Software License, Version 1.0.
+//    (See accompanying file LICENSE_1_0.txt or copy at
+//          http://www.boost.org/LICENSE_1_0.txt)
+
+#ifndef BOOST_NETWORK_TRAITS_ISTRINGSTREAM_INC
+# define BOOST_NETWORK_TRAITS_ISTRINGSTREAM_INC
+
+# include <sstream>
+# include <boost/network/tags.hpp>
+
+namespace boost { namespace network {
+
+    template <class Tag>
+    struct unsupported_tag;
+
+    template <class Tag>
+    struct istringstream {
+        typedef unsupported_tag<Tag> type;
+    };
+
+    template <>
+    struct istringstream<tags::default_string> {
+        typedef std::istringstream type;
+    };
+
+
+    template <>
+    struct istringstream<tags::default_wstring> {
+        typedef std::wistringstream type;
+    };
+
+    template <>
+    struct istringstream<tags::http_default_8bit_tcp_resolve> {
+        typedef std::istringstream type;
+    };
+
+    template <>
+    struct istringstream<tags::http_default_8bit_udp_resolve> {
+        typedef std::istringstream type;
+    };
+
+    template <>
+    struct istringstream<tags::http_keepalive_8bit_tcp_resolve> {
+        typedef std::istringstream type;
+    };
+
+    template <>
+    struct istringstream<tags::http_keepalive_8bit_udp_resolve> {
+        typedef std::istringstream type;
+    };
+
+} // namespace network
+
+} // namespace boost
+
+
+#endif // BOOST_NETWORK_TRAITS_ISTRINGSTREAM_INC
+

libs/network/example/http_client.cpp

@@ -57,6 +57,7 @@ int main(int argc, char * argv[]) {
         using namespace boost::network;
 
         http::basic_client<tags::http_keepalive_8bit_udp_resolve,1,1>::request request(source);
+        request << header("Connection", "close");
         http::basic_client<tags::http_keepalive_8bit_udp_resolve,1,1> client(
             http::basic_client<tags::http_keepalive_8bit_udp_resolve,1,1>::follow_redirects
             );

libs/network/example/simple_wget.cpp

@@ -47,8 +47,8 @@ main(int argc, char *argv[]) {
     try {
         http::client client;
         const char *uri = argv[1];
-        http::request request(uri);
-        http::response response = client.get(request);
+        http::client::request request(uri);
+        http::client::response response = client.get(request);
 
         std::string filename = get_filename(uri);
         std::cout << "Saving to: " << filename << std::endl;