From bdf1b2282166751de3ee08bdcb751247123b2034 Mon Sep 17 00:00:00 2001 From: chenzhaoyu Date: Thu, 1 Dec 2016 12:08:28 +0900 Subject: [PATCH 01/35] add client size close --- .../http/client/connection/async_normal.hpp | 73 +++++++++++++++++-- .../connection/async_protocol_handler.hpp | 16 ++++ 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp index d7016c764..67d6417d2 100644 --- a/boost/network/protocol/http/client/connection/async_normal.hpp +++ b/boost/network/protocol/http/client/connection/async_normal.hpp @@ -363,7 +363,17 @@ struct http_async_connection // Here we handle the body data ourself and append to an // ever-growing string buffer. auto self = this->shared_from_this(); - this->parse_body( + typename protocol_base::buffer_type::const_iterator begin = + this->part_begin; + typename protocol_base::buffer_type::const_iterator end = begin; + std::advance(end, remainder); + string_type body_string(begin, end); + if (check_parse_body_complete(body_string)) { + this->append_body(remainder); + handle_received_data(body, get_body, callback, + boost::asio::error::eof, bytes_transferred); + } else { + this->parse_body( delegate_, request_strand_.wrap([=] (boost::system::error_code const &ec, std::size_t bytes_transferred) { @@ -371,6 +381,7 @@ struct http_async_connection ec, bytes_transferred); }), remainder); + } } return; case body: @@ -428,10 +439,17 @@ struct http_async_connection ec, bytes_transferred); })); } else { - // Here we don't have a body callback. Let's make sure that we - // deal with the remainder from the headers part in case we do - // have data that's still in the buffer. - this->parse_body( + string_type body_string(this->partial_parsed); + body_string.append(this->part.begin(), bytes_transferred); + if (check_parse_body_complete (body_string)) { + this->append_body(bytes_transferred); + handle_received_data(body, get_body, callback, + boost::asio::error::eof, bytes_transferred); + } else { + // Here we don't have a body callback. Let's make sure that we + // deal with the remainder from the headers part in case we do + // have data that's still in the buffer. + this->parse_body( delegate_, request_strand_.wrap([=] (boost::system::error_code const &ec, std::size_t bytes_transferred) { @@ -439,6 +457,7 @@ struct http_async_connection ec, bytes_transferred); }), bytes_transferred); + } } } return; @@ -476,6 +495,50 @@ struct http_async_connection } } + inline bool check_parse_body_complete(string_type& body_string) + { + if (this->is_chunk_encoding) { + return parse_chunk_encoding_complete(body_string); + } + if (this->is_content_length && this->content_length >= 0) { + return parse_content_length_complete(body_string, this->content_length); + } + return false; + } + + inline bool parse_content_length_complete(string_type& body_string, size_t content_length){ + return body_string.length() >= content_length; + } + + bool parse_chunk_encoding_complete(string_type& body_string) { + string_type body; + string_type crlf = "\r\n"; + + typename string_type::iterator begin = body_string.begin(); + for (typename string_type::iterator iter = + std::search(begin, body_string.end(), crlf.begin(), crlf.end()); + iter != body_string.end(); + iter = + std::search(begin, body_string.end(), crlf.begin(), crlf.end())) { + string_type line(begin, iter); + if (line.empty()) { + std::advance(iter, 2); + begin = iter; + continue; + } + std::stringstream stream(line); + int len; + stream >> std::hex >> len; + std::advance(iter, 2); + if (!len) return true; + if (len <= body_string.end() - iter) { + std::advance(iter, len + 2); + } + begin = iter; + } + return false; + } + string_type parse_chunk_encoding(string_type& body_string) { string_type body; string_type crlf = "\r\n"; diff --git a/boost/network/protocol/http/client/connection/async_protocol_handler.hpp b/boost/network/protocol/http/client/connection/async_protocol_handler.hpp index e940862e3..c2c4e4ad6 100644 --- a/boost/network/protocol/http/client/connection/async_protocol_handler.hpp +++ b/boost/network/protocol/http/client/connection/async_protocol_handler.hpp @@ -246,6 +246,10 @@ struct http_async_protocol_handler { response_parser_type::http_header_line_done); typename headers_container::type headers; std::pair header_pair; + //init params + is_content_length = false; + content_length = -1; + is_chunk_end = false; while (!boost::empty(input_range)) { std::tie(parsed_ok, result_range) = headers_parser.parse_until( response_parser_type::http_header_colon, input_range); @@ -266,6 +270,10 @@ struct http_async_protocol_handler { } trim(header_pair.second); headers.insert(header_pair); + if (boost::iequals(header_pair.first, "Content-Length")) { + is_content_length = true; + content_length = std::stoi(header_pair.second); + } } // determine if the body parser will need to handle chunked encoding typename headers_range >::type transfer_encoding_range = @@ -325,6 +333,11 @@ struct http_async_protocol_handler { parsed_ok, std::distance(std::end(result_range), part_end)); } + void append_body(size_t bytes) { + partial_parsed.append(part_begin, part_begin + bytes); + part_begin = part.begin(); + } + template void parse_body(Delegate& delegate_, Callback callback, size_t bytes) { // TODO(dberris): we should really not use a string for the partial body @@ -351,6 +364,9 @@ struct http_async_protocol_handler { typename buffer_type::const_iterator part_begin; string_type partial_parsed; bool is_chunk_encoding; + bool is_chunk_end; + bool is_content_length; + std::size_t content_length; }; } // namespace impl From b950ea54019f18844047f20ab0234d85ab6b6807 Mon Sep 17 00:00:00 2001 From: chenzhaoyu Date: Thu, 1 Dec 2016 17:14:28 +0900 Subject: [PATCH 02/35] remove extra body buffer copy --- .../http/client/connection/async_normal.hpp | 73 ++----------------- .../connection/async_protocol_handler.hpp | 52 +++++++++++-- 2 files changed, 52 insertions(+), 73 deletions(-) diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp index 67d6417d2..d7016c764 100644 --- a/boost/network/protocol/http/client/connection/async_normal.hpp +++ b/boost/network/protocol/http/client/connection/async_normal.hpp @@ -363,17 +363,7 @@ struct http_async_connection // Here we handle the body data ourself and append to an // ever-growing string buffer. auto self = this->shared_from_this(); - typename protocol_base::buffer_type::const_iterator begin = - this->part_begin; - typename protocol_base::buffer_type::const_iterator end = begin; - std::advance(end, remainder); - string_type body_string(begin, end); - if (check_parse_body_complete(body_string)) { - this->append_body(remainder); - handle_received_data(body, get_body, callback, - boost::asio::error::eof, bytes_transferred); - } else { - this->parse_body( + this->parse_body( delegate_, request_strand_.wrap([=] (boost::system::error_code const &ec, std::size_t bytes_transferred) { @@ -381,7 +371,6 @@ struct http_async_connection ec, bytes_transferred); }), remainder); - } } return; case body: @@ -439,17 +428,10 @@ struct http_async_connection ec, bytes_transferred); })); } else { - string_type body_string(this->partial_parsed); - body_string.append(this->part.begin(), bytes_transferred); - if (check_parse_body_complete (body_string)) { - this->append_body(bytes_transferred); - handle_received_data(body, get_body, callback, - boost::asio::error::eof, bytes_transferred); - } else { - // Here we don't have a body callback. Let's make sure that we - // deal with the remainder from the headers part in case we do - // have data that's still in the buffer. - this->parse_body( + // Here we don't have a body callback. Let's make sure that we + // deal with the remainder from the headers part in case we do + // have data that's still in the buffer. + this->parse_body( delegate_, request_strand_.wrap([=] (boost::system::error_code const &ec, std::size_t bytes_transferred) { @@ -457,7 +439,6 @@ struct http_async_connection ec, bytes_transferred); }), bytes_transferred); - } } } return; @@ -495,50 +476,6 @@ struct http_async_connection } } - inline bool check_parse_body_complete(string_type& body_string) - { - if (this->is_chunk_encoding) { - return parse_chunk_encoding_complete(body_string); - } - if (this->is_content_length && this->content_length >= 0) { - return parse_content_length_complete(body_string, this->content_length); - } - return false; - } - - inline bool parse_content_length_complete(string_type& body_string, size_t content_length){ - return body_string.length() >= content_length; - } - - bool parse_chunk_encoding_complete(string_type& body_string) { - string_type body; - string_type crlf = "\r\n"; - - typename string_type::iterator begin = body_string.begin(); - for (typename string_type::iterator iter = - std::search(begin, body_string.end(), crlf.begin(), crlf.end()); - iter != body_string.end(); - iter = - std::search(begin, body_string.end(), crlf.begin(), crlf.end())) { - string_type line(begin, iter); - if (line.empty()) { - std::advance(iter, 2); - begin = iter; - continue; - } - std::stringstream stream(line); - int len; - stream >> std::hex >> len; - std::advance(iter, 2); - if (!len) return true; - if (len <= body_string.end() - iter) { - std::advance(iter, len + 2); - } - begin = iter; - } - return false; - } - string_type parse_chunk_encoding(string_type& body_string) { string_type body; string_type crlf = "\r\n"; diff --git a/boost/network/protocol/http/client/connection/async_protocol_handler.hpp b/boost/network/protocol/http/client/connection/async_protocol_handler.hpp index c2c4e4ad6..1d2fd4445 100644 --- a/boost/network/protocol/http/client/connection/async_protocol_handler.hpp +++ b/boost/network/protocol/http/client/connection/async_protocol_handler.hpp @@ -333,9 +333,47 @@ struct http_async_protocol_handler { parsed_ok, std::distance(std::end(result_range), part_end)); } - void append_body(size_t bytes) { - partial_parsed.append(part_begin, part_begin + bytes); - part_begin = part.begin(); + inline bool check_parse_body_complete() const { + if (this->is_chunk_encoding) { + return parse_chunk_encoding_complete(); + } + if (this->is_content_length && this->content_length >= 0) { + return parse_content_length_complete(); + } + return false; + } + + inline bool parse_content_length_complete() const { + return this->partial_parsed.length() >= this-> content_length; + } + + bool parse_chunk_encoding_complete() const { + string_type body; + string_type crlf = "\r\n"; + + typename string_type::const_iterator begin = partial_parsed.begin(); + for (typename string_type::const_iterator iter = + std::search(begin, partial_parsed.end(), crlf.begin(), crlf.end()); + iter != partial_parsed.end(); + iter = + std::search(begin, partial_parsed.end(), crlf.begin(), crlf.end())) { + string_type line(begin, iter); + if (line.empty()) { + std::advance(iter, 2); + begin = iter; + continue; + } + std::stringstream stream(line); + int len; + stream >> std::hex >> len; + std::advance(iter, 2); + if (!len) return true; + if (len <= partial_parsed.end() - iter) { + std::advance(iter, len + 2); + } + begin = iter; + } + return false; } template @@ -344,8 +382,12 @@ struct http_async_protocol_handler { // buffer. partial_parsed.append(part_begin, part_begin + bytes); part_begin = part.begin(); - delegate_->read_some( - boost::asio::mutable_buffers_1(part.data(), part.size()), callback); + if (check_parse_body_complete()) { + callback(boost::asio::error::eof, bytes); + } else { + delegate_->read_some( + boost::asio::mutable_buffers_1(part.data(), part.size()), callback); + } } typedef response_parser response_parser_type; From 673b85301ed5a090c6374c29c5450de3432e0432 Mon Sep 17 00:00:00 2001 From: Uche Mennel Date: Wed, 14 Dec 2016 10:11:42 +0100 Subject: [PATCH 03/35] Introduced new chunked transfer encoding parser to remove chunk markers in streaming clients. --- .../protocol/http/client/async_impl.hpp | 3 +- .../http/client/connection/async_base.hpp | 4 +- .../http/client/connection/async_normal.hpp | 171 ++++++++++++++---- boost/network/protocol/http/client/facade.hpp | 3 +- .../network/protocol/http/client/options.hpp | 16 +- boost/network/protocol/http/client/pimpl.hpp | 10 +- .../http/policies/async_connection.hpp | 14 +- 7 files changed, 168 insertions(+), 53 deletions(-) diff --git a/boost/network/protocol/http/client/async_impl.hpp b/boost/network/protocol/http/client/async_impl.hpp index ab54dfa61..b752fe51e 100644 --- a/boost/network/protocol/http/client/async_impl.hpp +++ b/boost/network/protocol/http/client/async_impl.hpp @@ -39,6 +39,7 @@ struct async_client async_client(bool cache_resolved, bool follow_redirect, bool always_verify_peer, int timeout, + bool remove_chunk_markers, std::shared_ptr service, optional certificate_filename, optional verify_path, @@ -46,7 +47,7 @@ struct async_client optional private_key_file, optional ciphers, optional sni_hostname, long ssl_options) - : connection_base(cache_resolved, follow_redirect, timeout), + : connection_base(cache_resolved, follow_redirect, timeout, remove_chunk_markers), service_ptr(service.get() ? service : std::make_shared()), service_(*service_ptr), diff --git a/boost/network/protocol/http/client/connection/async_base.hpp b/boost/network/protocol/http/client/connection/async_base.hpp index a31ad30fd..d3e3531e8 100644 --- a/boost/network/protocol/http/client/connection/async_base.hpp +++ b/boost/network/protocol/http/client/connection/async_base.hpp @@ -43,7 +43,7 @@ struct async_connection_base { // tag. static connection_ptr new_connection( resolve_function resolve, resolver_type &resolver, bool follow_redirect, - bool always_verify_peer, bool https, int timeout, + bool always_verify_peer, bool https, int timeout, bool remove_chunk_markers, optional certificate_filename = optional(), optional const &verify_path = optional(), optional certificate_file = optional(), @@ -59,7 +59,7 @@ struct async_connection_base { certificate_filename, verify_path, certificate_file, private_key_file, ciphers, sni_hostname, ssl_options); auto temp = std::make_shared( - resolver, resolve, follow_redirect, timeout, std::move(delegate)); + resolver, resolve, follow_redirect, timeout, remove_chunk_markers, std::move(delegate)); BOOST_ASSERT(temp != nullptr); return temp; } diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp index d7016c764..6b99c1469 100644 --- a/boost/network/protocol/http/client/connection/async_normal.hpp +++ b/boost/network/protocol/http/client/connection/async_normal.hpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -37,6 +38,77 @@ namespace network { namespace http { namespace impl { +template +struct chunk_encoding_parser { + + chunk_encoding_parser() : state(state_t::header), chunk_size(0) {} + + enum state_t { header, header_end, data, data_end }; + + state_t state; + size_t chunk_size; + std::array::type, 1024> buffer; + + void update_chunk_size(boost::iterator_range::type, 1024>::const_iterator> const& range) { + if (range.empty()) + return; + std::stringstream ss; + ss << std::hex << range; + size_t size; + ss >> size; + chunk_size = (chunk_size << (range.size()*4)) + size; + } + + boost::iterator_range::type, 1024>::const_iterator> operator()(boost::iterator_range::type, 1024>::const_iterator> const& range) { + auto iter = boost::begin(range); + auto begin = iter; + auto pos = boost::begin(buffer); + + while (iter != boost::end(range)) + switch(state) { + case state_t::header: + iter = std::find(iter, boost::end(range), '\r'); + update_chunk_size(boost::make_iterator_range(begin, iter)); + if (iter != boost::end(range)) { + state = state_t::header_end; + ++iter; + } + break; + + case state_t::header_end: + BOOST_ASSERT(*iter == '\n'); + ++iter; + state = state_t::data; + break; + + case state_t::data: + if (chunk_size == 0) { + BOOST_ASSERT(*iter == '\r'); + ++iter; + state = state_t::data_end; + } else { + auto len = std::min(chunk_size, (size_t)std::distance(iter, boost::end(range))); + begin = iter; + iter = std::next(iter, len); + pos = std::copy(begin, iter, pos); + chunk_size -= len; + } + break; + + case state_t::data_end: + BOOST_ASSERT (*iter == '\n'); + ++iter; + begin = iter; + state = state_t::header; + break; + + default: + BOOST_ASSERT(false && "Bug, report this to the developers!"); + } + return boost::make_iterator_range(boost::begin(buffer), pos); + } +}; + template struct async_connection_base; @@ -72,8 +144,10 @@ struct http_async_connection http_async_connection(resolver_type& resolver, resolve_function resolve, bool follow_redirect, int timeout, + bool remove_chunk_markers, connection_delegate_ptr delegate) : timeout_(timeout), + remove_chunk_markers_(remove_chunk_markers), timer_(resolver.get_io_service()), is_timedout_(false), follow_redirect_(follow_redirect), @@ -348,8 +422,11 @@ struct http_async_connection // The invocation of the callback is synchronous to allow us to // wait before scheduling another read. - callback(make_iterator_range(begin, end), ec); - + if (this->is_chunk_encoding && remove_chunk_markers_) { + callback(parse_chunk_encoding(make_iterator_range(begin, end)), ec); + } else { + callback(make_iterator_range(begin, end), ec); + } auto self = this->shared_from_this(); delegate_->read_some( boost::asio::mutable_buffers_1(this->part.data(), @@ -388,14 +465,28 @@ struct http_async_connection // We call the callback function synchronously passing the error // condition (in this case, end of file) so that it can handle it // appropriately. - callback(make_iterator_range(begin, end), ec); + if (this->is_chunk_encoding && remove_chunk_markers_) { + callback(parse_chunk_encoding(make_iterator_range(begin, end)), ec); + } else { + callback(make_iterator_range(begin, end), ec); + } } else { string_type body_string; - std::swap(body_string, this->partial_parsed); - body_string.append(this->part.begin(), this->part.begin() + bytes_transferred); - if (this->is_chunk_encoding) { - this->body_promise.set_value(parse_chunk_encoding(body_string)); + if (this->is_chunk_encoding && remove_chunk_markers_) { + for (size_t i = 0; i < this->partial_parsed.size(); i += 1024) { + auto range = parse_chunk_encoding( + boost::make_iterator_range(this->partial_parsed.data() + i, + this->partial_parsed.data() + std::min(i+1024, this->partial_parsed.size()))); + body_string.append(boost::begin(range), boost::end(range)); + } + this->partial_parsed.clear(); + auto range = parse_chunk_encoding(boost::make_iterator_range(this->part.begin(), + this->part.begin() + bytes_transferred)); + body_string.append(boost::begin(range), boost::end(range)); + this->body_promise.set_value(body_string); } else { + std::swap(body_string, this->partial_parsed); + body_string.append(this->part.begin(), this->part.begin() + bytes_transferred); this->body_promise.set_value(body_string); } } @@ -417,7 +508,11 @@ struct http_async_connection this->part.begin(); typename protocol_base::buffer_type::const_iterator end = begin; std::advance(end, bytes_transferred); - callback(make_iterator_range(begin, end), ec); + if (this->is_chunk_encoding && remove_chunk_markers_) { + callback(parse_chunk_encoding(make_iterator_range(begin, end)), ec); + } else { + callback(make_iterator_range(begin, end), ec); + } auto self = this->shared_from_this(); delegate_->read_some( boost::asio::mutable_buffers_1(this->part.data(), @@ -476,38 +571,39 @@ struct http_async_connection } } - string_type parse_chunk_encoding(string_type& body_string) { - string_type body; - string_type crlf = "\r\n"; - - typename string_type::iterator begin = body_string.begin(); - for (typename string_type::iterator iter = - std::search(begin, body_string.end(), crlf.begin(), crlf.end()); - iter != body_string.end(); - iter = - std::search(begin, body_string.end(), crlf.begin(), crlf.end())) { - string_type line(begin, iter); - if (line.empty()) { - break; - } - std::stringstream stream(line); - int len; - stream >> std::hex >> len; - std::advance(iter, 2); - if (len == 0) { - break; - } - if (len <= body_string.end() - iter) { - body.insert(body.end(), iter, iter + len); - std::advance(iter, len + 2); - } - begin = iter; - } + // string_type parse_chunk_encoding(string_type& body_string) { + // string_type body; + // string_type crlf = "\r\n"; - return body; - } + // typename string_type::iterator begin = body_string.begin(); + // for (typename string_type::iterator iter = + // std::search(begin, body_string.end(), crlf.begin(), crlf.end()); + // iter != body_string.end(); + // iter = + // std::search(begin, body_string.end(), crlf.begin(), crlf.end())) { + // string_type line(begin, iter); + // if (line.empty()) { + // break; + // } + // std::stringstream stream(line); + // int len; + // stream >> std::hex >> len; + // std::advance(iter, 2); + // if (len == 0) { + // break; + // } + // if (len <= body_string.end() - iter) { + // body.insert(body.end(), iter, iter + len); + // std::advance(iter, len + 2); + // } + // begin = iter; + // } + + // return body; + // } int timeout_; + bool remove_chunk_markers_; boost::asio::steady_timer timer_; bool is_timedout_; bool follow_redirect_; @@ -517,6 +613,7 @@ struct http_async_connection connection_delegate_ptr delegate_; boost::asio::streambuf command_streambuf; string_type method; + chunk_encoding_parser parse_chunk_encoding; }; } // namespace impl diff --git a/boost/network/protocol/http/client/facade.hpp b/boost/network/protocol/http/client/facade.hpp index 37a3eab49..be6fa45e6 100644 --- a/boost/network/protocol/http/client/facade.hpp +++ b/boost/network/protocol/http/client/facade.hpp @@ -303,7 +303,8 @@ class basic_client_facade { options.openssl_verify_path(), options.openssl_certificate_file(), options.openssl_private_key_file(), options.openssl_ciphers(), options.openssl_sni_hostname(), options.openssl_options(), - options.io_service(), options.timeout())); + options.io_service(), options.timeout(), + options.remove_chunk_markers())); } }; diff --git a/boost/network/protocol/http/client/options.hpp b/boost/network/protocol/http/client/options.hpp index 23f7a134d..e73567099 100644 --- a/boost/network/protocol/http/client/options.hpp +++ b/boost/network/protocol/http/client/options.hpp @@ -34,7 +34,8 @@ class client_options { openssl_options_(0), io_service_(), always_verify_peer_(true), - timeout_(0) {} + timeout_(0), + remove_chunk_markers_(false) {} client_options(client_options const& other) : cache_resolved_(other.cache_resolved_), @@ -48,7 +49,8 @@ class client_options { openssl_options_(other.openssl_options_), io_service_(other.io_service_), always_verify_peer_(other.always_verify_peer_), - timeout_(other.timeout_) {} + timeout_(other.timeout_), + remove_chunk_markers_(other.remove_chunk_markers) {} client_options& operator=(client_options other) { other.swap(*this); @@ -69,6 +71,7 @@ class client_options { swap(io_service_, other.io_service_); swap(always_verify_peer_, other.always_verify_peer_); swap(timeout_, other.timeout_); + swap(remove_chunk_markers_, other.remove_chunk_markers_); } /// Specify whether the client should cache resolved endpoints. @@ -154,6 +157,12 @@ class client_options { return *this; } + /// Set an overall timeout for HTTP requests. + client_options& remove_chunk_markers(bool v) { + remove_chunk_markers_ = v; + return *this; + } + bool cache_resolved() const { return cache_resolved_; } bool follow_redirects() const { return follow_redirects_; } @@ -190,6 +199,8 @@ class client_options { int timeout() const { return timeout_; } + bool remove_chunk_markers() const { return remove_chunk_markers_; } + private: bool cache_resolved_; bool follow_redirects_; @@ -203,6 +214,7 @@ class client_options { std::shared_ptr io_service_; bool always_verify_peer_; int timeout_; + bool remove_chunk_markers_; }; template diff --git a/boost/network/protocol/http/client/pimpl.hpp b/boost/network/protocol/http/client/pimpl.hpp index d62be32ce..01c77ca70 100644 --- a/boost/network/protocol/http/client/pimpl.hpp +++ b/boost/network/protocol/http/client/pimpl.hpp @@ -74,10 +74,12 @@ struct basic_client_impl optional const& private_key_file, optional const& ciphers, optional const& sni_hostname, long ssl_options, - std::shared_ptr service, int timeout) - : base_type(cache_resolved, follow_redirect, always_verify_peer, timeout, - service, certificate_filename, verify_path, certificate_file, - private_key_file, ciphers, sni_hostname, ssl_options) {} + std::shared_ptr service, int timeout, + bool remove_chunk_markers) + : base_type(cache_resolved, follow_redirect, always_verify_peer, timeout, + remove_chunk_markers, service, certificate_filename, verify_path, + certificate_file, private_key_file, ciphers, sni_hostname, + ssl_options) {} ~basic_client_impl() = default; }; diff --git a/boost/network/protocol/http/policies/async_connection.hpp b/boost/network/protocol/http/policies/async_connection.hpp index c3f0d131a..c83cd3542 100644 --- a/boost/network/protocol/http/policies/async_connection.hpp +++ b/boost/network/protocol/http/policies/async_connection.hpp @@ -38,7 +38,7 @@ struct async_connection_policy : resolver_policy::type { struct connection_impl { connection_impl( bool follow_redirect, bool always_verify_peer, resolve_function resolve, - resolver_type& resolver, bool https, int timeout, + resolver_type& resolver, bool https, int timeout, bool remove_chunk_markers, optional /*unused*/ const& certificate_filename, optional const& verify_path, optional const& certificate_file, @@ -47,8 +47,8 @@ struct async_connection_policy : resolver_policy::type { optional const& sni_hostname, long ssl_options) { pimpl = impl::async_connection_base:: new_connection(resolve, resolver, follow_redirect, always_verify_peer, - https, timeout, certificate_filename, verify_path, - certificate_file, private_key_file, ciphers, + https, timeout, remove_chunk_markers, certificate_filename, + verify_path, certificate_file, private_key_file, ciphers, sni_hostname, ssl_options); } @@ -85,7 +85,7 @@ struct async_connection_policy : resolver_policy::type { std::uint16_t port, resolve_completion_function once_resolved) { this->resolve(resolver, host, port, once_resolved); }, - resolver, boost::iequals(protocol_, string_type("https")), timeout_, + resolver, boost::iequals(protocol_, string_type("https")), timeout_, remove_chunk_markers_, certificate_filename, verify_path, certificate_file, private_key_file, ciphers, sni_hostname, ssl_options); } @@ -93,13 +93,15 @@ struct async_connection_policy : resolver_policy::type { void cleanup() {} async_connection_policy(bool cache_resolved, bool follow_redirect, - int timeout) + int timeout, bool remove_chunk_markers) : resolver_base(cache_resolved), follow_redirect_(follow_redirect), - timeout_(timeout) {} + timeout_(timeout), + remove_chunk_markers_(remove_chunk_markers) {} bool follow_redirect_; int timeout_; + bool remove_chunk_markers_; }; } // namespace http From c4d758a6855f5b669a210a0be4c4d85e07b96427 Mon Sep 17 00:00:00 2001 From: Uche Mennel Date: Wed, 14 Dec 2016 10:12:47 +0100 Subject: [PATCH 04/35] Applied new remove chunk markers option to tests. --- .../test/http/client_get_different_port_test.cpp | 9 ++++++--- libs/network/test/http/client_get_streaming_test.cpp | 5 ++++- libs/network/test/http/client_get_test.cpp | 12 +++++++++--- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/libs/network/test/http/client_get_different_port_test.cpp b/libs/network/test/http/client_get_different_port_test.cpp index 99a524874..a94839827 100644 --- a/libs/network/test/http/client_get_different_port_test.cpp +++ b/libs/network/test/http/client_get_different_port_test.cpp @@ -15,9 +15,12 @@ namespace http = boost::network::http; TYPED_TEST_CASE(HTTPClientTest, ClientTypes); TYPED_TEST(HTTPClientTest, GetDifferentPort) { - TypeParam client; - typename TypeParam::request r("http://www.boost.org/"); - auto response_ = client.get(r); + using client = TypeParam; + typename client::options options; + options.remove_chunk_markers(true); + client client_; + typename TypeParam::request request("http://www.boost.org/"); + auto response_ = client_.get(request); auto range = headers(response_)["Content-Type"]; EXPECT_TRUE(std::begin(range) != std::end(range)); EXPECT_NE(0, body(response_).size()); diff --git a/libs/network/test/http/client_get_streaming_test.cpp b/libs/network/test/http/client_get_streaming_test.cpp index 9934f6dc5..ae08f26e0 100644 --- a/libs/network/test/http/client_get_streaming_test.cpp +++ b/libs/network/test/http/client_get_streaming_test.cpp @@ -32,7 +32,10 @@ TYPED_TEST(HTTPClientTest, GetStreamingTest) { typename TypeParam::string_type dummy_body; body_handler handler_instance(body_string); { - TypeParam client_; + using client = TypeParam; + typename client::options options; + options.remove_chunk_markers(true); + client client_(options); ASSERT_NO_THROW(response = client_.get(request, handler_instance)); auto range = headers(response)["Content-Type"]; ASSERT_TRUE(!boost::empty(range)); diff --git a/libs/network/test/http/client_get_test.cpp b/libs/network/test/http/client_get_test.cpp index 75b437e24..779a2480e 100644 --- a/libs/network/test/http/client_get_test.cpp +++ b/libs/network/test/http/client_get_test.cpp @@ -15,7 +15,9 @@ TYPED_TEST_CASE(HTTPClientTest, ClientTypes); TYPED_TEST(HTTPClientTest, GetTest) { using client = TypeParam; typename client::request request("http://cpp-netlib.org/"); - client client_; + typename client::options options; + options.remove_chunk_markers(true); + client client_(options); typename client::response response; ASSERT_NO_THROW(response = client_.get(request)); try { @@ -34,7 +36,9 @@ TYPED_TEST(HTTPClientTest, GetTest) { TYPED_TEST(HTTPClientTest, GetHTTPSTest) { using client = TypeParam; typename client::request request("https://www.github.com/"); - client client_; + typename client::options options; + options.remove_chunk_markers(true); + client client_(options); typename client::response response = client_.get(request); EXPECT_TRUE(response.status() == 200 || (response.status() >= 300 && response.status() < 400)); @@ -52,7 +56,9 @@ TYPED_TEST(HTTPClientTest, TemporaryClientObjectTest) { using client = TypeParam; typename client::request request("http://cpp-netlib.org/"); typename client::response response; - ASSERT_NO_THROW(response = client().get(request)); + typename client::options options; + options.remove_chunk_markers(true); + ASSERT_NO_THROW(response = client(options).get(request)); auto range = headers(response); ASSERT_TRUE(!boost::empty(range)); try { From f8ff77d3ee795c32c51b904583e1b04a8b7a071e Mon Sep 17 00:00:00 2001 From: chenzhaoyu Date: Thu, 15 Dec 2016 18:46:30 +0900 Subject: [PATCH 05/35] fix content-length parse error --- .../client/connection/async_protocol_handler.hpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/boost/network/protocol/http/client/connection/async_protocol_handler.hpp b/boost/network/protocol/http/client/connection/async_protocol_handler.hpp index 1d2fd4445..44f01dac1 100644 --- a/boost/network/protocol/http/client/connection/async_protocol_handler.hpp +++ b/boost/network/protocol/http/client/connection/async_protocol_handler.hpp @@ -270,9 +270,15 @@ struct http_async_protocol_handler { } trim(header_pair.second); headers.insert(header_pair); - if (boost::iequals(header_pair.first, "Content-Length")) { - is_content_length = true; - content_length = std::stoi(header_pair.second); + if (!is_content_length && + boost::iequals(header_pair.first, "Content-Length")) { + try { + content_length = std::stoll(header_pair.second); + is_content_length = true; + } + catch (std::exception&) { + //is_content_length = false; + } } } // determine if the body parser will need to handle chunked encoding @@ -408,7 +414,7 @@ struct http_async_protocol_handler { bool is_chunk_encoding; bool is_chunk_end; bool is_content_length; - std::size_t content_length; + long long content_length; }; } // namespace impl From e9bf5f7402664c1ae1673d31f1e59f4c10c7cb79 Mon Sep 17 00:00:00 2001 From: Uche Mennel Date: Thu, 15 Dec 2016 23:43:17 +0100 Subject: [PATCH 06/35] Nicer formatting --- .../protocol/http/client/async_impl.hpp | 6 ++--- .../http/client/connection/async_base.hpp | 6 +++-- .../http/client/connection/async_normal.hpp | 24 +++++++++++-------- .../http/policies/async_connection.hpp | 6 ++--- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/boost/network/protocol/http/client/async_impl.hpp b/boost/network/protocol/http/client/async_impl.hpp index b752fe51e..fe8d2adae 100644 --- a/boost/network/protocol/http/client/async_impl.hpp +++ b/boost/network/protocol/http/client/async_impl.hpp @@ -38,8 +38,7 @@ struct async_client typedef std::function body_generator_function_type; async_client(bool cache_resolved, bool follow_redirect, - bool always_verify_peer, int timeout, - bool remove_chunk_markers, + bool always_verify_peer, int timeout, bool remove_chunk_markers, std::shared_ptr service, optional certificate_filename, optional verify_path, @@ -47,7 +46,8 @@ struct async_client optional private_key_file, optional ciphers, optional sni_hostname, long ssl_options) - : connection_base(cache_resolved, follow_redirect, timeout, remove_chunk_markers), + : connection_base(cache_resolved, follow_redirect, timeout, + remove_chunk_markers), service_ptr(service.get() ? service : std::make_shared()), service_(*service_ptr), diff --git a/boost/network/protocol/http/client/connection/async_base.hpp b/boost/network/protocol/http/client/connection/async_base.hpp index d3e3531e8..b1c1aa67a 100644 --- a/boost/network/protocol/http/client/connection/async_base.hpp +++ b/boost/network/protocol/http/client/connection/async_base.hpp @@ -43,7 +43,8 @@ struct async_connection_base { // tag. static connection_ptr new_connection( resolve_function resolve, resolver_type &resolver, bool follow_redirect, - bool always_verify_peer, bool https, int timeout, bool remove_chunk_markers, + bool always_verify_peer, bool https, int timeout, + bool remove_chunk_markers, optional certificate_filename = optional(), optional const &verify_path = optional(), optional certificate_file = optional(), @@ -59,7 +60,8 @@ struct async_connection_base { certificate_filename, verify_path, certificate_file, private_key_file, ciphers, sni_hostname, ssl_options); auto temp = std::make_shared( - resolver, resolve, follow_redirect, timeout, remove_chunk_markers, std::move(delegate)); + resolver, resolve, follow_redirect, timeout, remove_chunk_markers, + std::move(delegate)); BOOST_ASSERT(temp != nullptr); return temp; } diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp index 6b99c1469..f97b3e1ce 100644 --- a/boost/network/protocol/http/client/connection/async_normal.hpp +++ b/boost/network/protocol/http/client/connection/async_normal.hpp @@ -40,7 +40,6 @@ namespace impl { template struct chunk_encoding_parser { - chunk_encoding_parser() : state(state_t::header), chunk_size(0) {} enum state_t { header, header_end, data, data_end }; @@ -49,23 +48,27 @@ struct chunk_encoding_parser { size_t chunk_size; std::array::type, 1024> buffer; - void update_chunk_size(boost::iterator_range::type, 1024>::const_iterator> const& range) { - if (range.empty()) - return; + void update_chunk_size(boost::iterator_range::type, 1024>::const_iterator> const &range) { + if (range.empty()) + return; std::stringstream ss; ss << std::hex << range; size_t size; ss >> size; - chunk_size = (chunk_size << (range.size()*4)) + size; + chunk_size = (chunk_size << (range.size() * 4)) + size; } - boost::iterator_range::type, 1024>::const_iterator> operator()(boost::iterator_range::type, 1024>::const_iterator> const& range) { + boost::iterator_range< + typename std::array::type, 1024>::const_iterator> + operator()(boost::iterator_range::type, 1024>::const_iterator> const &range) { auto iter = boost::begin(range); auto begin = iter; auto pos = boost::begin(buffer); - while (iter != boost::end(range)) - switch(state) { + while (iter != boost::end(range)) + switch (state) { case state_t::header: iter = std::find(iter, boost::end(range), '\r'); update_chunk_size(boost::make_iterator_range(begin, iter)); @@ -87,7 +90,8 @@ struct chunk_encoding_parser { ++iter; state = state_t::data_end; } else { - auto len = std::min(chunk_size, (size_t)std::distance(iter, boost::end(range))); + auto len = std::min(chunk_size, + (size_t)std::distance(iter, boost::end(range))); begin = iter; iter = std::next(iter, len); pos = std::copy(begin, iter, pos); @@ -96,7 +100,7 @@ struct chunk_encoding_parser { break; case state_t::data_end: - BOOST_ASSERT (*iter == '\n'); + BOOST_ASSERT(*iter == '\n'); ++iter; begin = iter; state = state_t::header; diff --git a/boost/network/protocol/http/policies/async_connection.hpp b/boost/network/protocol/http/policies/async_connection.hpp index c83cd3542..6f6b1e809 100644 --- a/boost/network/protocol/http/policies/async_connection.hpp +++ b/boost/network/protocol/http/policies/async_connection.hpp @@ -85,9 +85,9 @@ struct async_connection_policy : resolver_policy::type { std::uint16_t port, resolve_completion_function once_resolved) { this->resolve(resolver, host, port, once_resolved); }, - resolver, boost::iequals(protocol_, string_type("https")), timeout_, remove_chunk_markers_, - certificate_filename, verify_path, certificate_file, private_key_file, - ciphers, sni_hostname, ssl_options); + resolver, boost::iequals(protocol_, string_type("https")), timeout_, + remove_chunk_markers_, certificate_filename, verify_path, + certificate_file, private_key_file, ciphers, sni_hostname, ssl_options); } void cleanup() {} From 9586a81e1c49c157078679cea5fea6bc965a505a Mon Sep 17 00:00:00 2001 From: Uche Mennel Date: Thu, 15 Dec 2016 23:48:19 +0100 Subject: [PATCH 07/35] Use strongly typed enum. --- boost/network/protocol/http/client/connection/async_normal.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp index f97b3e1ce..257c5afe7 100644 --- a/boost/network/protocol/http/client/connection/async_normal.hpp +++ b/boost/network/protocol/http/client/connection/async_normal.hpp @@ -42,7 +42,7 @@ template struct chunk_encoding_parser { chunk_encoding_parser() : state(state_t::header), chunk_size(0) {} - enum state_t { header, header_end, data, data_end }; + enum class state_t { header, header_end, data, data_end }; state_t state; size_t chunk_size; From c70f774f96fd9d2e82d4fdffff6bd6a7bfbfd827 Mon Sep 17 00:00:00 2001 From: Uche Mennel Date: Fri, 16 Dec 2016 01:03:41 +0100 Subject: [PATCH 08/35] Clarified and commented expression. --- boost/network/protocol/http/client/connection/async_normal.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp index 257c5afe7..59481b468 100644 --- a/boost/network/protocol/http/client/connection/async_normal.hpp +++ b/boost/network/protocol/http/client/connection/async_normal.hpp @@ -56,7 +56,8 @@ struct chunk_encoding_parser { ss << std::hex << range; size_t size; ss >> size; - chunk_size = (chunk_size << (range.size() * 4)) + size; + // New digits are appended as LSBs + chunk_size = (chunk_size << (range.size() * 4)) | size; } boost::iterator_range< From ff5c7c8fb2af00b2ea52f6c8b410380922865152 Mon Sep 17 00:00:00 2001 From: Uche Mennel Date: Mon, 19 Dec 2016 00:22:59 +0100 Subject: [PATCH 09/35] Removed commented lines. --- .../http/client/connection/async_normal.hpp | 31 ------------------- 1 file changed, 31 deletions(-) diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp index 59481b468..82df0e8f8 100644 --- a/boost/network/protocol/http/client/connection/async_normal.hpp +++ b/boost/network/protocol/http/client/connection/async_normal.hpp @@ -576,37 +576,6 @@ struct http_async_connection } } - // string_type parse_chunk_encoding(string_type& body_string) { - // string_type body; - // string_type crlf = "\r\n"; - - // typename string_type::iterator begin = body_string.begin(); - // for (typename string_type::iterator iter = - // std::search(begin, body_string.end(), crlf.begin(), crlf.end()); - // iter != body_string.end(); - // iter = - // std::search(begin, body_string.end(), crlf.begin(), crlf.end())) { - // string_type line(begin, iter); - // if (line.empty()) { - // break; - // } - // std::stringstream stream(line); - // int len; - // stream >> std::hex >> len; - // std::advance(iter, 2); - // if (len == 0) { - // break; - // } - // if (len <= body_string.end() - iter) { - // body.insert(body.end(), iter, iter + len); - // std::advance(iter, len + 2); - // } - // begin = iter; - // } - - // return body; - // } - int timeout_; bool remove_chunk_markers_; boost::asio::steady_timer timer_; From 06e23a7d34ead324d45b00d60bdfa9d0acb9bc2c Mon Sep 17 00:00:00 2001 From: anonimal Date: Sun, 8 Jan 2017 04:22:24 +0000 Subject: [PATCH 10/35] Build: allow options to use Boost + OpenSSL static libs cherry-pick'd from 0.13-release --- CMakeLists.txt | 11 +++++++++-- libs/network/src/CMakeLists.txt | 5 +++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6972594ae..461eead34 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,8 @@ option( CPP-NETLIB_BUILD_SHARED_LIBS "Build cpp-netlib as shared libraries." OFF option( CPP-NETLIB_BUILD_TESTS "Build the cpp-netlib project tests." ON) option( CPP-NETLIB_BUILD_EXAMPLES "Build the cpp-netlib project examples." ON) option( CPP-NETLIB_ENABLE_HTTPS "Build cpp-netlib with support for https if OpenSSL is found." ON) +option( CPP-NETLIB_STATIC_OPENSSL "Build cpp-netlib using static OpenSSL" OFF) +option( CPP-NETLIB_STATIC_BOOST "Build cpp-netlib using static Boost" OFF) include(GNUInstallDirs) @@ -36,8 +38,10 @@ else() set(BUILD_SHARED_LIBS OFF) endif() -# Always use Boost's shared libraries. -set(Boost_USE_STATIC_LIBS OFF) +# Use Boost's static libraries +if (CPP-NETLIB_STATIC_BOOST) + set(Boost_USE_STATIC_LIBS ON) +endif() # We need this for all tests to use the dynamic version. add_definitions(-DBOOST_TEST_DYN_LINK) @@ -64,6 +68,9 @@ if (CPP-NETLIB_ENABLE_HTTPS) endif() endif() endif() + if (CPP-NETLIB_STATIC_OPENSSL) + set(CMAKE_FIND_LIBRARY_SUFFIXES .a) + endif() find_package(OpenSSL) endif() diff --git a/libs/network/src/CMakeLists.txt b/libs/network/src/CMakeLists.txt index 0ad880075..44617f827 100644 --- a/libs/network/src/CMakeLists.txt +++ b/libs/network/src/CMakeLists.txt @@ -32,6 +32,11 @@ set_target_properties(cppnetlib-client-connections target_link_libraries(cppnetlib-client-connections ${Boost_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) if (OPENSSL_FOUND) target_link_libraries(cppnetlib-client-connections ${OPENSSL_LIBRARIES}) + if (CPP-NETLIB_STATIC_OPENSSL) + if (NOT MSVC AND NOT MINGW AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD") # dynlinker functions are built into libc on FreeBSD + target_link_libraries(cppnetlib-client-connections "-ldl") + endif() + endif() endif () install(TARGETS cppnetlib-client-connections EXPORT cppnetlibTargets From c9f1ee5d654540ecee282cbea6cbc7fd7d1a2a6d Mon Sep 17 00:00:00 2001 From: Daniel Nachbaur Date: Wed, 1 Feb 2017 11:21:30 +0100 Subject: [PATCH 11/35] Expose OS-chosen address and port from async_server --- .../protocol/http/server/async_server.hpp | 26 +++++++++++++++++-- .../network/protocol/http/server/options.hpp | 14 ++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/boost/network/protocol/http/server/async_server.hpp b/boost/network/protocol/http/server/async_server.hpp index 4cbefe0d2..c1e3c662e 100644 --- a/boost/network/protocol/http/server/async_server.hpp +++ b/boost/network/protocol/http/server/async_server.hpp @@ -35,13 +35,17 @@ struct async_server_base : server_storage_base, socket_options_base { /// Defines the type for the connection pointer. typedef std::shared_ptr connection_ptr; + /// Defines the type for the options. + typedef server_options options; + /// Constructs and initializes the asynchronous server core. - explicit async_server_base(server_options const &options) + explicit async_server_base(options const &options) : server_storage_base(options), socket_options_base(options), handler(options.handler()), address_(options.address()), port_(options.port()), + protocol_family(options.protocol_family()), thread_pool(options.thread_pool() ? options.thread_pool() : std::make_shared()), @@ -108,11 +112,19 @@ struct async_server_base : server_storage_base, socket_options_base { } } + /// Returns the server socket address, either IPv4 or IPv6 depending on + /// options.protocol_family() + const string_type& address() const { return address_; } + + /// Returns the server socket port + const string_type& port() const { return port_; } + private: typedef std::unique_lock scoped_mutex_lock; Handler &handler; string_type address_, port_; + typename options::protocol_family_t protocol_family; std::shared_ptr thread_pool; boost::asio::ip::tcp::acceptor acceptor; bool stopping; @@ -165,7 +177,15 @@ struct async_server_base : server_storage_base, socket_options_base { // this allows repeated cycles of run -> stop -> run service_.reset(); tcp::resolver resolver(service_); - tcp::resolver::query query(address_, port_); + tcp::resolver::query query( [&]{ + switch(protocol_family) { + case options::ipv4: + return tcp::resolver::query(tcp::v4(), address_, port_); + case options::ipv6: + return tcp::resolver::query(tcp::v6(), address_, port_); + default: + return tcp::resolver::query(address_, port_); + }}()); tcp::resolver::iterator endpoint_iterator = resolver.resolve(query, error); if (error) { BOOST_NETWORK_MESSAGE("Error resolving '" << address_ << ':' << port_); @@ -185,6 +205,8 @@ struct async_server_base : server_storage_base, socket_options_base { << port_); return; } + address_ = acceptor.local_endpoint().address().to_string(); + port_ = std::to_string(acceptor.local_endpoint().port()); acceptor.listen(boost::asio::socket_base::max_connections, error); if (error) { BOOST_NETWORK_MESSAGE("Error listening on socket: '" diff --git a/boost/network/protocol/http/server/options.hpp b/boost/network/protocol/http/server/options.hpp index 2e0f5761f..e84188bf3 100644 --- a/boost/network/protocol/http/server/options.hpp +++ b/boost/network/protocol/http/server/options.hpp @@ -34,6 +34,7 @@ struct server_options { handler_(handler), address_("localhost"), port_("80"), + protocol_family_(undefined), reuse_address_(false), report_aborted_(false), non_blocking_io_(true), @@ -88,6 +89,14 @@ struct server_options { return *this; } + enum protocol_family_t { ipv4, ipv6, undefined }; + + /// Set the protocol family for address resolving. Default is AF_UNSPEC. + server_options &protocol_family(protocol_family_t v) { + protocol_family_ = v; + return *this; + } + /// Set whether to reuse the address (SO_REUSE_ADDR). Default is false. server_options &reuse_address(bool v) { reuse_address_ = v; @@ -159,6 +168,9 @@ struct server_options { /// Returns the port to listen on. string_type port() const { return port_; } + /// Returns the protocol family used for address resolving. + protocol_family_t protocol_family() const { return protocol_family_; } + /// Returns a reference to the provided handler. Handler &handler() const { return handler_; } @@ -215,6 +227,7 @@ struct server_options { swap(io_service_, other.io_service_); swap(address_, other.address_); swap(port_, other.port_); + swap(protocol_family_, other.protocol_family_); swap(reuse_address_, other.reuse_address_); swap(report_aborted_, other.report_aborted_); swap(non_blocking_io_, other.non_blocking_io_); @@ -233,6 +246,7 @@ struct server_options { Handler &handler_; string_type address_; string_type port_; + protocol_family_t protocol_family_; bool reuse_address_; bool report_aborted_; bool non_blocking_io_; From 192d520d3a49ac0c59f6fcfd06b1416e0ed0d937 Mon Sep 17 00:00:00 2001 From: Uche Mennel Date: Thu, 23 Mar 2017 08:38:01 +0100 Subject: [PATCH 12/35] Nicer formatting and clean ups --- .../http/client/connection/async_normal.hpp | 34 ++++++++++--------- .../http/policies/async_connection.hpp | 6 ++-- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp index 82df0e8f8..ccf68ef37 100644 --- a/boost/network/protocol/http/client/connection/async_normal.hpp +++ b/boost/network/protocol/http/client/connection/async_normal.hpp @@ -11,7 +11,6 @@ #include #include -#include #include #include #include @@ -48,10 +47,10 @@ struct chunk_encoding_parser { size_t chunk_size; std::array::type, 1024> buffer; - void update_chunk_size(boost::iterator_range::type, 1024>::const_iterator> const &range) { - if (range.empty()) - return; + void update_chunk_size( + boost::iterator_range::type, 1024>::const_iterator> const &range) { + if (range.empty()) return; std::stringstream ss; ss << std::hex << range; size_t size; @@ -61,15 +60,15 @@ struct chunk_encoding_parser { } boost::iterator_range< - typename std::array::type, 1024>::const_iterator> - operator()(boost::iterator_range::type, 1024>::const_iterator> const &range) { + typename std::array::type, 1024>::const_iterator> + operator()( + boost::iterator_range::type, 1024>::const_iterator> const &range) { auto iter = boost::begin(range); auto begin = iter; auto pos = boost::begin(buffer); - while (iter != boost::end(range)) - switch (state) { + while (iter != boost::end(range)) switch (state) { case state_t::header: iter = std::find(iter, boost::end(range), '\r'); update_chunk_size(boost::make_iterator_range(begin, iter)); @@ -479,19 +478,22 @@ struct http_async_connection string_type body_string; if (this->is_chunk_encoding && remove_chunk_markers_) { for (size_t i = 0; i < this->partial_parsed.size(); i += 1024) { - auto range = parse_chunk_encoding( - boost::make_iterator_range(this->partial_parsed.data() + i, - this->partial_parsed.data() + std::min(i+1024, this->partial_parsed.size()))); + auto range = parse_chunk_encoding(boost::make_iterator_range( + this->partial_parsed.data() + i, + this->partial_parsed.data() + + std::min(i + 1024, this->partial_parsed.size()))); body_string.append(boost::begin(range), boost::end(range)); } this->partial_parsed.clear(); - auto range = parse_chunk_encoding(boost::make_iterator_range(this->part.begin(), - this->part.begin() + bytes_transferred)); + auto range = parse_chunk_encoding(boost::make_iterator_range( + this->part.begin(), + this->part.begin() + bytes_transferred)); body_string.append(boost::begin(range), boost::end(range)); this->body_promise.set_value(body_string); } else { std::swap(body_string, this->partial_parsed); - body_string.append(this->part.begin(), this->part.begin() + bytes_transferred); + body_string.append(this->part.begin(), + this->part.begin() + bytes_transferred); this->body_promise.set_value(body_string); } } diff --git a/boost/network/protocol/http/policies/async_connection.hpp b/boost/network/protocol/http/policies/async_connection.hpp index 6f6b1e809..900a37e59 100644 --- a/boost/network/protocol/http/policies/async_connection.hpp +++ b/boost/network/protocol/http/policies/async_connection.hpp @@ -47,9 +47,9 @@ struct async_connection_policy : resolver_policy::type { optional const& sni_hostname, long ssl_options) { pimpl = impl::async_connection_base:: new_connection(resolve, resolver, follow_redirect, always_verify_peer, - https, timeout, remove_chunk_markers, certificate_filename, - verify_path, certificate_file, private_key_file, ciphers, - sni_hostname, ssl_options); + https, timeout, remove_chunk_markers, + certificate_filename, verify_path, certificate_file, + private_key_file, ciphers, sni_hostname, ssl_options); } basic_response send_request(string_type /*unused*/ const& method, From 7a5388b1c8eeb0afbbe49addd0b6983e34b85db1 Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Tue, 4 Apr 2017 15:04:40 -0400 Subject: [PATCH 13/35] Make this work with Boost 1.55. --- CMakeLists.txt | 2 +- libs/network/src/server_request_parsers_impl.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 461eead34..8b29d78b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,7 +49,7 @@ add_definitions(-DBOOST_TEST_DYN_LINK) # Always use multi-threaded Boost libraries. set(Boost_USE_MULTI_THREADED ON) -find_package(Boost 1.58.0 REQUIRED COMPONENTS system thread) +find_package(Boost 1.55.0 REQUIRED COMPONENTS system thread) if (CPP-NETLIB_ENABLE_HTTPS) if (APPLE) diff --git a/libs/network/src/server_request_parsers_impl.cpp b/libs/network/src/server_request_parsers_impl.cpp index 9351983ca..d62fd2e94 100644 --- a/libs/network/src/server_request_parsers_impl.cpp +++ b/libs/network/src/server_request_parsers_impl.cpp @@ -10,7 +10,7 @@ #define BOOST_SPIRIT_UNICODE #include -#include +#include #include namespace boost { From 9e76e5fb3d04d4c459d8f280728f2701906fa363 Mon Sep 17 00:00:00 2001 From: Dean Michael Berris Date: Thu, 6 Apr 2017 16:21:06 +1000 Subject: [PATCH 14/35] Use boost-1.55 from boost-latest ppa (#751) * Use boost-1.55 from boost-latest ppa * squash: use correct boost package --- .travis.yml | 18 +++++++++--------- build.sh | 2 -- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5c2d163e5..6da551409 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,8 +34,6 @@ install: ln -s `which clang++-3.6` ${HOME}/bin/clang++; fi - export BOOST_VERSION=${BOOST_VER//./_} - export PATH=${HOME}/bin:${PATH} -- travis_wait ./install-boost.sh -- export BOOST_ROOT=${HOME}/${CC}-boost_${BOOST_VER//./_} - "${CXX} --version" cache: directories: @@ -48,14 +46,16 @@ after_failure: addons: apt: sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.6 - - kalakris-cmake + - ubuntu-toolchain-r-test + - llvm-toolchain-precise-3.6 + - kalakris-cmake + - boost-latest packages: - - gcc-4.8 - - g++-4.8 - - clang-3.6 - - cmake + - libboost1.55-all-dev + - gcc-4.8 + - g++-4.8 + - clang-3.6 + - cmake notifications: slack: secure: Y7lLjqZ83+b/jaJ5+EKwvgCDeERi4bVbDn9tLp8sieTdu+ENsPI+JmLYSXZXPpe7JrItrXW6uJJXN2wG1h7au4mpVVTghd31HBzuzrqVxDphWPhp16NYzvbAgQQRBXvFVvfSdW/Kb/n2fX6xDApY0t6vNREb/GKg0GyzESb4ZjU= diff --git a/build.sh b/build.sh index b3e3bb880..d7fca5333 100755 --- a/build.sh +++ b/build.sh @@ -11,8 +11,6 @@ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \ -DUri_BUILD_TESTS=$BUILD_TESTS \ -DUri_BUILD_DOCS=$BUILD_DOCS \ -DUri_DISABLE_LIBCXX=$Uri_DISABLE_LIBCXX \ - -DBOOST_INCLUDEDIR="${HOME}/${CC}-boost_${BOOST_VERSION}/include" \ - -DBOOST_LIBRARYDIR="${HOME}/${CC}-boost_${BOOST_VERSION}/lib" \ -DCMAKE_CXX_FLAGS="-std=c++11 ${CMAKE_CXX_FLAGS}" \ .. make -j8 From f87285973b3d54801aedc356834e41729c3d0948 Mon Sep 17 00:00:00 2001 From: Dean Michael Berris Date: Thu, 6 Apr 2017 21:53:10 +1000 Subject: [PATCH 15/35] Use EXPECT_FALSE instead of EXPECT_EQ(false, X) (#752) This fixes the failures we're finding from the CI builds. We also take the opportunity to improve the build system, to use ccache if it's available, and also to simplify the travis config. Now that we can just test with Boost 1.55 in the meantime, let's set the appropriate flags that will make the clang build/support work. --- .travis.yml | 71 +++++++++++-------- CMakeLists.txt | 14 +++- .../http/client/connection/async_normal.hpp | 4 +- build.sh | 2 +- .../http/request_incremental_parser_test.cpp | 10 +-- .../http/response_incremental_parser_test.cpp | 58 +++++++-------- 6 files changed, 91 insertions(+), 68 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6da551409..2048428be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,40 +4,49 @@ compiler: - g++ - clang env: -- BOOST_VER=1.59.0 BUILD_SHARED_LIBS="ON" CMAKE_BUILD_TYPE="Release" ENABLE_HTTPS="ON" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES -- BOOST_VER=1.59.0 BUILD_SHARED_LIBS="ON" CMAKE_BUILD_TYPE="Release" ENABLE_HTTPS="OFF" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES -- BOOST_VER=1.59.0 BUILD_SHARED_LIBS="ON" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES -- BOOST_VER=1.59.0 BUILD_SHARED_LIBS="ON" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="OFF" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES -- BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Release" ENABLE_HTTPS="ON" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES -- BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Release" ENABLE_HTTPS="OFF" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES -- BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES -- BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="OFF" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES +- BUILD_SHARED_LIBS="ON" CMAKE_BUILD_TYPE="Release" ENABLE_HTTPS="ON" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES +- BUILD_SHARED_LIBS="ON" CMAKE_BUILD_TYPE="Release" ENABLE_HTTPS="OFF" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES +- BUILD_SHARED_LIBS="ON" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES +- BUILD_SHARED_LIBS="ON" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="OFF" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES +- BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Release" ENABLE_HTTPS="ON" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES +- BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Release" ENABLE_HTTPS="OFF" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES +- BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES +- BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="OFF" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES # Support the sanitizers in clang only -# - BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=thread" -# - BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=address" +- BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=thread" Uri_BUILD_TEST=OFF Uri_DISABLE_LIBCXX=YES +- BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=address" Uri_BUILD_TEST=OFF Uri_DISABLE_LIBCXX=YES # TODO(deanberris): It seems Boost is not msan-clean yet; report bugs and maybe fix? #- BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2" -# matrix: -# exclude: -# - compiler: g++ -# env: BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=thread" -# - compiler: g++ -# env: BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=address" +cache: + - apt + - ccache +matrix: + exclude: + - compiler: g++ + env: BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=thread" Uri_BUILD_TEST=OFF Uri_DISABLE_LIBCXX=YES + - compiler: g++ + env: BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=address" Uri_BUILD_TEST=OFF Uri_DISABLE_LIBCXX=YES # TODO(deanberris): It seems Boost is not msan-clean yet; report bugs and maybe fix? # - compiler: g++ # env: BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2" install: -- mkdir -p ${HOME}/bin -- if [ "${CC}" = "gcc" ]; then export TOOLSET="gcc"; ln -s `which g++-4.8` ${HOME}/bin/g++; - ln -s `which gcc-4.8` ${HOME}/bin/gcc; fi -- if [ "${CC}" = "clang" ]; then export TOOLSET="clang"; ln -s `which clang-3.6` ${HOME}/bin/clang; - ln -s `which clang++-3.6` ${HOME}/bin/clang++; fi -- export BOOST_VERSION=${BOOST_VER//./_} -- export PATH=${HOME}/bin:${PATH} +- pwd +- export CUR_DIR=`pwd` +- mkdir -p ${CUR_DIR}/bin +- if [ "${CC}" = "gcc" ]; then + export CXX=g++-4.9; + export CC=gcc-4.9; + fi +- if [ "${CC}" = "clang" ]; then + export CXX=clang++-3.8; + export CC=clang-3.8; + export CCACHE_CPP2=yes; + fi +before_script: +- echo ${CXX} - "${CXX} --version" -cache: - directories: - - "${HOME}/${CC}-boost_${BOOST_VER//./_}" +- echo ${CC} +- "${CC} --version" script: - pwd - sh -x build.sh @@ -47,15 +56,17 @@ addons: apt: sources: - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.6 + - llvm-toolchain-precise-3.8 - kalakris-cmake - boost-latest + - george-edison55-precise-backports packages: - libboost1.55-all-dev - - gcc-4.8 - - g++-4.8 - - clang-3.6 + - gcc-4.9 + - g++-4.9 + - clang-3.8 - cmake + - cmake-data notifications: slack: secure: Y7lLjqZ83+b/jaJ5+EKwvgCDeERi4bVbDn9tLp8sieTdu+ENsPI+JmLYSXZXPpe7JrItrXW6uJJXN2wG1h7au4mpVVTghd31HBzuzrqVxDphWPhp16NYzvbAgQQRBXvFVvfSdW/Kb/n2fX6xDApY0t6vNREb/GKg0GyzESb4ZjU= diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b29d78b0..56534f32f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -97,11 +97,16 @@ if (${CMAKE_CXX_COMPILER_ID} MATCHES GNU) elseif (${CMAKE_CXX_COMPILER_ID} MATCHES Clang) # We want to link in C++11 mode in Clang too, but also set a high enough # template depth for the template metaprogramming. - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -ftemplate-depth=256 -std=c++11") + set (CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -Wall -ftemplate-depth=256 -std=c++11 -DBOOST_ASIO_HAS_STD_CHRONO -DBOOST_ASIO_HAS_STD_ARRAY -DBOOST_ASIO_HAS_STD_SHARED_PTR -DBOOST_ASIO_HAS_STD_ATOMIC -DBOOST_ASIO_HAS_VARIADIC_TEMPLATES -DBOOST_ASIO_HAS_MOVE -DBOOST_THREAD_VERSION=3") if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") # Use libc++ only in OS X. set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lc++") + elseif (${CMAKE_SYSTEM_NAME} MATCHES "Linux") + # Use libstdc++ for Linux. + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libstdc++") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lstdc++") endif() endif() @@ -140,6 +145,13 @@ if (MSVC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /bigobj") endif() +# See whether we can find the ccache program -- if we can, then use it for the build. +find_program(CCACHE_FOUND ccache) +if(CCACHE_FOUND) + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) + set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache) +endif(CCACHE_FOUND) + enable_testing() install(DIRECTORY boost DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp index ccf68ef37..ab99579f9 100644 --- a/boost/network/protocol/http/client/connection/async_normal.hpp +++ b/boost/network/protocol/http/client/connection/async_normal.hpp @@ -147,7 +147,7 @@ struct http_async_connection connection_delegate_ptr; http_async_connection(resolver_type& resolver, resolve_function resolve, - bool follow_redirect, int timeout, + bool follow_redirect, int64_t timeout, bool remove_chunk_markers, connection_delegate_ptr delegate) : timeout_(timeout), @@ -578,7 +578,7 @@ struct http_async_connection } } - int timeout_; + int64_t timeout_; bool remove_chunk_markers_; boost::asio::steady_timer timer_; bool is_timedout_; diff --git a/build.sh b/build.sh index d7fca5333..8e3e31d7e 100755 --- a/build.sh +++ b/build.sh @@ -13,6 +13,6 @@ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \ -DUri_DISABLE_LIBCXX=$Uri_DISABLE_LIBCXX \ -DCMAKE_CXX_FLAGS="-std=c++11 ${CMAKE_CXX_FLAGS}" \ .. -make -j8 +make make test cd .. diff --git a/libs/network/test/http/request_incremental_parser_test.cpp b/libs/network/test/http/request_incremental_parser_test.cpp index 9120294f5..f8c8f5577 100644 --- a/libs/network/test/http/request_incremental_parser_test.cpp +++ b/libs/network/test/http/request_incremental_parser_test.cpp @@ -52,7 +52,7 @@ TEST(IncrementalRequestParserTest, ParseMethod) { p.reset(); std::tie(parsed_ok, result_range) = p.parse_until(request_parser_type::method_done, invalid_http_method); - EXPECT_EQ(false, parsed_ok); + EXPECT_FALSE(parsed_ok); parsed.assign(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " [state:" << p.state() << "] " << std::endl; @@ -68,7 +68,7 @@ TEST(IncrementalRequestParserTest, ParseURI) { std::string valid_http_request = "GET / HTTP/1.1\r\n"; std::tie(parsed_ok, result_range) = p.parse_until(request_parser_type::uri_done, valid_http_request); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_FALSE(boost::empty(result_range)); std::string parsed(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " [state:" << p.state() << "] " @@ -78,7 +78,7 @@ TEST(IncrementalRequestParserTest, ParseURI) { p.reset(); std::tie(parsed_ok, result_range) = p.parse_until(request_parser_type::uri_done, invalid_http_request); - EXPECT_EQ(false, parsed_ok); + EXPECT_FALSE(parsed_ok); parsed.assign(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " [state:" << p.state() << "] " << std::endl; @@ -104,7 +104,7 @@ TEST(IncrementalRequestParserTest, ParseHTTPVersion) { p.reset(); std::tie(parsed_ok, result_range) = p.parse_until(request_parser_type::version_done, invalid_http_request); - EXPECT_EQ(false, parsed_ok); + EXPECT_FALSE(parsed_ok); parsed.assign(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " [state:" << p.state() << "] " << std::endl; @@ -121,7 +121,7 @@ TEST(IncrementalRequestParserTest, ParseHTTPHeaders) { "GET / HTTP/1.1\r\nHost: cpp-netlib.org\r\n\r\n"; std::tie(parsed_ok, result_range) = p.parse_until(request_parser_type::headers_done, valid_http_request); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_FALSE(boost::empty(result_range)); std::string parsed(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " [state:" << p.state() << "] " diff --git a/libs/network/test/http/response_incremental_parser_test.cpp b/libs/network/test/http/response_incremental_parser_test.cpp index 0642ce803..aeaa88cc5 100644 --- a/libs/network/test/http/response_incremental_parser_test.cpp +++ b/libs/network/test/http/response_incremental_parser_test.cpp @@ -82,7 +82,7 @@ TEST(IncrementalResponseParserTest, ParseHTTPVersion) { std::string valid_http_version = "HTTP/1.0 "; std::tie(parsed_ok, result_range) = p.parse_until( response_parser_type::http_version_done, valid_http_version); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_FALSE(boost::empty(result_range)); std::string parsed(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; @@ -90,7 +90,7 @@ TEST(IncrementalResponseParserTest, ParseHTTPVersion) { valid_http_version = "HTTP/1.1 "; std::tie(parsed_ok, result_range) = p.parse_until( response_parser_type::http_version_done, valid_http_version); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_FALSE(boost::empty(result_range)); parsed.assign(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; @@ -100,7 +100,7 @@ TEST(IncrementalResponseParserTest, ParseHTTPVersion) { parsed_ok = logic::indeterminate; std::tie(parsed_ok, result_range) = p.parse_until( response_parser_type::http_version_done, invalid_http_version); - EXPECT_EQ(false, parsed_ok); + EXPECT_FALSE(parsed_ok); parsed.assign(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; @@ -109,7 +109,7 @@ TEST(IncrementalResponseParserTest, ParseHTTPVersion) { parsed_ok = logic::indeterminate; std::tie(parsed_ok, result_range) = p.parse_until( response_parser_type::http_version_done, valid_http_version); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed.assign(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; } @@ -137,7 +137,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseStatus) { range_type result_range; std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_status_done, valid_status); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); std::string parsed = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; @@ -153,7 +153,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseStatus) { valid_status = "200" + TypeParam::literal; std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_status_done, valid_status); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; } @@ -171,7 +171,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseStatusMessage) { range_type result_range; std::tie(parsed_ok, result_range) = p.parse_until( response_parser_type::http_status_message_done, valid_status_message); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); std::string parsed = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; @@ -180,7 +180,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseStatusMessage) { valid_status_message = "OK" + TypeParam::literal; std::tie(parsed_ok, result_range) = p.parse_until( response_parser_type::http_status_message_done, valid_status_message); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; @@ -188,7 +188,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseStatusMessage) { valid_status_message = "Internal Server Error" + TypeParam::literal; std::tie(parsed_ok, result_range) = p.parse_until( response_parser_type::http_status_message_done, valid_status_message); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; @@ -196,7 +196,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseStatusMessage) { valid_status_message = TypeParam::literal; std::tie(parsed_ok, result_range) = p.parse_until( response_parser_type::http_status_message_done, valid_status_message); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; @@ -204,7 +204,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseStatusMessage) { valid_status_message = "한글메시지" + TypeParam::literal; std::tie(parsed_ok, result_range) = p.parse_until( response_parser_type::http_status_message_done, valid_status_message); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; } @@ -224,7 +224,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { range_type result_range; std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); std::string parsed1 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed1 << " state=" << p.state() << std::endl; @@ -233,7 +233,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { valid_headers.assign(std::end(result_range), end); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); std::string parsed2 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed2 << " state=" << p.state() << std::endl; @@ -241,7 +241,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { p.reset(response_parser_type::http_status_message_done); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_headers_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_NE(parsed1, parsed2); p.reset(response_parser_type::http_status_message_done); @@ -250,7 +250,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { eol::literal; std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed1 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed1 << " state=" << p.state() << std::endl; p.reset(response_parser_type::http_status_message_done); @@ -258,14 +258,14 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { valid_headers.assign(std::end(result_range), end); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed2 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed2 << " state=" << p.state() << std::endl; valid_headers.assign(std::end(result_range), end); p.reset(response_parser_type::http_status_message_done); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_headers_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_NE(parsed1, parsed2); p.reset(response_parser_type::http_status_message_done); @@ -274,7 +274,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { eol::literal; std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed1 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed1 << " state=" << p.state() << std::endl; p.reset(response_parser_type::http_status_message_done); @@ -282,14 +282,14 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { valid_headers.assign(std::end(result_range), end); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed2 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed2 << " state=" << p.state() << std::endl; valid_headers.assign(std::end(result_range), end); p.reset(response_parser_type::http_status_message_done); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_headers_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_NE(parsed1, parsed2); p.reset(response_parser_type::http_status_message_done); @@ -297,7 +297,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { eol::literal + eol::literal; std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed1 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed1 << " state=" << p.state() << std::endl; p.reset(response_parser_type::http_status_message_done); @@ -305,14 +305,14 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { valid_headers.assign(std::end(result_range), end); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed2 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed2 << " state=" << p.state() << std::endl; valid_headers.assign(std::end(result_range), end); p.reset(response_parser_type::http_status_message_done); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_headers_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_NE(parsed1, parsed2); p.reset(response_parser_type::http_status_message_done); @@ -321,7 +321,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { eol::literal; std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed1 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed1 << " state=" << p.state() << std::endl; p.reset(response_parser_type::http_status_message_done); @@ -329,14 +329,14 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { valid_headers.assign(std::end(result_range), end); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed2 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed2 << " state=" << p.state() << std::endl; valid_headers.assign(std::end(result_range), end); p.reset(response_parser_type::http_status_message_done); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_headers_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_NE(parsed1, parsed2); p.reset(response_parser_type::http_status_message_done); @@ -344,7 +344,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { eol::literal + eol::literal; std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed1 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed1 << " state=" << p.state() << std::endl; p.reset(response_parser_type::http_status_message_done); @@ -352,13 +352,13 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { valid_headers.assign(std::end(result_range), end); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed2 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed2 << " state=" << p.state() << std::endl; valid_headers.assign(std::end(result_range), end); p.reset(response_parser_type::http_status_message_done); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_headers_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_NE(parsed1, parsed2); } From fa76515abd4af2e80380d0f5fe4f003072f5f77b Mon Sep 17 00:00:00 2001 From: Edward Kigwana Date: Fri, 7 Apr 2017 00:33:01 +0000 Subject: [PATCH 16/35] content_length type was std::size_t and was changed to long long in (#753) commit f8ff77d3ee795c32c51b904583e1b04a8b7a071e. It is also initialized to -1 in parse_headers_real(...). Signed-off-by: Edward Kigwana --- .../http/client/connection/async_protocol_handler.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/boost/network/protocol/http/client/connection/async_protocol_handler.hpp b/boost/network/protocol/http/client/connection/async_protocol_handler.hpp index 44f01dac1..1be0f5584 100644 --- a/boost/network/protocol/http/client/connection/async_protocol_handler.hpp +++ b/boost/network/protocol/http/client/connection/async_protocol_handler.hpp @@ -348,9 +348,9 @@ struct http_async_protocol_handler { } return false; } - + inline bool parse_content_length_complete() const { - return this->partial_parsed.length() >= this-> content_length; + return static_cast(this->partial_parsed.length()) >= this->content_length; } bool parse_chunk_encoding_complete() const { From 610c8cbbc4c25fc4df242e6e4e6c7e2551a6c682 Mon Sep 17 00:00:00 2001 From: Dean Michael Berris Date: Fri, 7 Apr 2017 17:19:15 +1000 Subject: [PATCH 17/35] Fix exception propaation for client errors (#754) We change the cross-thread exception propagation given the asynchronous client implementation. The previous implementation caused double-wrapping of exceptions which works with the std::shared_future<...> implementation, but not with boost::shared_future<...> -- since Boost.Thread still uses Boost.Exception for the APIs. While we're here we make the exception propagation more explicit using boost::rethrow_exception(...), so we can unwrap the exception that's transported via an exception_ptr. Fixes #749. --- .../http/client/connection/async_normal.hpp | 22 +++--- .../protocol/http/message/async_message.hpp | 67 ++++++++++++++----- libs/network/test/http/client_get_test.cpp | 11 +++ 3 files changed, 70 insertions(+), 30 deletions(-) diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp index ab99579f9..355ead023 100644 --- a/boost/network/protocol/http/client/connection/async_normal.hpp +++ b/boost/network/protocol/http/client/connection/async_normal.hpp @@ -160,11 +160,9 @@ struct http_async_connection request_strand_(resolver.get_io_service()), delegate_(std::move(delegate)) {} - // This is the main entry point for the connection/request pipeline. - // We're - // overriding async_connection_base<...>::start(...) here which is - // called - // by the client. + // This is the main entry point for the connection/request pipeline. We're + // overriding async_connection_base<...>::start(...) here which is called by + // the client. virtual response start(request const& request, string_type const& method, bool get_body, body_callback_function_type callback, body_generator_function_type generator) { @@ -198,13 +196,13 @@ struct http_async_connection private: void set_errors(boost::system::error_code const& ec, body_callback_function_type callback) { boost::system::system_error error(ec); - this->version_promise.set_exception(std::make_exception_ptr(error)); - this->status_promise.set_exception(std::make_exception_ptr(error)); - this->status_message_promise.set_exception(std::make_exception_ptr(error)); - this->headers_promise.set_exception(std::make_exception_ptr(error)); - this->source_promise.set_exception(std::make_exception_ptr(error)); - this->destination_promise.set_exception(std::make_exception_ptr(error)); - this->body_promise.set_exception(std::make_exception_ptr(error)); + this->version_promise.set_exception(boost::copy_exception(error)); + this->status_promise.set_exception(boost::copy_exception(error)); + this->status_message_promise.set_exception(boost::copy_exception(error)); + this->headers_promise.set_exception(boost::copy_exception(error)); + this->source_promise.set_exception(boost::copy_exception(error)); + this->destination_promise.set_exception(boost::copy_exception(error)); + this->body_promise.set_exception(boost::copy_exception(error)); if ( callback ) callback( boost::iterator_range::type, 1024>::const_iterator>(), ec ); this->timer_.cancel(); diff --git a/boost/network/protocol/http/message/async_message.hpp b/boost/network/protocol/http/message/async_message.hpp index 861f3adf6..747a962ed 100644 --- a/boost/network/protocol/http/message/async_message.hpp +++ b/boost/network/protocol/http/message/async_message.hpp @@ -9,13 +9,13 @@ // (See accompanying file LICENSE_1_0.txt or copy at // http://www.boost.org/LICENSE_1_0.txt) -#include #include +#include // FIXME move this out to a trait -#include #include #include +#include namespace boost { namespace network { @@ -26,12 +26,11 @@ namespace impl { template struct ready_wrapper; -} // namespace impl - /* impl */ +} // namespace impl + /* impl */ template struct async_message { - typedef typename string::type string_type; typedef typename headers_container::type headers_container_type; typedef typename headers_container_type::value_type header_type; @@ -54,31 +53,56 @@ struct async_message { headers_(other.headers_), body_(other.body_) {} - string_type const status_message() const { return status_message_.get(); } + string_type status_message() const { + status_message_.wait(); + if (status_message_.has_exception()) + boost::rethrow_exception(status_message_.get_exception_ptr()); + return status_message_.get(); + } void status_message(boost::shared_future const& future) const { status_message_ = future; } - string_type const version() const { return version_.get(); } + string_type version() const { + version_.wait(); + if (version_.has_exception()) + boost::rethrow_exception(version_.get_exception_ptr()); + return version_.get(); + } void version(boost::shared_future const& future) const { version_ = future; } - std::uint16_t status() const { return status_.get(); } + std::uint16_t status() const { + status_.wait(); + if (status_.has_exception()) + boost::rethrow_exception(status_.get_exception_ptr()); + return status_.get(); + } void status(boost::shared_future const& future) const { status_ = future; } - string_type const source() const { return source_.get(); } + string_type source() const { + source_.wait(); + if (source_.has_exception()) + boost::rethrow_exception(source_.get_exception_ptr()); + return source_.get(); + } void source(boost::shared_future const& future) const { source_ = future; } - string_type const destination() const { return destination_.get(); } + string_type destination() const { + destination_.wait(); + if (destination_.has_exception()) + boost::rethrow_exception(source_.get_exception_ptr()); + return destination_.get(); + } void destination(boost::shared_future const& future) const { destination_ = future; @@ -86,31 +110,38 @@ struct async_message { headers_container_type const& headers() const { if (retrieved_headers_) return *retrieved_headers_; + if (headers_.has_exception()) + boost::rethrow_exception(headers_.get_exception_ptr()); headers_container_type raw_headers = headers_.get(); raw_headers.insert(added_headers.begin(), added_headers.end()); - for (string_type const & key : removed_headers) { + for (string_type const& key : removed_headers) { raw_headers.erase(key); } retrieved_headers_ = raw_headers; return *retrieved_headers_; } - void headers(boost::shared_future const& future) - const { + void headers( + boost::shared_future const& future) const { headers_ = future; } - void add_header(typename headers_container_type::value_type const& pair_) - const { + void add_header( + typename headers_container_type::value_type const& pair_) const { added_headers.insert(added_headers.end(), pair_); } - void remove_header(typename headers_container_type::key_type const& key_) - const { + void remove_header( + typename headers_container_type::key_type const& key_) const { removed_headers.insert(key_); } - string_type const body() const { return body_.get(); } + string_type body() const { + body_.wait(); + if (body_.has_exception()) + boost::rethrow_exception(body_.get_exception_ptr()); + return body_.get(); + } void body(boost::shared_future const& future) const { body_ = future; diff --git a/libs/network/test/http/client_get_test.cpp b/libs/network/test/http/client_get_test.cpp index 779a2480e..15997bd83 100644 --- a/libs/network/test/http/client_get_test.cpp +++ b/libs/network/test/http/client_get_test.cpp @@ -1,4 +1,5 @@ // Copyright 2010 Dean Michael Berris. +// Copyright 2017 Google, Inc. // Distributed under the Boost Software License, Version 1.0. // (See accompanying file LICENSE_1_0.txt or copy at // http://www.boost.org/LICENSE_1_0.txt) @@ -71,3 +72,13 @@ TYPED_TEST(HTTPClientTest, TemporaryClientObjectTest) { EXPECT_TRUE(response.status() == 200u || (response.status() >= 300 && response.status() < 400)); } + +TYPED_TEST(HTTPClientTest, PropagatesResolutionErrorsTest) { + using client = TypeParam; + typename client::request request("http://malformed.google.comq/"); + typename client::response response; + typename client::options options; + typename client::string_type response_body; + ASSERT_NO_THROW(response = client(options).get(request)); + EXPECT_THROW(response_body = body(response), std::exception); +} From 092cd570fb179d029d1865aade9f25aae90d97b9 Mon Sep 17 00:00:00 2001 From: chenzhaoyu Date: Wed, 17 May 2017 22:33:23 +0800 Subject: [PATCH 18/35] fix http client body content (#760) --- .../protocol/http/client/connection/async_protocol_handler.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boost/network/protocol/http/client/connection/async_protocol_handler.hpp b/boost/network/protocol/http/client/connection/async_protocol_handler.hpp index 1be0f5584..a0ce75bce 100644 --- a/boost/network/protocol/http/client/connection/async_protocol_handler.hpp +++ b/boost/network/protocol/http/client/connection/async_protocol_handler.hpp @@ -389,7 +389,7 @@ struct http_async_protocol_handler { partial_parsed.append(part_begin, part_begin + bytes); part_begin = part.begin(); if (check_parse_body_complete()) { - callback(boost::asio::error::eof, bytes); + callback(boost::asio::error::eof, 0); } else { delegate_->read_some( boost::asio::mutable_buffers_1(part.data(), part.size()), callback); From 216613c58ca5dc87c87391c5ffdb94d0656c5153 Mon Sep 17 00:00:00 2001 From: chenzhaoyu Date: Tue, 17 Oct 2017 22:27:23 +0800 Subject: [PATCH 19/35] fix http client timeout (#763) --- .../protocol/http/client/connection/async_normal.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp index 355ead023..1d5401a31 100644 --- a/boost/network/protocol/http/client/connection/async_normal.hpp +++ b/boost/network/protocol/http/client/connection/async_normal.hpp @@ -177,6 +177,12 @@ struct http_async_connection std::uint16_t source_port = request.source_port(); auto self = this->shared_from_this(); + if (timeout_ > 0) { + timer_.expires_from_now(std::chrono::seconds(timeout_)); + timer_.async_wait(request_strand_.wrap([=] (boost::system::error_code const &ec) { + self->handle_timeout(ec); + })); + } resolve_(resolver_, host_, port_, request_strand_.wrap( [=] (boost::system::error_code const &ec, @@ -184,12 +190,6 @@ struct http_async_connection self->handle_resolved(host_, port_, source_port, get_body, callback, generator, ec, endpoint_range); })); - if (timeout_ > 0) { - timer_.expires_from_now(std::chrono::seconds(timeout_)); - timer_.async_wait(request_strand_.wrap([=] (boost::system::error_code const &ec) { - self->handle_timeout(ec); - })); - } return response_; } From 827867fb0a133ddebfc2d96bdafb83e49c68ff6c Mon Sep 17 00:00:00 2001 From: umennel Date: Tue, 17 Oct 2017 16:28:33 +0200 Subject: [PATCH 20/35] Fix compilation errors with VS2015 (#762) --- .../protocol/http/client/connection/async_normal.hpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp index 1d5401a31..7550cfea0 100644 --- a/boost/network/protocol/http/client/connection/async_normal.hpp +++ b/boost/network/protocol/http/client/connection/async_normal.hpp @@ -30,6 +30,7 @@ #include #include #include +#include #include namespace boost { @@ -477,8 +478,10 @@ struct http_async_connection if (this->is_chunk_encoding && remove_chunk_markers_) { for (size_t i = 0; i < this->partial_parsed.size(); i += 1024) { auto range = parse_chunk_encoding(boost::make_iterator_range( - this->partial_parsed.data() + i, - this->partial_parsed.data() + + static_cast::type, 1024>::const_iterator>( + this->partial_parsed.data()) + i, + static_cast::type, 1024>::const_iterator>( + this->partial_parsed.data()) + std::min(i + 1024, this->partial_parsed.size()))); body_string.append(boost::begin(range), boost::end(range)); } From b17bc59b0b7e6df962fe123280d4b2ad286bf7ba Mon Sep 17 00:00:00 2001 From: Stefan Eilemann Date: Tue, 17 Oct 2017 16:32:06 +0200 Subject: [PATCH 21/35] ICC fixes (#4) (#737) --- boost/network/protocol/http/client/facade.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/boost/network/protocol/http/client/facade.hpp b/boost/network/protocol/http/client/facade.hpp index be6fa45e6..07ddbd8af 100644 --- a/boost/network/protocol/http/client/facade.hpp +++ b/boost/network/protocol/http/client/facade.hpp @@ -122,7 +122,7 @@ class basic_client_facade { } else { if (boost::empty(content_type_headers)) { typedef typename char_::type char_type; - static char_type content_type[] = "x-application/octet-stream"; + static char_type* content_type = "x-application/octet-stream"; request << header("Content-Type", content_type); } } @@ -227,7 +227,7 @@ class basic_client_facade { } else { if (boost::empty(content_type_headers)) { typedef typename char_::type char_type; - static char_type content_type[] = "x-application/octet-stream"; + static char_type* content_type = "x-application/octet-stream"; request << header("Content-Type", content_type); } } From a5252b9559466c67bb502c7b9985ff067a64bf63 Mon Sep 17 00:00:00 2001 From: Gregor Jasny Date: Wed, 29 Nov 2017 21:54:28 +0100 Subject: [PATCH 22/35] Work around ambiguous true_type Visual Studio 2017 cannot decide if it is boost::integral_constant boost::true_type or boost::spirit::true_type --- .../src/server_request_parsers_impl.cpp | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/libs/network/src/server_request_parsers_impl.cpp b/libs/network/src/server_request_parsers_impl.cpp index d62fd2e94..9a539b1ae 100644 --- a/libs/network/src/server_request_parsers_impl.cpp +++ b/libs/network/src/server_request_parsers_impl.cpp @@ -39,22 +39,20 @@ namespace http { void parse_version( std::string const& partial_parsed, std::tuple& version_pair) { - using namespace boost::spirit::qi; - parse(partial_parsed.begin(), partial_parsed.end(), - (lit("HTTP/") >> ushort_ >> '.' >> ushort_), version_pair); + boost::spirit::qi::parse(partial_parsed.begin(), partial_parsed.end(), + (boost::spirit::qi::lit("HTTP/") >> boost::spirit::qi::ushort_ >> '.' >> boost::spirit::qi::ushort_), version_pair); } void parse_headers( std::string const& input, std::vector& container) { u8_to_u32_iterator begin = input.begin(), end = input.end(); - using namespace boost::spirit::qi; - typedef as as_u32_string; - parse(begin, end, - *(+((alnum | punct) - ':') >> lit(": ") >> - as_u32_string()[+((unicode::alnum | space | punct) - '\r' - '\n')] >> - lit("\r\n")) >> - lit("\r\n"), + typedef boost::spirit::qi::as as_u32_string; + boost::spirit::qi::parse(begin, end, + *(+((boost::spirit::qi::alnum | boost::spirit::qi::punct) - ':') >> boost::spirit::qi::lit(": ") >> + as_u32_string()[+((boost::spirit::qi::unicode::alnum | boost::spirit::qi::space | boost::spirit::qi::punct) - '\r' - '\n')] >> + boost::spirit::qi::lit("\r\n")) >> + boost::spirit::qi::lit("\r\n"), container); } From 702dd54353c7285bd97b3fea8ac8e442f828eda1 Mon Sep 17 00:00:00 2001 From: Gregor Jasny Date: Fri, 1 Dec 2017 12:23:54 +0100 Subject: [PATCH 23/35] Make compatible with boost::chrono In Visual Studio 2013 the std::chrono implementation is quite buggy. Therefore we fall back to boost::chrono by specifying -DBOOST_ASIO_DISABLE_STD_CHRONO. This change makes cppnetlib compatible with that flag. --- .../protocol/http/client/connection/async_normal.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp index 7550cfea0..ac10407c0 100644 --- a/boost/network/protocol/http/client/connection/async_normal.hpp +++ b/boost/network/protocol/http/client/connection/async_normal.hpp @@ -179,7 +179,13 @@ struct http_async_connection auto self = this->shared_from_this(); if (timeout_ > 0) { +#if defined(BOOST_ASIO_HAS_STD_CHRONO) timer_.expires_from_now(std::chrono::seconds(timeout_)); +#elif defined(BOOST_ASIO_HAS_BOOST_CHRONO) + timer_.expires_from_now(boost::chrono::seconds(timeout_)); +#else +#error Need a chrono implementation +#endif timer_.async_wait(request_strand_.wrap([=] (boost::system::error_code const &ec) { self->handle_timeout(ec); })); From cda826e958829707595ff775e37a8e647d567416 Mon Sep 17 00:00:00 2001 From: Gregor Jasny Date: Fri, 1 Dec 2017 16:42:56 +0100 Subject: [PATCH 24/35] Make compatible with boost::chrono (this fixes two more places that 702dd54353c missed) --- .../network/protocol/http/client/connection/sync_normal.hpp | 6 ++++++ boost/network/protocol/http/client/connection/sync_ssl.hpp | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/boost/network/protocol/http/client/connection/sync_normal.hpp b/boost/network/protocol/http/client/connection/sync_normal.hpp index 1f3775474..c5909e131 100644 --- a/boost/network/protocol/http/client/connection/sync_normal.hpp +++ b/boost/network/protocol/http/client/connection/sync_normal.hpp @@ -77,7 +77,13 @@ struct http_sync_connection } } if (timeout_ > 0) { +#if defined(BOOST_ASIO_HAS_STD_CHRONO) timer_.expires_from_now(std::chrono::seconds(timeout_)); +#elif defined(BOOST_ASIO_HAS_BOOST_CHRONO) + timer_.expires_from_now(boost::chrono::seconds(timeout_)); +#else +#error Need a chrono implementation +#endif auto self = this->shared_from_this(); timer_.async_wait([=] (boost::system::error_code const &ec) { self->handle_timeout(ec); diff --git a/boost/network/protocol/http/client/connection/sync_ssl.hpp b/boost/network/protocol/http/client/connection/sync_ssl.hpp index 349449512..c7d5ac72e 100644 --- a/boost/network/protocol/http/client/connection/sync_ssl.hpp +++ b/boost/network/protocol/http/client/connection/sync_ssl.hpp @@ -119,7 +119,13 @@ struct https_sync_connection } } if (timeout_ > 0) { +#if defined(BOOST_ASIO_HAS_STD_CHRONO) timer_.expires_from_now(std::chrono::seconds(timeout_)); +#elif defined(BOOST_ASIO_HAS_BOOST_CHRONO) + timer_.expires_from_now(boost::chrono::seconds(timeout_)); +#else +#error Need a chrono implementation +#endif auto self = this->shared_from_this(); timer_.async_wait( [=](boost::system::error_code const& ec) { self->handle_timeout(ec); }); From baa2bbd6efd2beada3e8dc8280ad4ba22e027ba9 Mon Sep 17 00:00:00 2001 From: Nabagata Date: Sun, 24 Dec 2017 17:34:38 +0530 Subject: [PATCH 25/35] Update RATIONALE.txt --- RATIONALE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RATIONALE.txt b/RATIONALE.txt index c9e8960f0..55ed22204 100644 --- a/RATIONALE.txt +++ b/RATIONALE.txt @@ -35,7 +35,7 @@ Goals * Implement an efficient easy to use URI class/parser. * Implement a fully compliant cross-platform asynchronous DNS resolver - either as a wrapper to external (C) libraries, or as hand-rolled + either as a wrapper to external (C) libraries or as hand-rolled implementation. * Implement a MIME handler which builds message objects from either From 8211ce68416c50ed546fcefbc11cafd2762bbb63 Mon Sep 17 00:00:00 2001 From: Nabagata Date: Sun, 24 Dec 2017 17:30:30 +0530 Subject: [PATCH 26/35] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 8eb2abdd2..deefdd796 100644 --- a/README.rst +++ b/README.rst @@ -104,7 +104,7 @@ you will need. These are: Hacking on cpp-netlib --------------------- -cpp-netlib uses git_ for tracking work, and is hosted on GitHub_. +cpp-netlib uses git_ for tracking work and is hosted on GitHub_. cpp-netlib is hosted on GitHub_ following the GitHub recommended practice of forking the repository and submitting pull requests to the source repository. You can read more about the forking_ process and submitting `pull requests`_ if From a57c9fcb64cf56779407fbf0c5d6125653ef5ac1 Mon Sep 17 00:00:00 2001 From: Nabagata Date: Sun, 24 Dec 2017 17:24:23 +0530 Subject: [PATCH 27/35] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index deefdd796..a4293e6d3 100644 --- a/README.rst +++ b/README.rst @@ -39,7 +39,7 @@ geared towards providing a robust cross-platform networking library. cpp-netlib offers the following implementations: * Common Message Type -- A generic message type which can be used - to encapsulate and store message related information, used by all + to encapsulate and store message-related information, used by all network implementations as the primary means of data exchange. * Network protocol message parsers -- A collection of parsers which generate message objects from strings. From a38221c43df456331cac8d7fbb170d7df55c6cc0 Mon Sep 17 00:00:00 2001 From: Nabagata Date: Sun, 24 Dec 2017 17:22:35 +0530 Subject: [PATCH 28/35] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a4293e6d3..5d351980b 100644 --- a/README.rst +++ b/README.rst @@ -34,7 +34,7 @@ follow these instructions for cloning the project repository:: Introduction ------------ -cpp-netlib is a collection of network related routines/implementations +cpp-netlib is a collection of network-related routines/implementations geared towards providing a robust cross-platform networking library. cpp-netlib offers the following implementations: From e80404ec3222b0bd516ea4ad475c66c7222a4350 Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Tue, 6 Mar 2018 14:28:24 -0500 Subject: [PATCH 29/35] Updated uri submodule to pick up GCC 6+ fixes. --- deps/uri | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/uri b/deps/uri index c16a46ecb..f732ef834 160000 --- a/deps/uri +++ b/deps/uri @@ -1 +1 @@ -Subproject commit c16a46ecb6bf0c936179e324891a74b5e6d8f4d3 +Subproject commit f732ef834b6f9bc65802b8b5ac7ec9d5892c1853 From 540ed7622be3f9534709036522f86bde1e84829f Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Thu, 15 Mar 2018 14:12:32 -0400 Subject: [PATCH 30/35] Enable remove_chunk_markers by default. Fix a bug and add a comment. --- boost/network/protocol/http/client/options.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/boost/network/protocol/http/client/options.hpp b/boost/network/protocol/http/client/options.hpp index e73567099..bcc266789 100644 --- a/boost/network/protocol/http/client/options.hpp +++ b/boost/network/protocol/http/client/options.hpp @@ -35,7 +35,7 @@ class client_options { io_service_(), always_verify_peer_(true), timeout_(0), - remove_chunk_markers_(false) {} + remove_chunk_markers_(true) {} client_options(client_options const& other) : cache_resolved_(other.cache_resolved_), @@ -50,7 +50,7 @@ class client_options { io_service_(other.io_service_), always_verify_peer_(other.always_verify_peer_), timeout_(other.timeout_), - remove_chunk_markers_(other.remove_chunk_markers) {} + remove_chunk_markers_(other.remove_chunk_markers_) {} client_options& operator=(client_options other) { other.swap(*this); @@ -157,7 +157,7 @@ class client_options { return *this; } - /// Set an overall timeout for HTTP requests. + /// Set whether we process chunked-encoded streams. client_options& remove_chunk_markers(bool v) { remove_chunk_markers_ = v; return *this; From b9a864e160a604def450ecfb28c2c8c454f39db2 Mon Sep 17 00:00:00 2001 From: David VandeBunte Date: Mon, 4 Jun 2018 19:44:39 -0500 Subject: [PATCH 31/35] bugfix: libs: Update 'breathe' submodule pointer (#846) This project recently had its history rewritten, and the SHA we referred to no longer exists. See discussion here: https://github.com/cpp-netlib/cpp-netlib/pull/582 --- libs/network/doc/_ext/breathe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/network/doc/_ext/breathe b/libs/network/doc/_ext/breathe index 853385ef4..a564b991b 160000 --- a/libs/network/doc/_ext/breathe +++ b/libs/network/doc/_ext/breathe @@ -1 +1 @@ -Subproject commit 853385ef4f0c3dd126887750e20d5f7456065998 +Subproject commit a564b991b5eb2dc220ca29ef21b9c68bdc2f8ac6 From 604d61146d844131964c1181b7c122e25ae8f7c1 Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Tue, 12 Jun 2018 01:50:41 +0200 Subject: [PATCH 32/35] cherry-pick: updating breathe because their history has been rewritten (#847) --- libs/network/doc/_ext/breathe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/network/doc/_ext/breathe b/libs/network/doc/_ext/breathe index a564b991b..65440b267 160000 --- a/libs/network/doc/_ext/breathe +++ b/libs/network/doc/_ext/breathe @@ -1 +1 @@ -Subproject commit a564b991b5eb2dc220ca29ef21b9c68bdc2f8ac6 +Subproject commit 65440b2673ebcc5c6ee2792b0b25754effba1183 From ed6ff63a24df3fe9d7f9eda21dc2b0f6295ab5c7 Mon Sep 17 00:00:00 2001 From: Yuval Hager Date: Thu, 17 May 2018 09:25:18 -0700 Subject: [PATCH 33/35] fix client_get_streaming_test (#840) * fix client_get_streaming_test * perform client_get_streaming_test only if SSL is enabled * test for BOOST_NETWORK_ENABLE_HTTPS for https streaming test --- libs/network/test/http/client_get_streaming_test.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libs/network/test/http/client_get_streaming_test.cpp b/libs/network/test/http/client_get_streaming_test.cpp index ae08f26e0..f08d24a6a 100644 --- a/libs/network/test/http/client_get_streaming_test.cpp +++ b/libs/network/test/http/client_get_streaming_test.cpp @@ -25,8 +25,9 @@ struct body_handler { TYPED_TEST_CASE(HTTPClientTest, ClientTypes); +#ifdef BOOST_NETWORK_ENABLE_HTTPS TYPED_TEST(HTTPClientTest, GetStreamingTest) { - typename TypeParam::request request("http://www.boost.org/"); + typename TypeParam::request request("https://www.boost.org/"); typename TypeParam::response response; typename TypeParam::string_type body_string; typename TypeParam::string_type dummy_body; @@ -47,3 +48,4 @@ TYPED_TEST(HTTPClientTest, GetStreamingTest) { } EXPECT_EQ(dummy_body, typename TypeParam::string_type()); } +#endif From b9a53771ea0ca51f9b71e5b36153edc7ec9a6952 Mon Sep 17 00:00:00 2001 From: Dean Michael Berris Date: Tue, 19 Jun 2018 13:36:39 +1000 Subject: [PATCH 34/35] Update submodule for URI to use C++11 features --- deps/uri | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/uri b/deps/uri index f732ef834..ae076acc0 160000 --- a/deps/uri +++ b/deps/uri @@ -1 +1 @@ -Subproject commit f732ef834b6f9bc65802b8b5ac7ec9d5892c1853 +Subproject commit ae076acc01b2a0c2f973aadbe9a5e4b080fdc88c From b284b144c334437dc14eef8959fe826c2b8a4482 Mon Sep 17 00:00:00 2001 From: oneiric <33737923+coneiric@users.noreply.github.com> Date: Thu, 21 Jun 2018 23:46:01 +0000 Subject: [PATCH 35/35] Directives: remove parens to silence gcc8 warning (#844) --- boost/network/message/directives/detail/string_directive.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/boost/network/message/directives/detail/string_directive.hpp b/boost/network/message/directives/detail/string_directive.hpp index db07eceb4..3c1e652db 100644 --- a/boost/network/message/directives/detail/string_directive.hpp +++ b/boost/network/message/directives/detail/string_directive.hpp @@ -34,8 +34,8 @@ #define BOOST_NETWORK_STRING_DIRECTIVE(name, value, body, pod_body) \ template \ struct name##_directive { \ - ValueType const&((value)); \ - explicit name##_directive(ValueType const& value_) : value(value_) {} \ + ValueType const& value; \ + explicit name##_directive(ValueType const& v) : value(v) {} \ name##_directive(name##_directive const& other) : value(other.value) {} \ template class Message> \ typename enable_if, void>::type operator()( \