diff --git a/.travis.yml b/.travis.yml index 5c2d163e5..2048428be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,42 +4,49 @@ compiler: - g++ - clang env: -- BOOST_VER=1.59.0 BUILD_SHARED_LIBS="ON" CMAKE_BUILD_TYPE="Release" ENABLE_HTTPS="ON" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES -- BOOST_VER=1.59.0 BUILD_SHARED_LIBS="ON" CMAKE_BUILD_TYPE="Release" ENABLE_HTTPS="OFF" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES -- BOOST_VER=1.59.0 BUILD_SHARED_LIBS="ON" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES -- BOOST_VER=1.59.0 BUILD_SHARED_LIBS="ON" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="OFF" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES -- BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Release" ENABLE_HTTPS="ON" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES -- BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Release" ENABLE_HTTPS="OFF" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES -- BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES -- BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="OFF" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES +- BUILD_SHARED_LIBS="ON" CMAKE_BUILD_TYPE="Release" ENABLE_HTTPS="ON" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES +- BUILD_SHARED_LIBS="ON" CMAKE_BUILD_TYPE="Release" ENABLE_HTTPS="OFF" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES +- BUILD_SHARED_LIBS="ON" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES +- BUILD_SHARED_LIBS="ON" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="OFF" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES +- BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Release" ENABLE_HTTPS="ON" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES +- BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Release" ENABLE_HTTPS="OFF" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES +- BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES +- BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="OFF" Uri_BUILD_TESTS=OFF Uri_DISABLE_LIBCXX=YES # Support the sanitizers in clang only -# - BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=thread" -# - BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=address" +- BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=thread" Uri_BUILD_TEST=OFF Uri_DISABLE_LIBCXX=YES +- BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=address" Uri_BUILD_TEST=OFF Uri_DISABLE_LIBCXX=YES # TODO(deanberris): It seems Boost is not msan-clean yet; report bugs and maybe fix? #- BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2" -# matrix: -# exclude: -# - compiler: g++ -# env: BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=thread" -# - compiler: g++ -# env: BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=address" +cache: + - apt + - ccache +matrix: + exclude: + - compiler: g++ + env: BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=thread" Uri_BUILD_TEST=OFF Uri_DISABLE_LIBCXX=YES + - compiler: g++ + env: BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=address" Uri_BUILD_TEST=OFF Uri_DISABLE_LIBCXX=YES # TODO(deanberris): It seems Boost is not msan-clean yet; report bugs and maybe fix? # - compiler: g++ # env: BOOST_VER=1.59.0 BUILD_SHARED_LIBS="OFF" CMAKE_BUILD_TYPE="Debug" ENABLE_HTTPS="ON" CMAKE_CXX_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2" install: -- mkdir -p ${HOME}/bin -- if [ "${CC}" = "gcc" ]; then export TOOLSET="gcc"; ln -s `which g++-4.8` ${HOME}/bin/g++; - ln -s `which gcc-4.8` ${HOME}/bin/gcc; fi -- if [ "${CC}" = "clang" ]; then export TOOLSET="clang"; ln -s `which clang-3.6` ${HOME}/bin/clang; - ln -s `which clang++-3.6` ${HOME}/bin/clang++; fi -- export BOOST_VERSION=${BOOST_VER//./_} -- export PATH=${HOME}/bin:${PATH} -- travis_wait ./install-boost.sh -- export BOOST_ROOT=${HOME}/${CC}-boost_${BOOST_VER//./_} +- pwd +- export CUR_DIR=`pwd` +- mkdir -p ${CUR_DIR}/bin +- if [ "${CC}" = "gcc" ]; then + export CXX=g++-4.9; + export CC=gcc-4.9; + fi +- if [ "${CC}" = "clang" ]; then + export CXX=clang++-3.8; + export CC=clang-3.8; + export CCACHE_CPP2=yes; + fi +before_script: +- echo ${CXX} - "${CXX} --version" -cache: - directories: - - "${HOME}/${CC}-boost_${BOOST_VER//./_}" +- echo ${CC} +- "${CC} --version" script: - pwd - sh -x build.sh @@ -48,14 +55,18 @@ after_failure: addons: apt: sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.6 - - kalakris-cmake + - ubuntu-toolchain-r-test + - llvm-toolchain-precise-3.8 + - kalakris-cmake + - boost-latest + - george-edison55-precise-backports packages: - - gcc-4.8 - - g++-4.8 - - clang-3.6 - - cmake + - libboost1.55-all-dev + - gcc-4.9 + - g++-4.9 + - clang-3.8 + - cmake + - cmake-data notifications: slack: secure: Y7lLjqZ83+b/jaJ5+EKwvgCDeERi4bVbDn9tLp8sieTdu+ENsPI+JmLYSXZXPpe7JrItrXW6uJJXN2wG1h7au4mpVVTghd31HBzuzrqVxDphWPhp16NYzvbAgQQRBXvFVvfSdW/Kb/n2fX6xDApY0t6vNREb/GKg0GyzESb4ZjU= diff --git a/CMakeLists.txt b/CMakeLists.txt index 6972594ae..56534f32f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,8 @@ option( CPP-NETLIB_BUILD_SHARED_LIBS "Build cpp-netlib as shared libraries." OFF option( CPP-NETLIB_BUILD_TESTS "Build the cpp-netlib project tests." ON) option( CPP-NETLIB_BUILD_EXAMPLES "Build the cpp-netlib project examples." ON) option( CPP-NETLIB_ENABLE_HTTPS "Build cpp-netlib with support for https if OpenSSL is found." ON) +option( CPP-NETLIB_STATIC_OPENSSL "Build cpp-netlib using static OpenSSL" OFF) +option( CPP-NETLIB_STATIC_BOOST "Build cpp-netlib using static Boost" OFF) include(GNUInstallDirs) @@ -36,8 +38,10 @@ else() set(BUILD_SHARED_LIBS OFF) endif() -# Always use Boost's shared libraries. -set(Boost_USE_STATIC_LIBS OFF) +# Use Boost's static libraries +if (CPP-NETLIB_STATIC_BOOST) + set(Boost_USE_STATIC_LIBS ON) +endif() # We need this for all tests to use the dynamic version. add_definitions(-DBOOST_TEST_DYN_LINK) @@ -45,7 +49,7 @@ add_definitions(-DBOOST_TEST_DYN_LINK) # Always use multi-threaded Boost libraries. set(Boost_USE_MULTI_THREADED ON) -find_package(Boost 1.58.0 REQUIRED COMPONENTS system thread) +find_package(Boost 1.55.0 REQUIRED COMPONENTS system thread) if (CPP-NETLIB_ENABLE_HTTPS) if (APPLE) @@ -64,6 +68,9 @@ if (CPP-NETLIB_ENABLE_HTTPS) endif() endif() endif() + if (CPP-NETLIB_STATIC_OPENSSL) + set(CMAKE_FIND_LIBRARY_SUFFIXES .a) + endif() find_package(OpenSSL) endif() @@ -90,11 +97,16 @@ if (${CMAKE_CXX_COMPILER_ID} MATCHES GNU) elseif (${CMAKE_CXX_COMPILER_ID} MATCHES Clang) # We want to link in C++11 mode in Clang too, but also set a high enough # template depth for the template metaprogramming. - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -ftemplate-depth=256 -std=c++11") + set (CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -Wall -ftemplate-depth=256 -std=c++11 -DBOOST_ASIO_HAS_STD_CHRONO -DBOOST_ASIO_HAS_STD_ARRAY -DBOOST_ASIO_HAS_STD_SHARED_PTR -DBOOST_ASIO_HAS_STD_ATOMIC -DBOOST_ASIO_HAS_VARIADIC_TEMPLATES -DBOOST_ASIO_HAS_MOVE -DBOOST_THREAD_VERSION=3") if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") # Use libc++ only in OS X. set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lc++") + elseif (${CMAKE_SYSTEM_NAME} MATCHES "Linux") + # Use libstdc++ for Linux. + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libstdc++") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lstdc++") endif() endif() @@ -133,6 +145,13 @@ if (MSVC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /bigobj") endif() +# See whether we can find the ccache program -- if we can, then use it for the build. +find_program(CCACHE_FOUND ccache) +if(CCACHE_FOUND) + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) + set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache) +endif(CCACHE_FOUND) + enable_testing() install(DIRECTORY boost DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) diff --git a/RATIONALE.txt b/RATIONALE.txt index c9e8960f0..55ed22204 100644 --- a/RATIONALE.txt +++ b/RATIONALE.txt @@ -35,7 +35,7 @@ Goals * Implement an efficient easy to use URI class/parser. * Implement a fully compliant cross-platform asynchronous DNS resolver - either as a wrapper to external (C) libraries, or as hand-rolled + either as a wrapper to external (C) libraries or as hand-rolled implementation. * Implement a MIME handler which builds message objects from either diff --git a/README.rst b/README.rst index 8eb2abdd2..5d351980b 100644 --- a/README.rst +++ b/README.rst @@ -34,12 +34,12 @@ follow these instructions for cloning the project repository:: Introduction ------------ -cpp-netlib is a collection of network related routines/implementations +cpp-netlib is a collection of network-related routines/implementations geared towards providing a robust cross-platform networking library. cpp-netlib offers the following implementations: * Common Message Type -- A generic message type which can be used - to encapsulate and store message related information, used by all + to encapsulate and store message-related information, used by all network implementations as the primary means of data exchange. * Network protocol message parsers -- A collection of parsers which generate message objects from strings. @@ -104,7 +104,7 @@ you will need. These are: Hacking on cpp-netlib --------------------- -cpp-netlib uses git_ for tracking work, and is hosted on GitHub_. +cpp-netlib uses git_ for tracking work and is hosted on GitHub_. cpp-netlib is hosted on GitHub_ following the GitHub recommended practice of forking the repository and submitting pull requests to the source repository. You can read more about the forking_ process and submitting `pull requests`_ if diff --git a/boost/network/message/directives/detail/string_directive.hpp b/boost/network/message/directives/detail/string_directive.hpp index db07eceb4..3c1e652db 100644 --- a/boost/network/message/directives/detail/string_directive.hpp +++ b/boost/network/message/directives/detail/string_directive.hpp @@ -34,8 +34,8 @@ #define BOOST_NETWORK_STRING_DIRECTIVE(name, value, body, pod_body) \ template \ struct name##_directive { \ - ValueType const&((value)); \ - explicit name##_directive(ValueType const& value_) : value(value_) {} \ + ValueType const& value; \ + explicit name##_directive(ValueType const& v) : value(v) {} \ name##_directive(name##_directive const& other) : value(other.value) {} \ template class Message> \ typename enable_if, void>::type operator()( \ diff --git a/boost/network/protocol/http/client/async_impl.hpp b/boost/network/protocol/http/client/async_impl.hpp index ab54dfa61..fe8d2adae 100644 --- a/boost/network/protocol/http/client/async_impl.hpp +++ b/boost/network/protocol/http/client/async_impl.hpp @@ -38,7 +38,7 @@ struct async_client typedef std::function body_generator_function_type; async_client(bool cache_resolved, bool follow_redirect, - bool always_verify_peer, int timeout, + bool always_verify_peer, int timeout, bool remove_chunk_markers, std::shared_ptr service, optional certificate_filename, optional verify_path, @@ -46,7 +46,8 @@ struct async_client optional private_key_file, optional ciphers, optional sni_hostname, long ssl_options) - : connection_base(cache_resolved, follow_redirect, timeout), + : connection_base(cache_resolved, follow_redirect, timeout, + remove_chunk_markers), service_ptr(service.get() ? service : std::make_shared()), service_(*service_ptr), diff --git a/boost/network/protocol/http/client/connection/async_base.hpp b/boost/network/protocol/http/client/connection/async_base.hpp index a31ad30fd..b1c1aa67a 100644 --- a/boost/network/protocol/http/client/connection/async_base.hpp +++ b/boost/network/protocol/http/client/connection/async_base.hpp @@ -44,6 +44,7 @@ struct async_connection_base { static connection_ptr new_connection( resolve_function resolve, resolver_type &resolver, bool follow_redirect, bool always_verify_peer, bool https, int timeout, + bool remove_chunk_markers, optional certificate_filename = optional(), optional const &verify_path = optional(), optional certificate_file = optional(), @@ -59,7 +60,8 @@ struct async_connection_base { certificate_filename, verify_path, certificate_file, private_key_file, ciphers, sni_hostname, ssl_options); auto temp = std::make_shared( - resolver, resolve, follow_redirect, timeout, std::move(delegate)); + resolver, resolve, follow_redirect, timeout, remove_chunk_markers, + std::move(delegate)); BOOST_ASSERT(temp != nullptr); return temp; } diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp index d7016c764..ac10407c0 100644 --- a/boost/network/protocol/http/client/connection/async_normal.hpp +++ b/boost/network/protocol/http/client/connection/async_normal.hpp @@ -30,6 +30,7 @@ #include #include #include +#include #include namespace boost { @@ -37,6 +38,82 @@ namespace network { namespace http { namespace impl { +template +struct chunk_encoding_parser { + chunk_encoding_parser() : state(state_t::header), chunk_size(0) {} + + enum class state_t { header, header_end, data, data_end }; + + state_t state; + size_t chunk_size; + std::array::type, 1024> buffer; + + void update_chunk_size( + boost::iterator_range::type, 1024>::const_iterator> const &range) { + if (range.empty()) return; + std::stringstream ss; + ss << std::hex << range; + size_t size; + ss >> size; + // New digits are appended as LSBs + chunk_size = (chunk_size << (range.size() * 4)) | size; + } + + boost::iterator_range< + typename std::array::type, 1024>::const_iterator> + operator()( + boost::iterator_range::type, 1024>::const_iterator> const &range) { + auto iter = boost::begin(range); + auto begin = iter; + auto pos = boost::begin(buffer); + + while (iter != boost::end(range)) switch (state) { + case state_t::header: + iter = std::find(iter, boost::end(range), '\r'); + update_chunk_size(boost::make_iterator_range(begin, iter)); + if (iter != boost::end(range)) { + state = state_t::header_end; + ++iter; + } + break; + + case state_t::header_end: + BOOST_ASSERT(*iter == '\n'); + ++iter; + state = state_t::data; + break; + + case state_t::data: + if (chunk_size == 0) { + BOOST_ASSERT(*iter == '\r'); + ++iter; + state = state_t::data_end; + } else { + auto len = std::min(chunk_size, + (size_t)std::distance(iter, boost::end(range))); + begin = iter; + iter = std::next(iter, len); + pos = std::copy(begin, iter, pos); + chunk_size -= len; + } + break; + + case state_t::data_end: + BOOST_ASSERT(*iter == '\n'); + ++iter; + begin = iter; + state = state_t::header; + break; + + default: + BOOST_ASSERT(false && "Bug, report this to the developers!"); + } + return boost::make_iterator_range(boost::begin(buffer), pos); + } +}; + template struct async_connection_base; @@ -71,9 +148,11 @@ struct http_async_connection connection_delegate_ptr; http_async_connection(resolver_type& resolver, resolve_function resolve, - bool follow_redirect, int timeout, + bool follow_redirect, int64_t timeout, + bool remove_chunk_markers, connection_delegate_ptr delegate) : timeout_(timeout), + remove_chunk_markers_(remove_chunk_markers), timer_(resolver.get_io_service()), is_timedout_(false), follow_redirect_(follow_redirect), @@ -82,11 +161,9 @@ struct http_async_connection request_strand_(resolver.get_io_service()), delegate_(std::move(delegate)) {} - // This is the main entry point for the connection/request pipeline. - // We're - // overriding async_connection_base<...>::start(...) here which is - // called - // by the client. + // This is the main entry point for the connection/request pipeline. We're + // overriding async_connection_base<...>::start(...) here which is called by + // the client. virtual response start(request const& request, string_type const& method, bool get_body, body_callback_function_type callback, body_generator_function_type generator) { @@ -101,6 +178,18 @@ struct http_async_connection std::uint16_t source_port = request.source_port(); auto self = this->shared_from_this(); + if (timeout_ > 0) { +#if defined(BOOST_ASIO_HAS_STD_CHRONO) + timer_.expires_from_now(std::chrono::seconds(timeout_)); +#elif defined(BOOST_ASIO_HAS_BOOST_CHRONO) + timer_.expires_from_now(boost::chrono::seconds(timeout_)); +#else +#error Need a chrono implementation +#endif + timer_.async_wait(request_strand_.wrap([=] (boost::system::error_code const &ec) { + self->handle_timeout(ec); + })); + } resolve_(resolver_, host_, port_, request_strand_.wrap( [=] (boost::system::error_code const &ec, @@ -108,25 +197,19 @@ struct http_async_connection self->handle_resolved(host_, port_, source_port, get_body, callback, generator, ec, endpoint_range); })); - if (timeout_ > 0) { - timer_.expires_from_now(std::chrono::seconds(timeout_)); - timer_.async_wait(request_strand_.wrap([=] (boost::system::error_code const &ec) { - self->handle_timeout(ec); - })); - } return response_; } private: void set_errors(boost::system::error_code const& ec, body_callback_function_type callback) { boost::system::system_error error(ec); - this->version_promise.set_exception(std::make_exception_ptr(error)); - this->status_promise.set_exception(std::make_exception_ptr(error)); - this->status_message_promise.set_exception(std::make_exception_ptr(error)); - this->headers_promise.set_exception(std::make_exception_ptr(error)); - this->source_promise.set_exception(std::make_exception_ptr(error)); - this->destination_promise.set_exception(std::make_exception_ptr(error)); - this->body_promise.set_exception(std::make_exception_ptr(error)); + this->version_promise.set_exception(boost::copy_exception(error)); + this->status_promise.set_exception(boost::copy_exception(error)); + this->status_message_promise.set_exception(boost::copy_exception(error)); + this->headers_promise.set_exception(boost::copy_exception(error)); + this->source_promise.set_exception(boost::copy_exception(error)); + this->destination_promise.set_exception(boost::copy_exception(error)); + this->body_promise.set_exception(boost::copy_exception(error)); if ( callback ) callback( boost::iterator_range::type, 1024>::const_iterator>(), ec ); this->timer_.cancel(); @@ -348,8 +431,11 @@ struct http_async_connection // The invocation of the callback is synchronous to allow us to // wait before scheduling another read. - callback(make_iterator_range(begin, end), ec); - + if (this->is_chunk_encoding && remove_chunk_markers_) { + callback(parse_chunk_encoding(make_iterator_range(begin, end)), ec); + } else { + callback(make_iterator_range(begin, end), ec); + } auto self = this->shared_from_this(); delegate_->read_some( boost::asio::mutable_buffers_1(this->part.data(), @@ -388,14 +474,33 @@ struct http_async_connection // We call the callback function synchronously passing the error // condition (in this case, end of file) so that it can handle it // appropriately. - callback(make_iterator_range(begin, end), ec); + if (this->is_chunk_encoding && remove_chunk_markers_) { + callback(parse_chunk_encoding(make_iterator_range(begin, end)), ec); + } else { + callback(make_iterator_range(begin, end), ec); + } } else { string_type body_string; - std::swap(body_string, this->partial_parsed); - body_string.append(this->part.begin(), this->part.begin() + bytes_transferred); - if (this->is_chunk_encoding) { - this->body_promise.set_value(parse_chunk_encoding(body_string)); + if (this->is_chunk_encoding && remove_chunk_markers_) { + for (size_t i = 0; i < this->partial_parsed.size(); i += 1024) { + auto range = parse_chunk_encoding(boost::make_iterator_range( + static_cast::type, 1024>::const_iterator>( + this->partial_parsed.data()) + i, + static_cast::type, 1024>::const_iterator>( + this->partial_parsed.data()) + + std::min(i + 1024, this->partial_parsed.size()))); + body_string.append(boost::begin(range), boost::end(range)); + } + this->partial_parsed.clear(); + auto range = parse_chunk_encoding(boost::make_iterator_range( + this->part.begin(), + this->part.begin() + bytes_transferred)); + body_string.append(boost::begin(range), boost::end(range)); + this->body_promise.set_value(body_string); } else { + std::swap(body_string, this->partial_parsed); + body_string.append(this->part.begin(), + this->part.begin() + bytes_transferred); this->body_promise.set_value(body_string); } } @@ -417,7 +522,11 @@ struct http_async_connection this->part.begin(); typename protocol_base::buffer_type::const_iterator end = begin; std::advance(end, bytes_transferred); - callback(make_iterator_range(begin, end), ec); + if (this->is_chunk_encoding && remove_chunk_markers_) { + callback(parse_chunk_encoding(make_iterator_range(begin, end)), ec); + } else { + callback(make_iterator_range(begin, end), ec); + } auto self = this->shared_from_this(); delegate_->read_some( boost::asio::mutable_buffers_1(this->part.data(), @@ -476,38 +585,8 @@ struct http_async_connection } } - string_type parse_chunk_encoding(string_type& body_string) { - string_type body; - string_type crlf = "\r\n"; - - typename string_type::iterator begin = body_string.begin(); - for (typename string_type::iterator iter = - std::search(begin, body_string.end(), crlf.begin(), crlf.end()); - iter != body_string.end(); - iter = - std::search(begin, body_string.end(), crlf.begin(), crlf.end())) { - string_type line(begin, iter); - if (line.empty()) { - break; - } - std::stringstream stream(line); - int len; - stream >> std::hex >> len; - std::advance(iter, 2); - if (len == 0) { - break; - } - if (len <= body_string.end() - iter) { - body.insert(body.end(), iter, iter + len); - std::advance(iter, len + 2); - } - begin = iter; - } - - return body; - } - - int timeout_; + int64_t timeout_; + bool remove_chunk_markers_; boost::asio::steady_timer timer_; bool is_timedout_; bool follow_redirect_; @@ -517,6 +596,7 @@ struct http_async_connection connection_delegate_ptr delegate_; boost::asio::streambuf command_streambuf; string_type method; + chunk_encoding_parser parse_chunk_encoding; }; } // namespace impl diff --git a/boost/network/protocol/http/client/connection/async_protocol_handler.hpp b/boost/network/protocol/http/client/connection/async_protocol_handler.hpp index e940862e3..a0ce75bce 100644 --- a/boost/network/protocol/http/client/connection/async_protocol_handler.hpp +++ b/boost/network/protocol/http/client/connection/async_protocol_handler.hpp @@ -246,6 +246,10 @@ struct http_async_protocol_handler { response_parser_type::http_header_line_done); typename headers_container::type headers; std::pair header_pair; + //init params + is_content_length = false; + content_length = -1; + is_chunk_end = false; while (!boost::empty(input_range)) { std::tie(parsed_ok, result_range) = headers_parser.parse_until( response_parser_type::http_header_colon, input_range); @@ -266,6 +270,16 @@ struct http_async_protocol_handler { } trim(header_pair.second); headers.insert(header_pair); + if (!is_content_length && + boost::iequals(header_pair.first, "Content-Length")) { + try { + content_length = std::stoll(header_pair.second); + is_content_length = true; + } + catch (std::exception&) { + //is_content_length = false; + } + } } // determine if the body parser will need to handle chunked encoding typename headers_range >::type transfer_encoding_range = @@ -325,14 +339,61 @@ struct http_async_protocol_handler { parsed_ok, std::distance(std::end(result_range), part_end)); } + inline bool check_parse_body_complete() const { + if (this->is_chunk_encoding) { + return parse_chunk_encoding_complete(); + } + if (this->is_content_length && this->content_length >= 0) { + return parse_content_length_complete(); + } + return false; + } + + inline bool parse_content_length_complete() const { + return static_cast(this->partial_parsed.length()) >= this->content_length; + } + + bool parse_chunk_encoding_complete() const { + string_type body; + string_type crlf = "\r\n"; + + typename string_type::const_iterator begin = partial_parsed.begin(); + for (typename string_type::const_iterator iter = + std::search(begin, partial_parsed.end(), crlf.begin(), crlf.end()); + iter != partial_parsed.end(); + iter = + std::search(begin, partial_parsed.end(), crlf.begin(), crlf.end())) { + string_type line(begin, iter); + if (line.empty()) { + std::advance(iter, 2); + begin = iter; + continue; + } + std::stringstream stream(line); + int len; + stream >> std::hex >> len; + std::advance(iter, 2); + if (!len) return true; + if (len <= partial_parsed.end() - iter) { + std::advance(iter, len + 2); + } + begin = iter; + } + return false; + } + template void parse_body(Delegate& delegate_, Callback callback, size_t bytes) { // TODO(dberris): we should really not use a string for the partial body // buffer. partial_parsed.append(part_begin, part_begin + bytes); part_begin = part.begin(); - delegate_->read_some( - boost::asio::mutable_buffers_1(part.data(), part.size()), callback); + if (check_parse_body_complete()) { + callback(boost::asio::error::eof, 0); + } else { + delegate_->read_some( + boost::asio::mutable_buffers_1(part.data(), part.size()), callback); + } } typedef response_parser response_parser_type; @@ -351,6 +412,9 @@ struct http_async_protocol_handler { typename buffer_type::const_iterator part_begin; string_type partial_parsed; bool is_chunk_encoding; + bool is_chunk_end; + bool is_content_length; + long long content_length; }; } // namespace impl diff --git a/boost/network/protocol/http/client/connection/sync_normal.hpp b/boost/network/protocol/http/client/connection/sync_normal.hpp index 1f3775474..c5909e131 100644 --- a/boost/network/protocol/http/client/connection/sync_normal.hpp +++ b/boost/network/protocol/http/client/connection/sync_normal.hpp @@ -77,7 +77,13 @@ struct http_sync_connection } } if (timeout_ > 0) { +#if defined(BOOST_ASIO_HAS_STD_CHRONO) timer_.expires_from_now(std::chrono::seconds(timeout_)); +#elif defined(BOOST_ASIO_HAS_BOOST_CHRONO) + timer_.expires_from_now(boost::chrono::seconds(timeout_)); +#else +#error Need a chrono implementation +#endif auto self = this->shared_from_this(); timer_.async_wait([=] (boost::system::error_code const &ec) { self->handle_timeout(ec); diff --git a/boost/network/protocol/http/client/connection/sync_ssl.hpp b/boost/network/protocol/http/client/connection/sync_ssl.hpp index 349449512..c7d5ac72e 100644 --- a/boost/network/protocol/http/client/connection/sync_ssl.hpp +++ b/boost/network/protocol/http/client/connection/sync_ssl.hpp @@ -119,7 +119,13 @@ struct https_sync_connection } } if (timeout_ > 0) { +#if defined(BOOST_ASIO_HAS_STD_CHRONO) timer_.expires_from_now(std::chrono::seconds(timeout_)); +#elif defined(BOOST_ASIO_HAS_BOOST_CHRONO) + timer_.expires_from_now(boost::chrono::seconds(timeout_)); +#else +#error Need a chrono implementation +#endif auto self = this->shared_from_this(); timer_.async_wait( [=](boost::system::error_code const& ec) { self->handle_timeout(ec); }); diff --git a/boost/network/protocol/http/client/facade.hpp b/boost/network/protocol/http/client/facade.hpp index 37a3eab49..07ddbd8af 100644 --- a/boost/network/protocol/http/client/facade.hpp +++ b/boost/network/protocol/http/client/facade.hpp @@ -122,7 +122,7 @@ class basic_client_facade { } else { if (boost::empty(content_type_headers)) { typedef typename char_::type char_type; - static char_type content_type[] = "x-application/octet-stream"; + static char_type* content_type = "x-application/octet-stream"; request << header("Content-Type", content_type); } } @@ -227,7 +227,7 @@ class basic_client_facade { } else { if (boost::empty(content_type_headers)) { typedef typename char_::type char_type; - static char_type content_type[] = "x-application/octet-stream"; + static char_type* content_type = "x-application/octet-stream"; request << header("Content-Type", content_type); } } @@ -303,7 +303,8 @@ class basic_client_facade { options.openssl_verify_path(), options.openssl_certificate_file(), options.openssl_private_key_file(), options.openssl_ciphers(), options.openssl_sni_hostname(), options.openssl_options(), - options.io_service(), options.timeout())); + options.io_service(), options.timeout(), + options.remove_chunk_markers())); } }; diff --git a/boost/network/protocol/http/client/options.hpp b/boost/network/protocol/http/client/options.hpp index 23f7a134d..bcc266789 100644 --- a/boost/network/protocol/http/client/options.hpp +++ b/boost/network/protocol/http/client/options.hpp @@ -34,7 +34,8 @@ class client_options { openssl_options_(0), io_service_(), always_verify_peer_(true), - timeout_(0) {} + timeout_(0), + remove_chunk_markers_(true) {} client_options(client_options const& other) : cache_resolved_(other.cache_resolved_), @@ -48,7 +49,8 @@ class client_options { openssl_options_(other.openssl_options_), io_service_(other.io_service_), always_verify_peer_(other.always_verify_peer_), - timeout_(other.timeout_) {} + timeout_(other.timeout_), + remove_chunk_markers_(other.remove_chunk_markers_) {} client_options& operator=(client_options other) { other.swap(*this); @@ -69,6 +71,7 @@ class client_options { swap(io_service_, other.io_service_); swap(always_verify_peer_, other.always_verify_peer_); swap(timeout_, other.timeout_); + swap(remove_chunk_markers_, other.remove_chunk_markers_); } /// Specify whether the client should cache resolved endpoints. @@ -154,6 +157,12 @@ class client_options { return *this; } + /// Set whether we process chunked-encoded streams. + client_options& remove_chunk_markers(bool v) { + remove_chunk_markers_ = v; + return *this; + } + bool cache_resolved() const { return cache_resolved_; } bool follow_redirects() const { return follow_redirects_; } @@ -190,6 +199,8 @@ class client_options { int timeout() const { return timeout_; } + bool remove_chunk_markers() const { return remove_chunk_markers_; } + private: bool cache_resolved_; bool follow_redirects_; @@ -203,6 +214,7 @@ class client_options { std::shared_ptr io_service_; bool always_verify_peer_; int timeout_; + bool remove_chunk_markers_; }; template diff --git a/boost/network/protocol/http/client/pimpl.hpp b/boost/network/protocol/http/client/pimpl.hpp index d62be32ce..01c77ca70 100644 --- a/boost/network/protocol/http/client/pimpl.hpp +++ b/boost/network/protocol/http/client/pimpl.hpp @@ -74,10 +74,12 @@ struct basic_client_impl optional const& private_key_file, optional const& ciphers, optional const& sni_hostname, long ssl_options, - std::shared_ptr service, int timeout) - : base_type(cache_resolved, follow_redirect, always_verify_peer, timeout, - service, certificate_filename, verify_path, certificate_file, - private_key_file, ciphers, sni_hostname, ssl_options) {} + std::shared_ptr service, int timeout, + bool remove_chunk_markers) + : base_type(cache_resolved, follow_redirect, always_verify_peer, timeout, + remove_chunk_markers, service, certificate_filename, verify_path, + certificate_file, private_key_file, ciphers, sni_hostname, + ssl_options) {} ~basic_client_impl() = default; }; diff --git a/boost/network/protocol/http/message/async_message.hpp b/boost/network/protocol/http/message/async_message.hpp index 861f3adf6..747a962ed 100644 --- a/boost/network/protocol/http/message/async_message.hpp +++ b/boost/network/protocol/http/message/async_message.hpp @@ -9,13 +9,13 @@ // (See accompanying file LICENSE_1_0.txt or copy at // http://www.boost.org/LICENSE_1_0.txt) -#include #include +#include // FIXME move this out to a trait -#include #include #include +#include namespace boost { namespace network { @@ -26,12 +26,11 @@ namespace impl { template struct ready_wrapper; -} // namespace impl - /* impl */ +} // namespace impl + /* impl */ template struct async_message { - typedef typename string::type string_type; typedef typename headers_container::type headers_container_type; typedef typename headers_container_type::value_type header_type; @@ -54,31 +53,56 @@ struct async_message { headers_(other.headers_), body_(other.body_) {} - string_type const status_message() const { return status_message_.get(); } + string_type status_message() const { + status_message_.wait(); + if (status_message_.has_exception()) + boost::rethrow_exception(status_message_.get_exception_ptr()); + return status_message_.get(); + } void status_message(boost::shared_future const& future) const { status_message_ = future; } - string_type const version() const { return version_.get(); } + string_type version() const { + version_.wait(); + if (version_.has_exception()) + boost::rethrow_exception(version_.get_exception_ptr()); + return version_.get(); + } void version(boost::shared_future const& future) const { version_ = future; } - std::uint16_t status() const { return status_.get(); } + std::uint16_t status() const { + status_.wait(); + if (status_.has_exception()) + boost::rethrow_exception(status_.get_exception_ptr()); + return status_.get(); + } void status(boost::shared_future const& future) const { status_ = future; } - string_type const source() const { return source_.get(); } + string_type source() const { + source_.wait(); + if (source_.has_exception()) + boost::rethrow_exception(source_.get_exception_ptr()); + return source_.get(); + } void source(boost::shared_future const& future) const { source_ = future; } - string_type const destination() const { return destination_.get(); } + string_type destination() const { + destination_.wait(); + if (destination_.has_exception()) + boost::rethrow_exception(source_.get_exception_ptr()); + return destination_.get(); + } void destination(boost::shared_future const& future) const { destination_ = future; @@ -86,31 +110,38 @@ struct async_message { headers_container_type const& headers() const { if (retrieved_headers_) return *retrieved_headers_; + if (headers_.has_exception()) + boost::rethrow_exception(headers_.get_exception_ptr()); headers_container_type raw_headers = headers_.get(); raw_headers.insert(added_headers.begin(), added_headers.end()); - for (string_type const & key : removed_headers) { + for (string_type const& key : removed_headers) { raw_headers.erase(key); } retrieved_headers_ = raw_headers; return *retrieved_headers_; } - void headers(boost::shared_future const& future) - const { + void headers( + boost::shared_future const& future) const { headers_ = future; } - void add_header(typename headers_container_type::value_type const& pair_) - const { + void add_header( + typename headers_container_type::value_type const& pair_) const { added_headers.insert(added_headers.end(), pair_); } - void remove_header(typename headers_container_type::key_type const& key_) - const { + void remove_header( + typename headers_container_type::key_type const& key_) const { removed_headers.insert(key_); } - string_type const body() const { return body_.get(); } + string_type body() const { + body_.wait(); + if (body_.has_exception()) + boost::rethrow_exception(body_.get_exception_ptr()); + return body_.get(); + } void body(boost::shared_future const& future) const { body_ = future; diff --git a/boost/network/protocol/http/policies/async_connection.hpp b/boost/network/protocol/http/policies/async_connection.hpp index c3f0d131a..900a37e59 100644 --- a/boost/network/protocol/http/policies/async_connection.hpp +++ b/boost/network/protocol/http/policies/async_connection.hpp @@ -38,7 +38,7 @@ struct async_connection_policy : resolver_policy::type { struct connection_impl { connection_impl( bool follow_redirect, bool always_verify_peer, resolve_function resolve, - resolver_type& resolver, bool https, int timeout, + resolver_type& resolver, bool https, int timeout, bool remove_chunk_markers, optional /*unused*/ const& certificate_filename, optional const& verify_path, optional const& certificate_file, @@ -47,9 +47,9 @@ struct async_connection_policy : resolver_policy::type { optional const& sni_hostname, long ssl_options) { pimpl = impl::async_connection_base:: new_connection(resolve, resolver, follow_redirect, always_verify_peer, - https, timeout, certificate_filename, verify_path, - certificate_file, private_key_file, ciphers, - sni_hostname, ssl_options); + https, timeout, remove_chunk_markers, + certificate_filename, verify_path, certificate_file, + private_key_file, ciphers, sni_hostname, ssl_options); } basic_response send_request(string_type /*unused*/ const& method, @@ -86,20 +86,22 @@ struct async_connection_policy : resolver_policy::type { this->resolve(resolver, host, port, once_resolved); }, resolver, boost::iequals(protocol_, string_type("https")), timeout_, - certificate_filename, verify_path, certificate_file, private_key_file, - ciphers, sni_hostname, ssl_options); + remove_chunk_markers_, certificate_filename, verify_path, + certificate_file, private_key_file, ciphers, sni_hostname, ssl_options); } void cleanup() {} async_connection_policy(bool cache_resolved, bool follow_redirect, - int timeout) + int timeout, bool remove_chunk_markers) : resolver_base(cache_resolved), follow_redirect_(follow_redirect), - timeout_(timeout) {} + timeout_(timeout), + remove_chunk_markers_(remove_chunk_markers) {} bool follow_redirect_; int timeout_; + bool remove_chunk_markers_; }; } // namespace http diff --git a/boost/network/protocol/http/server/async_server.hpp b/boost/network/protocol/http/server/async_server.hpp index 4cbefe0d2..c1e3c662e 100644 --- a/boost/network/protocol/http/server/async_server.hpp +++ b/boost/network/protocol/http/server/async_server.hpp @@ -35,13 +35,17 @@ struct async_server_base : server_storage_base, socket_options_base { /// Defines the type for the connection pointer. typedef std::shared_ptr connection_ptr; + /// Defines the type for the options. + typedef server_options options; + /// Constructs and initializes the asynchronous server core. - explicit async_server_base(server_options const &options) + explicit async_server_base(options const &options) : server_storage_base(options), socket_options_base(options), handler(options.handler()), address_(options.address()), port_(options.port()), + protocol_family(options.protocol_family()), thread_pool(options.thread_pool() ? options.thread_pool() : std::make_shared()), @@ -108,11 +112,19 @@ struct async_server_base : server_storage_base, socket_options_base { } } + /// Returns the server socket address, either IPv4 or IPv6 depending on + /// options.protocol_family() + const string_type& address() const { return address_; } + + /// Returns the server socket port + const string_type& port() const { return port_; } + private: typedef std::unique_lock scoped_mutex_lock; Handler &handler; string_type address_, port_; + typename options::protocol_family_t protocol_family; std::shared_ptr thread_pool; boost::asio::ip::tcp::acceptor acceptor; bool stopping; @@ -165,7 +177,15 @@ struct async_server_base : server_storage_base, socket_options_base { // this allows repeated cycles of run -> stop -> run service_.reset(); tcp::resolver resolver(service_); - tcp::resolver::query query(address_, port_); + tcp::resolver::query query( [&]{ + switch(protocol_family) { + case options::ipv4: + return tcp::resolver::query(tcp::v4(), address_, port_); + case options::ipv6: + return tcp::resolver::query(tcp::v6(), address_, port_); + default: + return tcp::resolver::query(address_, port_); + }}()); tcp::resolver::iterator endpoint_iterator = resolver.resolve(query, error); if (error) { BOOST_NETWORK_MESSAGE("Error resolving '" << address_ << ':' << port_); @@ -185,6 +205,8 @@ struct async_server_base : server_storage_base, socket_options_base { << port_); return; } + address_ = acceptor.local_endpoint().address().to_string(); + port_ = std::to_string(acceptor.local_endpoint().port()); acceptor.listen(boost::asio::socket_base::max_connections, error); if (error) { BOOST_NETWORK_MESSAGE("Error listening on socket: '" diff --git a/boost/network/protocol/http/server/options.hpp b/boost/network/protocol/http/server/options.hpp index 2e0f5761f..e84188bf3 100644 --- a/boost/network/protocol/http/server/options.hpp +++ b/boost/network/protocol/http/server/options.hpp @@ -34,6 +34,7 @@ struct server_options { handler_(handler), address_("localhost"), port_("80"), + protocol_family_(undefined), reuse_address_(false), report_aborted_(false), non_blocking_io_(true), @@ -88,6 +89,14 @@ struct server_options { return *this; } + enum protocol_family_t { ipv4, ipv6, undefined }; + + /// Set the protocol family for address resolving. Default is AF_UNSPEC. + server_options &protocol_family(protocol_family_t v) { + protocol_family_ = v; + return *this; + } + /// Set whether to reuse the address (SO_REUSE_ADDR). Default is false. server_options &reuse_address(bool v) { reuse_address_ = v; @@ -159,6 +168,9 @@ struct server_options { /// Returns the port to listen on. string_type port() const { return port_; } + /// Returns the protocol family used for address resolving. + protocol_family_t protocol_family() const { return protocol_family_; } + /// Returns a reference to the provided handler. Handler &handler() const { return handler_; } @@ -215,6 +227,7 @@ struct server_options { swap(io_service_, other.io_service_); swap(address_, other.address_); swap(port_, other.port_); + swap(protocol_family_, other.protocol_family_); swap(reuse_address_, other.reuse_address_); swap(report_aborted_, other.report_aborted_); swap(non_blocking_io_, other.non_blocking_io_); @@ -233,6 +246,7 @@ struct server_options { Handler &handler_; string_type address_; string_type port_; + protocol_family_t protocol_family_; bool reuse_address_; bool report_aborted_; bool non_blocking_io_; diff --git a/build.sh b/build.sh index b3e3bb880..8e3e31d7e 100755 --- a/build.sh +++ b/build.sh @@ -11,10 +11,8 @@ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \ -DUri_BUILD_TESTS=$BUILD_TESTS \ -DUri_BUILD_DOCS=$BUILD_DOCS \ -DUri_DISABLE_LIBCXX=$Uri_DISABLE_LIBCXX \ - -DBOOST_INCLUDEDIR="${HOME}/${CC}-boost_${BOOST_VERSION}/include" \ - -DBOOST_LIBRARYDIR="${HOME}/${CC}-boost_${BOOST_VERSION}/lib" \ -DCMAKE_CXX_FLAGS="-std=c++11 ${CMAKE_CXX_FLAGS}" \ .. -make -j8 +make make test cd .. diff --git a/deps/uri b/deps/uri index c16a46ecb..ae076acc0 160000 --- a/deps/uri +++ b/deps/uri @@ -1 +1 @@ -Subproject commit c16a46ecb6bf0c936179e324891a74b5e6d8f4d3 +Subproject commit ae076acc01b2a0c2f973aadbe9a5e4b080fdc88c diff --git a/libs/network/doc/_ext/breathe b/libs/network/doc/_ext/breathe index 853385ef4..65440b267 160000 --- a/libs/network/doc/_ext/breathe +++ b/libs/network/doc/_ext/breathe @@ -1 +1 @@ -Subproject commit 853385ef4f0c3dd126887750e20d5f7456065998 +Subproject commit 65440b2673ebcc5c6ee2792b0b25754effba1183 diff --git a/libs/network/src/CMakeLists.txt b/libs/network/src/CMakeLists.txt index 0ad880075..44617f827 100644 --- a/libs/network/src/CMakeLists.txt +++ b/libs/network/src/CMakeLists.txt @@ -32,6 +32,11 @@ set_target_properties(cppnetlib-client-connections target_link_libraries(cppnetlib-client-connections ${Boost_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) if (OPENSSL_FOUND) target_link_libraries(cppnetlib-client-connections ${OPENSSL_LIBRARIES}) + if (CPP-NETLIB_STATIC_OPENSSL) + if (NOT MSVC AND NOT MINGW AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD") # dynlinker functions are built into libc on FreeBSD + target_link_libraries(cppnetlib-client-connections "-ldl") + endif() + endif() endif () install(TARGETS cppnetlib-client-connections EXPORT cppnetlibTargets diff --git a/libs/network/src/server_request_parsers_impl.cpp b/libs/network/src/server_request_parsers_impl.cpp index 9351983ca..9a539b1ae 100644 --- a/libs/network/src/server_request_parsers_impl.cpp +++ b/libs/network/src/server_request_parsers_impl.cpp @@ -10,7 +10,7 @@ #define BOOST_SPIRIT_UNICODE #include -#include +#include #include namespace boost { @@ -39,22 +39,20 @@ namespace http { void parse_version( std::string const& partial_parsed, std::tuple& version_pair) { - using namespace boost::spirit::qi; - parse(partial_parsed.begin(), partial_parsed.end(), - (lit("HTTP/") >> ushort_ >> '.' >> ushort_), version_pair); + boost::spirit::qi::parse(partial_parsed.begin(), partial_parsed.end(), + (boost::spirit::qi::lit("HTTP/") >> boost::spirit::qi::ushort_ >> '.' >> boost::spirit::qi::ushort_), version_pair); } void parse_headers( std::string const& input, std::vector& container) { u8_to_u32_iterator begin = input.begin(), end = input.end(); - using namespace boost::spirit::qi; - typedef as as_u32_string; - parse(begin, end, - *(+((alnum | punct) - ':') >> lit(": ") >> - as_u32_string()[+((unicode::alnum | space | punct) - '\r' - '\n')] >> - lit("\r\n")) >> - lit("\r\n"), + typedef boost::spirit::qi::as as_u32_string; + boost::spirit::qi::parse(begin, end, + *(+((boost::spirit::qi::alnum | boost::spirit::qi::punct) - ':') >> boost::spirit::qi::lit(": ") >> + as_u32_string()[+((boost::spirit::qi::unicode::alnum | boost::spirit::qi::space | boost::spirit::qi::punct) - '\r' - '\n')] >> + boost::spirit::qi::lit("\r\n")) >> + boost::spirit::qi::lit("\r\n"), container); } diff --git a/libs/network/test/http/client_get_different_port_test.cpp b/libs/network/test/http/client_get_different_port_test.cpp index 99a524874..a94839827 100644 --- a/libs/network/test/http/client_get_different_port_test.cpp +++ b/libs/network/test/http/client_get_different_port_test.cpp @@ -15,9 +15,12 @@ namespace http = boost::network::http; TYPED_TEST_CASE(HTTPClientTest, ClientTypes); TYPED_TEST(HTTPClientTest, GetDifferentPort) { - TypeParam client; - typename TypeParam::request r("http://www.boost.org/"); - auto response_ = client.get(r); + using client = TypeParam; + typename client::options options; + options.remove_chunk_markers(true); + client client_; + typename TypeParam::request request("http://www.boost.org/"); + auto response_ = client_.get(request); auto range = headers(response_)["Content-Type"]; EXPECT_TRUE(std::begin(range) != std::end(range)); EXPECT_NE(0, body(response_).size()); diff --git a/libs/network/test/http/client_get_streaming_test.cpp b/libs/network/test/http/client_get_streaming_test.cpp index 9934f6dc5..f08d24a6a 100644 --- a/libs/network/test/http/client_get_streaming_test.cpp +++ b/libs/network/test/http/client_get_streaming_test.cpp @@ -25,14 +25,18 @@ struct body_handler { TYPED_TEST_CASE(HTTPClientTest, ClientTypes); +#ifdef BOOST_NETWORK_ENABLE_HTTPS TYPED_TEST(HTTPClientTest, GetStreamingTest) { - typename TypeParam::request request("http://www.boost.org/"); + typename TypeParam::request request("https://www.boost.org/"); typename TypeParam::response response; typename TypeParam::string_type body_string; typename TypeParam::string_type dummy_body; body_handler handler_instance(body_string); { - TypeParam client_; + using client = TypeParam; + typename client::options options; + options.remove_chunk_markers(true); + client client_(options); ASSERT_NO_THROW(response = client_.get(request, handler_instance)); auto range = headers(response)["Content-Type"]; ASSERT_TRUE(!boost::empty(range)); @@ -44,3 +48,4 @@ TYPED_TEST(HTTPClientTest, GetStreamingTest) { } EXPECT_EQ(dummy_body, typename TypeParam::string_type()); } +#endif diff --git a/libs/network/test/http/client_get_test.cpp b/libs/network/test/http/client_get_test.cpp index 75b437e24..15997bd83 100644 --- a/libs/network/test/http/client_get_test.cpp +++ b/libs/network/test/http/client_get_test.cpp @@ -1,4 +1,5 @@ // Copyright 2010 Dean Michael Berris. +// Copyright 2017 Google, Inc. // Distributed under the Boost Software License, Version 1.0. // (See accompanying file LICENSE_1_0.txt or copy at // http://www.boost.org/LICENSE_1_0.txt) @@ -15,7 +16,9 @@ TYPED_TEST_CASE(HTTPClientTest, ClientTypes); TYPED_TEST(HTTPClientTest, GetTest) { using client = TypeParam; typename client::request request("http://cpp-netlib.org/"); - client client_; + typename client::options options; + options.remove_chunk_markers(true); + client client_(options); typename client::response response; ASSERT_NO_THROW(response = client_.get(request)); try { @@ -34,7 +37,9 @@ TYPED_TEST(HTTPClientTest, GetTest) { TYPED_TEST(HTTPClientTest, GetHTTPSTest) { using client = TypeParam; typename client::request request("https://www.github.com/"); - client client_; + typename client::options options; + options.remove_chunk_markers(true); + client client_(options); typename client::response response = client_.get(request); EXPECT_TRUE(response.status() == 200 || (response.status() >= 300 && response.status() < 400)); @@ -52,7 +57,9 @@ TYPED_TEST(HTTPClientTest, TemporaryClientObjectTest) { using client = TypeParam; typename client::request request("http://cpp-netlib.org/"); typename client::response response; - ASSERT_NO_THROW(response = client().get(request)); + typename client::options options; + options.remove_chunk_markers(true); + ASSERT_NO_THROW(response = client(options).get(request)); auto range = headers(response); ASSERT_TRUE(!boost::empty(range)); try { @@ -65,3 +72,13 @@ TYPED_TEST(HTTPClientTest, TemporaryClientObjectTest) { EXPECT_TRUE(response.status() == 200u || (response.status() >= 300 && response.status() < 400)); } + +TYPED_TEST(HTTPClientTest, PropagatesResolutionErrorsTest) { + using client = TypeParam; + typename client::request request("http://malformed.google.comq/"); + typename client::response response; + typename client::options options; + typename client::string_type response_body; + ASSERT_NO_THROW(response = client(options).get(request)); + EXPECT_THROW(response_body = body(response), std::exception); +} diff --git a/libs/network/test/http/request_incremental_parser_test.cpp b/libs/network/test/http/request_incremental_parser_test.cpp index 9120294f5..f8c8f5577 100644 --- a/libs/network/test/http/request_incremental_parser_test.cpp +++ b/libs/network/test/http/request_incremental_parser_test.cpp @@ -52,7 +52,7 @@ TEST(IncrementalRequestParserTest, ParseMethod) { p.reset(); std::tie(parsed_ok, result_range) = p.parse_until(request_parser_type::method_done, invalid_http_method); - EXPECT_EQ(false, parsed_ok); + EXPECT_FALSE(parsed_ok); parsed.assign(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " [state:" << p.state() << "] " << std::endl; @@ -68,7 +68,7 @@ TEST(IncrementalRequestParserTest, ParseURI) { std::string valid_http_request = "GET / HTTP/1.1\r\n"; std::tie(parsed_ok, result_range) = p.parse_until(request_parser_type::uri_done, valid_http_request); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_FALSE(boost::empty(result_range)); std::string parsed(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " [state:" << p.state() << "] " @@ -78,7 +78,7 @@ TEST(IncrementalRequestParserTest, ParseURI) { p.reset(); std::tie(parsed_ok, result_range) = p.parse_until(request_parser_type::uri_done, invalid_http_request); - EXPECT_EQ(false, parsed_ok); + EXPECT_FALSE(parsed_ok); parsed.assign(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " [state:" << p.state() << "] " << std::endl; @@ -104,7 +104,7 @@ TEST(IncrementalRequestParserTest, ParseHTTPVersion) { p.reset(); std::tie(parsed_ok, result_range) = p.parse_until(request_parser_type::version_done, invalid_http_request); - EXPECT_EQ(false, parsed_ok); + EXPECT_FALSE(parsed_ok); parsed.assign(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " [state:" << p.state() << "] " << std::endl; @@ -121,7 +121,7 @@ TEST(IncrementalRequestParserTest, ParseHTTPHeaders) { "GET / HTTP/1.1\r\nHost: cpp-netlib.org\r\n\r\n"; std::tie(parsed_ok, result_range) = p.parse_until(request_parser_type::headers_done, valid_http_request); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_FALSE(boost::empty(result_range)); std::string parsed(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " [state:" << p.state() << "] " diff --git a/libs/network/test/http/response_incremental_parser_test.cpp b/libs/network/test/http/response_incremental_parser_test.cpp index 0642ce803..aeaa88cc5 100644 --- a/libs/network/test/http/response_incremental_parser_test.cpp +++ b/libs/network/test/http/response_incremental_parser_test.cpp @@ -82,7 +82,7 @@ TEST(IncrementalResponseParserTest, ParseHTTPVersion) { std::string valid_http_version = "HTTP/1.0 "; std::tie(parsed_ok, result_range) = p.parse_until( response_parser_type::http_version_done, valid_http_version); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_FALSE(boost::empty(result_range)); std::string parsed(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; @@ -90,7 +90,7 @@ TEST(IncrementalResponseParserTest, ParseHTTPVersion) { valid_http_version = "HTTP/1.1 "; std::tie(parsed_ok, result_range) = p.parse_until( response_parser_type::http_version_done, valid_http_version); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_FALSE(boost::empty(result_range)); parsed.assign(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; @@ -100,7 +100,7 @@ TEST(IncrementalResponseParserTest, ParseHTTPVersion) { parsed_ok = logic::indeterminate; std::tie(parsed_ok, result_range) = p.parse_until( response_parser_type::http_version_done, invalid_http_version); - EXPECT_EQ(false, parsed_ok); + EXPECT_FALSE(parsed_ok); parsed.assign(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; @@ -109,7 +109,7 @@ TEST(IncrementalResponseParserTest, ParseHTTPVersion) { parsed_ok = logic::indeterminate; std::tie(parsed_ok, result_range) = p.parse_until( response_parser_type::http_version_done, valid_http_version); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed.assign(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; } @@ -137,7 +137,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseStatus) { range_type result_range; std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_status_done, valid_status); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); std::string parsed = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; @@ -153,7 +153,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseStatus) { valid_status = "200" + TypeParam::literal; std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_status_done, valid_status); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; } @@ -171,7 +171,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseStatusMessage) { range_type result_range; std::tie(parsed_ok, result_range) = p.parse_until( response_parser_type::http_status_message_done, valid_status_message); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); std::string parsed = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; @@ -180,7 +180,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseStatusMessage) { valid_status_message = "OK" + TypeParam::literal; std::tie(parsed_ok, result_range) = p.parse_until( response_parser_type::http_status_message_done, valid_status_message); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; @@ -188,7 +188,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseStatusMessage) { valid_status_message = "Internal Server Error" + TypeParam::literal; std::tie(parsed_ok, result_range) = p.parse_until( response_parser_type::http_status_message_done, valid_status_message); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; @@ -196,7 +196,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseStatusMessage) { valid_status_message = TypeParam::literal; std::tie(parsed_ok, result_range) = p.parse_until( response_parser_type::http_status_message_done, valid_status_message); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; @@ -204,7 +204,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseStatusMessage) { valid_status_message = "한글메시지" + TypeParam::literal; std::tie(parsed_ok, result_range) = p.parse_until( response_parser_type::http_status_message_done, valid_status_message); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed << " state=" << p.state() << std::endl; } @@ -224,7 +224,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { range_type result_range; std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); std::string parsed1 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed1 << " state=" << p.state() << std::endl; @@ -233,7 +233,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { valid_headers.assign(std::end(result_range), end); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); std::string parsed2 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed2 << " state=" << p.state() << std::endl; @@ -241,7 +241,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { p.reset(response_parser_type::http_status_message_done); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_headers_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_NE(parsed1, parsed2); p.reset(response_parser_type::http_status_message_done); @@ -250,7 +250,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { eol::literal; std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed1 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed1 << " state=" << p.state() << std::endl; p.reset(response_parser_type::http_status_message_done); @@ -258,14 +258,14 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { valid_headers.assign(std::end(result_range), end); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed2 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed2 << " state=" << p.state() << std::endl; valid_headers.assign(std::end(result_range), end); p.reset(response_parser_type::http_status_message_done); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_headers_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_NE(parsed1, parsed2); p.reset(response_parser_type::http_status_message_done); @@ -274,7 +274,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { eol::literal; std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed1 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed1 << " state=" << p.state() << std::endl; p.reset(response_parser_type::http_status_message_done); @@ -282,14 +282,14 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { valid_headers.assign(std::end(result_range), end); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed2 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed2 << " state=" << p.state() << std::endl; valid_headers.assign(std::end(result_range), end); p.reset(response_parser_type::http_status_message_done); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_headers_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_NE(parsed1, parsed2); p.reset(response_parser_type::http_status_message_done); @@ -297,7 +297,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { eol::literal + eol::literal; std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed1 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed1 << " state=" << p.state() << std::endl; p.reset(response_parser_type::http_status_message_done); @@ -305,14 +305,14 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { valid_headers.assign(std::end(result_range), end); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed2 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed2 << " state=" << p.state() << std::endl; valid_headers.assign(std::end(result_range), end); p.reset(response_parser_type::http_status_message_done); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_headers_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_NE(parsed1, parsed2); p.reset(response_parser_type::http_status_message_done); @@ -321,7 +321,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { eol::literal; std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed1 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed1 << " state=" << p.state() << std::endl; p.reset(response_parser_type::http_status_message_done); @@ -329,14 +329,14 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { valid_headers.assign(std::end(result_range), end); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed2 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed2 << " state=" << p.state() << std::endl; valid_headers.assign(std::end(result_range), end); p.reset(response_parser_type::http_status_message_done); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_headers_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_NE(parsed1, parsed2); p.reset(response_parser_type::http_status_message_done); @@ -344,7 +344,7 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { eol::literal + eol::literal; std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed1 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed1 << " state=" << p.state() << std::endl; p.reset(response_parser_type::http_status_message_done); @@ -352,13 +352,13 @@ TYPED_TEST(IncrementalResponseEOLTypeTest, ParseHTTPHeaders) { valid_headers.assign(std::end(result_range), end); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_header_line_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); parsed2 = std::string(std::begin(result_range), std::end(result_range)); std::cout << "PARSED: " << parsed2 << " state=" << p.state() << std::endl; valid_headers.assign(std::end(result_range), end); p.reset(response_parser_type::http_status_message_done); std::tie(parsed_ok, result_range) = p.parse_until(response_parser_type::http_headers_done, valid_headers); - EXPECT_EQ(true, parsed_ok); + EXPECT_TRUE(parsed_ok); EXPECT_NE(parsed1, parsed2); }