Skip to content

Commit 03f6d3b

Browse files
deanberrisdeanberris
authored
Merge pull request cpp-netlib#720 from umennel/master-integration
Introduced new chunked transfer encoding parser to remove chunk markers
2 parents 44f2089 + 192d520 commit 03f6d3b

File tree

10 files changed

+169
-64
lines changed

10 files changed

+169
-64
lines changed

‎boost/network/protocol/http/client/async_impl.hpp

+3-2
Original file line numberDiff line numberDiff line change
@@ -38,15 +38,16 @@ struct async_client
3838
typedef std::function<bool(string_type&)> body_generator_function_type;
3939

4040
async_client(bool cache_resolved, bool follow_redirect,
41-
bool always_verify_peer, int timeout,
41+
bool always_verify_peer, int timeout, bool remove_chunk_markers,
4242
std::shared_ptr<boost::asio::io_service> service,
4343
optional<string_type> certificate_filename,
4444
optional<string_type> verify_path,
4545
optional<string_type> certificate_file,
4646
optional<string_type> private_key_file,
4747
optional<string_type> ciphers,
4848
optional<string_type> sni_hostname, long ssl_options)
49-
: connection_base(cache_resolved, follow_redirect, timeout),
49+
: connection_base(cache_resolved, follow_redirect, timeout,
50+
remove_chunk_markers),
5051
service_ptr(service.get() ? service
5152
: std::make_shared<boost::asio::io_service>()),
5253
service_(*service_ptr),

‎boost/network/protocol/http/client/connection/async_base.hpp

+3-1
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ struct async_connection_base {
4444
static connection_ptr new_connection(
4545
resolve_function resolve, resolver_type &resolver, bool follow_redirect,
4646
bool always_verify_peer, bool https, int timeout,
47+
bool remove_chunk_markers,
4748
optional<string_type> certificate_filename = optional<string_type>(),
4849
optional<string_type> const &verify_path = optional<string_type>(),
4950
optional<string_type> certificate_file = optional<string_type>(),
@@ -59,7 +60,8 @@ struct async_connection_base {
5960
certificate_filename, verify_path, certificate_file, private_key_file,
6061
ciphers, sni_hostname, ssl_options);
6162
auto temp = std::make_shared<async_connection>(
62-
resolver, resolve, follow_redirect, timeout, std::move(delegate));
63+
resolver, resolve, follow_redirect, timeout, remove_chunk_markers,
64+
std::move(delegate));
6365
BOOST_ASSERT(temp != nullptr);
6466
return temp;
6567
}

‎boost/network/protocol/http/client/connection/async_normal.hpp

+112-39
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,82 @@ namespace network {
3737
namespace http {
3838
namespace impl {
3939

40+
template <class Tag>
41+
struct chunk_encoding_parser {
42+
chunk_encoding_parser() : state(state_t::header), chunk_size(0) {}
43+
44+
enum class state_t { header, header_end, data, data_end };
45+
46+
state_t state;
47+
size_t chunk_size;
48+
std::array<typename char_<Tag>::type, 1024> buffer;
49+
50+
void update_chunk_size(
51+
boost::iterator_range<typename std::array<
52+
typename char_<Tag>::type, 1024>::const_iterator> const &range) {
53+
if (range.empty()) return;
54+
std::stringstream ss;
55+
ss << std::hex << range;
56+
size_t size;
57+
ss >> size;
58+
// New digits are appended as LSBs
59+
chunk_size = (chunk_size << (range.size() * 4)) | size;
60+
}
61+
62+
boost::iterator_range<
63+
typename std::array<typename char_<Tag>::type, 1024>::const_iterator>
64+
operator()(
65+
boost::iterator_range<typename std::array<
66+
typename char_<Tag>::type, 1024>::const_iterator> const &range) {
67+
auto iter = boost::begin(range);
68+
auto begin = iter;
69+
auto pos = boost::begin(buffer);
70+
71+
while (iter != boost::end(range)) switch (state) {
72+
case state_t::header:
73+
iter = std::find(iter, boost::end(range), '\r');
74+
update_chunk_size(boost::make_iterator_range(begin, iter));
75+
if (iter != boost::end(range)) {
76+
state = state_t::header_end;
77+
++iter;
78+
}
79+
break;
80+
81+
case state_t::header_end:
82+
BOOST_ASSERT(*iter == '\n');
83+
++iter;
84+
state = state_t::data;
85+
break;
86+
87+
case state_t::data:
88+
if (chunk_size == 0) {
89+
BOOST_ASSERT(*iter == '\r');
90+
++iter;
91+
state = state_t::data_end;
92+
} else {
93+
auto len = std::min(chunk_size,
94+
(size_t)std::distance(iter, boost::end(range)));
95+
begin = iter;
96+
iter = std::next(iter, len);
97+
pos = std::copy(begin, iter, pos);
98+
chunk_size -= len;
99+
}
100+
break;
101+
102+
case state_t::data_end:
103+
BOOST_ASSERT(*iter == '\n');
104+
++iter;
105+
begin = iter;
106+
state = state_t::header;
107+
break;
108+
109+
default:
110+
BOOST_ASSERT(false && "Bug, report this to the developers!");
111+
}
112+
return boost::make_iterator_range(boost::begin(buffer), pos);
113+
}
114+
};
115+
40116
template <class Tag, unsigned version_major, unsigned version_minor>
41117
struct async_connection_base;
42118

@@ -72,8 +148,10 @@ struct http_async_connection
72148

73149
http_async_connection(resolver_type& resolver, resolve_function resolve,
74150
bool follow_redirect, int timeout,
151+
bool remove_chunk_markers,
75152
connection_delegate_ptr delegate)
76153
: timeout_(timeout),
154+
remove_chunk_markers_(remove_chunk_markers),
77155
timer_(resolver.get_io_service()),
78156
is_timedout_(false),
79157
follow_redirect_(follow_redirect),
@@ -348,8 +426,11 @@ struct http_async_connection
348426

349427
// The invocation of the callback is synchronous to allow us to
350428
// wait before scheduling another read.
351-
callback(make_iterator_range(begin, end), ec);
352-
429+
if (this->is_chunk_encoding && remove_chunk_markers_) {
430+
callback(parse_chunk_encoding(make_iterator_range(begin, end)), ec);
431+
} else {
432+
callback(make_iterator_range(begin, end), ec);
433+
}
353434
auto self = this->shared_from_this();
354435
delegate_->read_some(
355436
boost::asio::mutable_buffers_1(this->part.data(),
@@ -388,14 +469,31 @@ struct http_async_connection
388469
// We call the callback function synchronously passing the error
389470
// condition (in this case, end of file) so that it can handle it
390471
// appropriately.
391-
callback(make_iterator_range(begin, end), ec);
472+
if (this->is_chunk_encoding && remove_chunk_markers_) {
473+
callback(parse_chunk_encoding(make_iterator_range(begin, end)), ec);
474+
} else {
475+
callback(make_iterator_range(begin, end), ec);
476+
}
392477
} else {
393478
string_type body_string;
394-
std::swap(body_string, this->partial_parsed);
395-
body_string.append(this->part.begin(), this->part.begin() + bytes_transferred);
396-
if (this->is_chunk_encoding) {
397-
this->body_promise.set_value(parse_chunk_encoding(body_string));
479+
if (this->is_chunk_encoding && remove_chunk_markers_) {
480+
for (size_t i = 0; i < this->partial_parsed.size(); i += 1024) {
481+
auto range = parse_chunk_encoding(boost::make_iterator_range(
482+
this->partial_parsed.data() + i,
483+
this->partial_parsed.data() +
484+
std::min(i + 1024, this->partial_parsed.size())));
485+
body_string.append(boost::begin(range), boost::end(range));
486+
}
487+
this->partial_parsed.clear();
488+
auto range = parse_chunk_encoding(boost::make_iterator_range(
489+
this->part.begin(),
490+
this->part.begin() + bytes_transferred));
491+
body_string.append(boost::begin(range), boost::end(range));
492+
this->body_promise.set_value(body_string);
398493
} else {
494+
std::swap(body_string, this->partial_parsed);
495+
body_string.append(this->part.begin(),
496+
this->part.begin() + bytes_transferred);
399497
this->body_promise.set_value(body_string);
400498
}
401499
}
@@ -417,7 +515,11 @@ struct http_async_connection
417515
this->part.begin();
418516
typename protocol_base::buffer_type::const_iterator end = begin;
419517
std::advance(end, bytes_transferred);
420-
callback(make_iterator_range(begin, end), ec);
518+
if (this->is_chunk_encoding && remove_chunk_markers_) {
519+
callback(parse_chunk_encoding(make_iterator_range(begin, end)), ec);
520+
} else {
521+
callback(make_iterator_range(begin, end), ec);
522+
}
421523
auto self = this->shared_from_this();
422524
delegate_->read_some(
423525
boost::asio::mutable_buffers_1(this->part.data(),
@@ -476,38 +578,8 @@ struct http_async_connection
476578
}
477579
}
478580

479-
string_type parse_chunk_encoding(string_type& body_string) {
480-
string_type body;
481-
string_type crlf = "\r\n";
482-
483-
typename string_type::iterator begin = body_string.begin();
484-
for (typename string_type::iterator iter =
485-
std::search(begin, body_string.end(), crlf.begin(), crlf.end());
486-
iter != body_string.end();
487-
iter =
488-
std::search(begin, body_string.end(), crlf.begin(), crlf.end())) {
489-
string_type line(begin, iter);
490-
if (line.empty()) {
491-
break;
492-
}
493-
std::stringstream stream(line);
494-
int len;
495-
stream >> std::hex >> len;
496-
std::advance(iter, 2);
497-
if (len == 0) {
498-
break;
499-
}
500-
if (len <= body_string.end() - iter) {
501-
body.insert(body.end(), iter, iter + len);
502-
std::advance(iter, len + 2);
503-
}
504-
begin = iter;
505-
}
506-
507-
return body;
508-
}
509-
510581
int timeout_;
582+
bool remove_chunk_markers_;
511583
boost::asio::steady_timer timer_;
512584
bool is_timedout_;
513585
bool follow_redirect_;
@@ -517,6 +589,7 @@ struct http_async_connection
517589
connection_delegate_ptr delegate_;
518590
boost::asio::streambuf command_streambuf;
519591
string_type method;
592+
chunk_encoding_parser<Tag> parse_chunk_encoding;
520593
};
521594

522595
} // namespace impl

‎boost/network/protocol/http/client/facade.hpp

+2-1
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,8 @@ class basic_client_facade {
303303
options.openssl_verify_path(), options.openssl_certificate_file(),
304304
options.openssl_private_key_file(), options.openssl_ciphers(),
305305
options.openssl_sni_hostname(), options.openssl_options(),
306-
options.io_service(), options.timeout()));
306+
options.io_service(), options.timeout(),
307+
options.remove_chunk_markers()));
307308
}
308309
};
309310

‎boost/network/protocol/http/client/options.hpp

+14-2
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ class client_options {
3434
openssl_options_(0),
3535
io_service_(),
3636
always_verify_peer_(true),
37-
timeout_(0) {}
37+
timeout_(0),
38+
remove_chunk_markers_(false) {}
3839

3940
client_options(client_options const& other)
4041
: cache_resolved_(other.cache_resolved_),
@@ -48,7 +49,8 @@ class client_options {
4849
openssl_options_(other.openssl_options_),
4950
io_service_(other.io_service_),
5051
always_verify_peer_(other.always_verify_peer_),
51-
timeout_(other.timeout_) {}
52+
timeout_(other.timeout_),
53+
remove_chunk_markers_(other.remove_chunk_markers) {}
5254

5355
client_options& operator=(client_options other) {
5456
other.swap(*this);
@@ -69,6 +71,7 @@ class client_options {
6971
swap(io_service_, other.io_service_);
7072
swap(always_verify_peer_, other.always_verify_peer_);
7173
swap(timeout_, other.timeout_);
74+
swap(remove_chunk_markers_, other.remove_chunk_markers_);
7275
}
7376

7477
/// Specify whether the client should cache resolved endpoints.
@@ -154,6 +157,12 @@ class client_options {
154157
return *this;
155158
}
156159

160+
/// Set an overall timeout for HTTP requests.
161+
client_options& remove_chunk_markers(bool v) {
162+
remove_chunk_markers_ = v;
163+
return *this;
164+
}
165+
157166
bool cache_resolved() const { return cache_resolved_; }
158167

159168
bool follow_redirects() const { return follow_redirects_; }
@@ -190,6 +199,8 @@ class client_options {
190199

191200
int timeout() const { return timeout_; }
192201

202+
bool remove_chunk_markers() const { return remove_chunk_markers_; }
203+
193204
private:
194205
bool cache_resolved_;
195206
bool follow_redirects_;
@@ -203,6 +214,7 @@ class client_options {
203214
std::shared_ptr<boost::asio::io_service> io_service_;
204215
bool always_verify_peer_;
205216
int timeout_;
217+
bool remove_chunk_markers_;
206218
};
207219

208220
template <class Tag>

‎boost/network/protocol/http/client/pimpl.hpp

+6-4
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,12 @@ struct basic_client_impl
7474
optional<string_type> const& private_key_file,
7575
optional<string_type> const& ciphers,
7676
optional<string_type> const& sni_hostname, long ssl_options,
77-
std::shared_ptr<boost::asio::io_service> service, int timeout)
78-
: base_type(cache_resolved, follow_redirect, always_verify_peer, timeout,
79-
service, certificate_filename, verify_path, certificate_file,
80-
private_key_file, ciphers, sni_hostname, ssl_options) {}
77+
std::shared_ptr<boost::asio::io_service> service, int timeout,
78+
bool remove_chunk_markers)
79+
: base_type(cache_resolved, follow_redirect, always_verify_peer, timeout,
80+
remove_chunk_markers, service, certificate_filename, verify_path,
81+
certificate_file, private_key_file, ciphers, sni_hostname,
82+
ssl_options) {}
8183

8284
~basic_client_impl() = default;
8385
};

‎boost/network/protocol/http/policies/async_connection.hpp

+10-8
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ struct async_connection_policy : resolver_policy<Tag>::type {
3838
struct connection_impl {
3939
connection_impl(
4040
bool follow_redirect, bool always_verify_peer, resolve_function resolve,
41-
resolver_type& resolver, bool https, int timeout,
41+
resolver_type& resolver, bool https, int timeout, bool remove_chunk_markers,
4242
optional<string_type> /*unused*/ const& certificate_filename,
4343
optional<string_type> const& verify_path,
4444
optional<string_type> const& certificate_file,
@@ -47,9 +47,9 @@ struct async_connection_policy : resolver_policy<Tag>::type {
4747
optional<string_type> const& sni_hostname, long ssl_options) {
4848
pimpl = impl::async_connection_base<Tag, version_major, version_minor>::
4949
new_connection(resolve, resolver, follow_redirect, always_verify_peer,
50-
https, timeout, certificate_filename, verify_path,
51-
certificate_file, private_key_file, ciphers,
52-
sni_hostname, ssl_options);
50+
https, timeout, remove_chunk_markers,
51+
certificate_filename, verify_path, certificate_file,
52+
private_key_file, ciphers, sni_hostname, ssl_options);
5353
}
5454

5555
basic_response<Tag> send_request(string_type /*unused*/ const& method,
@@ -86,20 +86,22 @@ struct async_connection_policy : resolver_policy<Tag>::type {
8686
this->resolve(resolver, host, port, once_resolved);
8787
},
8888
resolver, boost::iequals(protocol_, string_type("https")), timeout_,
89-
certificate_filename, verify_path, certificate_file, private_key_file,
90-
ciphers, sni_hostname, ssl_options);
89+
remove_chunk_markers_, certificate_filename, verify_path,
90+
certificate_file, private_key_file, ciphers, sni_hostname, ssl_options);
9191
}
9292

9393
void cleanup() {}
9494

9595
async_connection_policy(bool cache_resolved, bool follow_redirect,
96-
int timeout)
96+
int timeout, bool remove_chunk_markers)
9797
: resolver_base(cache_resolved),
9898
follow_redirect_(follow_redirect),
99-
timeout_(timeout) {}
99+
timeout_(timeout),
100+
remove_chunk_markers_(remove_chunk_markers) {}
100101

101102
bool follow_redirect_;
102103
int timeout_;
104+
bool remove_chunk_markers_;
103105
};
104106

105107
} // namespace http

‎libs/network/test/http/client_get_different_port_test.cpp

+6-3
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,12 @@ namespace http = boost::network::http;
1515
TYPED_TEST_CASE(HTTPClientTest, ClientTypes);
1616

1717
TYPED_TEST(HTTPClientTest, GetDifferentPort) {
18-
TypeParam client;
19-
typename TypeParam::request r("http://www.boost.org:80/");
20-
auto response_ = client.get(r);
18+
using client = TypeParam;
19+
typename client::options options;
20+
options.remove_chunk_markers(true);
21+
client client_;
22+
typename TypeParam::request request("http://www.boost.org:80/");
23+
auto response_ = client_.get(request);
2124
auto range = headers(response_)["Content-Type"];
2225
EXPECT_TRUE(std::begin(range) != std::end(range));
2326
EXPECT_NE(0, body(response_).size());

0 commit comments

Comments
 (0)