Skip to content

Commit 90a2ee5

Browse files
SergeyRyabininSergey Ryabinin
authored andcommitted
Use curl_multi_handle
1 parent 1deaebf commit 90a2ee5

File tree

6 files changed

+1548
-2
lines changed

6 files changed

+1548
-2
lines changed
Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/**
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0.
4+
*/
5+
6+
#pragma once
7+
8+
#include <aws/core/utils/ResourceManager.h>
9+
#include <aws/core/http/Version.h>
10+
11+
#include <utility>
12+
#include <curl/curl.h>
13+
14+
namespace Aws
15+
{
16+
namespace Http
17+
{
18+
19+
/**
20+
* Simple Connection pool manager for Curl. It maintains connections in a thread safe manner. You
21+
* can call into acquire a handle, then put it back when finished. It is assumed that reusing an already
22+
* initialized handle is preferable (especially for synchronous clients). The pool doubles in capacity as
23+
* needed up to the maximum amount of connections.
24+
*/
25+
class CurlMultiHandleContainer
26+
{
27+
public:
28+
/**
29+
* Initializes an empty stack of CURL handles. If you are only making synchronous calls via your http client
30+
* then a small size is best. For async support, a good value would be 6 * number of Processors. *
31+
*/
32+
CurlMultiHandleContainer(unsigned maxSize = 50, long httpRequestTimeout = 0, long connectTimeout = 1000, bool tcpKeepAlive = true,
33+
unsigned long tcpKeepAliveIntervalMs = 30000, long lowSpeedTime = 3000, unsigned long lowSpeedLimit = 1,
34+
Version version = Version::HTTP_VERSION_2TLS);
35+
~CurlMultiHandleContainer();
36+
37+
/**
38+
* Blocks until a curl handle from the pool is available for use.
39+
*/
40+
CURL* AcquireCurlHandle();
41+
/**
42+
* Returns a handle to the pool for reuse. It is imperative that this is called
43+
* after you are finished with the handle.
44+
*/
45+
void ReleaseCurlHandle(CURL* handle);
46+
47+
/**
48+
* When the handle has bad DNS entries, problematic live connections, we need to destroy the handle from pool.
49+
*/
50+
void DestroyCurlHandle(CURL* handle);
51+
52+
inline CURLM* AccessCurlMultiHandle()
53+
{
54+
return m_curlMultiHandle;
55+
}
56+
57+
private:
58+
CurlMultiHandleContainer(const CurlMultiHandleContainer&) = delete;
59+
const CurlMultiHandleContainer& operator = (const CurlMultiHandleContainer&) = delete;
60+
CurlMultiHandleContainer(const CurlMultiHandleContainer&&) = delete;
61+
const CurlMultiHandleContainer& operator = (const CurlMultiHandleContainer&&) = delete;
62+
63+
CURL* CreateCurlHandleInPool();
64+
bool CheckAndGrowPool();
65+
void SetDefaultOptionsOnHandle(CURL* handle);
66+
static long ConvertHttpVersion(Version version);
67+
68+
Aws::Utils::ExclusiveOwnershipResourceManager<CURL*> m_handleContainer;
69+
CURLM* m_curlMultiHandle = nullptr;
70+
71+
unsigned m_maxPoolSize;
72+
unsigned long m_httpRequestTimeout;
73+
unsigned long m_connectTimeout;
74+
bool m_enableTcpKeepAlive;
75+
unsigned long m_tcpKeepAliveIntervalMs;
76+
unsigned long m_lowSpeedTime;
77+
unsigned long m_lowSpeedLimit;
78+
unsigned m_poolSize;
79+
std::mutex m_containerLock;
80+
Version m_version;
81+
};
82+
83+
} // namespace Http
84+
} // namespace Aws
85+
Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/**
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0.
4+
*/
5+
6+
7+
#pragma once
8+
9+
#include <aws/core/Core_EXPORTS.h>
10+
#include <aws/core/http/HttpClient.h>
11+
#include <aws/core/http/curl-multi/CurlMultiHandleContainer.h>
12+
#include <aws/core/client/ClientConfiguration.h>
13+
#include <aws/core/utils/memory/stl/AWSString.h>
14+
#include <aws/core/utils/memory/stl/AWSQueue.h>
15+
#include <atomic>
16+
#include <thread>
17+
#include <queue>
18+
19+
namespace Aws
20+
{
21+
namespace Http
22+
{
23+
namespace Standard
24+
{
25+
class StandardHttpResponse;
26+
}
27+
28+
//Curl implementation of an http client. Right now it is only synchronous.
29+
class AWS_CORE_API CurlMultiHttpClient: public HttpClient
30+
{
31+
public:
32+
struct CurlMultiHttpClientConfig
33+
{
34+
struct ProxyConfig
35+
{
36+
bool isEnabled = false;
37+
Aws::String userName;
38+
Aws::String password;
39+
Aws::String scheme;
40+
Aws::String host;
41+
Aws::String sslCertPath;
42+
Aws::String sslCertType;
43+
Aws::String sslKeyPath;
44+
Aws::String sslKeyType;
45+
Aws::String keyPasswd;
46+
unsigned port = 0;
47+
Aws::String nonProxyHosts;
48+
};
49+
50+
struct SslConfig
51+
{
52+
bool verifySSL = true;
53+
Aws::String caPath;
54+
Aws::String caFile;
55+
};
56+
57+
ProxyConfig proxyConfig;
58+
SslConfig sslConfig;
59+
60+
bool disableExpectHeader = false;
61+
bool allowRedirects = false;
62+
};
63+
struct CurlEasyHandleContext;
64+
65+
using Base = HttpClient;
66+
67+
//Creates client, initializes curl handle if it hasn't been created already.
68+
CurlMultiHttpClient(const Aws::Client::ClientConfiguration& clientConfig);
69+
virtual ~CurlMultiHttpClient();
70+
71+
//Makes request and receives response synchronously
72+
std::shared_ptr<HttpResponse> MakeRequest(const std::shared_ptr<HttpRequest>& request,
73+
Aws::Utils::RateLimits::RateLimiterInterface* readLimiter = nullptr,
74+
Aws::Utils::RateLimits::RateLimiterInterface* writeLimiter = nullptr) const override;
75+
76+
static void InitGlobalState();
77+
static void CleanupGlobalState();
78+
79+
protected:
80+
/**
81+
* Override any configuration on CURL handle for each request before sending.
82+
* The usage is to have a subclass of CurlMultiHttpClient and have your own implementation of this function to configure whatever you want on CURL handle.
83+
*/
84+
virtual void OverrideOptionsOnConnectionHandle(CURL*) const {}
85+
86+
private:
87+
void SubmitTask(std::shared_ptr<CurlEasyHandleContext> pEasyHandleCtx) const;
88+
89+
static std::shared_ptr<HttpResponse> HandleCurlResponse(std::shared_ptr<CurlEasyHandleContext> pEasyHandleCtx);
90+
static void CurlMultiPerformThread(CurlMultiHttpClient* pClient);
91+
92+
std::thread m_multiHandleThread;
93+
std::atomic<bool> m_isRunning;
94+
mutable std::mutex m_signalMutex;
95+
mutable std::condition_variable m_signalRunning;
96+
97+
// mutable std::mutex m_tasksMutex;
98+
mutable std::atomic<size_t> m_tasksQueued;
99+
mutable std::mutex m_tasksMutex;
100+
mutable Aws::UnorderedMap<CURL*, std::shared_ptr<CurlEasyHandleContext>> m_tasks;
101+
102+
CurlMultiHttpClientConfig m_config;
103+
104+
mutable CurlMultiHandleContainer m_curlMultiHandleContainer;
105+
106+
static std::atomic<bool> isGlobalStateInit;
107+
std::shared_ptr<smithy::components::tracing::TelemetryProvider> m_telemetryProvider;
108+
};
109+
110+
} // namespace Http
111+
} // namespace Aws
112+

src/aws-cpp-sdk-core/include/aws/core/utils/ResourceManager.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ namespace Aws
1515
namespace Utils
1616
{
1717
/**
18-
* Generic resource manager with Acquire/Release semantics. Acquire will block waiting on a an available resource. Release will
18+
* Generic resource manager with Acquire/Release semantics. Acquire will block waiting on an available resource. Release will
1919
* cause one blocked acquisition to unblock.
2020
*
2121
* You must call ShutdownAndWait() when finished with this container, this unblocks the listening thread and gives you a chance to

src/aws-cpp-sdk-core/source/http/HttpClientFactory.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#endif
1313
#if ENABLE_CURL_CLIENT
1414
#include <aws/core/http/curl/CurlHttpClient.h>
15+
#include <aws/core/http/curl-multi/CurlMultiHttpClient.h>
1516
#include <signal.h>
1617

1718
#elif ENABLE_WINDOWS_CLIENT
@@ -93,7 +94,7 @@ namespace Aws
9394
}
9495
#endif // ENABLE_WINDOWS_IXML_HTTP_REQUEST_2_CLIENT
9596
#elif ENABLE_CURL_CLIENT
96-
return Aws::MakeShared<CurlHttpClient>(HTTP_CLIENT_FACTORY_ALLOCATION_TAG, clientConfiguration);
97+
return Aws::MakeShared<CurlMultiHttpClient>(HTTP_CLIENT_FACTORY_ALLOCATION_TAG, clientConfiguration);
9798
#else
9899
// When neither of these clients is enabled, gcc gives a warning (converted
99100
// to error by -Werror) about the unused clientConfiguration parameter. We

0 commit comments

Comments
 (0)