From 2326e09595ba62f76fff12a2ef2e4f798dbd8c93 Mon Sep 17 00:00:00 2001 From: James Roper Date: Mon, 19 Aug 2013 14:42:49 +1000 Subject: [PATCH] Provided ability to use JDK ProxySelector Fixes #360. * Adds a useProxySelector option that tells async-http-client to use the JDK default ProxySelector. * Adds a ProxyServerSelector interface that AsyncHttpClientConfig and ProxyUtils now use. --- .../AsyncHttpClientConfig.java | 63 +++++++++--- .../AsyncHttpClientConfigBean.java | 14 ++- .../asynchttpclient/ProxyServerSelector.java | 27 ++++++ .../org/asynchttpclient/util/ProxyUtils.java | 96 +++++++++++++++++-- .../org/asynchttpclient/async/ProxyTest.java | 80 +++++++++++++++- .../ProxyAuthorizationHandler.java | 3 +- site/src/site/apt/proxy.apt | 23 +++++ 7 files changed, 277 insertions(+), 29 deletions(-) create mode 100644 api/src/main/java/org/asynchttpclient/ProxyServerSelector.java diff --git a/api/src/main/java/org/asynchttpclient/AsyncHttpClientConfig.java b/api/src/main/java/org/asynchttpclient/AsyncHttpClientConfig.java index 2b8ca1c84d..79143d1b00 100644 --- a/api/src/main/java/org/asynchttpclient/AsyncHttpClientConfig.java +++ b/api/src/main/java/org/asynchttpclient/AsyncHttpClientConfig.java @@ -92,7 +92,7 @@ public class AsyncHttpClientConfig { protected boolean allowPoolingConnection; protected ScheduledExecutorService reaper; protected ExecutorService applicationThreadPool; - protected ProxyServer proxyServer; + protected ProxyServerSelector proxyServerSelector; protected SSLContext sslContext; protected SSLEngineFactory sslEngineFactory; protected AsyncHttpProviderConfig providerConfig; @@ -136,7 +136,7 @@ private AsyncHttpClientConfig(int maxTotalConnections, boolean keepAlive, ScheduledExecutorService reaper, ExecutorService applicationThreadPool, - ProxyServer proxyServer, + ProxyServerSelector proxyServerSelector, SSLContext sslContext, SSLEngineFactory sslEngineFactory, AsyncHttpProviderConfig providerConfig, @@ -196,7 +196,7 @@ private AsyncHttpClientConfig(int maxTotalConnections, } else { this.applicationThreadPool = applicationThreadPool; } - this.proxyServer = proxyServer; + this.proxyServerSelector = proxyServerSelector; this.useRawUrl = useRawUrl; this.spdyEnabled = spdyEnabled; this.spdyInitialWindowSize = spdyInitialWindowSize; @@ -362,8 +362,8 @@ public boolean isManagedExecutorService() { * * @return instance of {@link ProxyServer} */ - public ProxyServer getProxyServer() { - return proxyServer; + public ProxyServerSelector getProxyServerSelector() { + return proxyServerSelector; } /** @@ -613,11 +613,12 @@ public static class Builder { private boolean compressionEnabled = Boolean.getBoolean(ASYNC_CLIENT + "compressionEnabled"); private String userAgent = System.getProperty(ASYNC_CLIENT + "userAgent", "AsyncHttpClient/" + AHC_VERSION); private boolean useProxyProperties = Boolean.getBoolean(ASYNC_CLIENT + "useProxyProperties"); + private boolean useProxySelector = Boolean.getBoolean(ASYNC_CLIENT + "useProxySelector"); private boolean allowPoolingConnection = true; private boolean useRelativeURIsWithSSLProxies = Boolean.getBoolean(ASYNC_CLIENT + "useRelativeURIsWithSSLProxies"); private ScheduledExecutorService reaper; private ExecutorService applicationThreadPool; - private ProxyServer proxyServer = null; + private ProxyServerSelector proxyServerSelector = null; private SSLContext sslContext; private SSLEngineFactory sslEngineFactory; private AsyncHttpProviderConfig providerConfig; @@ -816,6 +817,17 @@ public Builder setExecutorService(ExecutorService applicationThreadPool) { return this; } + /** + * Set an instance of {@link ProxyServerSelector} used by an {@link AsyncHttpClient} + * + * @param proxyServerSelector instance of {@link ProxyServerSelector} + * @return a {@link Builder} + */ + public Builder setProxyServerSelector(ProxyServerSelector proxyServerSelector) { + this.proxyServerSelector = proxyServerSelector; + return this; + } + /** * Set an instance of {@link ProxyServer} used by an {@link AsyncHttpClient} * @@ -823,7 +835,7 @@ public Builder setExecutorService(ExecutorService applicationThreadPool) { * @return a {@link Builder} */ public Builder setProxyServer(ProxyServer proxyServer) { - this.proxyServer = proxyServer; + this.proxyServerSelector = ProxyUtils.createProxyServerSelector(proxyServer); return this; } @@ -1024,12 +1036,27 @@ public Builder setRemoveQueryParamsOnRedirect(boolean removeQueryParamOnRedirect return this; } + /** + * Sets whether AHC should use the default JDK ProxySelector to select a proxy server. + *

+ * If useProxySelector is set to true but {@link #setProxyServer(ProxyServer)} + * was used to explicitly set a proxy server, the latter is preferred. + *

+ * See http://docs.oracle.com/javase/7/docs/api/java/net/ProxySelector.html + */ + public Builder setUseProxySelector(boolean useProxySelector) { + this.useProxySelector = useProxySelector; + return this; + } + /** * Sets whether AHC should use the default http.proxy* system properties - * to obtain proxy information. + * to obtain proxy information. This differs from {@link #setUseProxySelector(boolean)} + * in that AsyncHttpClient will use its own logic to handle the system properties, + * potentially supporting other protocols that the the JDK ProxySelector doesn't. *

- * If useProxyProperties is set to true but {@link #setProxyServer(ProxyServer)} was used - * to explicitly set a proxy server, the latter is preferred. + * If useProxyProperties is set to true but {@link #setUseProxySelector(boolean)} + * was also set to true, the latter is preferred. *

* See http://download.oracle.com/javase/1.4.2/docs/guide/net/properties.html */ @@ -1182,7 +1209,7 @@ public Builder(AsyncHttpClientConfig prototype) { defaultMaxConnectionLifeTimeInMs = prototype.getMaxConnectionLifeTimeInMs(); maxDefaultRedirects = prototype.getMaxRedirects(); defaultMaxTotalConnections = prototype.getMaxTotalConnections(); - proxyServer = prototype.getProxyServer(); + proxyServerSelector = prototype.getProxyServerSelector(); realm = prototype.getRealm(); defaultRequestTimeoutInMs = prototype.getRequestTimeoutInMs(); sslContext = prototype.getSSLContext(); @@ -1248,8 +1275,16 @@ public Thread newThread(Runnable r) { throw new IllegalStateException("ExecutorServices closed"); } - if (proxyServer == null && useProxyProperties) { - proxyServer = ProxyUtils.createProxy(System.getProperties()); + if (proxyServerSelector == null && useProxySelector) { + proxyServerSelector = ProxyUtils.getJdkDefaultProxyServerSelector(); + } + + if (proxyServerSelector == null && useProxyProperties) { + proxyServerSelector = ProxyUtils.createProxyServerSelector(System.getProperties()); + } + + if (proxyServerSelector == null) { + proxyServerSelector = ProxyServerSelector.NO_PROXY_SELECTOR; } return new AsyncHttpClientConfig(defaultMaxTotalConnections, @@ -1267,7 +1302,7 @@ public Thread newThread(Runnable r) { allowPoolingConnection, reaper, applicationThreadPool, - proxyServer, + proxyServerSelector, sslContext, sslEngineFactory, providerConfig, diff --git a/api/src/main/java/org/asynchttpclient/AsyncHttpClientConfigBean.java b/api/src/main/java/org/asynchttpclient/AsyncHttpClientConfigBean.java index 4e48491b34..9a7d75bf47 100644 --- a/api/src/main/java/org/asynchttpclient/AsyncHttpClientConfigBean.java +++ b/api/src/main/java/org/asynchttpclient/AsyncHttpClientConfigBean.java @@ -55,9 +55,12 @@ void configureDefaults() { compressionEnabled = Boolean.getBoolean(ASYNC_CLIENT + "compressionEnabled"); userAgent = System.getProperty(ASYNC_CLIENT + "userAgent", "AsyncHttpClient/" + AHC_VERSION); + boolean useProxySelector = Boolean.getBoolean(ASYNC_CLIENT + "useProxySelector"); boolean useProxyProperties = Boolean.getBoolean(ASYNC_CLIENT + "useProxyProperties"); - if (useProxyProperties) { - proxyServer = ProxyUtils.createProxy(System.getProperties()); + if (useProxySelector) { + proxyServerSelector = ProxyUtils.getJdkDefaultProxyServerSelector(); + } else if (useProxyProperties) { + proxyServerSelector = ProxyUtils.createProxyServerSelector(System.getProperties()); } allowPoolingConnection = true; @@ -163,7 +166,12 @@ public AsyncHttpClientConfigBean setApplicationThreadPool(ExecutorService applic } public AsyncHttpClientConfigBean setProxyServer(ProxyServer proxyServer) { - this.proxyServer = proxyServer; + this.proxyServerSelector = ProxyUtils.createProxyServerSelector(proxyServer); + return this; + } + + public AsyncHttpClientConfigBean setProxyServerSelector(ProxyServerSelector proxyServerSelector) { + this.proxyServerSelector = proxyServerSelector; return this; } diff --git a/api/src/main/java/org/asynchttpclient/ProxyServerSelector.java b/api/src/main/java/org/asynchttpclient/ProxyServerSelector.java new file mode 100644 index 0000000000..b883b1fcf9 --- /dev/null +++ b/api/src/main/java/org/asynchttpclient/ProxyServerSelector.java @@ -0,0 +1,27 @@ +package org.asynchttpclient; + +import java.net.URI; + +/** + * Selector for a proxy server + */ +public interface ProxyServerSelector { + + /** + * Select a proxy server to use for the given URI. + * + * @param uri The URI to select a proxy server for. + * @return The proxy server to use, if any. May return null. + */ + ProxyServer select(URI uri); + + /** + * A selector that always selects no proxy. + */ + static final ProxyServerSelector NO_PROXY_SELECTOR = new ProxyServerSelector() { + @Override + public ProxyServer select(URI uri) { + return null; + } + }; +} diff --git a/api/src/main/java/org/asynchttpclient/util/ProxyUtils.java b/api/src/main/java/org/asynchttpclient/util/ProxyUtils.java index 6e33588f6e..612642b524 100644 --- a/api/src/main/java/org/asynchttpclient/util/ProxyUtils.java +++ b/api/src/main/java/org/asynchttpclient/util/ProxyUtils.java @@ -14,13 +14,21 @@ import static org.asynchttpclient.util.MiscUtil.isNonEmpty; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.net.ProxySelector; +import java.net.URI; import java.util.List; +import java.util.Locale; import java.util.Properties; import org.asynchttpclient.AsyncHttpClientConfig; import org.asynchttpclient.ProxyServer; import org.asynchttpclient.ProxyServer.Protocol; +import org.asynchttpclient.ProxyServerSelector; import org.asynchttpclient.Request; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Utilities for Proxy handling. @@ -29,6 +37,8 @@ */ public class ProxyUtils { + private final static Logger log = LoggerFactory.getLogger(ProxyUtils.class); + private static final String PROPERTY_PREFIX = "org.asynchttpclient.AsyncHttpClientConfig.proxy."; /** @@ -69,7 +79,10 @@ public class ProxyUtils { public static ProxyServer getProxyServer(AsyncHttpClientConfig config, Request request) { ProxyServer proxyServer = request.getProxyServer(); if (proxyServer == null) { - proxyServer = config.getProxyServer(); + ProxyServerSelector selector = config.getProxyServerSelector(); + if (selector != null) { + proxyServer = selector.select(request.getOriginalURI()); + } } return ProxyUtils.avoidProxy(proxyServer, request) ? null : proxyServer; } @@ -107,7 +120,10 @@ public static boolean avoidProxy(final ProxyServer proxyServer, final String tar if (isNonEmpty(nonProxyHosts)) { for (String nonProxyHost : nonProxyHosts) { if (nonProxyHost.startsWith("*") && nonProxyHost.length() > 1 - && targetHost.endsWith(nonProxyHost.substring(1).toLowerCase())) { + && targetHost.endsWith(nonProxyHost.substring(1).toLowerCase(Locale.ENGLISH))) { + return true; + } else if (nonProxyHost.endsWith("*") && nonProxyHost.length() > 1 + && targetHost.startsWith(nonProxyHost.substring(0, nonProxyHost.length() - 1).toLowerCase(Locale.ENGLISH))) { return true; } else if (nonProxyHost.equalsIgnoreCase(targetHost)) { return true; @@ -134,31 +150,91 @@ public static boolean avoidProxy(final ProxyServer proxyServer, final String tar * @see #PROXY_PROTOCOL * @see #PROXY_NONPROXYHOSTS */ - public static ProxyServer createProxy(Properties properties) { - String host = System.getProperty(PROXY_HOST); + public static ProxyServerSelector createProxyServerSelector(Properties properties) { + String host = properties.getProperty(PROXY_HOST); if (host != null) { - int port = Integer.valueOf(System.getProperty(PROXY_PORT, "80")); + int port = Integer.valueOf(properties.getProperty(PROXY_PORT, "80")); Protocol protocol; try { - protocol = Protocol.valueOf(System.getProperty(PROXY_PROTOCOL, "HTTP")); + protocol = Protocol.valueOf(properties.getProperty(PROXY_PROTOCOL, "HTTP")); } catch (IllegalArgumentException e) { protocol = Protocol.HTTP; } - ProxyServer proxyServer = new ProxyServer(protocol, host, port, System.getProperty(PROXY_USER), System.getProperty(PROXY_PASSWORD)); + ProxyServer proxyServer = new ProxyServer(protocol, host, port, properties.getProperty(PROXY_USER), + properties.getProperty(PROXY_PASSWORD)); - String nonProxyHosts = System.getProperties().getProperty(PROXY_NONPROXYHOSTS); + String nonProxyHosts = properties.getProperty(PROXY_NONPROXYHOSTS); if (nonProxyHosts != null) { for (String spec : nonProxyHosts.split("\\|")) { proxyServer.addNonProxyHost(spec); } } - return proxyServer; + return createProxyServerSelector(proxyServer); } - return null; + return ProxyServerSelector.NO_PROXY_SELECTOR; + } + + /** + * Get a proxy server selector based on the JDK default proxy selector. + * + * @return The proxy server selector. + */ + public static ProxyServerSelector getJdkDefaultProxyServerSelector() { + return createProxyServerSelector(ProxySelector.getDefault()); + } + + /** + * Create a proxy server selector based on the passed in JDK proxy selector. + * + * @param proxySelector The proxy selector to use. Must not be null. + * @return The proxy server selector. + */ + public static ProxyServerSelector createProxyServerSelector(final ProxySelector proxySelector) { + return new ProxyServerSelector() { + @Override + public ProxyServer select(URI uri) { + List proxies = proxySelector.select(uri); + if (proxies != null) { + // Loop through them until we find one that we know how to use + for (Proxy proxy : proxies) { + switch (proxy.type()) { + case HTTP: + if (!(proxy.address() instanceof InetSocketAddress)) { + log.warn("Don't know how to connect to address " + proxy.address()); + } else { + InetSocketAddress address = (InetSocketAddress) proxy.address(); + return new ProxyServer(Protocol.HTTP, address.getHostString(), address.getPort()); + } + case DIRECT: + return null; + default: + log.warn("ProxySelector returned proxy type that we don't know how to use: " + proxy.type()); + break; + } + } + } + return null; + } + }; + } + + /** + * Create a proxy server selector that always selects a single proxy server. + * + * @param proxyServer The proxy server to select. + * @return The proxy server selector. + */ + public static ProxyServerSelector createProxyServerSelector(final ProxyServer proxyServer) { + return new ProxyServerSelector() { + @Override + public ProxyServer select(URI uri) { + return proxyServer; + } + }; } } diff --git a/api/src/test/java/org/asynchttpclient/async/ProxyTest.java b/api/src/test/java/org/asynchttpclient/async/ProxyTest.java index 94ee870f84..1af6c073f1 100644 --- a/api/src/test/java/org/asynchttpclient/async/ProxyTest.java +++ b/api/src/test/java/org/asynchttpclient/async/ProxyTest.java @@ -16,6 +16,7 @@ package org.asynchttpclient.async; import static org.testng.Assert.*; +import static org.testng.Assert.assertEquals; import org.asynchttpclient.AsyncHttpClient; import org.asynchttpclient.AsyncHttpClientConfig; @@ -23,7 +24,9 @@ import org.asynchttpclient.Response; import java.io.IOException; -import java.net.ConnectException; +import java.net.*; +import java.util.Arrays; +import java.util.List; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -249,4 +252,79 @@ public void testProxyActivationProperty() throws IOException, ExecutionException } } + @Test(groups = { "standalone", "default_provider" }) + public void testWildcardNonProxyHosts() throws IOException, ExecutionException, TimeoutException, InterruptedException { + Properties originalProps = System.getProperties(); + try { + Properties props = new Properties(); + props.putAll(originalProps); + + System.setProperties(props); + + System.setProperty("http.proxyHost", "127.0.0.1"); + System.setProperty("http.proxyPort", String.valueOf(port1)); + System.setProperty("http.nonProxyHosts", "127.*"); + + AsyncHttpClientConfig cfg = new AsyncHttpClientConfig.Builder().setUseProxyProperties(true).build(); + AsyncHttpClient client = getAsyncHttpClient(cfg); + try { + String target = "http://127.0.0.1:1234/"; + Future f = client.prepareGet(target).execute(); + try { + f.get(3, TimeUnit.SECONDS); + fail("should not be able to connect"); + } catch (ExecutionException e) { + // ok, no proxy used + } + } finally { + client.close(); + } + } finally { + System.setProperties(originalProps); + } + } + + @Test(groups = { "standalone", "default_provider" }) + public void testUseProxySelector() throws IOException, ExecutionException, TimeoutException, InterruptedException { + ProxySelector originalProxySelector = ProxySelector.getDefault(); + try { + ProxySelector.setDefault(new ProxySelector() { + public List select(URI uri) { + if (uri.getHost().equals("127.0.0.1")) { + return Arrays.asList(new Proxy(Proxy.Type.HTTP, new InetSocketAddress("127.0.0.1", port1))); + } else { + return Arrays.asList(Proxy.NO_PROXY); + } + } + + public void connectFailed(URI uri, SocketAddress sa, IOException ioe) { + } + }); + + AsyncHttpClientConfig cfg = new AsyncHttpClientConfig.Builder().setUseProxySelector(true).build(); + AsyncHttpClient client = getAsyncHttpClient(cfg); + try { + String target = "http://127.0.0.1:1234/"; + Future f = client.prepareGet(target).execute(); + Response resp = f.get(3, TimeUnit.SECONDS); + assertNotNull(resp); + assertEquals(resp.getStatusCode(), HttpServletResponse.SC_OK); + assertEquals(resp.getHeader("target"), "/"); + + target = "http://localhost:1234/"; + f = client.prepareGet(target).execute(); + try { + f.get(3, TimeUnit.SECONDS); + fail("should not be able to connect"); + } catch (ExecutionException e) { + // ok, no proxy used + } + } finally { + client.close(); + } + } finally { + ProxySelector.setDefault(originalProxySelector); + } + } + } diff --git a/providers/grizzly/src/main/java/org/asynchttpclient/providers/grizzly/statushandler/ProxyAuthorizationHandler.java b/providers/grizzly/src/main/java/org/asynchttpclient/providers/grizzly/statushandler/ProxyAuthorizationHandler.java index f2a156592e..d65a596ac6 100644 --- a/providers/grizzly/src/main/java/org/asynchttpclient/providers/grizzly/statushandler/ProxyAuthorizationHandler.java +++ b/providers/grizzly/src/main/java/org/asynchttpclient/providers/grizzly/statushandler/ProxyAuthorizationHandler.java @@ -65,7 +65,8 @@ public boolean handleStatus(final HttpResponsePacket responsePacket, final Request req = httpTransactionContext.getRequest(); ProxyServer proxyServer = httpTransactionContext.getProvider() .getClientConfig() - .getProxyServer(); + .getProxyServerSelector() + .select(req.getOriginalURI()); String principal = proxyServer.getPrincipal(); String password = proxyServer.getPassword(); Realm realm = new Realm.RealmBuilder().setPrincipal(principal) diff --git a/site/src/site/apt/proxy.apt b/site/src/site/apt/proxy.apt index d62a0ebb48..a09932fc0e 100644 --- a/site/src/site/apt/proxy.apt +++ b/site/src/site/apt/proxy.apt @@ -61,3 +61,26 @@ Response r = responseFuture.get(); You can also set the <<>> at the <<>> level. In that case, all request will share the same proxy information. + +Using Java System Properties + + The AsyncHttpClient library supports the standard + {{{http://docs.oracle.com/javase/7/docs/api/java/net/doc-files/net-properties.html#Proxies}Java Proxy System Properties}}. + You can configure this at a global level using the <<>> method on the + <<>>, or by setting the <<>> + system property to true. + +Using JDK ProxySelectors + + The AsyncHttpClient library also supports using the default + {{{http://docs.oracle.com/javase/7/docs/api/java/net/ProxySelector.html}JDK ProxySelector}}. This allows for more + fine grained control over which proxies to use, for example, it can be used in combination with + {{{https://code.google.com/p/proxy-vole/}Proxy Vole}} to use OS configured proxies or to use a proxy.pac file. + + You configure this at a global level using the <<>> method on the + <<>>, or by setting the + <<>> system property to true. + + If you don't change the default JDK <<>>, this setting is very similar to the <<>> + setting, though the <<>> setting does allow more flexibility, such as the ability to use an + HTTPS proxy. \ No newline at end of file