Merge pull request #365 from typesafehub/proxy-selector-1.7.x

Backported #362 to 1.7.x branch

Author: jfarcand

src/main/java/com/ning/http/client/AsyncHttpClientConfig.java

@@ -67,7 +67,7 @@ public class AsyncHttpClientConfig {
     protected boolean allowPoolingConnection;
     protected ScheduledExecutorService reaper;
     protected ExecutorService applicationThreadPool;
-    protected ProxyServer proxyServer;
+    protected ProxyServerSelector proxyServerSelector;
     protected SSLContext sslContext;
     protected SSLEngineFactory sslEngineFactory;
     protected AsyncHttpProviderConfig<?, ?> providerConfig;
@@ -106,7 +106,7 @@ private AsyncHttpClientConfig(int maxTotalConnections,
                                   boolean keepAlive,
                                   ScheduledExecutorService reaper,
                                   ExecutorService applicationThreadPool,
-                                  ProxyServer proxyServer,
+                                  ProxyServerSelector proxyServerSelector,
                                   SSLContext sslContext,
                                   SSLEngineFactory sslEngineFactory,
                                   AsyncHttpProviderConfig<?, ?> providerConfig,
@@ -162,7 +162,7 @@ private AsyncHttpClientConfig(int maxTotalConnections,
         } else {
             this.applicationThreadPool = applicationThreadPool;
         }
-        this.proxyServer = proxyServer;
+        this.proxyServerSelector = proxyServerSelector;
         this.useRawUrl = useRawUrl;
     }
 
@@ -310,8 +310,8 @@ public ExecutorService executorService() {
      *
      * @return instance of {@link com.ning.http.client.ProxyServer}
      */
-    public ProxyServer getProxyServer() {
-        return proxyServer;
+    public ProxyServerSelector getProxyServerSelector() {
+        return proxyServerSelector;
     }
 
     /**
@@ -531,11 +531,12 @@ public static class Builder {
         private boolean compressionEnabled = Boolean.getBoolean(ASYNC_CLIENT + "compressionEnabled");
         private String userAgent = System.getProperty(ASYNC_CLIENT + "userAgent", "NING/1.0");
         private boolean useProxyProperties = Boolean.getBoolean(ASYNC_CLIENT + "useProxyProperties");
+        private boolean useProxySelector = Boolean.getBoolean(ASYNC_CLIENT + "useProxySelector");
         private boolean allowPoolingConnection = true;
         private boolean useRelativeURIsWithSSLProxies = Boolean.getBoolean(ASYNC_CLIENT + "useRelativeURIsWithSSLProxies");
         private ScheduledExecutorService reaper;
         private ExecutorService applicationThreadPool;
-        private ProxyServer proxyServer = null;
+        private ProxyServerSelector proxyServerSelector = null;
         private SSLContext sslContext;
         private SSLEngineFactory sslEngineFactory;
         private AsyncHttpProviderConfig<?, ?> providerConfig;
@@ -731,13 +732,24 @@ public Builder setExecutorService(ExecutorService applicationThreadPool) {
         }
 
         /**
-         * Set an instance of {@link com.ning.http.client.ProxyServer} used by an {@link AsyncHttpClient}
+         * Set an instance of {@link ProxyServerSelector} used by an {@link AsyncHttpClient}
+         *
+         * @param proxyServerSelector instance of {@link ProxyServerSelector}
+         * @return a {@link Builder}
+         */
+        public Builder setProxyServerSelector(ProxyServerSelector proxyServerSelector) {
+            this.proxyServerSelector = proxyServerSelector;
+            return this;
+        }
+
+        /**
+         * Set an instance of {@link ProxyServer} used by an {@link AsyncHttpClient}
          *
          * @param proxyServer instance of {@link com.ning.http.client.ProxyServer}
          * @return a {@link Builder}
          */
         public Builder setProxyServer(ProxyServer proxyServer) {
-            this.proxyServer = proxyServer;
+            this.proxyServerSelector = ProxyUtils.createProxyServerSelector(proxyServer);
             return this;
         }
 
@@ -938,12 +950,27 @@ public Builder setRemoveQueryParamsOnRedirect(boolean removeQueryParamOnRedirect
             return this;
         }
 
+        /**
+         * Sets whether AHC should use the default JDK ProxySelector to select a proxy server.
+         * <p/>
+         * If useProxySelector is set to <code>true</code> but {@link #setProxyServer(ProxyServer)}
+         * was used to explicitly set a proxy server, the latter is preferred.
+         * <p/>
+         * See http://docs.oracle.com/javase/7/docs/api/java/net/ProxySelector.html
+         */
+        public Builder setUseProxySelector(boolean useProxySelector) {
+            this.useProxySelector = useProxySelector;
+            return this;
+        }
+
         /**
          * Sets whether AHC should use the default http.proxy* system properties
-         * to obtain proxy information.
+         * to obtain proxy information.  This differs from {@link #setUseProxySelector(boolean)}
+         * in that AsyncHttpClient will use its own logic to handle the system properties,
+         * potentially supporting other protocols that the the JDK ProxySelector doesn't.
          * <p/>
-         * If useProxyProperties is set to <code>true</code> but {@link #setProxyServer(ProxyServer)} was used
-         * to explicitly set a proxy server, the latter is preferred.
+         * If useProxyProperties is set to <code>true</code> but {@link #setUseProxySelector(boolean)}
+         * was also set to true, the latter is preferred.
          * <p/>
          * See http://download.oracle.com/javase/1.4.2/docs/guide/net/properties.html
          */
@@ -1036,7 +1063,7 @@ public Builder(AsyncHttpClientConfig prototype) {
             defaultMaxConnectionLifeTimeInMs = prototype.getMaxConnectionLifeTimeInMs();
             maxDefaultRedirects = prototype.getMaxRedirects();
             defaultMaxTotalConnections = prototype.getMaxTotalConnections();
-            proxyServer = prototype.getProxyServer();
+            proxyServerSelector = prototype.getProxyServerSelector();
             realm = prototype.getRealm();
             defaultRequestTimeoutInMs = prototype.getRequestTimeoutInMs();
             sslContext = prototype.getSSLContext();
@@ -1100,8 +1127,16 @@ public Thread newThread(Runnable r) {
                 throw new IllegalStateException("ExecutorServices closed");
             }
 
-            if (proxyServer == null && useProxyProperties) {
-                proxyServer = ProxyUtils.createProxy(System.getProperties());
+            if (proxyServerSelector == null && useProxySelector) {
+                proxyServerSelector = ProxyUtils.getJdkDefaultProxyServerSelector();
+            }
+
+            if (proxyServerSelector == null && useProxyProperties) {
+                proxyServerSelector = ProxyUtils.createProxyServerSelector(System.getProperties());
+            }
+
+            if (proxyServerSelector == null) {
+                proxyServerSelector = ProxyServerSelector.NO_PROXY_SELECTOR;
             }
 
             return new AsyncHttpClientConfig(defaultMaxTotalConnections,
@@ -1119,7 +1154,7 @@ public Thread newThread(Runnable r) {
                     allowPoolingConnection,
                     reaper,
                     applicationThreadPool,
-                    proxyServer,
+                    proxyServerSelector,
                     sslContext,
                     sslEngineFactory,
                     providerConfig,

src/main/java/com/ning/http/client/AsyncHttpClientConfigBean.java

@@ -55,9 +55,12 @@ void configureDefaults() {
         compressionEnabled = Boolean.getBoolean(ASYNC_CLIENT + "compressionEnabled");
         userAgent = System.getProperty(ASYNC_CLIENT + "userAgent", "NING/1.0");
 
+        boolean useProxySelector = Boolean.getBoolean(ASYNC_CLIENT + "useProxySelector");
         boolean useProxyProperties = Boolean.getBoolean(ASYNC_CLIENT + "useProxyProperties");
-        if (useProxyProperties) {
-            proxyServer = ProxyUtils.createProxy(System.getProperties());
+        if (useProxySelector) {
+            proxyServerSelector = ProxyUtils.getJdkDefaultProxyServerSelector();
+        } else if (useProxyProperties) {
+            proxyServerSelector = ProxyUtils.createProxyServerSelector(System.getProperties());
         }
 
         allowPoolingConnection = true;
@@ -163,7 +166,12 @@ public AsyncHttpClientConfigBean setApplicationThreadPool(ExecutorService applic
     }
 
     public AsyncHttpClientConfigBean setProxyServer(ProxyServer proxyServer) {
-        this.proxyServer = proxyServer;
+        this.proxyServerSelector = ProxyUtils.createProxyServerSelector(proxyServer);
+        return this;
+    }
+
+    public AsyncHttpClientConfigBean setProxyServerSelector(ProxyServerSelector proxyServerSelector) {
+        this.proxyServerSelector = proxyServerSelector;
         return this;
     }
 

src/main/java/com/ning/http/client/ProxyServerSelector.java

@@ -0,0 +1,26 @@
+package com.ning.http.client;
+
+import java.net.URI;
+
+/**
+ * Selector for a proxy server
+ */
+public interface ProxyServerSelector {
+
+    /**
+     * Select a proxy server to use for the given URI.
+     *
+     * @param uri The URI to select a proxy server for.
+     * @return The proxy server to use, if any.  May return null.
+     */
+    ProxyServer select(URI uri);
+
+    /**
+     * A selector that always selects no proxy.
+     */
+    static final ProxyServerSelector NO_PROXY_SELECTOR = new ProxyServerSelector() {
+        public ProxyServer select(URI uri) {
+            return null;
+        }
+    };
+}

src/main/java/com/ning/http/util/ProxyUtils.java

@@ -12,24 +12,33 @@
  */
 package com.ning.http.util;
 
-import static com.ning.http.util.MiscUtil.isNonEmpty;
 
 import com.ning.http.client.AsyncHttpClientConfig;
 import com.ning.http.client.ProxyServer;
 import com.ning.http.client.ProxyServer.Protocol;
+import com.ning.http.client.ProxyServerSelector;
 import com.ning.http.client.Request;
 
+import java.net.InetSocketAddress;
+import java.net.Proxy;
+import java.net.ProxySelector;
+import java.net.URI;
 import java.util.List;
 import java.util.Locale;
 import java.util.Properties;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * Utilities for Proxy handling.
  *
  * @author cstamas
  */
 public class ProxyUtils {
 
+    private final static Logger log = LoggerFactory.getLogger(ProxyUtils.class);
+
     private static final String PROPERTY_PREFIX = "com.ning.http.client.AsyncHttpClientConfig.proxy.";
 
     /**
@@ -70,7 +79,10 @@ public class ProxyUtils {
     public static ProxyServer getProxyServer(AsyncHttpClientConfig config, Request request) {
         ProxyServer proxyServer = request.getProxyServer();
         if (proxyServer == null) {
-            proxyServer = config.getProxyServer();
+            ProxyServerSelector selector = config.getProxyServerSelector();
+            if (selector != null) {
+                proxyServer = selector.select(request.getOriginalURI());
+            }
         }
         return ProxyUtils.avoidProxy(proxyServer, request) ? null : proxyServer;
     }
@@ -110,6 +122,9 @@ public static boolean avoidProxy(final ProxyServer proxyServer, final String tar
                     if (nonProxyHost.startsWith("*") && nonProxyHost.length() > 1
                             && targetHost.endsWith(nonProxyHost.substring(1).toLowerCase(Locale.ENGLISH))) {
                         return true;
+                    } else if (nonProxyHost.endsWith("*") && nonProxyHost.length() > 1
+                            && targetHost.startsWith(nonProxyHost.substring(0, nonProxyHost.length() - 1).toLowerCase(Locale.ENGLISH))) {
+                        return true;
                     } else if (nonProxyHost.equalsIgnoreCase(targetHost)) {
                         return true;
                     }
@@ -135,31 +150,89 @@ public static boolean avoidProxy(final ProxyServer proxyServer, final String tar
      * @see #PROXY_PROTOCOL
      * @see #PROXY_NONPROXYHOSTS
      */
-    public static ProxyServer createProxy(Properties properties) {
-        String host = System.getProperty(PROXY_HOST);
+    public static ProxyServerSelector createProxyServerSelector(Properties properties) {
+        String host = properties.getProperty(PROXY_HOST);
 
         if (host != null) {
-            int port = Integer.valueOf(System.getProperty(PROXY_PORT, "80"));
+            int port = Integer.valueOf(properties.getProperty(PROXY_PORT, "80"));
 
             Protocol protocol;
             try {
-                protocol = Protocol.valueOf(System.getProperty(PROXY_PROTOCOL, "HTTP"));
+                protocol = Protocol.valueOf(properties.getProperty(PROXY_PROTOCOL, "HTTP"));
             } catch (IllegalArgumentException e) {
                 protocol = Protocol.HTTP;
             }
 
-            ProxyServer proxyServer = new ProxyServer(protocol, host, port, System.getProperty(PROXY_USER), System.getProperty(PROXY_PASSWORD));
+            ProxyServer proxyServer = new ProxyServer(protocol, host, port, properties.getProperty(PROXY_USER),
+                    properties.getProperty(PROXY_PASSWORD));
 
-            String nonProxyHosts = System.getProperties().getProperty(PROXY_NONPROXYHOSTS);
+            String nonProxyHosts = properties.getProperty(PROXY_NONPROXYHOSTS);
             if (nonProxyHosts != null) {
                 for (String spec : nonProxyHosts.split("\\|")) {
                     proxyServer.addNonProxyHost(spec);
                 }
             }
 
-            return proxyServer;
+            return createProxyServerSelector(proxyServer);
         }
 
-        return null;
+        return ProxyServerSelector.NO_PROXY_SELECTOR;
+    }
+
+    /**
+     * Get a proxy server selector based on the JDK default proxy selector.
+     *
+     * @return The proxy server selector.
+     */
+    public static ProxyServerSelector getJdkDefaultProxyServerSelector() {
+        return createProxyServerSelector(ProxySelector.getDefault());
+    }
+
+    /**
+     * Create a proxy server selector based on the passed in JDK proxy selector.
+     *
+     * @param proxySelector The proxy selector to use.  Must not be null.
+     * @return The proxy server selector.
+     */
+    public static ProxyServerSelector createProxyServerSelector(final ProxySelector proxySelector) {
+        return new ProxyServerSelector() {
+            public ProxyServer select(URI uri) {
+                List<Proxy> proxies = proxySelector.select(uri);
+                if (proxies != null) {
+                    // Loop through them until we find one that we know how to use
+                    for (Proxy proxy : proxies) {
+                        switch (proxy.type()) {
+                            case HTTP:
+                                if (!(proxy.address() instanceof InetSocketAddress)) {
+                                    log.warn("Don't know how to connect to address " + proxy.address());
+                                } else {
+                                    InetSocketAddress address = (InetSocketAddress) proxy.address();
+                                    return new ProxyServer(Protocol.HTTP, address.getHostString(), address.getPort());
+                                }
+                            case DIRECT:
+                                return null;
+                            default:
+                                log.warn("ProxySelector returned proxy type that we don't know how to use: " + proxy.type());
+                                break;
+                        }
+                    }
+                }
+                return null;
+            }
+        };
+    }
+
+    /**
+     * Create a proxy server selector that always selects a single proxy server.
+     *
+     * @param proxyServer The proxy server to select.
+     * @return The proxy server selector.
+     */
+    public static ProxyServerSelector createProxyServerSelector(final ProxyServer proxyServer) {
+        return new ProxyServerSelector() {
+            public ProxyServer select(URI uri) {
+                return proxyServer;
+            }
+        };
     }
 }

src/site/apt/proxy.apt

@@ -61,3 +61,26 @@ Response r = responseFuture.get();
 
   You can also set the <<<ProxyServer>>> at the <<<AsyncHttpClientConfig>>> level. In that case, all request will share
   the same proxy information.
+
+Using Java System Properties
+
+  The AsyncHttpClient library supports the standard
+  {{{http://docs.oracle.com/javase/7/docs/api/java/net/doc-files/net-properties.html#Proxies}Java Proxy System Properties}}.
+  You can configure this at a global level using the <<<setUseProxyProperties(true)>>> method on the
+  <<<AsyncHttpClientConfig.Builder>>>, or by setting the <<<com.ning.http.client.AsyncHttpClientConfig.useProxyProperties>>>
+  system property to true.
+
+Using JDK ProxySelectors
+
+  The AsyncHttpClient library also supports using the default
+  {{{http://docs.oracle.com/javase/7/docs/api/java/net/ProxySelector.html}JDK ProxySelector}}.  This allows for more
+  fine grained control over which proxies to use, for example, it can be used in combination with
+  {{{https://code.google.com/p/proxy-vole/}Proxy Vole}} to use OS configured proxies or to use a proxy.pac file.
+
+  You configure this at a global level using the <<<setUseProxySelector(true)>>> method on the
+  <<<AsyncHttpClientConfig.Builder>>>, or by setting the
+   <<<com.ning.http.client.AsyncHttpClientConfig.useProxySelector>>> system property to true.
+
+  If you don't change the default JDK <<<ProxySelector>>>, this setting is very similar to the <<<useProxyProperties>>>
+  setting, though the <<<useProxyProperties>>> setting does allow more flexibility, such as the ability to use an
+  HTTPS proxy.