HTTPCLIENT-2356: Extend Auth API and authentication Logic to enable m…

httpclient5-testing/src/test/java/org/apache/hc/client5/testing/extension/async/StandardTestClientBuilder.java

@@ -33,6 +33,7 @@
 import org.apache.hc.client5.http.HttpRequestRetryStrategy;
 import org.apache.hc.client5.http.UserTokenHandler;
 import org.apache.hc.client5.http.auth.AuthSchemeFactory;
+import org.apache.hc.client5.http.auth.CredentialsProvider;
 import org.apache.hc.client5.http.config.ConnectionConfig;
 import org.apache.hc.client5.http.config.TlsConfig;
 import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient;
@@ -159,6 +160,12 @@ public TestAsyncClientBuilder setDefaultAuthSchemeRegistry(final Lookup<AuthSche
         return this;
     }
 
+    @Override
+    public TestAsyncClientBuilder setDefaultCredentialsProvider(final CredentialsProvider credentialsProvider) {
+        this.clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+        return this;
+    }
+
     @Override
     public TestAsyncClient build() throws Exception {
         final PoolingAsyncClientConnectionManager connectionManager = connectionManagerBuilder

httpclient5-testing/src/test/java/org/apache/hc/client5/testing/extension/async/TestAsyncClientBuilder.java

@@ -33,6 +33,7 @@
 import org.apache.hc.client5.http.HttpRequestRetryStrategy;
 import org.apache.hc.client5.http.UserTokenHandler;
 import org.apache.hc.client5.http.auth.AuthSchemeFactory;
+import org.apache.hc.client5.http.auth.CredentialsProvider;
 import org.apache.hc.client5.http.config.TlsConfig;
 import org.apache.hc.core5.http.Header;
 import org.apache.hc.core5.http.HttpRequestInterceptor;
@@ -105,6 +106,10 @@ default TestAsyncClientBuilder setDefaultAuthSchemeRegistry(Lookup<AuthSchemeFac
         throw new UnsupportedOperationException("Operation not supported by " + getProtocolLevel());
     }
 
+    default TestAsyncClientBuilder setDefaultCredentialsProvider(CredentialsProvider credentialsProvider) {
+        throw new UnsupportedOperationException("Operation not supported by " + getProtocolLevel());
+    }
+
     TestAsyncClient build() throws Exception;
 
 }

httpclient5-testing/src/test/java/org/apache/hc/client5/testing/extension/sync/StandardTestClientBuilder.java

@@ -33,6 +33,7 @@
 import org.apache.hc.client5.http.HttpRequestRetryStrategy;
 import org.apache.hc.client5.http.UserTokenHandler;
 import org.apache.hc.client5.http.auth.AuthSchemeFactory;
+import org.apache.hc.client5.http.auth.CredentialsProvider;
 import org.apache.hc.client5.http.classic.ExecChainHandler;
 import org.apache.hc.client5.http.config.ConnectionConfig;
 import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
@@ -150,6 +151,12 @@ public TestClientBuilder addExecInterceptorLast(final String name, final ExecCha
         return this;
     }
 
+    @Override
+    public TestClientBuilder setDefaultCredentialsProvider(final CredentialsProvider credentialsProvider) {
+        this.clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+        return this;
+    }
+
     @Override
     public TestClient build() throws Exception {
         final HttpClientConnectionManager connectionManagerCopy = connectionManager != null ? connectionManager :

httpclient5-testing/src/test/java/org/apache/hc/client5/testing/extension/sync/TestClientBuilder.java

@@ -33,6 +33,7 @@
 import org.apache.hc.client5.http.HttpRequestRetryStrategy;
 import org.apache.hc.client5.http.UserTokenHandler;
 import org.apache.hc.client5.http.auth.AuthSchemeFactory;
+import org.apache.hc.client5.http.auth.CredentialsProvider;
 import org.apache.hc.client5.http.classic.ExecChainHandler;
 import org.apache.hc.client5.http.io.HttpClientConnectionManager;
 import org.apache.hc.core5.http.Header;
@@ -98,6 +99,10 @@ default TestClientBuilder addExecInterceptorLast(String name, ExecChainHandler i
         throw new UnsupportedOperationException("Operation not supported by " + getProtocolLevel());
     }
 
+    default TestClientBuilder setDefaultCredentialsProvider(CredentialsProvider credentialsProvider) {
+        throw new UnsupportedOperationException("Operation not supported by " + getProtocolLevel());
+    }
+
     TestClient build() throws Exception;
 
 }

httpclient5/src/main/java/org/apache/hc/client5/http/auth/AuthExchange.java

@@ -38,6 +38,9 @@
  */
 public class AuthExchange {
 
+    // This only tracks the server state. In particular, even if the state is SUCCESS,
+    // the authentication may still fail if the challenge sent with a response cannot
+    // be validated locally in case of a mutual authentication.
     public enum State {
 
         UNCHALLENGED, CHALLENGED, HANDSHAKE, FAILURE, SUCCESS

httpclient5/src/main/java/org/apache/hc/client5/http/auth/AuthScheme.java

@@ -111,22 +111,71 @@ public interface AuthScheme {
      * Processes the given auth challenge. Some authentication schemes may involve multiple
      * challenge-response exchanges. Such schemes must be able to maintain internal state
      * when dealing with sequential challenges
+     * <p>
+     * Please note auth schemes that perform mutual authentication must implement
+     * {@link #processChallenge(HttpHost, AuthChallenge, HttpContext, boolean)} and
+     * {@link #isChallengeExpected()} instead.
      *
      * @param authChallenge the auth challenge
      * @param context HTTP context
      * @throws MalformedChallengeException in case the auth challenge is incomplete,
      * malformed or otherwise invalid.
+     *
+     * @see #processChallenge(HttpHost, AuthChallenge, HttpContext, boolean)
+     *
      * @since 5.0
      */
     void processChallenge(
             AuthChallenge authChallenge,
             HttpContext context) throws MalformedChallengeException;
 
+    /**
+     * Indicates that the even authorized (i.e. not 401 or 407) responses must be processed
+     * by this scheme.
+     *
+     * @return true if responses with non 401/407 response codes must be processed by the scheme.
+     *
+     * @since 5.5
+     */
+    default boolean isChallengeExpected() {
+        return false;
+    }
+
+    /**
+     * Processes the given auth challenge. Some authentication schemes may involve multiple
+     * challenge-response exchanges. Such schemes must be able to maintain internal state
+     * when dealing with sequential challenges.
+     * <p>
+     * When {@link #isChallengeExpected()} returns true, but no challenge was sent, this method
+     * must be called with a null {@link AuthChallenge} so that the scheme can handle this situation.
+     *
+     * @param host HTTP host
+     * @param authChallenge the auth challenge or null if no challenge was received
+     * @param context HTTP context
+     * @param challenged true if the response was unauthorised (401/407)
+     *
+     * @throws MalformedChallengeException in case the auth challenge is incomplete,
+     * @throws AuthenticationException in case the authentication process is unsuccessful.
+     *
+     * @since 5.5
+     */
+    default void processChallenge(
+            HttpHost host,
+            AuthChallenge authChallenge,
+            HttpContext context,
+            boolean challenged) throws MalformedChallengeException, AuthenticationException {
+        processChallenge(authChallenge, context);
+    }
+
     /**
      * Authentication process may involve a series of challenge-response exchanges.
      * This method tests if the authorization process has been fully completed (either
      * successfully or unsuccessfully), that is, all the required authorization
      * challenges have been processed in their entirety.
+     * <p>
+     * Please note if the scheme returns {@code true} from this method in response
+     * to a challenge, it effectively implies a failure to respond to this challenge
+     * and termination of the authentication process.
      *
      * @return {@code true} if the authentication process has been completed,
      * {@code false} otherwise.

httpclient5/src/main/java/org/apache/hc/client5/http/impl/DefaultAuthenticationStrategy.java

@@ -72,6 +72,10 @@ public class DefaultAuthenticationStrategy implements AuthenticationStrategy {
                 StandardAuthScheme.DIGEST,
                 StandardAuthScheme.BASIC));
 
+    protected List<String> getSchemePriority() {
+        return DEFAULT_SCHEME_PRIORITY;
+    }
+
     @Override
     public List<AuthScheme> select(
             final ChallengeType challengeType,
@@ -95,7 +99,7 @@ public List<AuthScheme> select(
         Collection<String> authPrefs = challengeType == ChallengeType.TARGET ?
                 config.getTargetPreferredAuthSchemes() : config.getProxyPreferredAuthSchemes();
         if (authPrefs == null) {
-            authPrefs = DEFAULT_SCHEME_PRIORITY;
+            authPrefs = getSchemePriority();
         }
         if (LOG.isDebugEnabled()) {
             LOG.debug("{} Authentication schemes in the order of preference: {}", exchangeId, authPrefs);

httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/AsyncConnectExec.java

@@ -43,10 +43,12 @@
 import org.apache.hc.client5.http.async.AsyncExecChainHandler;
 import org.apache.hc.client5.http.async.AsyncExecRuntime;
 import org.apache.hc.client5.http.auth.AuthExchange;
+import org.apache.hc.client5.http.auth.AuthenticationException;
 import org.apache.hc.client5.http.auth.ChallengeType;
+import org.apache.hc.client5.http.auth.MalformedChallengeException;
 import org.apache.hc.client5.http.config.RequestConfig;
 import org.apache.hc.client5.http.impl.auth.AuthCacheKeeper;
-import org.apache.hc.client5.http.impl.auth.HttpAuthenticator;
+import org.apache.hc.client5.http.impl.auth.AuthenticationHandler;
 import org.apache.hc.client5.http.impl.routing.BasicRouteDirector;
 import org.apache.hc.client5.http.protocol.HttpClientContext;
 import org.apache.hc.client5.http.routing.HttpRouteDirector;
@@ -93,7 +95,7 @@ public final class AsyncConnectExec implements AsyncExecChainHandler {
 
     private final HttpProcessor proxyHttpProcessor;
     private final AuthenticationStrategy proxyAuthStrategy;
-    private final HttpAuthenticator authenticator;
+    private final AuthenticationHandler authenticator;
     private final AuthCacheKeeper authCacheKeeper;
     private final HttpRouteDirector routeDirector;
 
@@ -106,7 +108,7 @@ public AsyncConnectExec(
         Args.notNull(proxyAuthStrategy, "Proxy authentication strategy");
         this.proxyHttpProcessor = proxyHttpProcessor;
         this.proxyAuthStrategy = proxyAuthStrategy;
-        this.authenticator = new HttpAuthenticator();
+        this.authenticator = new AuthenticationHandler();
         this.authCacheKeeper = authCachingDisabled ? null : new AuthCacheKeeper(schemePortResolver);
         this.routeDirector = BasicRouteDirector.INSTANCE;
     }
@@ -516,10 +518,11 @@ private boolean needAuthentication(
             final AuthExchange proxyAuthExchange,
             final HttpHost proxy,
             final HttpResponse response,
-            final HttpClientContext context) {
+            final HttpClientContext context) throws AuthenticationException, MalformedChallengeException {
         final RequestConfig config = context.getRequestConfigOrDefault();
         if (config.isAuthenticationEnabled()) {
             final boolean proxyAuthRequested = authenticator.isChallenged(proxy, ChallengeType.PROXY, response, proxyAuthExchange, context);
+            final boolean proxyMutualAuthRequired = authenticator.isChallengeExpected(proxyAuthExchange);
 
             if (authCacheKeeper != null) {
                 if (proxyAuthRequested) {
@@ -529,8 +532,8 @@ private boolean needAuthentication(
                 }
             }
 
-            if (proxyAuthRequested) {
-                final boolean updated = authenticator.updateAuthState(proxy, ChallengeType.PROXY, response,
+            if (proxyAuthRequested || proxyMutualAuthRequired) {
+                final boolean updated = authenticator.handleResponse(proxy, ChallengeType.PROXY, response,
                         proxyAuthStrategy, proxyAuthExchange, context);
 
                 if (authCacheKeeper != null) {

httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/AsyncProtocolExec.java

@@ -38,12 +38,14 @@
 import org.apache.hc.client5.http.async.AsyncExecChainHandler;
 import org.apache.hc.client5.http.async.AsyncExecRuntime;
 import org.apache.hc.client5.http.auth.AuthExchange;
+import org.apache.hc.client5.http.auth.AuthenticationException;
 import org.apache.hc.client5.http.auth.ChallengeType;
+import org.apache.hc.client5.http.auth.MalformedChallengeException;
 import org.apache.hc.client5.http.config.RequestConfig;
 import org.apache.hc.client5.http.impl.DefaultSchemePortResolver;
 import org.apache.hc.client5.http.impl.RequestSupport;
 import org.apache.hc.client5.http.impl.auth.AuthCacheKeeper;
-import org.apache.hc.client5.http.impl.auth.HttpAuthenticator;
+import org.apache.hc.client5.http.impl.auth.AuthenticationHandler;
 import org.apache.hc.client5.http.protocol.HttpClientContext;
 import org.apache.hc.core5.annotation.Contract;
 import org.apache.hc.core5.annotation.Internal;
@@ -84,7 +86,7 @@ public final class AsyncProtocolExec implements AsyncExecChainHandler {
 
     private final AuthenticationStrategy targetAuthStrategy;
     private final AuthenticationStrategy proxyAuthStrategy;
-    private final HttpAuthenticator authenticator;
+    private final AuthenticationHandler authenticator;
     private final SchemePortResolver schemePortResolver;
     private final AuthCacheKeeper authCacheKeeper;
 
@@ -95,7 +97,7 @@ public final class AsyncProtocolExec implements AsyncExecChainHandler {
             final boolean authCachingDisabled) {
         this.targetAuthStrategy = Args.notNull(targetAuthStrategy, "Target authentication strategy");
         this.proxyAuthStrategy = Args.notNull(proxyAuthStrategy, "Proxy authentication strategy");
-        this.authenticator = new HttpAuthenticator();
+        this.authenticator = new AuthenticationHandler();
         this.schemePortResolver = schemePortResolver != null ? schemePortResolver : DefaultSchemePortResolver.INSTANCE;
         this.authCacheKeeper = authCachingDisabled ? null : new AuthCacheKeeper(this.schemePortResolver);
     }
@@ -305,11 +307,12 @@ private boolean needAuthentication(
             final HttpHost target,
             final String pathPrefix,
             final HttpResponse response,
-            final HttpClientContext context) {
+            final HttpClientContext context) throws AuthenticationException, MalformedChallengeException {
         final RequestConfig config = context.getRequestConfigOrDefault();
         if (config.isAuthenticationEnabled()) {
             final boolean targetAuthRequested = authenticator.isChallenged(
                     target, ChallengeType.TARGET, response, targetAuthExchange, context);
+            final boolean targetMutualAuthRequired = authenticator.isChallengeExpected(targetAuthExchange);
 
             if (authCacheKeeper != null) {
                 if (targetAuthRequested) {
@@ -321,6 +324,7 @@ private boolean needAuthentication(
 
             final boolean proxyAuthRequested = authenticator.isChallenged(
                     proxy, ChallengeType.PROXY, response, proxyAuthExchange, context);
+            final boolean proxyMutualAuthRequired = authenticator.isChallengeExpected(proxyAuthExchange);
 
             if (authCacheKeeper != null) {
                 if (proxyAuthRequested) {
@@ -330,8 +334,8 @@ private boolean needAuthentication(
                 }
             }
 
-            if (targetAuthRequested) {
-                final boolean updated = authenticator.updateAuthState(target, ChallengeType.TARGET, response,
+            if (targetAuthRequested || targetMutualAuthRequired) {
+                final boolean updated = authenticator.handleResponse(target, ChallengeType.TARGET, response,
                         targetAuthStrategy, targetAuthExchange, context);
 
                 if (authCacheKeeper != null) {
@@ -340,8 +344,8 @@ private boolean needAuthentication(
 
                 return updated;
             }
-            if (proxyAuthRequested) {
-                final boolean updated = authenticator.updateAuthState(proxy, ChallengeType.PROXY, response,
+            if (proxyAuthRequested || proxyMutualAuthRequired) {
+                final boolean updated = authenticator.handleResponse(proxy, ChallengeType.PROXY, response,
                         proxyAuthStrategy, proxyAuthExchange, context);
 
                 if (authCacheKeeper != null) {