Extended Apache HTTP Digest Auth. for use of YaCy encoded password

When programmatically requesting the local peer with Apache http client,
authentication credentials must be passed as clear-text values. 
This extension to the apache org.apache.http.impl.auth.DigestScheme
permits use of the YaCy encoded password stored in the
adminAccountBase64MD5 configuration property.

4 files changed, 643 insertions, 5 deletions

diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java
index 662cae4d7..17912027e 100644
--- a/source/net/yacy/cora/protocol/http/HTTPClient.java
+++ b/source/net/yacy/cora/protocol/http/HTTPClient.java
@@ -54,10 +54,13 @@ import org.apache.http.HttpEntityEnclosingRequest;
 import org.apache.http.HttpHeaders;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.auth.AuthSchemeProvider;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.config.AuthSchemes;
 import org.apache.http.client.config.CookieSpecs;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
@@ -66,6 +69,7 @@ import org.apache.http.client.methods.HttpHead;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.config.Lookup;
 import org.apache.http.config.Registry;
 import org.apache.http.config.RegistryBuilder;
 import org.apache.http.config.SocketConfig;
@@ -79,6 +83,7 @@ import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.entity.mime.MultipartEntityBuilder;
 import org.apache.http.entity.mime.content.ContentBody;
+import org.apache.http.impl.auth.BasicSchemeFactory;
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
@@ -96,6 +101,7 @@ import net.yacy.cora.protocol.ClientIdentification;
 import net.yacy.cora.protocol.ConnectionInfo;
 import net.yacy.cora.protocol.Domains;
 import net.yacy.cora.protocol.HeaderFramework;
+import net.yacy.cora.protocol.http.auth.YaCyDigestSchemeFactory;
 import net.yacy.cora.util.CommonPattern;
 import net.yacy.cora.util.Memory;
 import net.yacy.kelondro.util.NamePrefixThreadFactory;
@@ -296,6 +302,10 @@ public class HTTPClient {
      * This method GETs a page from the server.
      *
      * @param uri the url to get
+     * @param username user name for HTTP authentication : only sent requesting localhost
+     * @param pass password for HTTP authentication : only sent when requesting localhost
+     * @param concurrent whether a new thread should be created to handle the request. 
+     * Ignored when requesting localhost or when the authentication password is not null
      * @return content bytes
      * @throws IOException
      */
@@ -307,6 +317,10 @@ public class HTTPClient {
      * This method GETs a page from the server.
      *
      * @param uri the url to get
+     * @param username user name for HTTP authentication : only sent requesting localhost
+     * @param pass password for HTTP authentication : only sent when requesting localhost
+     * @param concurrent whether a new thread should be created to handle the request. 
+     * Ignored when requesting localhost or when the authentication password is not null
      * @return content bytes
      * @throws IOException
      */
@@ -318,7 +332,11 @@ public class HTTPClient {
      * This method GETs a page from the server.
      *
      * @param uri the url to get
+     * @param username user name for HTTP authentication : only sent requesting localhost
+     * @param pass password for HTTP authentication : only sent when requesting localhost
      * @param maxBytes to get
+     * @param concurrent whether a new thread should be created to handle the request. 
+     * Ignored when requesting localhost or when the authentication password is not null
      * @return content bytes
      * @throws IOException
      */
@@ -331,7 +349,11 @@ public class HTTPClient {
      * This method GETs a page from the server.
      *
      * @param uri the url to get
-     * @param maxBytes to get
+     * @param username user name for HTTP authentication : only sent requesting localhost
+     * @param pass password for HTTP authentication : only sent when requesting localhost
+     * @param maxBytes maximum response bytes to read
+     * @param concurrent whether a new thread should be created to handle the request. 
+     * Ignored when requesting localhost or when the authentication password is not null
      * @return content bytes
      * @throws IOException
      */
@@ -351,9 +373,18 @@ public class HTTPClient {
         
         CredentialsProvider credsProvider = new BasicCredentialsProvider();
         credsProvider.setCredentials(
-                AuthScope.ANY, // thats ok since we tested for localhost!
+        		new AuthScope("localhost", url.getPort()),
                 new UsernamePasswordCredentials(username, pass));
-        CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider).build();
+        
+        /* Use the custom YaCyDigestScheme for HTTP Digest Authentication */
+        final Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
+                .register(AuthSchemes.BASIC, new BasicSchemeFactory())
+                .register(AuthSchemes.DIGEST, new YaCyDigestSchemeFactory())
+                .build();
+        
+        
+		CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider)
+				.setDefaultAuthSchemeRegistry(authSchemeRegistry).build();
         byte[] content = null;
         try {
             this.httpResponse = httpclient.execute(httpGet);
@@ -502,6 +533,23 @@ public class HTTPClient {
      * @throws IOException
      */
     public byte[] POSTbytes(final MultiProtocolURL url, final String vhost, final Map<String, ContentBody> post, final boolean usegzip, final boolean concurrent) throws IOException {
+    	return POSTbytes(url, vhost, post, null, null, usegzip, concurrent);
+    }
+    
+    /**
+     * Send data using HTTP POST method to the server named by vhost
+     *
+     * @param url address to request on the server
+     * @param vhost name of the server at address which should respond. When null, localhost is assumed.
+     * @param post data to send (name-value-pairs)
+     * @param userName user name for HTTP authentication : only sent when requesting localhost
+     * @param password encoded password for HTTP authentication : only sent when requesting localhost
+     * @param usegzip if the body should be gzipped
+     * @return response body
+     * @throws IOException when an error occurred
+     */
+    public byte[] POSTbytes(final MultiProtocolURL url, final String vhost, final Map<String, ContentBody> post, 
+    		final String userName, final String password, final boolean usegzip, final boolean concurrent) throws IOException {
     	final HttpPost httpPost = new HttpPost(url.toNormalform(true));
     	final boolean localhost = Domains.isLocalhost(url.getHost());
         if (!localhost) setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
@@ -518,8 +566,46 @@ public class HTTPClient {
         } else {
             httpPost.setEntity(multipartEntity);
         }
-
-        return getContentBytes(httpPost, Integer.MAX_VALUE, concurrent);
+        
+        if (!localhost || password == null) {
+            return getContentBytes(httpPost, Integer.MAX_VALUE, concurrent);
+        }
+        
+        byte[] content = null;
+        
+        final CredentialsProvider credsProvider = new BasicCredentialsProvider();
+        credsProvider.setCredentials(
+                new AuthScope("localhost", url.getPort()),
+                new UsernamePasswordCredentials(userName, password));
+        
+        /* Use the custom YaCyDigestScheme for HTTP Digest Authentication */
+        final Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
+                .register(AuthSchemes.BASIC, new BasicSchemeFactory())
+                .register(AuthSchemes.DIGEST, new YaCyDigestSchemeFactory())
+                .build();
+        
+        
+		CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider)
+				.setDefaultAuthSchemeRegistry(authSchemeRegistry).build();
+		
+        try {
+            this.httpResponse = httpclient.execute(httpPost);
+            try {
+                HttpEntity httpEntity = this.httpResponse.getEntity();
+                if (httpEntity != null) {
+                    if (getStatusCode() == HttpStatus.SC_OK) {
+                        content = getByteArray(httpEntity, Integer.MAX_VALUE);
+                    }
+                    // Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
+                    EntityUtils.consume(httpEntity);
+                }
+            } finally {
+                this.httpResponse.close();
+            }
+        } finally {
+            httpclient.close();
+        }
+        return content;
     }
 
     /**
diff --git a/source/net/yacy/cora/protocol/http/auth/HttpEntityDigester.java b/source/net/yacy/cora/protocol/http/auth/HttpEntityDigester.java
new file mode 100644
index 000000000..3227af496
--- /dev/null
+++ b/source/net/yacy/cora/protocol/http/auth/HttpEntityDigester.java
@@ -0,0 +1,75 @@
+/*
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation.  For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ *
+ */
+package net.yacy.cora.protocol.http.auth;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.security.MessageDigest;
+
+class HttpEntityDigester extends OutputStream {
+
+    private final MessageDigest digester;
+    private boolean closed;
+    private byte[] digest;
+
+    HttpEntityDigester(final MessageDigest digester) {
+        super();
+        this.digester = digester;
+        this.digester.reset();
+    }
+
+    @Override
+    public void write(final int b) throws IOException {
+        if (this.closed) {
+            throw new IOException("Stream has been already closed");
+        }
+        this.digester.update((byte) b);
+    }
+
+    @Override
+    public void write(final byte[] b, final int off, final int len) throws IOException {
+        if (this.closed) {
+            throw new IOException("Stream has been already closed");
+        }
+        this.digester.update(b, off, len);
+    }
+
+    @Override
+    public void close() throws IOException {
+        if (this.closed) {
+            return;
+        }
+        this.closed = true;
+        this.digest = this.digester.digest();
+        super.close();
+    }
+
+    public byte[] getDigest() {
+        return this.digest;
+    }
+
+}
diff --git a/source/net/yacy/cora/protocol/http/auth/YaCyDigestScheme.java b/source/net/yacy/cora/protocol/http/auth/YaCyDigestScheme.java
new file mode 100644
index 000000000..7fced6bb4
--- /dev/null
+++ b/source/net/yacy/cora/protocol/http/auth/YaCyDigestScheme.java
@@ -0,0 +1,411 @@
+// YaCyDigestScheme.java
+// Copyright 2017 by luccioman; https://github.com/luccioman
+//
+// This is a part of YaCy, a peer-to-peer based web search engine
+//
+// LICENSE
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+package net.yacy.cora.protocol.http.auth;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.security.MessageDigest;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Formatter;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.StringTokenizer;
+
+import org.apache.http.Consts;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpEntityEnclosingRequest;
+import org.apache.http.HttpRequest;
+import org.apache.http.auth.AUTH;
+import org.apache.http.auth.AuthenticationException;
+import org.apache.http.auth.Credentials;
+import org.apache.http.auth.params.AuthPNames;
+import org.apache.http.impl.auth.DigestScheme;
+import org.apache.http.impl.auth.UnsupportedDigestAlgorithmException;
+import org.apache.http.message.BasicHeaderValueFormatter;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.message.BufferedHeader;
+import org.apache.http.protocol.HttpContext;
+import org.apache.http.util.Args;
+import org.apache.http.util.CharArrayBuffer;
+import org.apache.http.util.EncodingUtils;
+
+/**
+ * Overrides the Apache DigestScheme to support credentials with YaCy encoded
+ * passwords instead of clear-text ones. This implementation is directly derived
+ * from the {@link DigestScheme} from Apache HTTPClient 4.5.3.
+ */
+public class YaCyDigestScheme extends DigestScheme {
+
+    private static final long serialVersionUID = 3883908186234566916L;
+
+    /**
+     * Hexa values used when creating 32 character long digest in HTTP DigestScheme
+     * in case of authentication.
+     *
+     * @see #encode(byte[])
+     */
+    private static final char[] HEXADECIMAL = {
+        '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd',
+        'e', 'f'
+    };
+
+    /** Whether the digest authentication process is complete */
+    private boolean complete;
+
+    private static final int QOP_UNKNOWN = -1;
+    private static final int QOP_MISSING = 0;
+    private static final int QOP_AUTH_INT = 1;
+    private static final int QOP_AUTH = 2;
+
+    private String lastNonce;
+    private long nounceCount;
+    private String cnonce;
+    private String a2;
+
+    /**
+     * @since 4.3
+     */
+    public YaCyDigestScheme(final Charset credentialsCharset) {
+        super(credentialsCharset);
+        this.complete = false;
+    }
+
+    public YaCyDigestScheme() {
+        this(Consts.ASCII);
+    }
+
+    /**
+	 * Produces a digest authorization string for the given set of
+	 * {@link Credentials}, method name and URI. The credentials password is
+	 * supposed to be a YaCy encoded password (see adminAccountBase64MD5
+	 * property in yacy.conf) and not a clear-text value.
+	 *
+	 * @param credentials
+	 *            A set of credentials to be used for authentication
+	 * @param request
+	 *            The request being authenticated
+	 *
+	 * @throws org.apache.http.auth.InvalidCredentialsException
+	 *             if authentication credentials are not valid or not applicable
+	 *             for this authentication scheme
+	 * @throws AuthenticationException
+	 *             if authorization string cannot be generated due to an
+	 *             authentication failure
+	 *
+	 * @return a digest authorization string
+	 */
+    @Override
+    public Header authenticate(
+            final Credentials credentials,
+            final HttpRequest request,
+            final HttpContext context) throws AuthenticationException {
+
+        Args.notNull(credentials, "Credentials");
+        Args.notNull(request, "HTTP request");
+        if (getParameter("realm") == null) {
+            throw new AuthenticationException("missing realm in challenge");
+        }
+        if (getParameter("nonce") == null) {
+            throw new AuthenticationException("missing nonce in challenge");
+        }
+        // Add method name and request-URI to the parameter map
+        getParameters().put("methodname", request.getRequestLine().getMethod());
+        getParameters().put("uri", request.getRequestLine().getUri());
+        final String charset = getParameter("charset");
+        if (charset == null) {
+            getParameters().put("charset", getCredentialsCharset(request));
+        }
+        return createDigestHeader(credentials, request);
+    }
+    
+    String getCredentialsCharset(final HttpRequest request) {
+        String charset = (String) request.getParams().getParameter(AuthPNames.CREDENTIAL_CHARSET);
+        if (charset == null) {
+            charset = getCredentialsCharset().name();
+        }
+        return charset;
+    }
+
+    private static MessageDigest createMessageDigest(
+            final String digAlg) throws UnsupportedDigestAlgorithmException {
+        try {
+            return MessageDigest.getInstance(digAlg);
+        } catch (final Exception e) {
+            throw new UnsupportedDigestAlgorithmException(
+              "Unsupported algorithm in HTTP Digest authentication: "
+               + digAlg);
+        }
+    }
+
+    /**
+     * Creates digest-response header as defined in RFC2617.
+     *
+     * @param credentials User credentials
+     *
+     * @return The digest-response as String.
+     */
+    private Header createDigestHeader(
+            final Credentials credentials,
+            final HttpRequest request) throws AuthenticationException {
+        final String uri = getParameter("uri");
+        final String realm = getParameter("realm");
+        final String nonce = getParameter("nonce");
+        final String opaque = getParameter("opaque");
+        final String method = getParameter("methodname");
+        String algorithm = getParameter("algorithm");
+        // If an algorithm is not specified, default to MD5.
+        if (algorithm == null) {
+            algorithm = "MD5";
+        }
+
+        final Set<String> qopset = new HashSet<String>(8);
+        int qop = QOP_UNKNOWN;
+        final String qoplist = getParameter("qop");
+        if (qoplist != null) {
+            final StringTokenizer tok = new StringTokenizer(qoplist, ",");
+            while (tok.hasMoreTokens()) {
+                final String variant = tok.nextToken().trim();
+                qopset.add(variant.toLowerCase(Locale.ROOT));
+            }
+            if (request instanceof HttpEntityEnclosingRequest && qopset.contains("auth-int")) {
+                qop = QOP_AUTH_INT;
+            } else if (qopset.contains("auth")) {
+                qop = QOP_AUTH;
+            }
+        } else {
+            qop = QOP_MISSING;
+        }
+
+        if (qop == QOP_UNKNOWN) {
+            throw new AuthenticationException("None of the qop methods is supported: " + qoplist);
+        }
+
+        String charset = getParameter("charset");
+        if (charset == null) {
+            charset = "ISO-8859-1";
+        }
+
+        String digAlg = algorithm;
+        if (digAlg.equalsIgnoreCase("MD5-sess")) {
+            digAlg = "MD5";
+        }
+
+        final MessageDigest digester;
+        try {
+            digester = createMessageDigest(digAlg);
+        } catch (final UnsupportedDigestAlgorithmException ex) {
+            throw new AuthenticationException("Unsuppported digest algorithm: " + digAlg);
+        }
+
+        final String uname = credentials.getUserPrincipal().getName();
+        /* Modification for YaCy Digest authentication using the encoded password from yacy.conf */
+        String pwd = credentials.getPassword();
+        if(pwd.startsWith("MD5:")) {
+        	pwd = pwd.substring("MD5:".length());
+        }
+
+        if (nonce.equals(this.lastNonce)) {
+            nounceCount++;
+        } else {
+            nounceCount = 1;
+            cnonce = null;
+            lastNonce = nonce;
+        }
+        final StringBuilder sb = new StringBuilder(256);
+        final Formatter formatter = new Formatter(sb, Locale.US);
+        formatter.format("%08x", Long.valueOf(nounceCount));
+        formatter.close();
+        final String nc = sb.toString();
+
+        if (cnonce == null) {
+            cnonce = createCnonce();
+        }
+
+        a2 = null;
+        // 3.2.2.2: Calculating digest
+        String hasha1 ;
+        if (algorithm.equalsIgnoreCase("MD5-sess")) {
+            // H( unq(username-value) ":" unq(realm-value) ":" passwd )
+            //      ":" unq(nonce-value)
+            //      ":" unq(cnonce-value)
+
+            // calculated one per session
+            sb.setLength(0);
+            /* Modification for YaCy Digest Authentication : the pwd value is already the result of MD5(userName:realm:password)
+            sb.append(uname).append(':').append(realm).append(':').append(pwd);
+            final String checksum = encode(digester.digest(EncodingUtils.getBytes(sb.toString(), charset)));
+            sb.setLength(0);
+            sb.append(checksum).append(':').append(nonce).append(':').append(cnonce);
+            a1 = sb.toString();*/
+            sb.append(pwd).append(':').append(nonce).append(':').append(cnonce);
+            hasha1 = encode(digester.digest(EncodingUtils.getBytes(sb.toString(), charset)));;
+        } else {
+            // unq(username-value) ":" unq(realm-value) ":" passwd
+            /* Modification for YaCy Digest Authentication : the pwd value is already the result of MD5(userName:realm:password)
+            sb.setLength(0);
+            sb.append(uname).append(':').append(realm).append(':').append(pwd);
+            a1 = sb.toString();*/
+            hasha1 = pwd;
+        }
+        /*Modification for YaCy Digest Authentication
+        final String hasha1 = encode(digester.digest(EncodingUtils.getBytes(a1, charset)));*/
+
+        if (qop == QOP_AUTH) {
+            // Method ":" digest-uri-value
+            a2 = method + ':' + uri;
+        } else if (qop == QOP_AUTH_INT) {
+            // Method ":" digest-uri-value ":" H(entity-body)
+            HttpEntity entity = null;
+            if (request instanceof HttpEntityEnclosingRequest) {
+                entity = ((HttpEntityEnclosingRequest) request).getEntity();
+            }
+            if (entity != null && !entity.isRepeatable()) {
+                // If the entity is not repeatable, try falling back onto QOP_AUTH
+                if (qopset.contains("auth")) {
+                    qop = QOP_AUTH;
+                    a2 = method + ':' + uri;
+                } else {
+                    throw new AuthenticationException("Qop auth-int cannot be used with " +
+                            "a non-repeatable entity");
+                }
+            } else {
+                final HttpEntityDigester entityDigester = new HttpEntityDigester(digester);
+                try {
+                    if (entity != null) {
+                        entity.writeTo(entityDigester);
+                    }
+                    entityDigester.close();
+                } catch (final IOException ex) {
+                    throw new AuthenticationException("I/O error reading entity content", ex);
+                }
+                a2 = method + ':' + uri + ':' + encode(entityDigester.getDigest());
+            }
+        } else {
+            a2 = method + ':' + uri;
+        }
+
+        final String hasha2 = encode(digester.digest(EncodingUtils.getBytes(a2, charset)));
+
+        // 3.2.2.1
+
+        final String digestValue;
+        if (qop == QOP_MISSING) {
+            sb.setLength(0);
+            sb.append(hasha1).append(':').append(nonce).append(':').append(hasha2);
+            digestValue = sb.toString();
+        } else {
+            sb.setLength(0);
+            sb.append(hasha1).append(':').append(nonce).append(':').append(nc).append(':')
+                .append(cnonce).append(':').append(qop == QOP_AUTH_INT ? "auth-int" : "auth")
+                .append(':').append(hasha2);
+            digestValue = sb.toString();
+        }
+
+        final String digest = encode(digester.digest(EncodingUtils.getAsciiBytes(digestValue)));
+
+        final CharArrayBuffer buffer = new CharArrayBuffer(128);
+        if (isProxy()) {
+            buffer.append(AUTH.PROXY_AUTH_RESP);
+        } else {
+            buffer.append(AUTH.WWW_AUTH_RESP);
+        }
+        buffer.append(": Digest ");
+
+        final List<BasicNameValuePair> params = new ArrayList<BasicNameValuePair>(20);
+        params.add(new BasicNameValuePair("username", uname));
+        params.add(new BasicNameValuePair("realm", realm));
+        params.add(new BasicNameValuePair("nonce", nonce));
+        params.add(new BasicNameValuePair("uri", uri));
+        params.add(new BasicNameValuePair("response", digest));
+
+        if (qop != QOP_MISSING) {
+            params.add(new BasicNameValuePair("qop", qop == QOP_AUTH_INT ? "auth-int" : "auth"));
+            params.add(new BasicNameValuePair("nc", nc));
+            params.add(new BasicNameValuePair("cnonce", cnonce));
+        }
+        // algorithm cannot be null here
+        params.add(new BasicNameValuePair("algorithm", algorithm));
+        if (opaque != null) {
+            params.add(new BasicNameValuePair("opaque", opaque));
+        }
+
+        for (int i = 0; i < params.size(); i++) {
+            final BasicNameValuePair param = params.get(i);
+            if (i > 0) {
+                buffer.append(", ");
+            }
+            final String name = param.getName();
+            final boolean noQuotes = ("nc".equals(name) || "qop".equals(name)
+                    || "algorithm".equals(name));
+            BasicHeaderValueFormatter.INSTANCE.formatNameValuePair(buffer, param, !noQuotes);
+        }
+        return new BufferedHeader(buffer);
+    }
+
+    /**
+     * Encodes the 128 bit (16 bytes) MD5 digest into a 32 characters long
+     * <CODE>String</CODE> according to RFC 2617.
+     *
+     * @param binaryData array containing the digest
+     * @return encoded MD5, or <CODE>null</CODE> if encoding failed
+     */
+    static String encode(final byte[] binaryData) {
+        final int n = binaryData.length;
+        final char[] buffer = new char[n * 2];
+        for (int i = 0; i < n; i++) {
+            final int low = (binaryData[i] & 0x0f);
+            final int high = ((binaryData[i] & 0xf0) >> 4);
+            buffer[i * 2] = HEXADECIMAL[high];
+            buffer[(i * 2) + 1] = HEXADECIMAL[low];
+        }
+
+        return new String(buffer);
+    }
+
+
+    /**
+     * Creates a random cnonce value based on the current time.
+     *
+     * @return The cnonce value as String.
+     */
+    public static String createCnonce() {
+        final SecureRandom rnd = new SecureRandom();
+        final byte[] tmp = new byte[8];
+        rnd.nextBytes(tmp);
+        return encode(tmp);
+    }
+
+    @Override
+    public String toString() {
+        final StringBuilder builder = new StringBuilder();
+        builder.append("DIGEST [complete=").append(complete)
+                .append(", nonce=").append(lastNonce)
+                .append(", nc=").append(nounceCount)
+                .append("]");
+        return builder.toString();
+    }
+
+}
diff --git a/source/net/yacy/cora/protocol/http/auth/YaCyDigestSchemeFactory.java b/source/net/yacy/cora/protocol/http/auth/YaCyDigestSchemeFactory.java
new file mode 100644
index 000000000..9fa9abb40
--- /dev/null
+++ b/source/net/yacy/cora/protocol/http/auth/YaCyDigestSchemeFactory.java
@@ -0,0 +1,66 @@
+// YaCyDigestSchemeFactory.java
+// Copyright 2017 by luccioman; https://github.com/luccioman
+//
+// This is a part of YaCy, a peer-to-peer based web search engine
+//
+// LICENSE
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+package net.yacy.cora.protocol.http.auth;
+
+import java.nio.charset.Charset;
+
+import org.apache.http.annotation.Contract;
+import org.apache.http.annotation.ThreadingBehavior;
+import org.apache.http.auth.AuthScheme;
+import org.apache.http.auth.AuthSchemeFactory;
+import org.apache.http.auth.AuthSchemeProvider;
+import org.apache.http.params.HttpParams;
+import org.apache.http.protocol.HttpContext;
+
+/**
+ * {@link AuthSchemeProvider} implementation that creates and initializes
+ * {@link YaCyDigestScheme} instances that support use of YaCy encoded password
+ * instead of clear-text password.
+ */
+@Contract(threading = ThreadingBehavior.IMMUTABLE)
+public class YaCyDigestSchemeFactory implements AuthSchemeFactory, AuthSchemeProvider {
+
+    private final Charset charset;
+
+    /**
+     * @param charset characters set
+     */
+    public YaCyDigestSchemeFactory(final Charset charset) {
+        super();
+        this.charset = charset;
+    }
+
+    public YaCyDigestSchemeFactory() {
+        this(null);
+    }
+
+    @Override
+    public AuthScheme newInstance(final HttpParams params) {
+        return new YaCyDigestScheme();
+    }
+
+    @Override
+    public AuthScheme create(final HttpContext context) {
+        return new YaCyDigestScheme(this.charset);
+    }
+
+}