Merge branch 'master' of git@github.com:yacy/yacy_search_server.git

20 files changed, 542 insertions, 676 deletions

diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml
index 470372f26..477dbd6a6 100644
--- a/.github/FUNDING.yml
+++ b/.github/FUNDING.yml
@@ -1,2 +1,2 @@
 github: orbiter
-patreon: 0rb1t3r
+patreon: orbiterlab
diff --git a/defaults/solr/solrconfig.xml b/defaults/solr/solrconfig.xml
index 54b9afbca..9c536d3fa 100644
--- a/defaults/solr/solrconfig.xml
+++ b/defaults/solr/solrconfig.xml
@@ -180,12 +180,13 @@
          The default since Lucene 2.3 was the LogByteSizeMergePolicy,
          Even older versions of Lucene used LogDocMergePolicy.
       -->
-    <!--
-        <mergePolicyFactory class="solr.TieredMergePolicyFactory">
-          <int name="maxMergeAtOnce">10</int>
-          <int name="segmentsPerTier">10</int>
+        <mergePolicyFactory class="org.apache.solr.index.TieredMergePolicyFactory">
+          <int name="maxMergeAtOnce">8</int>
+          <int name="segmentsPerTier">8</int>
+          <int name="maxMergedSegmentMB">51200</int>
+          <int name="maxCFSSegmentSizeMB">1024</int>
+          <double name="noCFSRatio">0.1</double>
         </mergePolicyFactory>
-     -->
 
     <!-- Expert: Merge Scheduler
          The Merge Scheduler in Lucene controls how merges are
@@ -314,8 +315,8 @@
          have some sort of hard autoCommit to limit the log size.
       -->
      <autoCommit> 
-       <maxTime>${solr.autoCommit.maxTime:15000}</maxTime> 
-       <openSearcher>false</openSearcher> 
+       <maxTime>${solr.autoCommit.maxTime:180000}</maxTime> 
+       <openSearcher>true</openSearcher> 
      </autoCommit>
 
     <!-- softAutoCommit is like autoCommit except it causes a
@@ -442,8 +443,8 @@
                and old cache.  
       -->
     <filterCache class="solr.FastLRUCache"
-                 size="512"
-                 initialSize="512"
+                 size="100"
+                 initialSize="100"
                  autowarmCount="0"/>
 
     <!-- Query Result Cache
@@ -466,27 +467,26 @@
                    size="512"
                    initialSize="512"
                    autowarmCount="0"/>
-    
-    <!-- custom cache currently used by block join --> 
+
+    <!-- custom cache currently used by block join 
     <cache name="perSegFilter"
       class="solr.search.LRUCache"
       size="10"
       initialSize="0"
       autowarmCount="10"
-      regenerator="solr.NoOpRegenerator" />
-
+      regenerator="solr.NoOpRegenerator" /> -->
+      
     <!-- Field Value Cache
          
          Cache used to hold field values that are quickly accessible
          by document id.  The fieldValueCache is created by default
          even if not configured here.
       -->
-    <!--
        <fieldValueCache class="solr.FastLRUCache"
-                        size="512"
-                        autowarmCount="128"
+                        size="32"
+                        initialSize="0"
+                        autowarmCount="0"
                         showItems="32" />
-      -->
 
     <!-- Custom Cache
 
diff --git a/htroot/IndexImportMediawiki_p.java b/htroot/IndexImportMediawiki_p.java
index 0b95382d7..b45a76a5e 100644
--- a/htroot/IndexImportMediawiki_p.java
+++ b/htroot/IndexImportMediawiki_p.java
@@ -196,29 +196,29 @@ public class IndexImportMediawiki_p {
      * @return the last modified date for the file at fileURL, or 0L when unknown or when an error occurred
      */
 	private static long getLastModified(MultiProtocolURL fileURL) {
-		long lastModified = 0l;
 		try {
 			if (fileURL.isHTTP() || fileURL.isHTTPS()) {
 				/* http(s) : we do not use MultiprotocolURL.lastModified() which always returns 0L for these protocols */
-				HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
-				HttpResponse headResponse = httpClient.HEADResponse(fileURL, false);
-				if (headResponse != null && headResponse.getStatusLine() != null
-						&& headResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
-					Header lastModifiedHeader = headResponse
-							.getFirstHeader(HeaderFramework.LAST_MODIFIED);
-					if (lastModifiedHeader != null) {
-						Date lastModifiedDate = HeaderFramework.parseHTTPDate(lastModifiedHeader.getValue());
-						if(lastModifiedDate != null) {
-							lastModified = lastModifiedDate.getTime();
-						}
-					}
-				}
+				try (HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {
+    				HttpResponse headResponse = httpClient.HEADResponse(fileURL, false);
+    				if (headResponse != null && headResponse.getStatusLine() != null
+    						&& headResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+    					Header lastModifiedHeader = headResponse
+    							.getFirstHeader(HeaderFramework.LAST_MODIFIED);
+    					if (lastModifiedHeader != null) {
+    						Date lastModifiedDate = HeaderFramework.parseHTTPDate(lastModifiedHeader.getValue());
+    						if(lastModifiedDate != null) {
+    							return lastModifiedDate.getTime();
+    						}
+    					}
+    				}
+    			}
 			} else {
-				lastModified = fileURL.lastModified();
+				return fileURL.lastModified();
 			}
 		} catch (IOException ignored) {
 			ConcurrentLog.warn("IndexImportMediawiki_p", "Could not retrieve last modified date for dump file at " + fileURL);
 		}
-		return lastModified;
+		return 0l;
 	}
 }
diff --git a/htroot/env/templates/header.template b/htroot/env/templates/header.template
index 2ce0e2d3a..5bd7f9f3a 100644
--- a/htroot/env/templates/header.template
+++ b/htroot/env/templates/header.template
@@ -145,7 +145,7 @@
           <ul class="dropdown-menu">
           <li><a href="#">YaCy is free software, so we need the help of many to support the development.<br/><b>You</b> can help by joining a sponsoring plan:</a></li>
           <li><a href="https://github.com/users/Orbiter/sponsorship" target="_blank"><i>external</i>&nbsp;&nbsp;&nbsp;<b>become a Github Sponsor</b></a></li>
-          <li><a href="https://www.patreon.com/0rb1t3r" target="_blank"><i>external</i>&nbsp;&nbsp;&nbsp;<b>become a YaCy Patreon</b></a></li>
+          <li><a href="https://www.patreon.com/bePatron?u=185903" target="_blank"><i>external</i>&nbsp;&nbsp;&nbsp;<b>become a YaCy Patreon</b></a></li>
           <li><a href="#">Please help! We need financial help to move on with the development!</a></li>
           </ul>
         </li>
diff --git a/source/net/yacy/cora/document/id/MultiProtocolURL.java b/source/net/yacy/cora/document/id/MultiProtocolURL.java
index 7f4bf9a2c..92ab41a52 100644
--- a/source/net/yacy/cora/document/id/MultiProtocolURL.java
+++ b/source/net/yacy/cora/document/id/MultiProtocolURL.java
@@ -2538,7 +2538,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
             return new ByteArrayInputStream(b);
         }
         if (isHTTP() || isHTTPS()) {
-                final HTTPClient client = new HTTPClient(agent);
+            try (final HTTPClient client = new HTTPClient(agent)){
                 client.setHost(getHost());
                 client.GET(this, false);
                 if (client.getStatusCode() != HttpStatus.SC_OK) {
@@ -2546,6 +2546,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
                             "\nServer returned status: " + client.getHttpResponse().getStatusLine());
                 }
                 return new HTTPInputStream(client);
+            }
         }
 
         return null;
@@ -2562,9 +2563,10 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
             return b;
         }
         if (isHTTP() || isHTTPS()) {
-                final HTTPClient client = new HTTPClient(agent);
-                client.setHost(getHost());
-                return client.GETbytes(this, username, pass, false);
+                try (final HTTPClient client = new HTTPClient(agent)) {
+                    client.setHost(getHost());
+                    return client.GETbytes(this, username, pass, false);
+                }
         }
 
         return null;
diff --git a/source/net/yacy/cora/federate/opensearch/OpenSearchConnector.java b/source/net/yacy/cora/federate/opensearch/OpenSearchConnector.java
index 0bb06fbba..d61a9deed 100644
--- a/source/net/yacy/cora/federate/opensearch/OpenSearchConnector.java
+++ b/source/net/yacy/cora/federate/opensearch/OpenSearchConnector.java
@@ -297,10 +297,9 @@ public class OpenSearchConnector extends AbstractFederateSearchConnector impleme
         String searchurl = this.parseSearchTemplate(baseurl, searchTerms, startIndex, count);
         try {
         	DigestURL aurl = new DigestURL(searchurl);
-            try {
+        	try (final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {
                 this.lastaccesstime = System.currentTimeMillis();
                 
-                final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
                 byte[] result = httpClient.GETbytes(aurl, null, null, false);
                 
     			if(result == null) {
diff --git a/source/net/yacy/cora/federate/opensearch/SRURSSConnector.java b/source/net/yacy/cora/federate/opensearch/SRURSSConnector.java
index ffaadf458..2f2892267 100644
--- a/source/net/yacy/cora/federate/opensearch/SRURSSConnector.java
+++ b/source/net/yacy/cora/federate/opensearch/SRURSSConnector.java
@@ -121,8 +121,9 @@ public class SRURSSConnector {
             parts.put("resource", UTF8.StringBody(global ? "global" : "local"));
             parts.put("nav", UTF8.StringBody("none"));
             // result = HTTPConnector.getConnector(userAgent == null ? MultiProtocolURI.yacybotUserAgent : userAgent).post(new MultiProtocolURI(rssSearchServiceURL), (int) timeout, uri.getHost(), parts);
-            final HTTPClient httpClient = new HTTPClient(agent);
-            result = httpClient.POSTbytes(new MultiProtocolURL(rssSearchServiceURL), uri.getHost(), parts, false, false);
+            try (final HTTPClient httpClient = new HTTPClient(agent)) {
+                result = httpClient.POSTbytes(new MultiProtocolURL(rssSearchServiceURL), uri.getHost(), parts, false, false);
+            }
 
             final RSSReader reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result);
             if (reader == null) {
diff --git a/source/net/yacy/cora/federate/yacy/api/Network.java b/source/net/yacy/cora/federate/yacy/api/Network.java
index 876d1ffa7..486b1b917 100644
--- a/source/net/yacy/cora/federate/yacy/api/Network.java
+++ b/source/net/yacy/cora/federate/yacy/api/Network.java
@@ -49,7 +49,7 @@ public class Network {
      */
     public static Peers getNetwork(final String address) throws IOException {
         Peers peers = new Peers();
-        final HTTPClient httpclient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
+        try (final HTTPClient httpclient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {
             final byte[] content = httpclient.GETbytes("http://" + address  + "/Network.xml?page=1&maxCount=1000&ip=", null, null, false);
             ByteArrayInputStream bais = new ByteArrayInputStream(content);
             Document doc = null;
@@ -74,6 +74,7 @@ public class Network {
                   //log.info(peer.toString());
                }
             }
+        }
         return peers;
     }
     
diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java
index 06aead621..abb583f80 100644
--- a/source/net/yacy/cora/protocol/http/HTTPClient.java
+++ b/source/net/yacy/cora/protocol/http/HTTPClient.java
@@ -25,6 +25,7 @@
 
 package net.yacy.cora.protocol.http;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -92,7 +93,6 @@ import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
 import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.impl.client.HttpClients;
 import org.apache.http.impl.client.IdleConnectionEvictor;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.protocol.HTTP;
@@ -119,11 +119,12 @@ import net.yacy.kelondro.util.NamePrefixThreadFactory;
  * @author sixcooler
  *
  */
-public class HTTPClient {
+public class HTTPClient implements Closeable {
+    
+    private static final int default_timeout = 6000;
     
-    private final static int default_timeout = 6000;
     /** Maximum number of simultaneously open outgoing HTTP connections in the pool */
-	private final static int maxcon = 200;
+	private static final int maxcon = 200;
 	
 	/** Default sleep time in seconds between each run of the connection evictor */
 	private static final int DEFAULT_CONNECTION_EVICTOR_SLEEP_TIME = 5;
@@ -131,7 +132,13 @@ public class HTTPClient {
 	/** Default maximum time in seconds to keep alive an idle connection in the pool */
 	private static final int DEFAULT_POOLED_CONNECTION_TIME_TO_LIVE = 30;
 	
-	private final static RequestConfig dfltReqConf = initRequestConfig();
+	private static final RequestConfig DFLTREQUESTCONFIG = initRequestConfig();
+	
+	/** Use the custom YaCyDigestScheme for HTTP Digest Authentication */
+    private static final Lookup<AuthSchemeProvider> AUTHSCHEMEREGISTRY = RegistryBuilder.<AuthSchemeProvider>create()
+            .register(AuthSchemes.BASIC, new BasicSchemeFactory())
+            .register(AuthSchemes.DIGEST, new YaCyDigestSchemeFactory())
+            .build();
 	
 	/** The connection manager holding the configured connection pool for this client */
 	public static final PoolingHttpClientConnectionManager CONNECTION_MANAGER = initPoolingConnectionManager();
@@ -146,11 +153,7 @@ public class HTTPClient {
 			Boolean.parseBoolean(System.getProperty("jsse.enableSNIExtension", Boolean.toString(ENABLE_SNI_EXTENSION_DEFAULT))));
 	
 
-    // digest factories
-    private final BasicSchemeFactory BASIC_SCHEME_FACTORY = new BasicSchemeFactory();
-    private final YaCyDigestSchemeFactory YACY_DIGEST_SCHEME_FACTORY = new YaCyDigestSchemeFactory();
-
-	/**
+    /**
 	 * Background daemon thread evicting expired idle connections from the pool.
 	 * This may be eventually already done by the pool itself on connection request,
 	 * but this background task helps when no request is made to the pool for a long
@@ -167,19 +170,23 @@ public class HTTPClient {
 	private final static HttpClientBuilder clientBuilder = initClientBuilder();
 	private final RequestConfig.Builder reqConfBuilder;
 	private Set<Entry<String, String>> headers = null;
-	private CloseableHttpResponse httpResponse = null;
-	private HttpUriRequest currentRequest = null;
 	private long upbytes = 0L;
 	private String host = null;
 	private final long timeout;
 	private static ExecutorService executor = Executors
 			.newCachedThreadPool(new NamePrefixThreadFactory(HTTPClient.class.getSimpleName() + ".execute"));
+	
+	/** these are the main variable to hold information and to take care of closing: */
+	private CloseableHttpClient client = null;
+	private CloseableHttpResponse httpResponse = null;
+	private HttpUriRequest currentRequest = null;
 
+	
     public HTTPClient(final ClientIdentification.Agent agent) {
         super();
         this.timeout = agent.clientTimeout;
         clientBuilder.setUserAgent(agent.userAgent);
-        reqConfBuilder = RequestConfig.copy(dfltReqConf);
+        reqConfBuilder = RequestConfig.copy(DFLTREQUESTCONFIG);
         setTimout(agent.clientTimeout);
     }
     
@@ -187,13 +194,9 @@ public class HTTPClient {
         super();
         this.timeout = timeout;
         clientBuilder.setUserAgent(agent.userAgent);
-        reqConfBuilder = RequestConfig.copy(dfltReqConf);
+        reqConfBuilder = RequestConfig.copy(DFLTREQUESTCONFIG);
         setTimout(timeout);
     }
-
-    public static void setDefaultUserAgent(final String defaultAgent) {
-    	clientBuilder.setUserAgent(defaultAgent);
-    }
     
     private static RequestConfig initRequestConfig() {
     	final RequestConfig.Builder builder = RequestConfig.custom();
@@ -215,7 +218,9 @@ public class HTTPClient {
     	final HttpClientBuilder builder = HttpClientBuilder.create();
     	
     	builder.setConnectionManager(CONNECTION_MANAGER);
-		builder.setDefaultRequestConfig(dfltReqConf);
+    	builder.setConnectionManagerShared(true);
+    	
+		builder.setDefaultRequestConfig(DFLTREQUESTCONFIG);
 		
     	// UserAgent
 		builder.setUserAgent(ClientIdentification.yacyInternetCrawlerAgent.userAgent);
@@ -430,15 +435,15 @@ public class HTTPClient {
     public byte[] GETbytes(final MultiProtocolURL url, final String username, final String pass, final int maxBytes, final boolean concurrent) throws IOException {
         final boolean localhost = Domains.isLocalhost(url.getHost());
         final String urix = url.toNormalform(true);
-        HttpGet httpGet = null;
+        
         try {
-            httpGet = new HttpGet(urix);
+            this.currentRequest = new HttpGet(urix);
         } catch (IllegalArgumentException e) {
             throw new IOException(e.getMessage()); // can be caused  at java.net.URI.create()
         }
         if (!localhost) setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
         if (!localhost || pass == null) {
-            return getContentBytes(httpGet, maxBytes, concurrent);
+            return getContentBytes(maxBytes, concurrent);
         }
         
         CredentialsProvider credsProvider = new BasicCredentialsProvider();
@@ -446,48 +451,31 @@ public class HTTPClient {
         		new AuthScope("localhost", url.getPort()),
                 new UsernamePasswordCredentials(username, pass));
         
-        /* Use the custom YaCyDigestScheme for HTTP Digest Authentication */
-        final Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
-                .register(AuthSchemes.BASIC, BASIC_SCHEME_FACTORY)
-                .register(AuthSchemes.DIGEST, YACY_DIGEST_SCHEME_FACTORY)
-                .build();
-
-		CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider)
-				.setDefaultAuthSchemeRegistry(authSchemeRegistry).build();
-        byte[] content = null;
-        try {
-            this.httpResponse = httpclient.execute(httpGet);
-            try {
-                HttpEntity httpEntity = this.httpResponse.getEntity();
-                if (httpEntity != null) {
-                    if (getStatusCode() == HttpStatus.SC_OK) {
-            			if (maxBytes >= 0 && httpEntity.getContentLength() > maxBytes) {
-            				/* When anticipated content length is already known and exceed the specified limit : 
-            				 * throw an exception and abort the connection, consistently with getByteArray() implementation 
-            				 * Otherwise returning null and consuming fully the entity can be very long on large resources */
-            				throw new IOException("Content to download exceed maximum value of " + Formatter.bytesToString(maxBytes));
-            			}
-                        content = getByteArray(httpEntity, maxBytes);
-                    }
-                    // Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
-                    EntityUtils.consume(httpEntity);
+        try (final CloseableHttpClient httpclient = clientBuilder.setDefaultCredentialsProvider(credsProvider)
+                .setDefaultAuthSchemeRegistry(AUTHSCHEMEREGISTRY).build()) {
+            this.httpResponse = httpclient.execute(this.currentRequest);
+            HttpEntity httpEntity = this.httpResponse.getEntity();
+            if (httpEntity != null) {
+                if (getStatusCode() == HttpStatus.SC_OK) {
+        			if (maxBytes >= 0 && httpEntity.getContentLength() > maxBytes) {
+        				/* When anticipated content length is already known and exceed the specified limit : 
+        				 * throw an exception and abort the connection, consistently with getByteArray() implementation 
+        				 * Otherwise returning null and consuming fully the entity can be very long on large resources */
+        				throw new IOException("Content to download exceed maximum value of " + Formatter.bytesToString(maxBytes));
+        			}
+                    return getByteArray(httpEntity, maxBytes);
                 }
-            } catch (final IOException e) {
-            	httpGet.abort();
-                throw e;
-            } finally {
-                this.httpResponse.close();
             }
         } finally {
-            httpclient.close();
+            close();
         }
-        return content;
+        return null;
     }
     
     /**
      * This method GETs a page from the server.
      * to be used for streaming out
-     * Please take care to call finish()!
+     * Please take care to call close()!
      *
      * @param uri the url to get
      * @throws IOException
@@ -499,7 +487,7 @@ public class HTTPClient {
     /**
      * This method GETs a page from the server.
      * to be used for streaming out
-     * Please take care to call finish()!
+     * Please take care to call close()!
      *
      * @param url the url to get
      * @throws IOException
@@ -507,15 +495,15 @@ public class HTTPClient {
     public void GET(final MultiProtocolURL url, final boolean concurrent) throws IOException {
         if (this.currentRequest != null) throw new IOException("Client is in use!");
         final String urix = url.toNormalform(true);
-        HttpGet httpGet = null;
+        
         try {
-            httpGet = new HttpGet(urix);
+            this.currentRequest = new HttpGet(urix);
         } catch (IllegalArgumentException e) {
             throw new IOException(e.getMessage()); // can be caused  at java.net.URI.create()
         }
         setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
-        this.currentRequest = httpGet;
-        execute(httpGet, concurrent);
+        
+        execute(concurrent);
     }
 
     /**
@@ -537,18 +525,16 @@ public class HTTPClient {
      * @throws IOException
      */
     public HttpResponse HEADResponse(final MultiProtocolURL url, final boolean concurrent) throws IOException {
-        final HttpHead httpHead = new HttpHead(url.toNormalform(true));
+        this.currentRequest = new HttpHead(url.toNormalform(true));
         setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
-    	execute(httpHead, concurrent);
-    	finish();
-    	ConnectionInfo.removeConnection(httpHead.hashCode());
+    	execute(concurrent);
     	return this.httpResponse;
     }
 
     /**
      * This method POSTs a page from the server.
      * to be used for streaming out
-     * Please take care to call finish()!
+     * Please take care to call close()!
      *
      * @param uri the url to post
      * @param instream the input to post
@@ -564,7 +550,7 @@ public class HTTPClient {
     /**
      * This method POSTs a page from the server.
      * to be used for streaming out
-     * Please take care to call finish()!
+     * Please take care to call close()!
      *
      * @param url the url to post
      * @param instream the input to post
@@ -573,16 +559,15 @@ public class HTTPClient {
      */
     public void POST(final MultiProtocolURL url, final InputStream instream, final long length, final boolean concurrent) throws IOException {
     	if (this.currentRequest != null) throw new IOException("Client is in use!");
-        final HttpPost httpPost = new HttpPost(url.toNormalform(true));
+    	this.currentRequest = new HttpPost(url.toNormalform(true));
         String host = url.getHost();
         if (host == null) host = Domains.LOCALHOST;
         setHost(host); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
         final NonClosingInputStreamEntity inputStreamEntity = new NonClosingInputStreamEntity(instream, length);
     	// statistics
     	this.upbytes = length;
-    	httpPost.setEntity(inputStreamEntity);
-    	this.currentRequest = httpPost;
-    	execute(httpPost, concurrent);
+    	((HttpPost) this.currentRequest).setEntity(inputStreamEntity);
+    	execute(concurrent);
     }
 
     /**
@@ -628,7 +613,7 @@ public class HTTPClient {
      */
     public byte[] POSTbytes(final MultiProtocolURL url, final String vhost, final Map<String, ContentBody> post, 
     		final String userName, final String password, final boolean usegzip, final boolean concurrent) throws IOException {
-    	final HttpPost httpPost = new HttpPost(url.toNormalform(true));
+        this.currentRequest = new HttpPost(url.toNormalform(true));
     	final boolean localhost = Domains.isLocalhost(url.getHost());
         if (!localhost) setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
     	if (vhost == null) setHost(Domains.LOCALHOST);
@@ -640,49 +625,33 @@ public class HTTPClient {
         this.upbytes = multipartEntity.getContentLength();
 
         if (usegzip) {
-            httpPost.setEntity(new GzipCompressingEntity(multipartEntity));
+            ((HttpPost) this.currentRequest).setEntity(new GzipCompressingEntity(multipartEntity));
         } else {
-            httpPost.setEntity(multipartEntity);
+            ((HttpPost) this.currentRequest).setEntity(multipartEntity);
         }
         
         if (!localhost || password == null) {
-            return getContentBytes(httpPost, Integer.MAX_VALUE, concurrent);
+            return getContentBytes(Integer.MAX_VALUE, concurrent);
         }
         
-        byte[] content = null;
-        
         final CredentialsProvider credsProvider = new BasicCredentialsProvider();
         credsProvider.setCredentials(
                 new AuthScope("localhost", url.getPort()),
                 new UsernamePasswordCredentials(userName, password));
 
-        /* Use the custom YaCyDigestScheme for HTTP Digest Authentication */
-        final Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
-                .register(AuthSchemes.BASIC, BASIC_SCHEME_FACTORY)
-                .register(AuthSchemes.DIGEST, YACY_DIGEST_SCHEME_FACTORY)
-                .build();
-
-		CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider)
-				.setDefaultAuthSchemeRegistry(authSchemeRegistry).build();
-		
-        try {
-            this.httpResponse = httpclient.execute(httpPost);
-            try {
-                HttpEntity httpEntity = this.httpResponse.getEntity();
-                if (httpEntity != null) {
-                    if (getStatusCode() == HttpStatus.SC_OK) {
-                        content = getByteArray(httpEntity, Integer.MAX_VALUE);
-                    }
-                    // Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
-                    EntityUtils.consume(httpEntity);
+        try (final CloseableHttpClient httpclient = clientBuilder.setDefaultCredentialsProvider(credsProvider)
+                .setDefaultAuthSchemeRegistry(AUTHSCHEMEREGISTRY).build()) {
+            this.httpResponse = httpclient.execute(this.currentRequest);
+            HttpEntity httpEntity = this.httpResponse.getEntity();
+            if (httpEntity != null) {
+                if (getStatusCode() == HttpStatus.SC_OK) {
+                    return getByteArray(httpEntity, Integer.MAX_VALUE);
                 }
-            } finally {
-                this.httpResponse.close();
             }
         } finally {
-            httpclient.close();
+            close();
         }
-        return content;
+        return null;
     }
 
     /**
@@ -797,7 +766,7 @@ public class HTTPClient {
     /**
      * This method gets direct access to the content-stream
      * Since this way is uncontrolled by the Client think of using 'writeTo' instead!
-     * Please take care to call finish()!
+     * Please take care to call close()!
      *
      * @return the content as InputStream
      * @throws IOException
@@ -808,10 +777,7 @@ public class HTTPClient {
             if (httpEntity != null) try {
                     return httpEntity.getContent();
             } catch (final IOException e) {
-                ConnectionInfo.removeConnection(this.currentRequest.hashCode());
-                this.currentRequest.abort();
-                this.currentRequest = null;
-                this.httpResponse.close();
+                close();
                 throw e;
             }
         }
@@ -820,7 +786,7 @@ public class HTTPClient {
 
     /**
      * This method streams the content to the outputStream
-     * Please take care to call finish()!
+     * Please take care to call close()!
      *
      * @param outputStream
      * @throws IOException
@@ -831,82 +797,41 @@ public class HTTPClient {
             if (httpEntity != null) try {
                 httpEntity.writeTo(outputStream);
                 outputStream.flush();
-                // Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
-                EntityUtils.consume(httpEntity);
-                ConnectionInfo.removeConnection(this.currentRequest.hashCode());
-                this.currentRequest = null;
-            } catch (final IOException e) {
-                ConnectionInfo.removeConnection(this.currentRequest.hashCode());
-                this.currentRequest.abort();
-                this.currentRequest = null;
-                this.httpResponse.close();
-                throw e;
+            } finally {
+                close();
             }
         }
     }
 
     /**
-     * This method ensures correct finish of client-connections
+     * This method ensures correct close of client-connections
      * This method should be used after every use of GET or POST and writeTo or getContentstream!
      *
      * @throws IOException
      */
-	public void finish() throws IOException {
+	@Override
+    public void close() throws IOException {
 		try {
 			if (this.httpResponse != null) {
-				final HttpEntity httpEntity = this.httpResponse.getEntity();
-				if (httpEntity != null && httpEntity.isStreaming()) {
-					/*
-					 * Try to fully consume the eventual remaining of the
-					 * content stream : if too long abort the request. Not using
-					 * EntityUtils.consumeQuietly(httpEntity) because too long
-					 * to perform on large resources when calling this before
-					 * full stream processing end : for example on caller
-					 * exception handling .
-					 */
-					InputStream contentStream = null;
-					try {
-						contentStream = httpEntity.getContent();
-						if (contentStream != null) {
-							byte[] buffer = new byte[2048];
-							int count = 0;
-							int readNb = contentStream.read(buffer);
-							while (readNb >= 0 && count < 10) {
-								readNb = contentStream.read(buffer);
-								count++;
-							}
-							if (readNb >= 0) {
-								if (this.currentRequest != null) {
-									this.currentRequest.abort();
-								}
-							}
-						}
-					} catch(IOException e){
-						/* Silently ignore here IOException (for example caused by stream already closed) as in EntityUtils.consumeQuietly() */
-					} finally {
-						if (contentStream != null) {
-							try {
-								contentStream.close();
-							} catch(IOException ignored) {}
-						}
-						this.httpResponse.close();
-					}
-
-				}
-
+                // Ensures that the entity content Stream is closed.
+                EntityUtils.consumeQuietly(this.httpResponse.getEntity());
+				this.httpResponse.close();
+			}
+			if (this.client != null) {
+			    client.close();
 			}
 		} finally {
 			if (this.currentRequest != null) {
 				ConnectionInfo.removeConnection(this.currentRequest.hashCode());
+				this.currentRequest.abort();
 				this.currentRequest = null;
 			}
 		}
 	}
 
-    private byte[] getContentBytes(final HttpUriRequest httpUriRequest, final int maxBytes, final boolean concurrent) throws IOException {
-        byte[] content = null;
-    	try {
-            execute(httpUriRequest, concurrent);
+    private byte[] getContentBytes(final int maxBytes, final boolean concurrent) throws IOException {
+        try {
+            execute(concurrent);
             if (this.httpResponse == null) return null;
             // get the response body
             final HttpEntity httpEntity = this.httpResponse.getEntity();
@@ -918,33 +843,27 @@ public class HTTPClient {
         				 * Otherwise returning null and consuming fully the entity can be very long on large resources */
         				throw new IOException("Content to download exceed maximum value of " + Formatter.bytesToString(maxBytes));
         			}
-                    content = getByteArray(httpEntity, maxBytes);
+                    return getByteArray(httpEntity, maxBytes);
                 }
-                // Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
-            	EntityUtils.consume(httpEntity);
             }
-        } catch (final IOException e) {
-                httpUriRequest.abort();
-                throw e;
         } finally {
-        	if (this.httpResponse != null) this.httpResponse.close();
-        	ConnectionInfo.removeConnection(httpUriRequest.hashCode());
+        	close();
         }
-    	return content;
+    	return null;
     }
 
-    private void execute(final HttpUriRequest httpUriRequest, final boolean concurrent) throws IOException {
+    private void execute(final boolean concurrent) throws IOException {
     	final HttpClientContext context = HttpClientContext.create();
     	context.setRequestConfig(reqConfBuilder.build());
     	if (this.host != null)
     		context.setTargetHost(new HttpHost(this.host));
     	
-    	setHeaders(httpUriRequest);
+    	setHeaders();
     	// statistics
-    	storeConnectionInfo(httpUriRequest);
+    	storeConnectionInfo();
     	// execute the method; some asserts confirm that that the request can be send with Content-Length and is therefore not terminated by EOF
-	    if (httpUriRequest instanceof HttpEntityEnclosingRequest) {
-	        final HttpEntityEnclosingRequest hrequest = (HttpEntityEnclosingRequest) httpUriRequest;
+	    if (this.currentRequest instanceof HttpEntityEnclosingRequest) {
+	        final HttpEntityEnclosingRequest hrequest = (HttpEntityEnclosingRequest) this.currentRequest;
 	        final HttpEntity entity = hrequest.getEntity();
 	        assert entity != null;
 	        //assert !entity.isChunked();
@@ -953,16 +872,17 @@ public class HTTPClient {
 	    }
 
 	    final String initialThreadName = Thread.currentThread().getName();
-	    Thread.currentThread().setName("HTTPClient-" + httpUriRequest.getURI());
+	    final String uri = this.currentRequest.getURI().toString();
+	    Thread.currentThread().setName("HTTPClient-" + uri);
         final long time = System.currentTimeMillis();
 	    try {
 	        
+	        this.client = clientBuilder.build();
 	        if (concurrent) {
 	            FutureTask<CloseableHttpResponse> t = new FutureTask<CloseableHttpResponse>(new Callable<CloseableHttpResponse>() {
 	                @Override
                     public CloseableHttpResponse call() throws ClientProtocolException, IOException {
-	                    final CloseableHttpClient client = clientBuilder.build();
-	                    CloseableHttpResponse response = client.execute(httpUriRequest, context);
+	                    CloseableHttpResponse response = client.execute(currentRequest, context);
 	                    return response;
 	                }
 	            });
@@ -973,20 +893,18 @@ public class HTTPClient {
 	                throw e.getCause();
 	            } catch (Throwable e) {}
 	            try {t.cancel(true);} catch (Throwable e) {}
-	            if (this.httpResponse == null) throw new IOException("timout to client after " + this.timeout + "ms" + " for url " + httpUriRequest.getURI().toString());
+	            if (this.httpResponse == null) {
+	                throw new IOException("timout to client after " + this.timeout + "ms" + " for url " + uri);
+	            }
 	        } else {
-	            final CloseableHttpClient client = clientBuilder.build();
-	            this.httpResponse = client.execute(httpUriRequest, context);
+	            this.httpResponse = client.execute(this.currentRequest, context);
 	        }
             this.httpResponse.setHeader(HeaderFramework.RESPONSE_TIME_MILLIS, Long.toString(System.currentTimeMillis() - time));
         } catch (final Throwable e) {
-            ConnectionInfo.removeConnection(httpUriRequest.hashCode());
-            httpUriRequest.abort();
-            if (this.httpResponse != null) this.httpResponse.close();
-            //e.printStackTrace();
+            close();
             throw new IOException("Client can't execute: "
             		+ (e.getCause() == null ? e.getMessage() : e.getCause().getMessage())
-            		+ " duration=" + Long.toString(System.currentTimeMillis() - time) + " for url " + httpUriRequest.getURI().toString());
+            		+ " duration=" + Long.toString(System.currentTimeMillis() - time) + " for url " + uri);
         } finally {
         	/* Restore the thread initial name */
         	Thread.currentThread().setName(initialThreadName);
@@ -1001,11 +919,10 @@ public class HTTPClient {
      * @throws IOException when a read error occured or content length is over maxBytes
      */
 	public static byte[] getByteArray(final HttpEntity entity, int maxBytes) throws IOException {
-		final InputStream instream = entity.getContent();
-		if (instream == null) {
-			return null;
-		}
-		try {
+		try (final InputStream instream = entity.getContent()) {
+		    if (instream == null) {
+		        return null;
+		    }
 			long contentLength = entity.getContentLength();
 			/*
 			 * When no maxBytes is specified, the default limit is
@@ -1046,29 +963,30 @@ public class HTTPClient {
 		} catch (final OutOfMemoryError e) {
 			throw new IOException(e.toString());
 		} finally {
-			instream.close();
+		    // Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
+            EntityUtils.consume(entity);
 		}
 	}
 
-    private void setHeaders(final HttpUriRequest httpUriRequest) {
+    private void setHeaders() {
     	if (this.headers != null) {
             for (final Entry<String, String> entry : this.headers) {
-                    httpUriRequest.setHeader(entry.getKey(),entry.getValue());
+                this.currentRequest.setHeader(entry.getKey(),entry.getValue());
             }
     	}
-    	if (this.host != null) httpUriRequest.setHeader(HTTP.TARGET_HOST, this.host);
-        httpUriRequest.setHeader(HTTP.CONN_DIRECTIVE, "close"); // don't keep alive, prevent CLOSE_WAIT state
+    	if (this.host != null) this.currentRequest.setHeader(HTTP.TARGET_HOST, this.host);
+    	this.currentRequest.setHeader(HTTP.CONN_DIRECTIVE, "close"); // don't keep alive, prevent CLOSE_WAIT state
     }
 
-    private void storeConnectionInfo(final HttpUriRequest httpUriRequest) {
-    	final int port = httpUriRequest.getURI().getPort();
-    	final String thost = httpUriRequest.getURI().getHost();
+    private void storeConnectionInfo() {
+    	final int port = this.currentRequest.getURI().getPort();
+    	final String thost = this.currentRequest.getURI().getHost();
     	//assert thost != null : "uri = " + httpUriRequest.getURI().toString();
     	ConnectionInfo.addConnection(new ConnectionInfo(
-    			httpUriRequest.getURI().getScheme(),
+    	        this.currentRequest.getURI().getScheme(),
     			port == -1 ? thost : thost + ":" + port,
-    			httpUriRequest.getMethod() + " " + httpUriRequest.getURI().getPath(),
-    			httpUriRequest.hashCode(),
+    			        this.currentRequest.getMethod() + " " + this.currentRequest.getURI().getPath(),
+    			        this.currentRequest.hashCode(),
     			System.currentTimeMillis(),
     			this.upbytes));
     }
diff --git a/source/net/yacy/cora/util/HTTPInputStream.java b/source/net/yacy/cora/util/HTTPInputStream.java
index 035edfad9..94eb9e957 100755
--- a/source/net/yacy/cora/util/HTTPInputStream.java
+++ b/source/net/yacy/cora/util/HTTPInputStream.java
@@ -60,7 +60,7 @@ public class HTTPInputStream extends InputStream {
 	 */
 	@Override
 	public void close() throws IOException {
-		httpClient.finish();
+		httpClient.close();
 	}
 
 
diff --git a/source/net/yacy/crawler/retrieval/HTTPLoader.java b/source/net/yacy/crawler/retrieval/HTTPLoader.java
index 62edd0fbf..acdc452e6 100644
--- a/source/net/yacy/crawler/retrieval/HTTPLoader.java
+++ b/source/net/yacy/crawler/retrieval/HTTPLoader.java
@@ -140,123 +140,124 @@ public final class HTTPLoader {
         final RequestHeader requestHeader = createRequestheader(request, agent);

 

         // HTTP-Client

-        final HTTPClient client = new HTTPClient(agent);

-        client.setRedirecting(false); // we want to handle redirection

-                                        // ourselves, so we don't index pages

-                                        // twice

-        client.setTimout(this.socketTimeout);

-        client.setHeader(requestHeader.entrySet());

-

-        // send request

-        client.GET(url, false);

-        final StatusLine statusline = client.getHttpResponse().getStatusLine();

-        final int statusCode = statusline.getStatusCode();

-        final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());

-        String requestURLString = request.url().toNormalform(true);

-

-        // check redirection

-        if (statusCode > 299 && statusCode < 310) {

-            client.finish();

-

-            final DigestURL redirectionUrl = extractRedirectURL(request, profile, url, statusline, responseHeader, requestURLString);

-

-            if (this.sb.getConfigBool(SwitchboardConstants.CRAWLER_FOLLOW_REDIRECTS, true)) {

-                // we have two use cases here: loading from a crawl or just

-                // loading the url. Check this:

-                if (profile != null && !CrawlSwitchboard.DEFAULT_PROFILES.contains(profile.name())) {

-                    // put redirect url on the crawler queue to repeat a

-                    // double-check

-                    /* We have to clone the request instance and not to modify directly its URL, 

-                     * otherwise the stackCrawl() function would reject it, because detecting it as already in the activeWorkerEntries */

-                    Request redirectedRequest = new Request(request.initiator(),

-                            redirectionUrl,

-                            request.referrerhash(),

-                            request.name(),

-                            request.appdate(),

-                            request.profileHandle(),

-                            request.depth(),

-                            request.timezoneOffset());

-                    String rejectReason = this.sb.crawlStacker.stackCrawl(redirectedRequest);

-                    if(rejectReason != null) {

-                        throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " aborted. Reason : " + rejectReason);

+        try (final HTTPClient client = new HTTPClient(agent)) {

+            client.setRedirecting(false); // we want to handle redirection

+                                            // ourselves, so we don't index pages

+                                            // twice

+            client.setTimout(this.socketTimeout);

+            client.setHeader(requestHeader.entrySet());

+    

+            // send request

+            client.GET(url, false);

+            final StatusLine statusline = client.getHttpResponse().getStatusLine();

+            final int statusCode = statusline.getStatusCode();

+            final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());

+            String requestURLString = request.url().toNormalform(true);

+    

+            // check redirection

+            if (statusCode > 299 && statusCode < 310) {

+                client.close();

+    

+                final DigestURL redirectionUrl = extractRedirectURL(request, profile, url, statusline, responseHeader, requestURLString);

+    

+                if (this.sb.getConfigBool(SwitchboardConstants.CRAWLER_FOLLOW_REDIRECTS, true)) {

+                    // we have two use cases here: loading from a crawl or just

+                    // loading the url. Check this:

+                    if (profile != null && !CrawlSwitchboard.DEFAULT_PROFILES.contains(profile.name())) {

+                        // put redirect url on the crawler queue to repeat a

+                        // double-check

+                        /* We have to clone the request instance and not to modify directly its URL, 

+                         * otherwise the stackCrawl() function would reject it, because detecting it as already in the activeWorkerEntries */

+                        Request redirectedRequest = new Request(request.initiator(),

+                                redirectionUrl,

+                                request.referrerhash(),

+                                request.name(),

+                                request.appdate(),

+                                request.profileHandle(),

+                                request.depth(),

+                                request.timezoneOffset());

+                        String rejectReason = this.sb.crawlStacker.stackCrawl(redirectedRequest);

+                        if(rejectReason != null) {

+                            throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " aborted. Reason : " + rejectReason);

+                        }

+                        // in the end we must throw an exception (even if this is

+                        // not an error, just to abort the current process

+                        throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " to "

+                                + redirectionUrl.toNormalform(false) + " placed on crawler queue for double-check");

                     }

-                    // in the end we must throw an exception (even if this is

-                    // not an error, just to abort the current process

-                    throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " to "

-                            + redirectionUrl.toNormalform(false) + " placed on crawler queue for double-check");

-                }

-

-                // if we are already doing a shutdown we don't need to retry

-                // crawling

-                if (Thread.currentThread().isInterrupted()) {

-                    this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile,

-                            FailCategory.FINAL_LOAD_CONTEXT, "server shutdown", statusCode);

-                    throw new IOException(

-                            "CRAWLER Redirect of URL=" + requestURLString + " aborted because of server shutdown.$");

-                }

-

-                // check if the redirected URL is the same as the requested URL

-                // this shortcuts a time-out using retryCount

-                if (redirectionUrl.equals(url)) {

-                    this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, "redirect to same url", -1);

-                    throw new IOException( "retry counter exceeded for URL " + request.url().toString() + ". Processing aborted.$");

-                }

-

-                // retry crawling with new url

-                request.redirectURL(redirectionUrl);

-                return openInputStream(request, profile, retryCount - 1, maxFileSize, blacklistType, agent);

-            }

-            // we don't want to follow redirects

-            this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, "redirection not wanted", statusCode);

-            throw new IOException("REJECTED UNWANTED REDIRECTION '" + statusline + "' for URL '" + requestURLString + "'$");

-        } else if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_NON_AUTHORITATIVE_INFORMATION) {

-            // the transfer is ok

-

-            /*

-             * When content is not large (less than Response.CRAWLER_MAX_SIZE_TO_CACHE), we have better cache it if cache is enabled and url is not local

-             */

-            long contentLength = client.getHttpResponse().getEntity().getContentLength();

-            InputStream contentStream;

-            if (profile != null && profile.storeHTCache() && contentLength > 0 && contentLength < (Response.CRAWLER_MAX_SIZE_TO_CACHE) && !url.isLocal()) {

-                byte[] content = null;

-                try {

-                    content = HTTPClient.getByteArray(client.getHttpResponse().getEntity(), maxFileSize);

-                    Cache.store(url, responseHeader, content);

-                } catch (final IOException e) {

-                    this.log.warn("cannot write " + url + " to Cache (3): " + e.getMessage(), e);

-                } finally {

-                    client.finish();

-                }

-

-                contentStream = new ByteArrayInputStream(content);

-            } else {

-                /*

-                 * Content length may already be known now : check it before opening a stream

-                 */

-                if (maxFileSize >= 0 && contentLength > maxFileSize) {

-                    throw new IOException("Content to download exceed maximum value of " + maxFileSize + " bytes");

+    

+                    // if we are already doing a shutdown we don't need to retry

+                    // crawling

+                    if (Thread.currentThread().isInterrupted()) {

+                        this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile,

+                                FailCategory.FINAL_LOAD_CONTEXT, "server shutdown", statusCode);

+                        throw new IOException(

+                                "CRAWLER Redirect of URL=" + requestURLString + " aborted because of server shutdown.$");

+                    }

+    

+                    // check if the redirected URL is the same as the requested URL

+                    // this shortcuts a time-out using retryCount

+                    if (redirectionUrl.equals(url)) {

+                        this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, "redirect to same url", -1);

+                        throw new IOException( "retry counter exceeded for URL " + request.url().toString() + ". Processing aborted.$");

+                    }

+    

+                    // retry crawling with new url

+                    request.redirectURL(redirectionUrl);

+                    return openInputStream(request, profile, retryCount - 1, maxFileSize, blacklistType, agent);

                 }

+                // we don't want to follow redirects

+                this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, "redirection not wanted", statusCode);

+                throw new IOException("REJECTED UNWANTED REDIRECTION '" + statusline + "' for URL '" + requestURLString + "'$");

+            } else if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_NON_AUTHORITATIVE_INFORMATION) {

+                // the transfer is ok

+    

                 /*

-                 * Create a HTTPInputStream delegating to

-                 * client.getContentstream(). Close method will ensure client is

-                 * properly closed.

+                 * When content is not large (less than Response.CRAWLER_MAX_SIZE_TO_CACHE), we have better cache it if cache is enabled and url is not local

                  */

-                contentStream = new HTTPInputStream(client);

-                /* Anticipated content length may not be already known or incorrect : let's apply now the same eventual content size restriction as when loading in a byte array */

-                if(maxFileSize >= 0) {

-                    contentStream = new StrictLimitInputStream(contentStream, maxFileSize,

-                            "Content to download exceed maximum value of " + Formatter.bytesToString(maxFileSize));

+                long contentLength = client.getHttpResponse().getEntity().getContentLength();

+                InputStream contentStream;

+                if (profile != null && profile.storeHTCache() && contentLength > 0 && contentLength < (Response.CRAWLER_MAX_SIZE_TO_CACHE) && !url.isLocal()) {

+                    byte[] content = null;

+                    try {

+                        content = HTTPClient.getByteArray(client.getHttpResponse().getEntity(), maxFileSize);

+                        Cache.store(url, responseHeader, content);

+                    } catch (final IOException e) {

+                        this.log.warn("cannot write " + url + " to Cache (3): " + e.getMessage(), e);

+                    } finally {

+                        client.close();

+                    }

+    

+                    contentStream = new ByteArrayInputStream(content);

+                } else {

+                    /*

+                     * Content length may already be known now : check it before opening a stream

+                     */

+                    if (maxFileSize >= 0 && contentLength > maxFileSize) {

+                        throw new IOException("Content to download exceed maximum value of " + maxFileSize + " bytes");

+                    }

+                    /*

+                     * Create a HTTPInputStream delegating to

+                     * client.getContentstream(). Close method will ensure client is

+                     * properly closed.

+                     */

+                    contentStream = new HTTPInputStream(client);

+                    /* Anticipated content length may not be already known or incorrect : let's apply now the same eventual content size restriction as when loading in a byte array */

+                    if(maxFileSize >= 0) {

+                        contentStream = new StrictLimitInputStream(contentStream, maxFileSize,

+                                "Content to download exceed maximum value of " + Formatter.bytesToString(maxFileSize));

+                    }

                 }

+    

+                return new StreamResponse(new Response(request, requestHeader, responseHeader, profile, false, null), contentStream);

+            } else {

+                client.close();

+                // if the response has not the right response type then reject file

+                this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile,

+                        FailCategory.TEMPORARY_NETWORK_FAILURE, "wrong http status code", statusCode);

+                throw new IOException("REJECTED WRONG STATUS TYPE '" + statusline

+                        + "' for URL '" + requestURLString + "'$");

             }

-

-            return new StreamResponse(new Response(request, requestHeader, responseHeader, profile, false, null), contentStream);

-        } else {

-            client.finish();

-            // if the response has not the right response type then reject file

-            this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile,

-                    FailCategory.TEMPORARY_NETWORK_FAILURE, "wrong http status code", statusCode);

-            throw new IOException("REJECTED WRONG STATUS TYPE '" + statusline

-                    + "' for URL '" + requestURLString + "'$");

         }

     }

 

@@ -364,90 +365,91 @@ public final class HTTPLoader {
         final RequestHeader requestHeader = createRequestheader(request, agent);

 

         // HTTP-Client

-        final HTTPClient client = new HTTPClient(agent);

-        client.setRedirecting(false); // we want to handle redirection ourselves, so we don't index pages twice

-        client.setTimout(this.socketTimeout);

-        client.setHeader(requestHeader.entrySet());

-

-        // send request

-        final byte[] responseBody = client.GETbytes(url, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""), maxFileSize, false);

-        final int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();

-        final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());

-        String requestURLString = request.url().toNormalform(true);

-

-        // check redirection

-        if (statusCode > 299 && statusCode < 310) {

-

-            final DigestURL redirectionUrl = extractRedirectURL(request, profile, url, client.getHttpResponse().getStatusLine(),

-                    responseHeader, requestURLString);

-

-            if (this.sb.getConfigBool(SwitchboardConstants.CRAWLER_FOLLOW_REDIRECTS, true)) {

-                // we have two use cases here: loading from a crawl or just loading the url. Check this:

-                if (profile != null && !CrawlSwitchboard.DEFAULT_PROFILES.contains(profile.name())) {

-                    // put redirect url on the crawler queue to repeat a double-check

-                    /* We have to clone the request instance and not to modify directly its URL, 

-                     * otherwise the stackCrawl() function would reject it, because detecting it as already in the activeWorkerEntries */

-                    Request redirectedRequest = new Request(request.initiator(),

-                            redirectionUrl,

-                            request.referrerhash(),

-                            request.name(),

-                            request.appdate(),

-                            request.profileHandle(),

-                            request.depth(),

-                            request.timezoneOffset());

-                    String rejectReason = this.sb.crawlStacker.stackCrawl(redirectedRequest);

-                    // in the end we must throw an exception (even if this is not an error, just to abort the current process

-                    if(rejectReason != null) {

-                        throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " aborted. Reason : " + rejectReason);

+        try (final HTTPClient client = new HTTPClient(agent)) {

+            client.setRedirecting(false); // we want to handle redirection ourselves, so we don't index pages twice

+            client.setTimout(this.socketTimeout);

+            client.setHeader(requestHeader.entrySet());

+    

+            // send request

+            final byte[] responseBody = client.GETbytes(url, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""), maxFileSize, false);

+            final int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();

+            final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());

+            String requestURLString = request.url().toNormalform(true);

+    

+            // check redirection

+            if (statusCode > 299 && statusCode < 310) {

+    

+                final DigestURL redirectionUrl = extractRedirectURL(request, profile, url, client.getHttpResponse().getStatusLine(),

+                        responseHeader, requestURLString);

+    

+                if (this.sb.getConfigBool(SwitchboardConstants.CRAWLER_FOLLOW_REDIRECTS, true)) {

+                    // we have two use cases here: loading from a crawl or just loading the url. Check this:

+                    if (profile != null && !CrawlSwitchboard.DEFAULT_PROFILES.contains(profile.name())) {

+                        // put redirect url on the crawler queue to repeat a double-check

+                        /* We have to clone the request instance and not to modify directly its URL, 

+                         * otherwise the stackCrawl() function would reject it, because detecting it as already in the activeWorkerEntries */

+                        Request redirectedRequest = new Request(request.initiator(),

+                                redirectionUrl,

+                                request.referrerhash(),

+                                request.name(),

+                                request.appdate(),

+                                request.profileHandle(),

+                                request.depth(),

+                                request.timezoneOffset());

+                        String rejectReason = this.sb.crawlStacker.stackCrawl(redirectedRequest);

+                        // in the end we must throw an exception (even if this is not an error, just to abort the current process

+                        if(rejectReason != null) {

+                            throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " aborted. Reason : " + rejectReason);

+                        }

+                        throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " to " + redirectionUrl.toNormalform(false) + " placed on crawler queue for double-check");

                     }

-                    throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " to " + redirectionUrl.toNormalform(false) + " placed on crawler queue for double-check");

+    

+                    // if we are already doing a shutdown we don't need to retry crawling

+                    if (Thread.currentThread().isInterrupted()) {

+                        this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.FINAL_LOAD_CONTEXT, "server shutdown", statusCode);

+                        throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " aborted because of server shutdown.$");

+                    }

+    

+                    // retry crawling with new url

+                    request.redirectURL(redirectionUrl);

+                    return load(request, profile, retryCount - 1, maxFileSize, blacklistType, agent);

                 }

-

-                // if we are already doing a shutdown we don't need to retry crawling

-                if (Thread.currentThread().isInterrupted()) {

-                    this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.FINAL_LOAD_CONTEXT, "server shutdown", statusCode);

-                    throw new IOException("CRAWLER Redirect of URL=" + requestURLString + " aborted because of server shutdown.$");

+                // we don't want to follow redirects

+                this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, "redirection not wanted", statusCode);

+                throw new IOException("REJECTED UNWANTED REDIRECTION '" + client.getHttpResponse().getStatusLine() + "' for URL '" + requestURLString + "'$");

+            } else if (responseBody == null) {

+                // no response, reject file

+                this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, "no response body", statusCode);

+                throw new IOException("REJECTED EMPTY RESPONSE BODY '" + client.getHttpResponse().getStatusLine() + "' for URL '" + requestURLString + "'$");

+            } else if (statusCode == 200 || statusCode == 203) {

+                // the transfer is ok

+    

+                // we write the new cache entry to file system directly

+                final long contentLength = responseBody.length;

+                ByteCount.addAccountCount(ByteCount.CRAWLER, contentLength);

+    

+                // check length again in case it was not possible to get the length before loading

+                if (maxFileSize >= 0 && contentLength > maxFileSize) {

+                    this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, "file size limit exceeded", statusCode);

+                    throw new IOException("REJECTED URL " + request.url() + " because file size '" + contentLength + "' exceeds max filesize limit of " + maxFileSize + " bytes. (GET)$");

                 }

-

-                // retry crawling with new url

-                request.redirectURL(redirectionUrl);

-                return load(request, profile, retryCount - 1, maxFileSize, blacklistType, agent);

-            }

-            // we don't want to follow redirects

-            this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, "redirection not wanted", statusCode);

-            throw new IOException("REJECTED UNWANTED REDIRECTION '" + client.getHttpResponse().getStatusLine() + "' for URL '" + requestURLString + "'$");

-        } else if (responseBody == null) {

-            // no response, reject file

-            this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, "no response body", statusCode);

-            throw new IOException("REJECTED EMPTY RESPONSE BODY '" + client.getHttpResponse().getStatusLine() + "' for URL '" + requestURLString + "'$");

-        } else if (statusCode == 200 || statusCode == 203) {

-            // the transfer is ok

-

-            // we write the new cache entry to file system directly

-            final long contentLength = responseBody.length;

-            ByteCount.addAccountCount(ByteCount.CRAWLER, contentLength);

-

-            // check length again in case it was not possible to get the length before loading

-            if (maxFileSize >= 0 && contentLength > maxFileSize) {

-                this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, "file size limit exceeded", statusCode);

-                throw new IOException("REJECTED URL " + request.url() + " because file size '" + contentLength + "' exceeds max filesize limit of " + maxFileSize + " bytes. (GET)$");

+    

+                // create a new cache entry

+                response = new Response(

+                        request,

+                        requestHeader,

+                        responseHeader,

+                        profile,

+                        false,

+                        responseBody

+                );

+    

+                return response;

+            } else {

+                // if the response has not the right response type then reject file

+                this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, "wrong http status code", statusCode);

+                throw new IOException("REJECTED WRONG STATUS TYPE '" + client.getHttpResponse().getStatusLine() + "' for URL '" + requestURLString + "'$");

             }

-

-            // create a new cache entry

-            response = new Response(

-                    request,

-                    requestHeader,

-                    responseHeader,

-                    profile,

-                    false,

-                    responseBody

-            );

-

-            return response;

-        } else {

-            // if the response has not the right response type then reject file

-            this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, "wrong http status code", statusCode);

-            throw new IOException("REJECTED WRONG STATUS TYPE '" + client.getHttpResponse().getStatusLine() + "' for URL '" + requestURLString + "'$");

         }

     }

 

@@ -484,9 +486,9 @@ public final class HTTPLoader {
         requestHeader.put(HeaderFramework.ACCEPT_CHARSET, DEFAULT_CHARSET);

         requestHeader.put(HeaderFramework.ACCEPT_ENCODING, DEFAULT_ENCODING);

 

-        final HTTPClient client = new HTTPClient(agent);

-        client.setTimout(20000);

-        client.setHeader(requestHeader.entrySet());

+        try (final HTTPClient client = new HTTPClient(agent)) {

+            client.setTimout(20000);

+            client.setHeader(requestHeader.entrySet());

             final byte[] responseBody = client.GETbytes(request.url(), null, null, false);

             final int code = client.getHttpResponse().getStatusLine().getStatusCode();

             final ResponseHeader header = new ResponseHeader(code, client.getHttpResponse().getAllHeaders());

@@ -539,6 +541,7 @@ public final class HTTPLoader {
                 // if the response has not the right response type then reject file

                 throw new IOException("REJECTED WRONG STATUS TYPE '" + client.getHttpResponse().getStatusLine() + "' for URL " + request.url().toString());

             }

+        }

         return response;

     }

 

diff --git a/source/net/yacy/data/WorkTables.java b/source/net/yacy/data/WorkTables.java
index 679c33057..0af5baf55 100644
--- a/source/net/yacy/data/WorkTables.java
+++ b/source/net/yacy/data/WorkTables.java
@@ -327,50 +327,53 @@ public class WorkTables extends Tables {
      * @return a map of the called urls and the http status code of the api call or -1 if any other IOException occurred
      */
     public Map<String, Integer> execAPICalls(String host, int port, Collection<String> pks, final String username, final String pass) {
-        // now call the api URLs and store the result status
-        final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
-        client.setTimout(120000);
-        Tables.Row row;
         LinkedHashMap<String, Integer> l = new LinkedHashMap<String, Integer>();
-        for (final String pk: pks) {
-            row = null;
-            try {
-                row = select(WorkTables.TABLE_API_NAME, UTF8.getBytes(pk));
-            } catch (final IOException e) {
-                ConcurrentLog.logException(e);
-            } catch (final SpaceExceededException e) {
-                ConcurrentLog.logException(e);
-            }
-            if (row == null) continue;
-            String theapicall = UTF8.String(row.get(WorkTables.TABLE_API_COL_URL)) + "&" + WorkTables.TABLE_API_COL_APICALL_PK + "=" + UTF8.String(row.getPK());
-            try {
-                MultiProtocolURL url = new MultiProtocolURL("http", host, port, theapicall);
-                final Map<String, String> attributes = url.getAttributes();
-                final boolean isTokenProtectedAPI = attributes.containsKey(TransactionManager.TRANSACTION_TOKEN_PARAM);
-                // use 4 param MultiProtocolURL to allow api_row_url with searchpart (like url?p=a&p2=b ) in client.GETbytes()
-                if (theapicall.length() > 1000 || isTokenProtectedAPI) {
-                    // use a POST to execute the call
-                    execPostAPICall(host, port, username, pass, client, l, url, isTokenProtectedAPI);
-                } else {
-                    // use a GET to execute the call
-                    ConcurrentLog.info("WorkTables", "executing url: " + url.toNormalform(true));
-                    try {
-                        client.GETbytes(url, username, pass, false); // use GETbytes(MultiProtocolURL,..) form to allow url in parameter (&url=path%
-                        if(client.getStatusCode() == HttpStatus.SC_METHOD_NOT_ALLOWED) {
-                        	/* GET method not allowed (HTTP 450 status) : this may be an old API entry,
-                        	 * now restricted to HTTP POST and requiring a transaction token. We try now with POST. */
-                        	execPostAPICall(host, port, username, pass, client, l, url, true);
-                        } else {
-                        	l.put(url.toNormalform(true), client.getStatusCode());
+        // now call the api URLs and store the result status
+        try (final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {
+            client.setTimout(120000);
+            Tables.Row row;
+            for (final String pk: pks) {
+                row = null;
+                try {
+                    row = select(WorkTables.TABLE_API_NAME, UTF8.getBytes(pk));
+                } catch (final IOException e) {
+                    ConcurrentLog.logException(e);
+                } catch (final SpaceExceededException e) {
+                    ConcurrentLog.logException(e);
+                }
+                if (row == null) continue;
+                String theapicall = UTF8.String(row.get(WorkTables.TABLE_API_COL_URL)) + "&" + WorkTables.TABLE_API_COL_APICALL_PK + "=" + UTF8.String(row.getPK());
+                try {
+                    MultiProtocolURL url = new MultiProtocolURL("http", host, port, theapicall);
+                    final Map<String, String> attributes = url.getAttributes();
+                    final boolean isTokenProtectedAPI = attributes.containsKey(TransactionManager.TRANSACTION_TOKEN_PARAM);
+                    // use 4 param MultiProtocolURL to allow api_row_url with searchpart (like url?p=a&p2=b ) in client.GETbytes()
+                    if (theapicall.length() > 1000 || isTokenProtectedAPI) {
+                        // use a POST to execute the call
+                        execPostAPICall(host, port, username, pass, client, l, url, isTokenProtectedAPI);
+                    } else {
+                        // use a GET to execute the call
+                        ConcurrentLog.info("WorkTables", "executing url: " + url.toNormalform(true));
+                        try {
+                            client.GETbytes(url, username, pass, false); // use GETbytes(MultiProtocolURL,..) form to allow url in parameter (&url=path%
+                            if(client.getStatusCode() == HttpStatus.SC_METHOD_NOT_ALLOWED) {
+                            	/* GET method not allowed (HTTP 450 status) : this may be an old API entry,
+                            	 * now restricted to HTTP POST and requiring a transaction token. We try now with POST. */
+                            	execPostAPICall(host, port, username, pass, client, l, url, true);
+                            } else {
+                            	l.put(url.toNormalform(true), client.getStatusCode());
+                            }
+                        } catch (final IOException e) {
+                            ConcurrentLog.logException(e);
+                            l.put(url.toString(), -1);
                         }
-                    } catch (final IOException e) {
-                        ConcurrentLog.logException(e);
-                        l.put(url.toString(), -1);
                     }
+                } catch (MalformedURLException ex) {
+                    ConcurrentLog.warn("APICALL", "wrong url in apicall " + theapicall);
                 }
-            } catch (MalformedURLException ex) {
-                ConcurrentLog.warn("APICALL", "wrong url in apicall " + theapicall);
             }
+        } catch (IOException e) {
+            ConcurrentLog.logException(e);
         }
         return l;
     }
@@ -447,11 +450,10 @@ public class WorkTables extends Tables {
 	 */
     public static int execGetAPICall(String host, int port, String path, byte[] pk, final String username, final String pass) {
         // now call the api URLs and store the result status
-        final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
-        client.setTimout(120000);
         String url = "http://" + host + ":" + port + path;
         if (pk != null) url += "&" + WorkTables.TABLE_API_COL_APICALL_PK + "=" + UTF8.String(pk);
-        try {
+        try (final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {
+            client.setTimout(120000);
             client.GETbytes(url, username, pass, false);
             return client.getStatusCode();
         } catch (final IOException e) {
diff --git a/source/net/yacy/document/parser/sitemapParser.java b/source/net/yacy/document/parser/sitemapParser.java
index 5486635c4..be52f72e7 100644
--- a/source/net/yacy/document/parser/sitemapParser.java
+++ b/source/net/yacy/document/parser/sitemapParser.java
@@ -114,9 +114,8 @@ public class sitemapParser extends AbstractParser implements Parser {
     public static SitemapReader parse(final DigestURL sitemapURL, final ClientIdentification.Agent agent) throws IOException {
         // download document
         ConcurrentLog.info("SitemapReader", "loading sitemap from " + sitemapURL.toNormalform(true));
-        final HTTPClient client = new HTTPClient(agent);
         // client.setHeader(requestHeader.entrySet());
-        try {
+        try (final HTTPClient client = new HTTPClient(agent)) {
             client.GET(sitemapURL.toNormalform(false), false);
             if (client.getStatusCode() != 200) {
                 throw new IOException("Unable to download the sitemap file " + sitemapURL +
diff --git a/source/net/yacy/document/parser/xml/opensearchdescriptionReader.java b/source/net/yacy/document/parser/xml/opensearchdescriptionReader.java
index e6a20a064..37866ebc5 100644
--- a/source/net/yacy/document/parser/xml/opensearchdescriptionReader.java
+++ b/source/net/yacy/document/parser/xml/opensearchdescriptionReader.java
@@ -147,19 +147,12 @@ public class opensearchdescriptionReader extends DefaultHandler {
     public opensearchdescriptionReader(final String path, final ClientIdentification.Agent agent) {
         this();
         this.agent = agent;
-        HTTPClient www = new HTTPClient(agent);
-        try {
+        try (HTTPClient www = new HTTPClient(agent)) {
             www.GET(path, false);
             final SAXParser saxParser = getParser();
             saxParser.parse(www.getContentstream(), this);
         } catch (final Exception e) {
             ConcurrentLog.logException(e);
-        } finally {
-        	try {
-				www.finish();
-			} catch (final IOException e) {
-				ConcurrentLog.logException(e);
-			}
         }
     }
 
@@ -170,8 +163,7 @@ public class opensearchdescriptionReader extends DefaultHandler {
         this.parsingTextValue = false;
         this.rssurl = null;
         this.atomurl = null;
-        HTTPClient www = new HTTPClient(this.agent);
-        try {
+        try (HTTPClient www = new HTTPClient(this.agent)) {
             www.GET(path, false);
             final SAXParser saxParser = getParser();
             try {
@@ -185,12 +177,6 @@ public class opensearchdescriptionReader extends DefaultHandler {
         } catch (final Exception e) {
             ConcurrentLog.warn("opensearchdescriptionReader", "parse exception: " + e);
             return false;
-        } finally {
-        	try {
-				www.finish();
-			} catch (final IOException e) {
-				ConcurrentLog.logException(e);
-			}
         }
     }
 
diff --git a/source/net/yacy/http/ProxyHandler.java b/source/net/yacy/http/ProxyHandler.java
index 7dcfc438b..2892c0039 100644
--- a/source/net/yacy/http/ProxyHandler.java
+++ b/source/net/yacy/http/ProxyHandler.java
@@ -132,12 +132,11 @@ public class ProxyHandler extends AbstractRemoteHandler implements Handler {
         RequestHeader proxyHeaders = ProxyHandler.convertHeaderFromJetty(request);
         setProxyHeaderForClient(request, proxyHeaders);
 
-        final HTTPClient client = new HTTPClient(ClientIdentification.yacyProxyAgent);
-        client.setTimout(timeout);
-        client.setHeader(proxyHeaders.entrySet());
-        client.setRedirecting(false);
         // send request
-        try {
+        try (final HTTPClient client = new HTTPClient(ClientIdentification.yacyProxyAgent)) {
+            client.setTimout(timeout);
+            client.setHeader(proxyHeaders.entrySet());
+            client.setRedirecting(false);
             String queryString = request.getQueryString() != null ? "?" + request.getQueryString() : "";
             DigestURL digestURI = new DigestURL(request.getScheme(), request.getServerName(), request.getServerPort(), request.getRequestURI() + queryString);
             if (request.getMethod().equals(HeaderFramework.METHOD_GET)) {
@@ -219,8 +218,6 @@ public class ProxyHandler extends AbstractRemoteHandler implements Handler {
             }
             } catch (final SocketException se) {
                 throw new ServletException("Socket Exception: " + se.getMessage());
-            } finally {
-                client.finish();
             }
         
         // we handled this request, break out of handler chain
diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java
index 4260da7bf..5611b033e 100644
--- a/source/net/yacy/peers/Protocol.java
+++ b/source/net/yacy/peers/Protocol.java
@@ -159,11 +159,11 @@ public final class Protocol {
             final String path,

             final Map<String, ContentBody> parts,

             final int timeout) throws IOException {

-            final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);

-            httpClient.setTimout(timeout);

-            MultiProtocolURL targetURL = new MultiProtocolURL(targetBaseURL, path);

-			this.result = httpClient.POSTbytes(targetURL, Seed.b64Hash2hexHash(targetHash) + ".yacyh", parts, false,

-					true);

+            try (final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {

+                httpClient.setTimout(timeout);

+                MultiProtocolURL targetURL = new MultiProtocolURL(targetBaseURL, path);

+    			this.result = httpClient.POSTbytes(targetURL, Seed.b64Hash2hexHash(targetHash) + ".yacyh", parts, false, true);

+            }

         }

         

         /**

@@ -197,19 +197,16 @@ public final class Protocol {
         final String salt = crypt.randomSalt();

         long responseTime = Long.MAX_VALUE;

         byte[] content = null;

-        try {

+        try (final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 30000)) {

             // generate request

-            final Map<String, ContentBody> parts =

-                basicRequestParts(Switchboard.getSwitchboard(), null, salt);

+            final Map<String, ContentBody> parts = basicRequestParts(Switchboard.getSwitchboard(), null, salt);

             parts.put("count", UTF8.StringBody("20"));

             parts.put("magic", UTF8.StringBody(Long.toString(Network.magic)));

             parts.put("seed", UTF8.StringBody(mySeed.genSeedStr(salt)));

             // send request

             final long start = System.currentTimeMillis();

             // final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/hello.html"), 30000, yacySeed.b64Hash2hexHash(otherHash) + ".yacyh", parts);

-            final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 30000);

-            content =

-                httpClient.POSTbytes(

+            content = httpClient.POSTbytes(

                     new MultiProtocolURL(targetBaseURL, "/yacy/hello.html"),

                     Seed.b64Hash2hexHash(targetHash) + ".yacyh",

                     parts,

@@ -433,41 +430,44 @@ public final class Protocol {
         parts.put("count", UTF8.StringBody(Integer.toString(maxCount)));

         parts.put("time", UTF8.StringBody(Long.toString(maxTime)));

         // final byte[] result = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/urls.xml"), (int) maxTime, target.getHexHash() + ".yacyh", parts);

-        final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, (int) maxTime);

         RSSReader reader = null;

-        for (final String ip: target.getIPs()) {

-        	MultiProtocolURL targetBaseURL = null;

-            try {

-            	targetBaseURL = target.getPublicMultiprotocolURL(ip, preferHttps);

-            	byte[] result;

-            	try {

-            		result = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false, true);

-            	} catch(final IOException e) {

-            		if(targetBaseURL.isHTTPS()) {

-            			/* Failed with https : retry with http */

-            			targetBaseURL = target.getPublicMultiprotocolURL(ip, false);

-            			result = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false, true);

-            			if(result != null) {

-							/* Got something with http : mark peer SSL as unavailable on target peer */

-							markSSLUnavailableOnPeer(seedDB, target, ip, "yacyClient.queryRemoteCrawlURLs");

-            			}

-            		} else {

-            			throw e;

-            		}

-            	}

-                reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result);

-            } catch(MalformedURLException e) {

-				Network.log.warn("yacyClient.queryRemoteCrawlURLs malformed target URL for peer '" + target.getName()

-						+ "' on address : " + ip);

-            } catch (final IOException e ) {

-                reader = null;

-                Network.log.warn("yacyClient.queryRemoteCrawlURLs failed asking peer '" + target.getName() + "': probably bad response from remote peer (1), reader == null");

-            }

-            if (reader != null) {

-            	break;

+        try (final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, (int) maxTime)) {

+            for (final String ip: target.getIPs()) {

+            	MultiProtocolURL targetBaseURL = null;

+                try {

+                	targetBaseURL = target.getPublicMultiprotocolURL(ip, preferHttps);

+                	byte[] result;

+                	try {

+                		result = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false, true);

+                	} catch(final IOException e) {

+                		if(targetBaseURL.isHTTPS()) {

+                			/* Failed with https : retry with http */

+                			targetBaseURL = target.getPublicMultiprotocolURL(ip, false);

+                			result = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false, true);

+                			if(result != null) {

+    							/* Got something with http : mark peer SSL as unavailable on target peer */

+    							markSSLUnavailableOnPeer(seedDB, target, ip, "yacyClient.queryRemoteCrawlURLs");

+                			}

+                		} else {

+                			throw e;

+                		}

+                	}

+                    reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result);

+                } catch(MalformedURLException e) {

+    				Network.log.warn("yacyClient.queryRemoteCrawlURLs malformed target URL for peer '" + target.getName()

+    						+ "' on address : " + ip);

+                } catch (final IOException e ) {

+                    reader = null;

+                    Network.log.warn("yacyClient.queryRemoteCrawlURLs failed asking peer '" + target.getName() + "': probably bad response from remote peer (1), reader == null");

+                }

+                if (reader != null) {

+                	break;

+                }

+                target.put(Seed.RCOUNT, "0");

+                seedDB.peerActions.interfaceDeparture(target, ip);

             }

-            target.put(Seed.RCOUNT, "0");

-            seedDB.peerActions.interfaceDeparture(target, ip);

+        } catch (IOException e) {

+            Network.log.warn(e);

         }

         

         final RSSFeed feed = reader == null ? null : reader.getFeed();

@@ -962,13 +962,14 @@ public final class Protocol {
                 //resultMap = FileUtils.table(HTTPConnector.getConnector(MultiProtocolURI.crawlerUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/search.html"), 60000, target.getHexHash() + ".yacyh", parts));

             }

 

-            final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 8000);

-            byte[] a = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL + "/yacy/search.html"), hostname, parts, false, true);

-            if (a != null && a.length > 200000) {

-                // there is something wrong. This is too large, maybe a hack on the other side?

-                a = null;

+            try (final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 8000)) {

+                byte[] a = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL + "/yacy/search.html"), hostname, parts, false, true);

+                if (a != null && a.length > 200000) {

+                    // there is something wrong. This is too large, maybe a hack on the other side?

+                    a = null;

+                }

+                resultMap = FileUtils.table(a);

             }

-            resultMap = FileUtils.table(a);

 

             // evaluate request result

             if ( resultMap == null || resultMap.isEmpty() ) {

@@ -1628,25 +1629,26 @@ public final class Protocol {
 				}

 				parts.put("lurlEntry", UTF8.StringBody(crypt.simpleEncode(lurlstr, salt)));

 				// send request

-				final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 10000);

-				MultiProtocolURL targetBaseURL = target.getPublicMultiprotocolURL(ip, preferHttps);

 				byte[] content;

-				try {

-					content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/crawlReceipt.html"),

-							target.getHexHash() + ".yacyh", parts, false, true);

-				} catch(final IOException e) {

-					if(targetBaseURL.isHTTPS()) {

-						/* Failed using https : retry with http */

-						targetBaseURL = target.getPublicMultiprotocolURL(ip, false);

-						content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/crawlReceipt.html"),

-								target.getHexHash() + ".yacyh", parts, false, true);

-						if(content != null) {

-							/* Success with http : mark SSL as unavailable on the target peer */

-							markSSLUnavailableOnPeer(sb.peers, target, ip, "yacyClient.crawlReceipt");

-						}

-					} else {

-						throw e;

-					}

+				try (final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 10000)) {

+    				MultiProtocolURL targetBaseURL = target.getPublicMultiprotocolURL(ip, preferHttps);

+    				try {

+    					content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/crawlReceipt.html"),

+    							target.getHexHash() + ".yacyh", parts, false, true);

+    				} catch(final IOException e) {

+    					if(targetBaseURL.isHTTPS()) {

+    						/* Failed using https : retry with http */

+    						targetBaseURL = target.getPublicMultiprotocolURL(ip, false);

+    						content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/crawlReceipt.html"),

+    								target.getHexHash() + ".yacyh", parts, false, true);

+    						if(content != null) {

+    							/* Success with http : mark SSL as unavailable on the target peer */

+    							markSSLUnavailableOnPeer(sb.peers, target, ip, "yacyClient.crawlReceipt");

+    						}

+    					} else {

+    						throw e;

+    					}

+    				}

 				}

 				return FileUtils.table(content);

 			} catch (final Exception e ) {

@@ -1849,23 +1851,24 @@ public final class Protocol {
                 parts.put("wordc", UTF8.StringBody(Integer.toString(indexes.size())));

                 parts.put("entryc", UTF8.StringBody(Integer.toString(indexcount)));

                 parts.put("indexes", UTF8.StringBody(entrypost.toString()));

-                final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout);

                 byte[] content = null;

-                try {

-					content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferRWI.html"),

-							targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);

-                } catch(final IOException e) {

-                    if(targetBaseURL.isHTTPS()) {

-                    	targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, false);

-                    	/* Failed with https : retry with http on the same address */

-						content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferRWI.html"),

-								targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);

-						if(content != null) {

-							/* Success with http : mark SSL as unavailable on the target peer */

-            				markSSLUnavailableOnPeer(Switchboard.getSwitchboard().peers, targetSeed, ip, "yacyClient.transferRWI");

-						}

-                    } else {

-                    	throw e;

+                try (final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout)) {

+                    try {

+    					content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferRWI.html"),

+    							targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);

+                    } catch(final IOException e) {

+                        if(targetBaseURL.isHTTPS()) {

+                        	targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, false);

+                        	/* Failed with https : retry with http on the same address */

+    						content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferRWI.html"),

+    								targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);

+    						if(content != null) {

+    							/* Success with http : mark SSL as unavailable on the target peer */

+                				markSSLUnavailableOnPeer(Switchboard.getSwitchboard().peers, targetSeed, ip, "yacyClient.transferRWI");

+    						}

+                        } else {

+                        	throw e;

+                        }

                     }

                 }

                 final Iterator<String> v = FileUtils.strings(content);

@@ -1953,20 +1956,21 @@ public final class Protocol {
                 MultiProtocolURL targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, preferHttps);

                 

                 parts.put("urlc", UTF8.StringBody(Integer.toString(urlc)));

-                final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout);

                 byte[] content = null;

-                try {

-					content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferURL.html"),

-							targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);

-                } catch(final IOException e) {

-                	if(targetBaseURL.isHTTPS()) {

-                		targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, false);

-                		/* Failed with https : retry with http on the same address */

+                try (final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout)) {

+                    try {

     					content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferURL.html"),

-    							targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);                		

-                	} else {

-                		throw e;

-                	}

+    							targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);

+                    } catch(final IOException e) {

+                    	if(targetBaseURL.isHTTPS()) {

+                    		targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, false);

+                    		/* Failed with https : retry with http on the same address */

+        					content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferURL.html"),

+        							targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);                		

+                    	} else {

+                    		throw e;

+                    	}

+                    }

                 }

                 final Iterator<String> v = FileUtils.strings(content);

 

@@ -1998,10 +2002,8 @@ public final class Protocol {
 				SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT);

 

         for (final String ip : targetSeed.getIPs()) {

-            try {

-                final Map<String, ContentBody> parts =

-                    basicRequestParts(sb, targetSeed.hash, salt);

-                final HTTPClient httpclient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 15000);

+            try (final HTTPClient httpclient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 15000)) {

+                final Map<String, ContentBody> parts = basicRequestParts(sb, targetSeed.hash, salt);

                 MultiProtocolURL targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, preferHttps);

                 byte[] content;

                 try {

diff --git a/source/net/yacy/peers/SeedDB.java b/source/net/yacy/peers/SeedDB.java
index 8244a2e2a..ca0cd0a35 100644
--- a/source/net/yacy/peers/SeedDB.java
+++ b/source/net/yacy/peers/SeedDB.java
@@ -897,19 +897,20 @@ public final class SeedDB implements AlternativeDomainNames {
         reqHeader.put(HeaderFramework.CACHE_CONTROL, "no-cache, no-store"); // httpc uses HTTP/1.0 is this necessary?

         reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent);

 

-        final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);

-        client.setHeader(reqHeader.entrySet());

         byte[] content = null;

-        try {

-            // send request

-        	content = client.GETbytes(seedURL, null, null, false);

-        } catch (final Exception e) {

-        	throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage());

-        }

-

-        // check response code

-        if (client.getHttpResponse().getStatusLine().getStatusCode() != 200) {

-        	throw new IOException("Server returned status: " + client.getHttpResponse().getStatusLine());

+        try (final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {

+            client.setHeader(reqHeader.entrySet());

+            try {

+                // send request

+            	content = client.GETbytes(seedURL, null, null, false);

+            } catch (final Exception e) {

+            	throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage());

+            }

+    

+            // check response code

+            if (client.getHttpResponse().getStatusLine().getStatusCode() != 200) {

+            	throw new IOException("Server returned status: " + client.getHttpResponse().getStatusLine());

+            }

         }

 

         try {

@@ -1124,13 +1125,12 @@ public final class SeedDB implements AlternativeDomainNames {
             @Override

             public void run() {

                 // load the seed list

-                try {

+                try (final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout)) {

                     DigestURL url = new DigestURL(seedListFileURL);

                     //final long start = System.currentTimeMillis();

                     final RequestHeader reqHeader = new RequestHeader();

                     reqHeader.put(HeaderFramework.PRAGMA, "no-cache");

                     reqHeader.put(HeaderFramework.CACHE_CONTROL, "no-cache, no-store");

-                    final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout);

                     client.setHeader(reqHeader.entrySet());

 

                     client.HEADResponse(url.toNormalform(false), false);

diff --git a/source/net/yacy/server/http/HTTPDProxyHandler.java b/source/net/yacy/server/http/HTTPDProxyHandler.java
index c63b4ee9c..493cd7a6f 100644
--- a/source/net/yacy/server/http/HTTPDProxyHandler.java
+++ b/source/net/yacy/server/http/HTTPDProxyHandler.java
@@ -444,10 +444,10 @@ public final class HTTPDProxyHandler {
 

             requestHeader.remove(HeaderFramework.HOST);

 

-            final HTTPClient client = setupHttpClient(requestHeader, agent);

-

             // send request

-            try {

+            try (final HTTPClient client = new HTTPClient(agent, timeout)) {

+                client.setHeader(requestHeader.entrySet());

+                client.setRedirecting(false);

             	client.GET(getUrl, false);

                 if (log.isFinest()) log.finest(reqID +"    response status: "+ client.getHttpResponse().getStatusLine());

 

@@ -596,20 +596,7 @@ public final class HTTPDProxyHandler {
                     }

                 } // end hasBody

             } catch(final SocketException se) {

-                // if opened ...

-//                if(res != null) {

-//                    // client cut proxy connection, abort download

-//                    res.abort();

-//                }

-            	client.finish();

                 handleProxyException(se,conProp,respond,url);

-            } finally {

-                // if opened ...

-//                if(res != null) {

-//                    // ... close connection

-//                    res.closeStream();

-//                }

-            	client.finish();

             }

         } catch (final Exception e) {

             handleProxyException(e,conProp,respond,url);

@@ -760,20 +747,6 @@ public final class HTTPDProxyHandler {
     }

 

     /**

-     * creates a new HttpClient and sets parameters according to proxy needs

-     *

-     * @param requestHeader

-     * @return

-     */

-    private static HTTPClient setupHttpClient(final RequestHeader requestHeader, final ClientIdentification.Agent agent) {

-        // setup HTTP-client

-    	final HTTPClient client = new HTTPClient(agent, timeout);

-    	client.setHeader(requestHeader.entrySet());

-    	client.setRedirecting(false);

-        return client;

-    }

-

-    /**

      * determines in which form the response should be send and sets header accordingly

      * if the content length is not set we need to use chunked content encoding

      * Implemented:

diff --git a/source/net/yacy/server/serverSwitch.java b/source/net/yacy/server/serverSwitch.java
index 746a5900d..ab615e581 100644
--- a/source/net/yacy/server/serverSwitch.java
+++ b/source/net/yacy/server/serverSwitch.java
@@ -686,23 +686,13 @@ public class serverSwitch {
             final String[] uris = CommonPattern.COMMA.split(uri);

             for (String netdef : uris) {

                 netdef = netdef.trim();

-                try {

+                try (final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {

                     final RequestHeader reqHeader = new RequestHeader();

-                    reqHeader

-                            .put(HeaderFramework.USER_AGENT,

-                                    ClientIdentification.yacyInternetCrawlerAgent.userAgent);

-                    final HTTPClient client = new HTTPClient(

-                            ClientIdentification.yacyInternetCrawlerAgent);

+                    reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent);

                     client.setHeader(reqHeader.entrySet());

-                    byte[] data = client

-                            .GETbytes(

-                                    uri,

-                                    getConfig(

-                                            SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,

-                                            "admin"),

-                                    getConfig(

-                                            SwitchboardConstants.ADMIN_ACCOUNT_B64MD5,

-                                            ""), false);

+                    byte[] data = client.GETbytes(uri,

+                                    getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"),

+                                    getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""), false);

                     if (data == null || data.length == 0) {

                         continue;

                     }

diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java
index 7dcec8e14..4ed9b59ce 100644
--- a/source/net/yacy/yacy.java
+++ b/source/net/yacy/yacy.java
@@ -289,9 +289,6 @@ public final class yacy {
             final int deleteOldDownloadsAfterDays = (int) sb.getConfigLong("update.deleteOld", 30);

             yacyRelease.deleteOldDownloads(sb.releasePath, deleteOldDownloadsAfterDays );

 

-            // set user-agent

-            HTTPClient.setDefaultUserAgent(ClientIdentification.yacyInternetCrawlerAgent.userAgent);

-

             // start main threads

             final int port = sb.getLocalPort();

             try {

@@ -535,9 +532,7 @@ public final class yacy {
         final String adminUser = config.getProperty(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin");

 

         // send 'wget' to web interface

-        final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);

-        // con.setHeader(requestHeader.entrySet());

-        try {

+        try (final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {

             /* First get a valid transaction token using HTTP GET */

             con.GETbytes("http://localhost:"+ port +"/" + path, adminUser, encodedPassword, false);

 

@@ -603,9 +598,7 @@ public final class yacy {
         if (encodedPassword == null) encodedPassword = ""; // not defined

 

         // send 'wget' to web interface

-        final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);

-        // con.setHeader(requestHeader.entrySet());

-        try {

+        try (final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {

             con.GETbytes("http://localhost:"+ port +"/" + path, config.getProperty(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,"admin"), encodedPassword, false);

             if (con.getStatusCode() > 199 && con.getStatusCode() < 300) {

                 ConcurrentLog.config("COMMAND-STEERING", "YACY accepted steering command: " + processdescription);