Merge branch 'master' of https://github.com/yacy/yacy_search_server.git

5 files changed, 47 insertions, 19 deletions

diff --git a/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
index 5d317a0be..b2abffce5 100644
--- a/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
+++ b/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
@@ -125,20 +125,16 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
     public void clearCaches() {
         final SolrConfig solrConfig = this.core.getSolrConfig();
         @SuppressWarnings("unchecked")
-        final
-        SolrCache<String, ?> fieldValueCache = solrConfig.fieldValueCacheConfig == null ? null : solrConfig.fieldValueCacheConfig.newInstance();
+        final SolrCache<String, ?> fieldValueCache = solrConfig.fieldValueCacheConfig == null ? null : solrConfig.fieldValueCacheConfig.newInstance();
         if (fieldValueCache != null) fieldValueCache.clear();
         @SuppressWarnings("unchecked")
-        final
-        SolrCache<Query, DocSet> filterCache= solrConfig.filterCacheConfig == null ? null : solrConfig.filterCacheConfig.newInstance();
+        final SolrCache<Query, DocSet> filterCache= solrConfig.filterCacheConfig == null ? null : solrConfig.filterCacheConfig.newInstance();
         if (filterCache != null) filterCache.clear();
         @SuppressWarnings("unchecked")
-        final
-        SolrCache<QueryResultKey, DocList> queryResultCache = solrConfig.queryResultCacheConfig == null ? null : solrConfig.queryResultCacheConfig.newInstance();
+        final SolrCache<QueryResultKey, DocList> queryResultCache = solrConfig.queryResultCacheConfig == null ? null : solrConfig.queryResultCacheConfig.newInstance();
         if (queryResultCache != null) queryResultCache.clear();
         @SuppressWarnings("unchecked")
-        final
-        SolrCache<Integer, Document> documentCache = solrConfig.documentCacheConfig == null ? null : solrConfig.documentCacheConfig.newInstance();
+        final SolrCache<Integer, Document> documentCache = solrConfig.documentCacheConfig == null ? null : solrConfig.documentCacheConfig.newInstance();
         if (documentCache != null) documentCache.clear();
     }
 
@@ -332,7 +328,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
         //System.out.println("EmbeddedSolrConnector.getResponseByParams * QUERY: " + ql); System.out.println("STACKTRACE: " + ConcurrentLog.stackTrace());
         QueryResponse rsp;
         try {
-            System.out.println("*** PARAMS: " + params);
+            // System.out.println("*** PARAMS: " + params);
             rsp = this.server.query(params);
             Thread.currentThread().setName(threadname);
             if (rsp != null) if (log.isFine()) log.fine(rsp.getResults().getNumFound() + " results for " + ql);
diff --git a/source/net/yacy/crawler/HostBalancer.java b/source/net/yacy/crawler/HostBalancer.java
index cd0bcca87..a66969781 100644
--- a/source/net/yacy/crawler/HostBalancer.java
+++ b/source/net/yacy/crawler/HostBalancer.java
@@ -155,6 +155,7 @@ public class HostBalancer implements Balancer {
 
     @Override
     public synchronized void close() {
+        log.info("closing all HostBalancer queues (" + this.queues.size() + ") for hostPath " + this.hostsPath);
         if (depthCache != null) {
             depthCache.clear();
         }
diff --git a/source/net/yacy/crawler/HostQueue.java b/source/net/yacy/crawler/HostQueue.java
index dec1a420c..4594f2a4f 100644
--- a/source/net/yacy/crawler/HostQueue.java
+++ b/source/net/yacy/crawler/HostQueue.java
@@ -288,6 +288,7 @@ public class HostQueue implements Balancer {
 
     @Override
     public synchronized void close() {
+        log.info("closing HostQueue, closing " + this.depthStacks.size() + " depthStacks for host " + this.hostName);
         for (final Map.Entry<Integer, Index> entry: this.depthStacks.entrySet()) {
             final int size = entry.getValue().size();
             entry.getValue().close();
diff --git a/source/net/yacy/kelondro/index/BufferedObjectIndex.java b/source/net/yacy/kelondro/index/BufferedObjectIndex.java
index a20db7249..186f95bfc 100644
--- a/source/net/yacy/kelondro/index/BufferedObjectIndex.java
+++ b/source/net/yacy/kelondro/index/BufferedObjectIndex.java
@@ -61,7 +61,7 @@ public class BufferedObjectIndex implements Index, Iterable<Row.Entry> {
     public boolean isOnDemand() {
         return this.backend instanceof OnDemandOpenFileIndex;
     }
-    
+
     @Override
     public byte[] smallestKey() {
         if (this.buffer == null || this.buffer.isEmpty()) return this.backend.smallestKey();
@@ -78,8 +78,12 @@ public class BufferedObjectIndex implements Index, Iterable<Row.Entry> {
 
     private final void flushBuffer() throws IOException, SpaceExceededException {
         if (!this.buffer.isEmpty()) {
-            for (final Row.Entry e: this.buffer) {
-                this.backend.put(e);
+            if (this.backend instanceof OnDemandOpenFileIndex) {
+                ((OnDemandOpenFileIndex) this.backend).put(this.buffer);
+            } else {
+                for (final Row.Entry e: this.buffer) {
+                    this.backend.put(e);
+                }
             }
             this.buffer.clear();
         }
@@ -90,7 +94,7 @@ public class BufferedObjectIndex implements Index, Iterable<Row.Entry> {
         this.backend.optimize();
         this.buffer.optimize();
     }
-    
+
     @Override
     public long mem() {
         return this.backend.mem() + this.buffer.mem();
diff --git a/source/net/yacy/kelondro/index/OnDemandOpenFileIndex.java b/source/net/yacy/kelondro/index/OnDemandOpenFileIndex.java
index 27b60acce..db6aa5c8b 100644
--- a/source/net/yacy/kelondro/index/OnDemandOpenFileIndex.java
+++ b/source/net/yacy/kelondro/index/OnDemandOpenFileIndex.java
@@ -7,12 +7,12 @@
  *  modify it under the terms of the GNU Lesser General Public
  *  License as published by the Free Software Foundation; either
  *  version 2.1 of the License, or (at your option) any later version.
- *  
+ *
  *  This library is distributed in the hope that it will be useful,
  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  *  Lesser General Public License for more details.
- *  
+ *
  *  You should have received a copy of the GNU Lesser General Public License
  *  along with this program in the file lgpl21.txt
  *  If not, see <http://www.gnu.org/licenses/>.
@@ -58,7 +58,7 @@ public class OnDemandOpenFileIndex implements Index, Iterable<Row.Entry> {
 
     private Index getIndex() {
         try {
-            return new Table(file, rowdef, 1000, 0, false, exceed134217727, false);
+            return new Table(this.file, this.rowdef, 1000, 0, false, this.exceed134217727, false);
         } catch (kelondroException e) {
             ConcurrentLog.logException(e);
             return null;
@@ -67,7 +67,7 @@ public class OnDemandOpenFileIndex implements Index, Iterable<Row.Entry> {
             return null;
         }
     }
-    
+
     @Override
     public synchronized byte[] smallestKey() {
         Index index = getIndex();
@@ -93,7 +93,7 @@ public class OnDemandOpenFileIndex implements Index, Iterable<Row.Entry> {
         index.optimize();
         index.close();
     }
-    
+
     @Override
     public synchronized long mem() {
         Index index = getIndex();
@@ -133,6 +133,7 @@ public class OnDemandOpenFileIndex implements Index, Iterable<Row.Entry> {
 
     @Override
     public synchronized void close() {
+        // there is actually nothing to do here because this class does not hold any data
     }
 
     @Override
@@ -149,7 +150,7 @@ public class OnDemandOpenFileIndex implements Index, Iterable<Row.Entry> {
 
     @Override
     public synchronized int size() {
-        if (sizecache >= 0) return sizecache;
+        if (this.sizecache >= 0) return this.sizecache;
         Index index = getIndex();
         if (index == null) return 0;
         int i = index.size();
@@ -227,6 +228,31 @@ public class OnDemandOpenFileIndex implements Index, Iterable<Row.Entry> {
         }
     }
 
+    /**
+     * Mass-put method in case that a larger amount of rows should be stored.
+     * This is the case in the BufferedObjectIndex class where a write buffer is flushed at once.
+     * Without a mass-backend to store the data, the put would be called many times where each time the file is opened and closed.
+     * This should speed-up the process.
+     * @param rowset
+     * @throws IOException
+     * @throws SpaceExceededException
+     */
+    public synchronized void put(final RowSet rowset) throws IOException, SpaceExceededException {
+        Index index = getIndex();
+        if (index == null) return;
+        try {
+            for (final Row.Entry row: rowset) {
+                boolean b = index.put(row);
+                if (this.sizecache >= 0 && b) this.sizecache++;
+            }
+            return;
+        } catch (IOException e) {
+            throw e;
+        } finally {
+            index.close();
+        }
+    }
+
     @Override
     public synchronized Entry remove(final byte[] key) throws IOException {
         Index index = getIndex();