diff options
| author | orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542> | 2009-04-15 06:34:27 +0000 |
|---|---|---|
| committer | orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542> | 2009-04-15 06:34:27 +0000 |
| commit | 89ec3acb3e1e23e27b90d6bbbb7a84206c498229 (patch) | |
| tree | a9314007bb76608c9aa35303a7164e1b550cbf7d | |
| parent | 7a48090fcff69b9695b57de38b5a29e1d6fa8c9f (diff) | |
- full abstraction of index content type: the kelondro full text index may now also contain indexes about other content than text, i.e. navigation indexes or reverse linking indexes.
- during index joins all word positions are maintained: better ranking for word distance possible; exact phrase match can be implemented soundly
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5804 6c8d7289-2bf4-0310-a012-ef5d649a1542
38 files changed, 713 insertions, 491 deletions
diff --git a/build.properties b/build.properties index 2f1206b02..d4d024152 100644 --- a/build.properties +++ b/build.properties @@ -3,7 +3,7 @@ javacSource=1.5 javacTarget=1.5
# Release Configuration
-releaseVersion=0.74
+releaseVersion=0.75
stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
diff --git a/htroot/IndexControlRWIs_p.java b/htroot/IndexControlRWIs_p.java index 1585824b6..370d02258 100644 --- a/htroot/IndexControlRWIs_p.java +++ b/htroot/IndexControlRWIs_p.java @@ -42,7 +42,7 @@ import de.anomic.kelondro.text.Reference; import de.anomic.kelondro.text.ReferenceContainer; import de.anomic.kelondro.text.ReferenceContainerCache; import de.anomic.kelondro.text.metadataPrototype.URLMetadataRow; -import de.anomic.kelondro.text.referencePrototype.WordReferenceRow; +import de.anomic.kelondro.text.referencePrototype.WordReference; import de.anomic.plasma.plasmaSearchAPI; import de.anomic.plasma.plasmaSearchEvent; import de.anomic.plasma.plasmaSearchRankingProcess; @@ -124,9 +124,9 @@ public class IndexControlRWIs_p { if (post.containsKey("keyhashdeleteall")) try { if (delurl || delurlref) { // generate an urlx array - ReferenceContainer index = null; + ReferenceContainer<WordReference> index = null; index = sb.webIndex.index().get(keyhash, null); - final Iterator<WordReferenceRow> en = index.entries(); + final Iterator<WordReference> en = index.entries(); int i = 0; urlx = new String[index.size()]; while (en.hasNext()) { @@ -203,11 +203,11 @@ public class IndexControlRWIs_p { } // prepare index - ReferenceContainer index; + ReferenceContainer<WordReference> index; final long starttime = System.currentTimeMillis(); index = sb.webIndex.index().get(keyhash, null); // built urlCache - final Iterator<WordReferenceRow> urlIter = index.entries(); + final Iterator<WordReference> urlIter = index.entries(); final HashMap<String, URLMetadataRow> knownURLs = new HashMap<String, URLMetadataRow>(); final HashSet<String> unknownURLEntries = new HashSet<String>(); Reference iEntry; @@ -224,7 +224,7 @@ public class IndexControlRWIs_p { } // make an indexContainerCache - ReferenceContainerCache icc = new ReferenceContainerCache(index.rowdef, plasmaWordIndex.wordOrder); + ReferenceContainerCache<WordReference> icc = new ReferenceContainerCache<WordReference>(plasmaWordIndex.wordReferenceFactory, index.rowdef, plasmaWordIndex.wordOrder); icc.add(index); // transport to other peer @@ -244,8 +244,8 @@ public class IndexControlRWIs_p { // generate list if (post.containsKey("keyhashsimilar")) try { - final Iterator<ReferenceContainer> containerIt = sb.webIndex.index().references(keyhash, true, 256, false).iterator(); - ReferenceContainer container; + final Iterator<ReferenceContainer<WordReference>> containerIt = sb.webIndex.index().references(keyhash, true, 256, false).iterator(); + ReferenceContainer<WordReference> container; int i = 0; int rows = 0, cols = 0; prop.put("keyhashsimilar", "1"); diff --git a/htroot/api/timeline.java b/htroot/api/timeline.java index 79017a91d..1aa5e9724 100644 --- a/htroot/api/timeline.java +++ b/htroot/api/timeline.java @@ -31,10 +31,11 @@ import java.util.TreeSet; import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.text.ReferenceContainer;
-import de.anomic.kelondro.text.referencePrototype.WordReferenceRow;
+import de.anomic.kelondro.text.referencePrototype.WordReference;
import de.anomic.kelondro.util.DateFormatter;
import de.anomic.plasma.plasmaSearchQuery;
import de.anomic.plasma.plasmaSwitchboard;
+import de.anomic.plasma.plasmaWordIndex;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.tools.iso639;
@@ -78,15 +79,16 @@ public final class timeline { yacyCore.log.logInfo("INIT TIMELINE SEARCH: " + plasmaSearchQuery.anonymizedQueryHashes(query[0]) + " - " + count + " links");
// get the index container with the result vector
- HashMap<String, ReferenceContainer>[] localSearchContainerMaps = sb.webIndex.localSearchContainers(query[0], query[1], null);
- final ReferenceContainer index =
+ HashMap<String, ReferenceContainer<WordReference>>[] localSearchContainerMaps = sb.webIndex.localSearchContainers(query[0], query[1], null);
+ final ReferenceContainer<WordReference> index =
ReferenceContainer.joinExcludeContainers(
+ plasmaWordIndex.wordReferenceFactory,
localSearchContainerMaps[0].values(),
localSearchContainerMaps[1].values(),
maxdist);
- Iterator<WordReferenceRow> i = index.entries();
- WordReferenceRow entry;
+ Iterator<WordReference> i = index.entries();
+ WordReference entry;
int c = 0;
Date lm;
String lms;
diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index 8b752124f..d08959582 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -38,6 +38,7 @@ import de.anomic.http.httpRequestHeader; import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.Bitfield;
import de.anomic.kelondro.text.ReferenceContainer;
+import de.anomic.kelondro.text.referencePrototype.WordReference;
import de.anomic.kelondro.util.SortStack;
import de.anomic.net.natLib;
import de.anomic.plasma.plasmaProfiling;
@@ -208,17 +209,17 @@ public final class search { yacyCore.log.logInfo("INIT HASH SEARCH (abstracts only): " + plasmaSearchQuery.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links");
final long timer = System.currentTimeMillis();
- final Map<String, ReferenceContainer>[] containers = sb.webIndex.localSearchContainers(theQuery.queryHashes, theQuery.excludeHashes, plasmaSearchQuery.hashes2Set(urls));
+ final Map<String, ReferenceContainer<WordReference>>[] containers = sb.webIndex.localSearchContainers(theQuery.queryHashes, theQuery.excludeHashes, plasmaSearchQuery.hashes2Set(urls));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.COLLECTION, containers[0].size(), System.currentTimeMillis() - timer), false);
if (containers != null) {
- final Iterator<Map.Entry<String, ReferenceContainer>> ci = containers[0].entrySet().iterator();
- Map.Entry<String, ReferenceContainer> entry;
+ final Iterator<Map.Entry<String, ReferenceContainer<WordReference>>> ci = containers[0].entrySet().iterator();
+ Map.Entry<String, ReferenceContainer<WordReference>> entry;
String wordhash;
while (ci.hasNext()) {
entry = ci.next();
wordhash = entry.getKey();
- final ReferenceContainer container = entry.getValue();
+ final ReferenceContainer<WordReference> container = entry.getValue();
indexabstractContainercount += container.size();
indexabstract.append("indexabstract." + wordhash + "=").append(ReferenceContainer.compressIndex(container, null, 1000).toString()).append(serverCore.CRLF_STRING);
}
diff --git a/source/de/anomic/kelondro/blob/BLOBArray.java b/source/de/anomic/kelondro/blob/BLOBArray.java index f87d5c97e..91adcbca1 100755 --- a/source/de/anomic/kelondro/blob/BLOBArray.java +++ b/source/de/anomic/kelondro/blob/BLOBArray.java @@ -42,7 +42,9 @@ import de.anomic.kelondro.order.ByteOrder; import de.anomic.kelondro.order.CloneableIterator; import de.anomic.kelondro.order.NaturalOrder; import de.anomic.kelondro.order.MergeIterator; +import de.anomic.kelondro.text.Reference; import de.anomic.kelondro.text.ReferenceContainer; +import de.anomic.kelondro.text.ReferenceFactory; import de.anomic.kelondro.text.ReferenceContainerCache.blobFileEntries; import de.anomic.kelondro.util.DateFormatter; import de.anomic.kelondro.util.FileUtils; @@ -551,20 +553,20 @@ public class BLOBArray implements BLOB { blobs = null; } - public File mergeMount(File f1, File f2, Row payloadrow, File newFile) throws IOException { + public File mergeMount(File f1, File f2, ReferenceFactory<?> factory, Row payloadrow, File newFile) throws IOException { Log.logInfo("BLOBArray", "merging " + f1.getName() + " with " + f2.getName()); - File resultFile = mergeWorker(f1, f2, payloadrow, newFile); + File resultFile = mergeWorker(factory, this.keylength, this.ordering, f1, f2, payloadrow, newFile); if (resultFile == null) return null; mountBLOB(resultFile, false); Log.logInfo("BLOBArray", "merged " + f1.getName() + " with " + f2.getName() + " into " + resultFile); return resultFile; } - private File mergeWorker(File f1, File f2, Row payloadrow, File newFile) throws IOException { + private static <ReferenceType extends Reference> File mergeWorker(ReferenceFactory<ReferenceType> factory, int keylength, ByteOrder order, File f1, File f2, Row payloadrow, File newFile) throws IOException { // iterate both files and write a new one - CloneableIterator<ReferenceContainer> i1 = new blobFileEntries(f1, payloadrow); - CloneableIterator<ReferenceContainer> i2 = new blobFileEntries(f2, payloadrow); + CloneableIterator<ReferenceContainer<ReferenceType>> i1 = new blobFileEntries<ReferenceType>(f1, factory, payloadrow); + CloneableIterator<ReferenceContainer<ReferenceType>> i2 = new blobFileEntries<ReferenceType>(f2, factory, payloadrow); if (!i1.hasNext()) { if (i2.hasNext()) { FileUtils.deletedelete(f1); @@ -583,8 +585,8 @@ public class BLOBArray implements BLOB { assert i1.hasNext(); assert i2.hasNext(); File tmpFile = new File(newFile.getParentFile(), newFile.getName() + ".tmp"); - HeapWriter writer = new HeapWriter(tmpFile, newFile, this.keylength(), this.ordering()); - merge(i1, i2, this.ordering(), writer); + HeapWriter writer = new HeapWriter(tmpFile, newFile, keylength, order); + merge(i1, i2, order, writer); try { writer.close(true); // we don't need the old files any more @@ -599,10 +601,10 @@ public class BLOBArray implements BLOB { } } - private static void merge(CloneableIterator<ReferenceContainer> i1, CloneableIterator<ReferenceContainer> i2, ByteOrder ordering, HeapWriter writer) throws IOException { + private static <ReferenceType extends Reference> void merge(CloneableIterator<ReferenceContainer<ReferenceType>> i1, CloneableIterator<ReferenceContainer<ReferenceType>> i2, ByteOrder ordering, HeapWriter writer) throws IOException { assert i1.hasNext(); assert i2.hasNext(); - ReferenceContainer c1, c2, c1o, c2o; + ReferenceContainer<ReferenceType> c1, c2, c1o, c2o; c1 = i1.next(); c2 = i2.next(); int e; diff --git a/source/de/anomic/kelondro/text/AbstractBufferedIndex.java b/source/de/anomic/kelondro/text/AbstractBufferedIndex.java index 3d3ea9ed9..f87b229d9 100644 --- a/source/de/anomic/kelondro/text/AbstractBufferedIndex.java +++ b/source/de/anomic/kelondro/text/AbstractBufferedIndex.java @@ -33,17 +33,22 @@ import java.util.TreeSet; import de.anomic.kelondro.order.Order;
-public abstract class AbstractBufferedIndex extends AbstractIndex implements BufferedIndex {
+public abstract class AbstractBufferedIndex<ReferenceType extends Reference> extends AbstractIndex<ReferenceType> implements BufferedIndex<ReferenceType> {
- public synchronized TreeSet<ReferenceContainer> references(final String startHash, final boolean rot, int count, boolean ram) throws IOException {
+ public AbstractBufferedIndex(final ReferenceFactory<ReferenceType> factory) {
+ super(factory);
+ }
+
+ public synchronized TreeSet<ReferenceContainer<ReferenceType>> references(final String startHash, final boolean rot, int count, boolean ram) throws IOException {
// creates a set of indexContainers
// this does not use the cache
- final Order<ReferenceContainer> containerOrder = new ReferenceContainerOrder(this.ordering().clone());
- containerOrder.rotate(ReferenceContainer.emptyContainer(startHash, 0));
- final TreeSet<ReferenceContainer> containers = new TreeSet<ReferenceContainer>(containerOrder);
- final Iterator<ReferenceContainer> i = references(startHash, rot, ram);
+ final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, this.ordering().clone());
+ ReferenceContainer<ReferenceType> emptyContainer = ReferenceContainer.emptyContainer(factory, startHash, 0);
+ containerOrder.rotate(emptyContainer);
+ final TreeSet<ReferenceContainer<ReferenceType>> containers = new TreeSet<ReferenceContainer<ReferenceType>>(containerOrder);
+ final Iterator<ReferenceContainer<ReferenceType>> i = references(startHash, rot, ram);
if (ram) count = Math.min(size(), count);
- ReferenceContainer container;
+ ReferenceContainer<ReferenceType> container;
// this loop does not terminate using the i.hasNex() predicate when rot == true
// because then the underlying iterator is a rotating iterator without termination
// in this case a termination must be ensured with a counter
diff --git a/source/de/anomic/kelondro/text/AbstractIndex.java b/source/de/anomic/kelondro/text/AbstractIndex.java index bc04c5349..8c835de5a 100644 --- a/source/de/anomic/kelondro/text/AbstractIndex.java +++ b/source/de/anomic/kelondro/text/AbstractIndex.java @@ -34,7 +34,13 @@ import java.util.TreeSet; import de.anomic.kelondro.order.Order; -public abstract class AbstractIndex implements Index { +public abstract class AbstractIndex <ReferenceType extends Reference> implements Index<ReferenceType> { + + final protected ReferenceFactory<ReferenceType> factory; + + public AbstractIndex(final ReferenceFactory<ReferenceType> factory) { + this.factory = factory; + } public int remove(final Set<String> wordHashes, final String urlHash) throws IOException { // remove the same url hashes for multiple words @@ -56,15 +62,16 @@ public abstract class AbstractIndex implements Index { } } - public synchronized TreeSet<ReferenceContainer> references(final String startHash, final boolean rot, int count) throws IOException { + public synchronized TreeSet<ReferenceContainer<ReferenceType>> references(final String startHash, final boolean rot, int count) throws IOException { // creates a set of indexContainers // this does not use the cache - final Order<ReferenceContainer> containerOrder = new ReferenceContainerOrder(this.ordering().clone()); - containerOrder.rotate(ReferenceContainer.emptyContainer(startHash, 0)); - final TreeSet<ReferenceContainer> containers = new TreeSet<ReferenceContainer>(containerOrder); - final Iterator<ReferenceContainer> i = references(startHash, rot); + final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, this.ordering().clone()); + ReferenceContainer<ReferenceType> emptyContainer = ReferenceContainer.emptyContainer(factory, startHash, 0); + containerOrder.rotate(emptyContainer); + final TreeSet<ReferenceContainer<ReferenceType>> containers = new TreeSet<ReferenceContainer<ReferenceType>>(containerOrder); + final Iterator<ReferenceContainer<ReferenceType>> i = references(startHash, rot); //if (ram) count = Math.min(size(), count); - ReferenceContainer container; + ReferenceContainer<ReferenceType> container; // this loop does not terminate using the i.hasNex() predicate when rot == true // because then the underlying iterator is a rotating iterator without termination // in this case a termination must be ensured with a counter diff --git a/source/de/anomic/kelondro/text/AbstractReference.java b/source/de/anomic/kelondro/text/AbstractReference.java new file mode 100644 index 000000000..826b60d1f --- /dev/null +++ b/source/de/anomic/kelondro/text/AbstractReference.java @@ -0,0 +1,78 @@ +// AbstractReference.java +// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 14.04.2009 on http://yacy.net +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2009-04-07 11:34:41 +0200 (Di, 07 Apr 2009) $ +// $LastChangedRevision: 5783 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.kelondro.text; + +import java.util.ArrayList; + +public abstract class AbstractReference implements Reference { + + protected static ArrayList<Integer> a(int i) { + ArrayList<Integer> l = new ArrayList<Integer>(1); + l.add(i); + return l; + } + protected static int max(ArrayList<Integer> a) { + assert a.size() > 0; + if (a.size() == 1) return a.get(0); + if (a.size() == 2) return Math.max(a.get(0), a.get(1)); + int r = a.get(0); + for (int i = 1; i < a.size(); i++) if (a.get(i) > r) r = a.get(i); + return r; + } + protected static int min(ArrayList<Integer> a) { + assert a.size() > 0; + if (a.size() == 1) return a.get(0); + if (a.size() == 2) return Math.min(a.get(0), a.get(1)); + int r = a.get(0); + for (int i = 1; i < a.size(); i++) if (a.get(i) < r) r = a.get(i); + return r; + } + + public int maxposition() { + assert positions() > 0; + if (positions() == 1) return position(0); + int p = position(0); + for (int i = positions() - 1; i > 0; i--) if (position(i) > p) p = position(i); + return p; + } + + public int minposition() { + assert positions() > 0; + if (positions() == 1) return position(0); + int p = position(0); + for (int i = positions() - 1; i > 0; i--) if (position(i) < p) p = position(i); + return p; + } + + public int distance() { + int d = 0; + for (int i = 0; i < this.positions() - 1; i++) { + d += Math.abs(this.position(i) - this.position(i + 1)); + } + return d; + } +} diff --git a/source/de/anomic/kelondro/text/BufferedIndex.java b/source/de/anomic/kelondro/text/BufferedIndex.java index c75d5d987..72ea61a54 100644 --- a/source/de/anomic/kelondro/text/BufferedIndex.java +++ b/source/de/anomic/kelondro/text/BufferedIndex.java @@ -44,7 +44,7 @@ import de.anomic.kelondro.order.CloneableIterator; * (which integrates ReferenceContainerArray and ReferenceContainerCache) * to make it possible to switch between the old and new index data structure */ -public interface BufferedIndex extends Index { +public interface BufferedIndex<ReferenceType extends Reference> extends Index<ReferenceType> { /* * methods for monitoring of the buffer @@ -118,7 +118,7 @@ public interface BufferedIndex extends Index { * @return * @throws IOException */ - public CloneableIterator<ReferenceContainer> references( + public CloneableIterator<ReferenceContainer<ReferenceType>> references( String startHash, boolean rot, boolean buffer @@ -137,7 +137,7 @@ public interface BufferedIndex extends Index { * @return * @throws IOException */ - public TreeSet<ReferenceContainer> references( + public TreeSet<ReferenceContainer<ReferenceType>> references( String startHash, boolean rot, int count, diff --git a/source/de/anomic/kelondro/text/BufferedIndexCollection.java b/source/de/anomic/kelondro/text/BufferedIndexCollection.java index 9858a2790..70c28cd5e 100644 --- a/source/de/anomic/kelondro/text/BufferedIndexCollection.java +++ b/source/de/anomic/kelondro/text/BufferedIndexCollection.java @@ -50,7 +50,7 @@ import de.anomic.kelondro.util.MemoryControl; import de.anomic.kelondro.util.Log;
import de.anomic.server.serverProfiling;
-public final class BufferedIndexCollection extends AbstractBufferedIndex implements Index, BufferedIndex {
+public final class BufferedIndexCollection<ReferenceType extends Reference> extends AbstractBufferedIndex<ReferenceType> implements Index<ReferenceType>, BufferedIndex<ReferenceType> {
// environment constants
public static final long wCacheMaxAge = 1000 * 60 * 30; // milliseconds; 30 minutes
@@ -58,39 +58,42 @@ public final class BufferedIndexCollection extends AbstractBufferedIndex impleme public static final int lowcachedivisor = 900;
public static final int maxCollectionPartition = 7; // should be 7
- private final IndexBuffer buffer;
- private final IndexCollection collections;
+ private final IndexBuffer<ReferenceType> buffer;
+ private final IndexCollection<ReferenceType> collections;
public BufferedIndexCollection (
File indexPrimaryTextLocation,
+ final ReferenceFactory<ReferenceType> factory,
final ByteOrder wordOrdering,
final Row payloadrow,
final int entityCacheMaxSize,
final boolean useCommons,
final int redundancy,
Log log) throws IOException {
-
+ super(factory);
+
final File textindexcache = new File(indexPrimaryTextLocation, "RICACHE");
if (!(textindexcache.exists())) textindexcache.mkdirs();
if (new File(textindexcache, "index.dhtin.blob").exists()) {
// migration of the both caches into one
- this.buffer = new IndexBuffer(textindexcache, wordOrdering, payloadrow, entityCacheMaxSize, wCacheMaxChunk, wCacheMaxAge, "index.dhtout.blob", log);
- IndexBuffer dhtInCache = new IndexBuffer(textindexcache, wordOrdering, payloadrow, entityCacheMaxSize, wCacheMaxChunk, wCacheMaxAge, "index.dhtin.blob", log);
- for (ReferenceContainer c: dhtInCache) {
+ this.buffer = new IndexBuffer<ReferenceType>(textindexcache, factory, wordOrdering, payloadrow, entityCacheMaxSize, wCacheMaxChunk, wCacheMaxAge, "index.dhtout.blob", log);
+ IndexBuffer<ReferenceType> dhtInCache = new IndexBuffer<ReferenceType>(textindexcache, factory, wordOrdering, payloadrow, entityCacheMaxSize, wCacheMaxChunk, wCacheMaxAge, "index.dhtin.blob", log);
+ for (ReferenceContainer<ReferenceType> c: dhtInCache) {
this.buffer.add(c);
}
FileUtils.deletedelete(new File(textindexcache, "index.dhtin.blob"));
} else {
// read in new BLOB
- this.buffer = new IndexBuffer(textindexcache, wordOrdering, payloadrow, entityCacheMaxSize, wCacheMaxChunk, wCacheMaxAge, "index.dhtout.blob", log);
+ this.buffer = new IndexBuffer<ReferenceType>(textindexcache, factory, wordOrdering, payloadrow, entityCacheMaxSize, wCacheMaxChunk, wCacheMaxAge, "index.dhtout.blob", log);
}
// create collections storage path
final File textindexcollections = new File(indexPrimaryTextLocation, "RICOLLECTION");
if (!(textindexcollections.exists())) textindexcollections.mkdirs();
- this.collections = new IndexCollection(
+ this.collections = new IndexCollection<ReferenceType>(
textindexcollections,
"collection",
+ factory,
12,
Base64Order.enhancedCoder,
maxCollectionPartition,
@@ -100,7 +103,7 @@ public final class BufferedIndexCollection extends AbstractBufferedIndex impleme /* methods for interface Index */
- public void add(final ReferenceContainer entries) {
+ public void add(final ReferenceContainer<ReferenceType> entries) {
assert (entries.row().objectsize == WordReferenceRow.urlEntryRow.objectsize);
// add the entry
@@ -108,7 +111,7 @@ public final class BufferedIndexCollection extends AbstractBufferedIndex impleme cacheFlushControl();
}
- public void add(final String wordHash, final WordReferenceRow entry) throws IOException {
+ public void add(final String wordHash, final ReferenceType entry) throws IOException {
// add the entry
buffer.add(wordHash, entry);
cacheFlushControl();
@@ -124,14 +127,14 @@ public final class BufferedIndexCollection extends AbstractBufferedIndex impleme return buffer.count(key) + collections.count(key);
}
- public ReferenceContainer get(final String wordHash, final Set<String> urlselection) {
+ public ReferenceContainer<ReferenceType> get(final String wordHash, final Set<String> urlselection) {
if (wordHash == null) {
// wrong input
return null;
}
// get from cache
- ReferenceContainer container;
+ ReferenceContainer<ReferenceType> container;
container = buffer.get(wordHash, urlselection);
// get from collection index
@@ -169,8 +172,9 @@ public final class BufferedIndexCollection extends AbstractBufferedIndex impleme return container;
}
- public ReferenceContainer delete(final String wordHash) {
- final ReferenceContainer c = new ReferenceContainer(
+ public ReferenceContainer<ReferenceType> delete(final String wordHash) {
+ final ReferenceContainer<ReferenceType> c = new ReferenceContainer<ReferenceType>(
+ factory,
wordHash,
WordReferenceRow.urlEntryRow,
buffer.count(wordHash));
@@ -193,17 +197,18 @@ public final class BufferedIndexCollection extends AbstractBufferedIndex impleme return removed;
}
- public synchronized CloneableIterator<ReferenceContainer> references(final String startHash, final boolean rot, final boolean ram) throws IOException {
- final CloneableIterator<ReferenceContainer> i = wordContainers(startHash, ram);
+ public synchronized CloneableIterator<ReferenceContainer<ReferenceType>> references(final String startHash, final boolean rot, final boolean ram) throws IOException {
+ final CloneableIterator<ReferenceContainer<ReferenceType>> i = wordContainers(startHash, ram);
if (rot) {
- return new RotateIterator<ReferenceContainer>(i, new String(Base64Order.zero(startHash.length())), buffer.size() + ((ram) ? 0 : collections.size()));
+ return new RotateIterator<ReferenceContainer<ReferenceType>>(i, new String(Base64Order.zero(startHash.length())), buffer.size() + ((ram) ? 0 : collections.size()));
}
return i;
}
- private synchronized CloneableIterator<ReferenceContainer> wordContainers(final String startWordHash, final boolean ram) throws IOException {
- final Order<ReferenceContainer> containerOrder = new ReferenceContainerOrder(buffer.ordering().clone());
- containerOrder.rotate(ReferenceContainer.emptyContainer(startWordHash, 0));
+ private synchronized CloneableIterator<ReferenceContainer<ReferenceType>> wordContainers(final String startWordHash, final boolean ram) throws IOException {
+ final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, buffer.ordering().clone());
+ ReferenceContainer<ReferenceType> emptyContainer = ReferenceContainer.emptyContainer(factory, startWordHash, 0);
+ containerOrder.rotate(emptyContainer);
if (ram) {
return buffer.references(startWordHash, false);
}
@@ -298,13 +303,13 @@ public final class BufferedIndexCollection extends AbstractBufferedIndex impleme }
}
- private synchronized void flushCacheOne(final IndexBuffer ram) {
+ private synchronized void flushCacheOne(final IndexBuffer<ReferenceType> ram) {
if (ram.size() > 0) collections.add(flushContainer(ram));
}
- private ReferenceContainer flushContainer(final IndexBuffer ram) {
+ private ReferenceContainer<ReferenceType> flushContainer(final IndexBuffer<ReferenceType> ram) {
String wordHash;
- ReferenceContainer c;
+ ReferenceContainer<ReferenceType> c;
wordHash = ram.maxScoreWordHash();
c = ram.get(wordHash, null);
if ((c != null) && (c.size() > wCacheMaxChunk)) {
@@ -326,9 +331,9 @@ public final class BufferedIndexCollection extends AbstractBufferedIndex impleme return collections.ordering();
}
- public CloneableIterator<ReferenceContainer> references(String startWordHash, boolean rot) {
- final Order<ReferenceContainer> containerOrder = new ReferenceContainerOrder(this.buffer.ordering().clone());
- return new MergeIterator<ReferenceContainer>(
+ public CloneableIterator<ReferenceContainer<ReferenceType>> references(String startWordHash, boolean rot) {
+ final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, this.buffer.ordering().clone());
+ return new MergeIterator<ReferenceContainer<ReferenceType>>(
this.buffer.references(startWordHash, false),
this.collections.references(startWordHash, false),
containerOrder,
diff --git a/source/de/anomic/kelondro/text/IODispatcher.java b/source/de/anomic/kelondro/text/IODispatcher.java index 91cba793a..bf67778d0 100644 --- a/source/de/anomic/kelondro/text/IODispatcher.java +++ b/source/de/anomic/kelondro/text/IODispatcher.java @@ -43,15 +43,17 @@ import de.anomic.kelondro.index.Row; * of merging with a call to the start() - method. To shut down all mergings, call terminate()
* only once.
*/
-public class IODispatcher extends Thread {
+public class IODispatcher <ReferenceType extends Reference> extends Thread {
private final Boolean poison, vita;
private ArrayBlockingQueue<Boolean> controlQueue;
private ArrayBlockingQueue<MergeJob> mergeQueue;
private ArrayBlockingQueue<DumpJob> dumpQueue;
private ArrayBlockingQueue<Boolean> termQueue;
+ ReferenceFactory<ReferenceType> factory;
- public IODispatcher(int dumpQueueLength, int mergeQueueLength) {
+ public IODispatcher(ReferenceFactory<ReferenceType> factory, int dumpQueueLength, int mergeQueueLength) {
+ this.factory = factory;
this.poison = new Boolean(false);
this.vita = new Boolean(true);
this.controlQueue = new ArrayBlockingQueue<Boolean>(dumpQueueLength + mergeQueueLength + 1);
@@ -76,7 +78,7 @@ public class IODispatcher extends Thread { }
}
- public synchronized void dump(ReferenceContainerCache cache, File file, ReferenceContainerArray array) {
+ public synchronized void dump(ReferenceContainerCache<ReferenceType> cache, File file, ReferenceContainerArray<ReferenceType> array) {
if (dumpQueue == null || !this.isAlive()) {
cache.dump(file, true);
} else {
@@ -98,7 +100,7 @@ public class IODispatcher extends Thread { public synchronized void merge(File f1, File f2, BLOBArray array, Row payloadrow, File newFile) {
if (mergeQueue == null || !this.isAlive()) {
try {
- array.mergeMount(f1, f2, payloadrow, newFile);
+ array.mergeMount(f1, f2, factory, payloadrow, newFile);
} catch (IOException e) {
e.printStackTrace();
}
@@ -110,7 +112,7 @@ public class IODispatcher extends Thread { } catch (InterruptedException e) {
e.printStackTrace();
try {
- array.mergeMount(f1, f2, payloadrow, newFile);
+ array.mergeMount(f1, f2, factory, payloadrow, newFile);
} catch (IOException ee) {
ee.printStackTrace();
}
@@ -149,10 +151,10 @@ public class IODispatcher extends Thread { }
public class DumpJob {
- ReferenceContainerCache cache;
+ ReferenceContainerCache<ReferenceType> cache;
File file;
- ReferenceContainerArray array;
- public DumpJob(ReferenceContainerCache cache, File file, ReferenceContainerArray array) {
+ ReferenceContainerArray<ReferenceType> array;
+ public DumpJob(ReferenceContainerCache<ReferenceType> cache, File file, ReferenceContainerArray<ReferenceType> array) {
this.cache = cache;
this.file = file;
this.array = array;
@@ -183,7 +185,7 @@ public class IODispatcher extends Thread { public File merge() {
try {
- return array.mergeMount(f1, f2, payloadrow, newFile);
+ return array.mergeMount(f1, f2, factory, payloadrow, newFile);
} catch (IOException e) {
e.printStackTrace();
}
diff --git a/source/de/anomic/kelondro/text/Index.java b/source/de/anomic/kelondro/text/Index.java index f56dc13e0..ac4ea9e2a 100644 --- a/source/de/anomic/kelondro/text/Index.java +++ b/source/de/anomic/kelondro/text/Index.java @@ -34,9 +34,8 @@ import java.util.TreeSet; import de.anomic.kelondro.order.ByteOrder;
import de.anomic.kelondro.order.CloneableIterator;
-import de.anomic.kelondro.text.referencePrototype.WordReferenceRow;
-public interface Index {
+public interface Index <ReferenceType extends Reference> {
/**
* add references to the reverse index
@@ -46,7 +45,7 @@ public interface Index { * @param newEntries the References to be merged with existing references
* @throws IOException
*/
- public void add(ReferenceContainer newEntries) throws IOException;
+ public void add(ReferenceContainer<ReferenceType> newEntries) throws IOException;
/**
* add a single reference to the reverse index
@@ -57,7 +56,7 @@ public interface Index { * @param entry
* @throws IOException
*/
- public void add(final String termHash, final WordReferenceRow entry) throws IOException;
+ public void add(final String termHash, final ReferenceType entry) throws IOException;
/**
* check if there are references stored to the given word hash
@@ -84,7 +83,7 @@ public interface Index { * @return the references
* @throws IOException
*/
- public ReferenceContainer get(String termHash, Set<String> referenceselection) throws IOException;
+ public ReferenceContainer<ReferenceType> get(String termHash, Set<String> referenceselection) throws IOException;
/**
* delete all references for a word
@@ -92,7 +91,7 @@ public interface Index { * @return the deleted references
* @throws IOException
*/
- public ReferenceContainer delete(String termHash) throws IOException;
+ public ReferenceContainer<ReferenceType> delete(String termHash) throws IOException;
/**
* remove a specific reference entry
@@ -124,13 +123,13 @@ public interface Index { * @return
* @throws IOException
*/
- public CloneableIterator<ReferenceContainer> references(
+ public CloneableIterator<ReferenceContainer<ReferenceType>> references(
String startHash,
boolean rot
) throws IOException;
- public TreeSet<ReferenceContainer> references(
+ public TreeSet<ReferenceContainer<ReferenceType>> references(
String startHash,
boolean rot,
int count
diff --git a/source/de/anomic/kelondro/text/IndexBuffer.java b/source/de/anomic/kelondro/text/IndexBuffer.java index 32da6d50e..2a4c669b3 100644 --- a/source/de/anomic/kelondro/text/IndexBuffer.java +++ b/source/de/anomic/kelondro/text/IndexBuffer.java @@ -44,7 +44,7 @@ import de.anomic.kelondro.util.Log; * A IndexCache is a ReferenceContainerCache with an attached cache flush logic
*
*/
-public final class IndexBuffer extends AbstractIndex implements Index, IndexReader, Iterable<ReferenceContainer> {
+public final class IndexBuffer<ReferenceType extends Reference> extends AbstractIndex<ReferenceType> implements Index<ReferenceType>, IndexReader<ReferenceType>, Iterable<ReferenceContainer<ReferenceType>> {
// class variables
private final ScoreCluster<String> hashScore;
@@ -55,11 +55,12 @@ public final class IndexBuffer extends AbstractIndex implements Index, IndexRead public long cacheReferenceAgeLimit; // the maximum age (= time not changed) of a RWI entity
private final Log log;
private final File dumpFile;
- private ReferenceContainerCache heap;
+ private ReferenceContainerCache<ReferenceType> heap;
@SuppressWarnings("unchecked")
public IndexBuffer(
final File databaseRoot,
+ final ReferenceFactory<ReferenceType> factory,
final ByteOrder wordOrdering,
final Row payloadrow,
final int entityCacheMaxSize,
@@ -67,7 +68,8 @@ public final class IndexBuffer extends AbstractIndex implements Index, IndexRead final long wCacheReferenceAgeLimitInit,
final String newHeapName,
final Log log) {
-
+ super(factory);
+
// creates a new index cache
// the cache has a back-end where indexes that do not fit in the cache are flushed
this.hashScore = new ScoreCluster<String>();
@@ -78,7 +80,7 @@ public final class IndexBuffer extends AbstractIndex implements Index, IndexRead this.cacheReferenceAgeLimit = wCacheReferenceAgeLimitInit;
this.log = log;
this.dumpFile = new File(databaseRoot, newHeapName);
- this.heap = new ReferenceContainerCache(payloadrow, wordOrdering);
+ this.heap = new ReferenceContainerCache(factory, payloadrow, wordOrdering);
// read in dump of last session
boolean initFailed = false;
@@ -149,7 +151,7 @@ public final class IndexBuffer extends AbstractIndex implements Index, IndexRead return heap.size();
}
- public synchronized CloneableIterator<ReferenceContainer> references(final String startWordHash, final boolean rot) {
+ public synchronized CloneableIterator<ReferenceContainer<ReferenceType>> references(final String startWordHash, final boolean rot) {
// we return an iterator object that creates top-level-clones of the indexContainers
// in the cache, so that manipulations of the iterated objects do not change
// objects in the cache.
@@ -197,7 +199,7 @@ public final class IndexBuffer extends AbstractIndex implements Index, IndexRead hash = hashDate.getMinObject(); // flush oldest entries
}
if (hash == null) {
- final ReferenceContainer ic = heap.references(null, false).next();
+ final ReferenceContainer<ReferenceType> ic = heap.references(null, false).next();
if (ic != null) hash = ic.getTermHash();
}
return hash;
@@ -208,10 +210,10 @@ public final class IndexBuffer extends AbstractIndex implements Index, IndexRead return null;
}
- public synchronized ArrayList<ReferenceContainer> bestFlushContainers(final int count) {
- final ArrayList<ReferenceContainer> containerList = new ArrayList<ReferenceContainer>();
+ public synchronized ArrayList<ReferenceContainer<ReferenceType>> bestFlushContainers(final int count) {
+ final ArrayList<ReferenceContainer<ReferenceType>> containerList = new ArrayList<ReferenceContainer<ReferenceType>>();
String hash;
- ReferenceContainer container;
+ ReferenceContainer<ReferenceType> container;
for (int i = 0; i < count; i++) {
hash = bestFlushWordHash();
if (hash == null) return containerList;
@@ -241,11 +243,11 @@ public final class IndexBuffer extends AbstractIndex implements Index, IndexRead return this.heap.count(key);
}
- public synchronized ReferenceContainer get(final String wordHash, final Set<String> urlselection) {
+ public synchronized ReferenceContainer<ReferenceType> get(final String wordHash, final Set<String> urlselection) {
if (wordHash == null) return null;
// retrieve container
- ReferenceContainer container = heap.get(wordHash, null);
+ ReferenceContainer<ReferenceType> container = heap.get(wordHash, null);
// We must not use the container from cache to store everything we find,
// as that container remains linked to in the cache and might be changed later
@@ -259,10 +261,10 @@ public final class IndexBuffer extends AbstractIndex implements Index, IndexRead return container;
}
- public synchronized ReferenceContainer delete(final String wordHash) {
+ public synchronized ReferenceContainer<ReferenceType> delete(final String wordHash) {
// returns the index that had been deleted
if (wordHash == null || heap == null) return null;
- final ReferenceContainer container = heap.delete(wordHash);
+ final ReferenceContainer<ReferenceType> container = heap.delete(wordHash);
hashScore.deleteScore(wordHash);
hashDate.deleteScore(wordHash);
return container;
@@ -300,7 +302,7 @@ public final class IndexBuffer extends AbstractIndex implements Index, IndexRead return 0;
}
- public synchronized void add(final ReferenceContainer container) {
+ public synchronized void add(final ReferenceContainer<ReferenceType> container) {
if (container == null || container.size() == 0 || heap == null) return;
// put new words into cache
@@ -309,7 +311,7 @@ public final class IndexBuffer extends AbstractIndex implements Index, IndexRead hashDate.setScore(container.getTermHash(), intTime(System.currentTimeMillis()));
}
- public void add(final String wordHash, final WordReferenceRow entry) throws IOException {
+ public void add(final String wordHash, final ReferenceType entry) throws IOException {
if (entry == null || heap == null) return;
// put new words into cache
@@ -325,7 +327,7 @@ public final class IndexBuffer extends AbstractIndex implements Index, IndexRead hashDate.clear();
}
- public Iterator<ReferenceContainer> iterator() {
+ public Iterator<ReferenceContainer<ReferenceType>> iterator() {
return references(null, false);
}
@@ -337,7 +339,7 @@ public final class IndexBuffer extends AbstractIndex implements Index, IndexRead // calculate the real size in bytes of the index cache
long cacheBytes = 0;
final long entryBytes = WordReferenceRow.urlEntryRow.objectsize;
- final Iterator<ReferenceContainer> it = references(null, false);
+ final Iterator<ReferenceContainer<ReferenceType>> it = references(null, false);
while (it.hasNext()) cacheBytes += it.next().size() * entryBytes;
return cacheBytes;
}
diff --git a/source/de/anomic/kelondro/text/IndexCell.java b/source/de/anomic/kelondro/text/IndexCell.java index a30792daa..ccb0b4606 100644 --- a/source/de/anomic/kelondro/text/IndexCell.java +++ b/source/de/anomic/kelondro/text/IndexCell.java @@ -36,7 +36,6 @@ import de.anomic.kelondro.order.ByteOrder; import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.MergeIterator;
import de.anomic.kelondro.order.Order;
-import de.anomic.kelondro.text.referencePrototype.WordReferenceRow;
import de.anomic.kelondro.util.MemoryControl;
import de.anomic.server.serverProfiling;
@@ -51,30 +50,33 @@ import de.anomic.server.serverProfiling; * another BLOB file in the index array.
*/
-public final class IndexCell extends AbstractBufferedIndex implements BufferedIndex {
+public final class IndexCell<ReferenceType extends Reference> extends AbstractBufferedIndex<ReferenceType> implements BufferedIndex<ReferenceType> {
private static final long cleanupCycle = 10000;
// class variables
- private final ReferenceContainerArray array;
- private ReferenceContainerCache ram;
- private int maxRamEntries;
- private final IODispatcher merger;
- private long lastCleanup;
- private final long targetFileSize, maxFileSize;
+ private final ReferenceContainerArray<ReferenceType> array;
+ private ReferenceContainerCache<ReferenceType> ram;
+ private int maxRamEntries;
+ private final IODispatcher<ReferenceType> merger;
+ private long lastCleanup;
+ private final long targetFileSize, maxFileSize;
public IndexCell(
final File cellPath,
+ final ReferenceFactory<ReferenceType> factory,
final ByteOrder termOrder,
final Row payloadrow,
final int maxRamEntries,
final long targetFileSize,
final long maxFileSize,
- IODispatcher merger
+ IODispatcher<ReferenceType> merger
) throws IOException {
- this.array = new ReferenceContainerArray(cellPath, termOrder, payloadrow, merger);
- this.ram = new ReferenceContainerCache(payloadrow, termOrder);
+ super(factory);
+
+ this.array = new ReferenceContainerArray<ReferenceType>(cellPath, factory, termOrder, payloadrow, merger);
+ this.ram = new ReferenceContainerCache<ReferenceType>(factory, payloadrow, termOrder);
this.ram.initWriteMode();
this.maxRamEntries = maxRamEntries;
this.merger = merger;
@@ -94,13 +96,13 @@ public final class IndexCell extends AbstractBufferedIndex implements BufferedIn * @throws IOException
* @throws IOException
*/
- public synchronized void add(ReferenceContainer newEntries) throws IOException {
+ public synchronized void add(ReferenceContainer<ReferenceType> newEntries) throws IOException {
this.ram.add(newEntries);
serverProfiling.update("wordcache", Long.valueOf(this.ram.size()), true);
cleanCache();
}
- public synchronized void add(String hash, WordReferenceRow entry) throws IOException {
+ public synchronized void add(String hash, ReferenceType entry) throws IOException {
this.ram.add(hash, entry);
serverProfiling.update("wordcache", Long.valueOf(this.ram.size()), true);
cleanCache();
@@ -115,8 +117,8 @@ public final class IndexCell extends AbstractBufferedIndex implements BufferedIn }
public int count(String termHash) {
- ReferenceContainer c0 = this.ram.get(termHash, null);
- ReferenceContainer c1;
+ ReferenceContainer<ReferenceType> c0 = this.ram.get(termHash, null);
+ ReferenceContainer<ReferenceType> c1;
try {
c1 = this.array.get(termHash);
} catch (IOException e) {
@@ -134,9 +136,9 @@ public final class IndexCell extends AbstractBufferedIndex implements BufferedIn * all containers in the BLOBs and the RAM are merged and returned
* @throws IOException
*/
- public ReferenceContainer get(String termHash, Set<String> urlselection) throws IOException {
- ReferenceContainer c0 = this.ram.get(termHash, null);
- ReferenceContainer c1 = this.array.get(termHash);
+ public ReferenceContainer<ReferenceType> get(String termHash, Set<String> urlselection) throws IOException {
+ ReferenceContainer<ReferenceType> c0 = this.ram.get(termHash, null);
+ ReferenceContainer<ReferenceType> c1 = this.array.get(termHash);
if (c1 == null) {
if (c0 == null) return null;
return c0;
@@ -150,9 +152,9 @@ public final class IndexCell extends AbstractBufferedIndex implements BufferedIn * the deleted containers are merged and returned as result of the method
* @throws IOException
*/
- public ReferenceContainer delete(String termHash) throws IOException {
- ReferenceContainer c0 = this.ram.delete(termHash);
- ReferenceContainer c1 = this.array.get(termHash);
+ public ReferenceContainer<ReferenceType> delete(String termHash) throws IOException {
+ ReferenceContainer<ReferenceType> c0 = this.ram.delete(termHash);
+ ReferenceContainer<ReferenceType> c1 = this.array.get(termHash);
if (c1 == null) {
if (c0 == null) return null;
return c0;
@@ -171,16 +173,16 @@ public final class IndexCell extends AbstractBufferedIndex implements BufferedIn * @throws IOException
*/
public int remove(String termHash, Set<String> urlHashes) throws IOException {
- int reduced = this.array.replace(termHash, new RemoveRewriter(urlHashes));
+ int reduced = this.array.replace(termHash, new RemoveRewriter<ReferenceType>(urlHashes));
return reduced / this.array.rowdef().objectsize;
}
public boolean remove(String termHash, String urlHash) throws IOException {
- int reduced = this.array.replace(termHash, new RemoveRewriter(urlHash));
+ int reduced = this.array.replace(termHash, new RemoveRewriter<ReferenceType>(urlHash));
return reduced > 0;
}
- private static class RemoveRewriter implements ReferenceContainerArray.ContainerRewriter {
+ private static class RemoveRewriter<RT extends Reference> implements ReferenceContainerArray.ContainerRewriter<RT> {
Set<String> urlHashes;
@@ -193,19 +195,19 @@ public final class IndexCell extends AbstractBufferedIndex implements BufferedIn this.urlHashes.add(urlHash);
}
- public ReferenceContainer rewrite(ReferenceContainer container) {
+ public ReferenceContainer<RT> rewrite(ReferenceContainer<RT> container) {
container.removeEntries(urlHashes);
return container;
}
}
- public CloneableIterator<ReferenceContainer> references(String starttermHash, boolean rot) {
- final Order<ReferenceContainer> containerOrder = new ReferenceContainerOrder(this.ram.rowdef().getOrdering().clone());
- containerOrder.rotate(new ReferenceContainer(starttermHash, this.ram.rowdef(), 0));
- return new MergeIterator<ReferenceContainer>(
+ public CloneableIterator<ReferenceContainer<ReferenceType>> references(String starttermHash, boolean rot) {
+ final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, this.ram.rowdef().getOrdering().clone());
+ containerOrder.rotate(new ReferenceContainer<ReferenceType>(factory, starttermHash, this.ram.rowdef(), 0));
+ return new MergeIterator<ReferenceContainer<ReferenceType>>(
this.ram.references(starttermHash, rot),
- new MergeIterator<ReferenceContainer>(
+ new MergeIterator<ReferenceContainer<ReferenceType>>(
this.ram.references(starttermHash, false),
this.array.wordContainerIterator(starttermHash, false, false),
containerOrder,
@@ -216,13 +218,13 @@ public final class IndexCell extends AbstractBufferedIndex implements BufferedIn true);
}
- public CloneableIterator<ReferenceContainer> references(String startTermHash, boolean rot, boolean ram) {
- final Order<ReferenceContainer> containerOrder = new ReferenceContainerOrder(this.ram.rowdef().getOrdering().clone());
- containerOrder.rotate(new ReferenceContainer(startTermHash, this.ram.rowdef(), 0));
+ public CloneableIterator<ReferenceContainer<ReferenceType>> references(String startTermHash, boolean rot, boolean ram) {
+ final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, this.ram.rowdef().getOrdering().clone());
+ containerOrder.rotate(new ReferenceContainer<ReferenceType>(factory, startTermHash, this.ram.rowdef(), 0));
if (ram) {
return this.ram.references(startTermHash, rot);
}
- return new MergeIterator<ReferenceContainer>(
+ return new MergeIterator<ReferenceContainer<ReferenceType>>(
this.ram.references(startTermHash, false),
this.array.wordContainerIterator(startTermHash, false, false),
containerOrder,
@@ -292,7 +294,7 @@ public final class IndexCell extends AbstractBufferedIndex implements BufferedIn //this.array.mountBLOBContainer(dumpFile);
merger.dump(this.ram, dumpFile, array);
// get a fresh ram cache
- this.ram = new ReferenceContainerCache(this.array.rowdef(), this.array.ordering());
+ this.ram = new ReferenceContainerCache<ReferenceType>(factory, this.array.rowdef(), this.array.ordering());
this.ram.initWriteMode();
}
diff --git a/source/de/anomic/kelondro/text/IndexCollection.java b/source/de/anomic/kelondro/text/IndexCollection.java index 506d062e2..a1c141f70 100644 --- a/source/de/anomic/kelondro/text/IndexCollection.java +++ b/source/de/anomic/kelondro/text/IndexCollection.java @@ -54,15 +54,16 @@ import de.anomic.kelondro.order.RotateIterator; import de.anomic.kelondro.table.EcoTable; import de.anomic.kelondro.table.FixedWidthArray; import de.anomic.kelondro.table.FlexTable; -import de.anomic.kelondro.text.referencePrototype.WordReferenceRow; +import de.anomic.kelondro.text.referencePrototype.WordReference; import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.MemoryControl; import de.anomic.kelondro.util.kelondroException; import de.anomic.kelondro.util.kelondroOutOfLimitsException; import de.anomic.kelondro.util.Log; +import de.anomic.plasma.plasmaWordIndex; import de.anomic.yacy.yacyURL; -public class IndexCollection extends AbstractIndex implements Index { +public class IndexCollection<ReferenceType extends Reference> extends AbstractIndex<ReferenceType> implements Index<ReferenceType> { private static final int loadfactor = 4; private static final int serialNumber = 0; @@ -92,11 +93,14 @@ public class IndexCollection extends AbstractIndex implements Index { public IndexCollection( final File path, final String filenameStub, + final ReferenceFactory<ReferenceType> factory, final int keyLength, final ByteOrder wordOrder, final int maxpartitions, final Row payloadrow, boolean useCommons) throws IOException { + super(factory); + // the buffersize is number of bytes that are only used if the kelondroFlexTable is backed up with a kelondroTree indexErrors = 0; this.path = path; @@ -160,11 +164,11 @@ public class IndexCollection extends AbstractIndex implements Index { return index.row().objectOrder; } - public synchronized CloneableIterator<ReferenceContainer> references(final String startWordHash, final boolean rot) { + public synchronized CloneableIterator<ReferenceContainer<ReferenceType>> references(final String startWordHash, final boolean rot) { return new wordContainersIterator(startWordHash, rot); } - public class wordContainersIterator implements CloneableIterator<ReferenceContainer> { + public class wordContainersIterator implements CloneableIterator<ReferenceContainer<ReferenceType>> { private final Iterator<Object[]> wci; private final boolean rot; @@ -182,13 +186,13 @@ public class IndexCollection extends AbstractIndex implements Index { return wci.hasNext(); } - public ReferenceContainer next() { + public ReferenceContainer<ReferenceType> next() { final Object[] oo = wci.next(); if (oo == null) return null; final byte[] key = (byte[]) oo[0]; final RowSet collection = (RowSet) oo[1]; if (collection == null) return null; - return new ReferenceContainer(new String(key), collection); + return new ReferenceContainer<ReferenceType>(factory, new String(key), collection); } public void remove() { @@ -201,22 +205,22 @@ public class IndexCollection extends AbstractIndex implements Index { return this.has(wordHash.getBytes()); } - public ReferenceContainer get(final String wordHash, final Set<String> urlselection) { + public ReferenceContainer<ReferenceType> get(final String wordHash, final Set<String> urlselection) { try { final RowSet collection = this.get(wordHash.getBytes()); if (collection != null) collection.select(urlselection); if ((collection == null) || (collection.size() == 0)) return null; - return new ReferenceContainer(wordHash, collection); + return new ReferenceContainer<ReferenceType>(factory, wordHash, collection); } catch (final IOException e) { return null; } } - public ReferenceContainer delete(final String wordHash) { + public ReferenceContainer<ReferenceType> delete(final String wordHash) { try { final RowSet collection = this.delete(wordHash.getBytes()); if (collection == null) return null; - return new ReferenceContainer(wordHash, collection); + return new ReferenceContainer<ReferenceType>(factory, wordHash, collection); } catch (final IOException e) { return null; } @@ -240,7 +244,7 @@ public class IndexCollection extends AbstractIndex implements Index { } } - public void add(final ReferenceContainer newEntries) { + public void add(final ReferenceContainer<ReferenceType> newEntries) { if (newEntries == null) return; try { this.merge(newEntries); @@ -251,10 +255,10 @@ public class IndexCollection extends AbstractIndex implements Index { } } - public void add(String wordhash, WordReferenceRow entry) { + public void add(String wordhash, ReferenceType entry) { if (entry == null) return; try { - ReferenceContainer container = new ReferenceContainer(wordhash, this.payloadrow, 1); + ReferenceContainer<ReferenceType> container = new ReferenceContainer<ReferenceType>(factory, wordhash, this.payloadrow, 1); container.add(entry); this.merge(container); } catch (final kelondroOutOfLimitsException e) { @@ -705,7 +709,7 @@ public class IndexCollection extends AbstractIndex implements Index { index.put(indexrow); // write modified indexrow } - private synchronized void merge(final ReferenceContainer container) throws IOException, kelondroOutOfLimitsException { + private synchronized void merge(final ReferenceContainer<ReferenceType> container) throws IOException, kelondroOutOfLimitsException { if ((container == null) || (container.size() == 0)) return; final byte[] key = container.getTermHash().getBytes(); @@ -1059,8 +1063,10 @@ public class IndexCollection extends AbstractIndex implements Index { final String filenameStub = args[1]; try { // initialize collection index - final IndexCollection collectionIndex = new IndexCollection( - path, filenameStub, 9 /*keyLength*/, + final IndexCollection<WordReference> collectionIndex = new IndexCollection<WordReference>( + path, filenameStub, + plasmaWordIndex.wordReferenceFactory, + 9 /*keyLength*/, NaturalOrder.naturalOrder, 7, rowdef, false); @@ -1084,7 +1090,7 @@ public class IndexCollection extends AbstractIndex implements Index { for (int j = 0; j < i; j++) { collection.addUnique(rowdef.newEntry(new byte[][]{("def" + j).getBytes(), "xxx".getBytes()})); } - collectionIndex.merge(new ReferenceContainer("key-" + i, collection)); + collectionIndex.merge(new ReferenceContainer<WordReference>(plasmaWordIndex.wordReferenceFactory,"key-" + i, collection)); } // printout of index diff --git a/source/de/anomic/kelondro/text/IndexCollectionMigration.java b/source/de/anomic/kelondro/text/IndexCollectionMigration.java index 9e79e70f8..ac608256d 100644 --- a/source/de/anomic/kelondro/text/IndexCollectionMigration.java +++ b/source/de/anomic/kelondro/text/IndexCollectionMigration.java @@ -45,26 +45,29 @@ import de.anomic.kelondro.text.referencePrototype.WordReferenceRow; import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.Log;
-public final class IndexCollectionMigration extends AbstractBufferedIndex implements Index, BufferedIndex {
+public final class IndexCollectionMigration<ReferenceType extends Reference> extends AbstractBufferedIndex<ReferenceType> implements Index<ReferenceType>, BufferedIndex<ReferenceType> {
- private final IndexCell cell;
- private IndexCollection collections;
- private final IODispatcher merger;
+ private final IndexCell<ReferenceType> cell;
+ private IndexCollection<ReferenceType> collections;
+ private final IODispatcher<ReferenceType> merger;
public IndexCollectionMigration (
final File indexPrimaryTextLocation,
+ final ReferenceFactory<ReferenceType> factory,
final ByteOrder wordOrdering,
final Row payloadrow,
final int entityCacheMaxSize,
final long targetFileSize,
final long maxFileSize,
- final IODispatcher merger,
+ final IODispatcher<ReferenceType> merger,
final Log log) throws IOException {
-
+ super(factory);
+
this.merger = merger;
final File celldir = new File(indexPrimaryTextLocation, "RICELL");
- this.cell = new IndexCell(
+ this.cell = new IndexCell<ReferenceType>(
celldir,
+ factory,
wordOrdering,
WordReferenceRow.urlEntryRow,
entityCacheMaxSize,
@@ -98,9 +101,10 @@ public final class IndexCollectionMigration extends AbstractBufferedIndex implem // open collections, this is for migration only.
final File textindexcollections = new File(indexPrimaryTextLocation, "RICOLLECTION");
if (textindexcollections.exists()) {
- this.collections = new IndexCollection(
+ this.collections = new IndexCollection<ReferenceType>(
textindexcollections,
"collection",
+ factory,
12,
Base64Order.enhancedCoder,
BufferedIndexCollection.maxCollectionPartition,
@@ -125,11 +129,11 @@ public final class IndexCollectionMigration extends AbstractBufferedIndex implem /* methods for interface Index */
- public void add(final ReferenceContainer entries) throws IOException {
+ public void add(final ReferenceContainer<ReferenceType> entries) throws IOException {
assert (entries.row().objectsize == WordReferenceRow.urlEntryRow.objectsize);
if (this.collections != null) {
- ReferenceContainer e = this.collections.delete(entries.getTermHash());
+ ReferenceContainer<ReferenceType> e = this.collections.delete(entries.getTermHash());
if (e != null) {
e.merge(entries);
cell.add(e);
@@ -141,9 +145,9 @@ public final class IndexCollectionMigration extends AbstractBufferedIndex implem }
}
- public void add(final String wordHash, final WordReferenceRow entry) throws IOException {
+ public void add(final String wordHash, final ReferenceType entry) throws IOException {
if (this.collections != null) {
- ReferenceContainer e = this.collections.delete(wordHash);
+ ReferenceContainer<ReferenceType> e = this.collections.delete(wordHash);
if (e != null) {
e.add(entry);
cell.add(e);
@@ -157,7 +161,7 @@ public final class IndexCollectionMigration extends AbstractBufferedIndex implem public boolean has(final String wordHash) {
if (this.collections != null) {
- ReferenceContainer e = this.collections.delete(wordHash);
+ ReferenceContainer<ReferenceType> e = this.collections.delete(wordHash);
if (e != null) {
try {
cell.add(e);
@@ -175,7 +179,7 @@ public final class IndexCollectionMigration extends AbstractBufferedIndex implem public int count(String wordHash) {
if (this.collections != null) {
- ReferenceContainer e = this.collections.delete(wordHash);
+ ReferenceContainer<ReferenceType> e = this.collections.delete(wordHash);
if (e != null) {
try {
cell.add(e);
@@ -191,28 +195,28 @@ public final class IndexCollectionMigration extends AbstractBufferedIndex implem }
}
- public ReferenceContainer get(final String wordHash, final Set<String> urlselection) throws IOException {
+ public ReferenceContainer<ReferenceType> get(final String wordHash, final Set<String> urlselection) throws IOException {
if (wordHash == null) {
// wrong input
return null;
}
if (this.collections != null) {
- ReferenceContainer e = this.collections.delete(wordHash);
+ ReferenceContainer<ReferenceType> e = this.collections.delete(wordHash);
if (e != null) cell.add(e);
}
return this.cell.get(wordHash, urlselection);
}
- public ReferenceContainer delete(final String wordHash) throws IOException {
- ReferenceContainer cc = cell.delete(wordHash);
+ public ReferenceContainer<ReferenceType> delete(final String wordHash) throws IOException {
+ ReferenceContainer<ReferenceType> cc = cell.delete(wordHash);
if (cc == null) {
if (collections == null) return null;
return collections.delete(wordHash);
} else {
if (collections == null) return cc;
- ReferenceContainer cd = collections.delete(wordHash);
+ ReferenceContainer<ReferenceType> cd = collections.delete(wordHash);
if (cd == null) return cc;
return cc.merge(cd);
}
@@ -220,7 +224,7 @@ public final class IndexCollectionMigration extends AbstractBufferedIndex implem public boolean remove(final String wordHash, final String urlHash) throws IOException {
if (this.collections != null) {
- ReferenceContainer e = this.collections.delete(wordHash);
+ ReferenceContainer<ReferenceType> e = this.collections.delete(wordHash);
if (e != null) cell.add(e);
}
return cell.remove(wordHash, urlHash);
@@ -228,28 +232,29 @@ public final class IndexCollectionMigration extends AbstractBufferedIndex implem public int remove(final String wordHash, final Set<String> urlHashes) throws IOException {
if (this.collections != null) {
- ReferenceContainer e = this.collections.delete(wordHash);
+ ReferenceContainer<ReferenceType> e = this.collections.delete(wordHash);
if (e != null) cell.add(e);
}
return cell.remove(wordHash, urlHashes);
}
- public synchronized CloneableIterator<ReferenceContainer> references(final String startHash, final boolean rot, final boolean ram) throws IOException {
- final CloneableIterator<ReferenceContainer> i = wordContainers(startHash, ram);
+ public synchronized CloneableIterator<ReferenceContainer<ReferenceType>> references(final String startHash, final boolean rot, final boolean ram) throws IOException {
+ final CloneableIterator<ReferenceContainer<ReferenceType>> i = wordContainers(startHash, ram);
if (rot) {
- return new RotateIterator<ReferenceContainer>(i, new String(Base64Order.zero(startHash.length())), cell.size() + ((ram) ? 0 : collections.size()));
+ return new RotateIterator<ReferenceContainer<ReferenceType>>(i, new String(Base64Order.zero(startHash.length())), cell.size() + ((ram) ? 0 : collections.size()));
}
return i;
}
- private synchronized CloneableIterator<ReferenceContainer> wordContainers(final String startWordHash, final boolean ram) throws IOException {
- final Order<ReferenceContainer> containerOrder = new ReferenceContainerOrder(cell.ordering().clone());
- containerOrder.rotate(ReferenceContainer.emptyContainer(startWordHash, 0));
+ private synchronized CloneableIterator<ReferenceContainer<ReferenceType>> wordContainers(final String startWordHash, final boolean ram) throws IOException {
+ final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, cell.ordering().clone());
+ ReferenceContainer<ReferenceType> emptyContainer = ReferenceContainer.emptyContainer(factory, startWordHash, 0);
+ containerOrder.rotate(emptyContainer);
if (ram) {
return cell.references(startWordHash, true);
}
if (collections == null) return cell.references(startWordHash, false);
- return new MergeIterator<ReferenceContainer>(
+ return new MergeIterator<ReferenceContainer<ReferenceType>>(
cell.references(startWordHash, false),
collections.references(startWordHash, false),
containerOrder,
@@ -320,11 +325,11 @@ public final class IndexCollectionMigration extends AbstractBufferedIndex implem return cell.ordering();
}
- public CloneableIterator<ReferenceContainer> references(String startWordHash, boolean rot) {
- final Order<ReferenceContainer> containerOrder = new ReferenceContainerOrder(this.cell.ordering().clone());
+ public CloneableIterator<ReferenceContainer<ReferenceType>> references(String startWordHash, boolean rot) {
+ final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, this.cell.ordering().clone());
if (this.collections == null) return this.cell.references(startWordHash, rot);
//else
- return new MergeIterator<ReferenceContainer>(
+ return new MergeIterator<ReferenceContainer<ReferenceType>>(
this.cell.references(startWordHash, false),
this.collections.references(startWordHash, false),
containerOrder,
diff --git a/source/de/anomic/kelondro/text/IndexReader.java b/source/de/anomic/kelondro/text/IndexReader.java index f3cd005cb..2874990d0 100644 --- a/source/de/anomic/kelondro/text/IndexReader.java +++ b/source/de/anomic/kelondro/text/IndexReader.java @@ -30,12 +30,12 @@ import java.util.Set; import de.anomic.kelondro.order.CloneableIterator;
-public interface IndexReader {
+public interface IndexReader<ReferenceType extends Reference> {
public int size();
public boolean has(String wordHash); // should only be used if in case that true is returned the getContainer is NOT called
- public ReferenceContainer get(String wordHash, Set<String> urlselection);
- public CloneableIterator<ReferenceContainer> references(String startWordHash, boolean rot);
+ public ReferenceContainer<ReferenceType> get(String wordHash, Set<String> urlselection);
+ public CloneableIterator<ReferenceContainer<ReferenceType>> references(String startWordHash, boolean rot);
public void close();
}
diff --git a/source/de/anomic/kelondro/text/Reference.java b/source/de/anomic/kelondro/text/Reference.java index 62e4479cb..2b4c82be3 100644 --- a/source/de/anomic/kelondro/text/Reference.java +++ b/source/de/anomic/kelondro/text/Reference.java @@ -38,8 +38,6 @@ public interface Reference { public long lastModified();
- //public long freshUntil();
-
public String toString();
public boolean isNewer(Reference other);
@@ -47,4 +45,19 @@ public interface Reference { public boolean isOlder(Reference other);
public int hashCode();
+
+ public void join(final Reference oe);
+
+ public int positions();
+
+ public int maxposition();
+
+ public int minposition();
+
+ public int position(int p);
+
+ public void addPosition(int position);
+
+ public int distance();
+
}
diff --git a/source/de/anomic/kelondro/text/ReferenceContainer.java b/source/de/anomic/kelondro/text/ReferenceContainer.java index a76450f31..7f55a5c65 100644 --- a/source/de/anomic/kelondro/text/ReferenceContainer.java +++ b/source/de/anomic/kelondro/text/ReferenceContainer.java @@ -37,9 +37,7 @@ import java.util.TreeMap; import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.index.RowSet;
import de.anomic.kelondro.order.Base64Order;
-import de.anomic.kelondro.text.referencePrototype.WordReference;
import de.anomic.kelondro.text.referencePrototype.WordReferenceRow;
-import de.anomic.kelondro.text.referencePrototype.WordReferenceVars;
import de.anomic.kelondro.util.ByteBuffer;
/**
@@ -49,29 +47,32 @@ import de.anomic.kelondro.util.ByteBuffer; * This class extends the RowSet with methods for the handling of
* special ReferenceRow Row entry objects.
*/
-public class ReferenceContainer extends RowSet {
+public class ReferenceContainer<RT extends Reference> extends RowSet {
private String termHash;
-
- public ReferenceContainer(final String termHash, final RowSet collection) {
+ private ReferenceFactory<RT> factory;
+
+ public ReferenceContainer(final ReferenceFactory<RT> factory, final String termHash, final RowSet collection) {
super(collection);
+ this.factory = factory;
this.termHash = termHash;
}
- public ReferenceContainer(final String termHash, final Row rowdef, final int objectCount) {
+ public ReferenceContainer(final ReferenceFactory<RT> factory, final String termHash, final Row rowdef, final int objectCount) {
super(rowdef, objectCount);
this.termHash = termHash;
+ this.factory = factory;
this.lastTimeWrote = 0;
}
- public ReferenceContainer topLevelClone() {
- final ReferenceContainer newContainer = new ReferenceContainer(this.termHash, this.rowdef, this.size());
+ public ReferenceContainer<RT> topLevelClone() {
+ final ReferenceContainer<RT> newContainer = new ReferenceContainer<RT>(this.factory, this.termHash, this.rowdef, this.size());
newContainer.addAllUnique(this);
return newContainer;
}
- public static ReferenceContainer emptyContainer(final String wordHash, final int elementCount) {
- return new ReferenceContainer(wordHash, WordReferenceRow.urlEntryRow, elementCount);
+ public static <RT extends Reference> ReferenceContainer<RT> emptyContainer(final ReferenceFactory<RT> factory, final String wordHash, final int elementCount) {
+ return new ReferenceContainer<RT>(factory, wordHash, WordReferenceRow.urlEntryRow, elementCount);
}
public void setWordHash(final String newWordHash) {
@@ -86,42 +87,31 @@ public class ReferenceContainer extends RowSet { return termHash;
}
- public void add(final WordReferenceRow entry) {
+ public void add(final Reference entry) {
// add without double-occurrence test
assert entry.toKelondroEntry().objectsize() == super.rowdef.objectsize;
this.addUnique(entry.toKelondroEntry());
}
- public void add(final Reference entry, final long updateTime) {
- // add without double-occurrence test
- if (entry instanceof WordReferenceRow) {
- assert ((WordReferenceRow) entry).toKelondroEntry().objectsize() == super.rowdef.objectsize;
- this.add((WordReferenceRow) entry);
- } else {
- this.add(((WordReferenceVars) entry).toRowEntry());
- }
- this.lastTimeWrote = updateTime;
- }
-
- public ReferenceContainer merge(final ReferenceContainer c) {
- return new ReferenceContainer(this.termHash, super.merge(c));
+ public ReferenceContainer<RT> merge(final ReferenceContainer<RT> c) {
+ return new ReferenceContainer<RT>(this.factory, this.termHash, super.merge(c));
}
- public Reference put(final WordReferenceRow entry) {
+ public Reference put(final Reference entry) {
assert entry.toKelondroEntry().objectsize() == super.rowdef.objectsize;
final Row.Entry r = super.replace(entry.toKelondroEntry());
if (r == null) return null;
return new WordReferenceRow(r);
}
- public boolean putRecent(final WordReferenceRow entry) {
+ public boolean putRecent(final Reference entry) {
assert entry.toKelondroEntry().objectsize() == super.rowdef.objectsize;
// returns true if the new entry was added, false if it already existed
final Row.Entry oldEntryRow = this.replace(entry.toKelondroEntry());
if (oldEntryRow == null) {
return true;
}
- final WordReferenceRow oldEntry = new WordReferenceRow(oldEntryRow);
+ final Reference oldEntry = new WordReferenceRow(oldEntryRow);
if (entry.isOlder(oldEntry)) { // A more recent Entry is already in this container
this.replace(oldEntry.toKelondroEntry()); // put it back
return false;
@@ -129,13 +119,13 @@ public class ReferenceContainer extends RowSet { return true;
}
- public int putAllRecent(final ReferenceContainer c) {
+ public int putAllRecent(final ReferenceContainer<RT> c) {
// adds all entries in c and checks every entry for double-occurrence
// returns the number of new elements
if (c == null) return 0;
int x = 0;
synchronized (c) {
- final Iterator<WordReferenceRow> i = c.entries();
+ final Iterator<RT> i = c.entries();
while (i.hasNext()) {
try {
if (putRecent(i.next())) x++;
@@ -148,10 +138,10 @@ public class ReferenceContainer extends RowSet { return x;
}
- public WordReference get(final String urlHash) {
+ public RT get(final String urlHash) {
final Row.Entry entry = this.get(urlHash.getBytes());
if (entry == null) return null;
- return new WordReferenceRow(entry);
+ return this.factory.produce(entry, false);
}
/**
@@ -159,10 +149,10 @@ public class ReferenceContainer extends RowSet { * if the url hash was found, return the entry, but delete the entry from the container
* if the entry was not found, return null.
*/
- public Reference remove(final String urlHash) {
+ public RT remove(final String urlHash) {
final Row.Entry entry = remove(urlHash.getBytes());
if (entry == null) return null;
- return new WordReferenceRow(entry);
+ return this.factory.produce(entry, false);
}
public int removeEntries(final Set<String> urlHashes) {
@@ -172,12 +162,12 @@ public class ReferenceContainer extends RowSet { return count;
}
- public Iterator<WordReferenceRow> entries() {
+ public Iterator<RT> entries() {
// returns an iterator of indexRWIEntry objects
return new entryIterator();
}
- public class entryIterator implements Iterator<WordReferenceRow> {
+ public class entryIterator implements Iterator<RT> {
Iterator<Row.Entry> rowEntryIterator;
@@ -189,10 +179,10 @@ public class ReferenceContainer extends RowSet { return rowEntryIterator.hasNext();
}
- public WordReferenceRow next() {
+ public RT next() {
final Row.Entry rentry = rowEntryIterator.next();
if (rentry == null) return null;
- return new WordReferenceRow(rentry);
+ return factory.produce(rentry, false);
}
public void remove() {
@@ -202,8 +192,8 @@ public class ReferenceContainer extends RowSet { }
public static Object mergeUnique(final Object a, final Object b) {
- final ReferenceContainer c = (ReferenceContainer) a;
- c.addAllUnique((ReferenceContainer) b);
+ final ReferenceContainer<?> c = (ReferenceContainer<?>) a;
+ c.addAllUnique((ReferenceContainer<?>) b);
return c;
}
@@ -226,29 +216,33 @@ public class ReferenceContainer extends RowSet { containerMergeMethod = meth;
}
- public static ReferenceContainer joinExcludeContainers(
- final Collection<ReferenceContainer> includeContainers,
- final Collection<ReferenceContainer> excludeContainers,
+ public static <RT extends Reference> ReferenceContainer<RT> joinExcludeContainers(
+ final ReferenceFactory<RT> factory,
+ final Collection<ReferenceContainer<RT>> includeContainers,
+ final Collection<ReferenceContainer<RT>> excludeContainers,
final int maxDistance) {
// join a search result and return the joincount (number of pages after join)
// since this is a conjunction we return an empty entity if any word is not known
- if (includeContainers == null) return ReferenceContainer.emptyContainer(null, 0);
+ if (includeContainers == null) return ReferenceContainer.emptyContainer(factory, null, 0);
// join the result
- final ReferenceContainer rcLocal = ReferenceContainer.joinContainers(includeContainers, maxDistance);
- if (rcLocal == null) return ReferenceContainer.emptyContainer(null, 0);
+ final ReferenceContainer<RT> rcLocal = ReferenceContainer.joinContainers(factory, includeContainers, maxDistance);
+ if (rcLocal == null) return ReferenceContainer.emptyContainer(factory, null, 0);
excludeContainers(rcLocal, excludeContainers);
return rcLocal;
}
- public static ReferenceContainer joinContainers(final Collection<ReferenceContainer> containers, final int maxDistance) {
+ public static <RT extends Reference> ReferenceContainer<RT> joinContainers(
+ final ReferenceFactory<RT> factory,
+ final Collection<ReferenceContainer<RT>> containers,
+ final int maxDistance) {
// order entities by their size
- final TreeMap<Long, ReferenceContainer> map = new TreeMap<Long, ReferenceContainer>();
- ReferenceContainer singleContainer;
- final Iterator<ReferenceContainer> i = containers.iterator();
+ final TreeMap<Long, ReferenceContainer<RT>> map = new TreeMap<Long, ReferenceContainer<RT>>();
+ ReferenceContainer<RT> singleContainer;
+ final Iterator<ReferenceContainer<RT>> i = containers.iterator();
int count = 0;
while (i.hasNext()) {
// get next entity:
@@ -268,13 +262,13 @@ public class ReferenceContainer extends RowSet { // the map now holds the search results in order of number of hits per word
// we now must pairwise build up a conjunction of these sets
Long k = map.firstKey(); // the smallest, which means, the one with the least entries
- ReferenceContainer searchA, searchB, searchResult = map.remove(k);
+ ReferenceContainer<RT> searchA, searchB, searchResult = map.remove(k);
while ((map.size() > 0) && (searchResult.size() > 0)) {
// take the first element of map which is a result and combine it with result
k = map.firstKey(); // the next smallest...
searchA = searchResult;
searchB = map.remove(k);
- searchResult = ReferenceContainer.joinConstructive(searchA, searchB, maxDistance);
+ searchResult = ReferenceContainer.joinConstructive(factory, searchA, searchB, maxDistance);
// free resources
searchA = null;
searchB = null;
@@ -285,12 +279,12 @@ public class ReferenceContainer extends RowSet { return searchResult;
}
- public static ReferenceContainer excludeContainers(ReferenceContainer pivot, final Collection<ReferenceContainer> containers) {
+ public static <RT extends Reference> ReferenceContainer<RT> excludeContainers(ReferenceContainer<RT> pivot, final Collection<ReferenceContainer<RT>> containers) {
// check if there is any result
if ((containers == null) || (containers.size() == 0)) return pivot; // no result, nothing found
- final Iterator<ReferenceContainer> i = containers.iterator();
+ final Iterator<ReferenceContainer<RT>> i = containers.iterator();
while (i.hasNext()) {
pivot = excludeDestructive(pivot, i.next());
if ((pivot == null) || (pivot.size() == 0)) return null;
@@ -306,7 +300,11 @@ public class ReferenceContainer extends RowSet { return l;
}
- public static ReferenceContainer joinConstructive(final ReferenceContainer i1, final ReferenceContainer i2, final int maxDistance) {
+ public static <RT extends Reference> ReferenceContainer<RT> joinConstructive(
+ final ReferenceFactory<RT> factory,
+ final ReferenceContainer<RT> i1,
+ final ReferenceContainer<RT> i2,
+ final int maxDistance) {
if ((i1 == null) || (i2 == null)) return null;
if ((i1.size() == 0) || (i2.size() == 0)) return null;
@@ -319,50 +317,59 @@ public class ReferenceContainer extends RowSet { // start most efficient method
if (stepsEnum > stepsTest) {
if (i1.size() < i2.size())
- return joinConstructiveByTest(i1, i2, maxDistance);
+ return joinConstructiveByTest(factory, i1, i2, maxDistance);
else
- return joinConstructiveByTest(i2, i1, maxDistance);
+ return joinConstructiveByTest(factory, i2, i1, maxDistance);
}
- return joinConstructiveByEnumeration(i1, i2, maxDistance);
+ return joinConstructiveByEnumeration(factory, i1, i2, maxDistance);
}
- private static ReferenceContainer joinConstructiveByTest(final ReferenceContainer small, final ReferenceContainer large, final int maxDistance) {
+ private static <RT extends Reference> ReferenceContainer<RT> joinConstructiveByTest(
+ final ReferenceFactory<RT> factory,
+ final ReferenceContainer<RT> small,
+ final ReferenceContainer<RT> large,
+ final int maxDistance) {
System.out.println("DEBUG: JOIN METHOD BY TEST, maxdistance = " + maxDistance);
assert small.rowdef.equals(large.rowdef) : "small = " + small.rowdef.toString() + "; large = " + large.rowdef.toString();
final int keylength = small.rowdef.width(0);
assert (keylength == large.rowdef.width(0));
- final ReferenceContainer conj = new ReferenceContainer(null, small.rowdef, 0); // start with empty search result
- final Iterator<WordReferenceRow> se = small.entries();
- WordReferenceVars ie0;
- WordReference ie1;
+ final ReferenceContainer<RT> conj = new ReferenceContainer<RT>(factory, null, small.rowdef, 0); // start with empty search result
+ final Iterator<RT> se = small.entries();
+ RT ie0;
+ RT ie1;
while (se.hasNext()) {
- ie0 = new WordReferenceVars(se.next());
+ ie0 = se.next();
ie1 = large.get(ie0.metadataHash());
if ((ie0 != null) && (ie1 != null)) {
assert (ie0.metadataHash().length() == keylength) : "ie0.urlHash() = " + ie0.metadataHash();
assert (ie1.metadataHash().length() == keylength) : "ie1.urlHash() = " + ie1.metadataHash();
// this is a hit. Calculate word distance:
+
ie0.join(ie1);
- if (ie0.worddistance() <= maxDistance) conj.add(ie0.toRowEntry());
+ if (ie0.distance() <= maxDistance) conj.add(ie0);
}
}
return conj;
}
- private static ReferenceContainer joinConstructiveByEnumeration(final ReferenceContainer i1, final ReferenceContainer i2, final int maxDistance) {
+ private static <RT extends Reference> ReferenceContainer<RT> joinConstructiveByEnumeration(
+ final ReferenceFactory<RT> factory,
+ final ReferenceContainer<RT> i1,
+ final ReferenceContainer<RT> i2,
+ final int maxDistance) {
System.out.println("DEBUG: JOIN METHOD BY ENUMERATION, maxdistance = " + maxDistance);
assert i1.rowdef.equals(i2.rowdef) : "i1 = " + i1.rowdef.toString() + "; i2 = " + i2.rowdef.toString();
final int keylength = i1.rowdef.width(0);
assert (keylength == i2.rowdef.width(0));
- final ReferenceContainer conj = new ReferenceContainer(null, i1.rowdef, 0); // start with empty search result
+ final ReferenceContainer<RT> conj = new ReferenceContainer<RT>(factory, null, i1.rowdef, 0); // start with empty search result
if (!((i1.rowdef.getOrdering().signature().equals(i2.rowdef.getOrdering().signature())))) return conj; // ordering must be equal
- final Iterator<WordReferenceRow> e1 = i1.entries();
- final Iterator<WordReferenceRow> e2 = i2.entries();
+ final Iterator<RT> e1 = i1.entries();
+ final Iterator<RT> e2 = i2.entries();
int c;
if ((e1.hasNext()) && (e2.hasNext())) {
- WordReferenceVars ie1;
- WordReference ie2;
- ie1 = new WordReferenceVars(e1.next());
+ RT ie1;
+ RT ie2;
+ ie1 = e1.next();
ie2 = e2.next();
while (true) {
@@ -371,14 +378,14 @@ public class ReferenceContainer extends RowSet { c = i1.rowdef.getOrdering().compare(ie1.metadataHash().getBytes(), ie2.metadataHash().getBytes());
//System.out.println("** '" + ie1.getUrlHash() + "'.compareTo('" + ie2.getUrlHash() + "')="+c);
if (c < 0) {
- if (e1.hasNext()) ie1 = new WordReferenceVars(e1.next()); else break;
+ if (e1.hasNext()) ie1 = e1.next(); else break;
} else if (c > 0) {
if (e2.hasNext()) ie2 = e2.next(); else break;
} else {
// we have found the same urls in different searches!
ie1.join(ie2);
- if (ie1.worddistance() <= maxDistance) conj.add(ie1.toRowEntry());
- if (e1.hasNext()) ie1 = new WordReferenceVars(e1.next()); else break;
+ if (ie1.distance() <= maxDistance) conj.add(ie1);
+ if (e1.hasNext()) ie1 = e1.next(); else break;
if (e2.hasNext()) ie2 = e2.next(); else break;
}
}
@@ -386,7 +393,9 @@ public class ReferenceContainer extends RowSet { return conj;
}
- public static ReferenceContainer excludeDestructive(final ReferenceContainer pivot, final ReferenceContainer excl) {
+ public static <RT extends Reference> ReferenceContainer<RT> excludeDestructive(
+ final ReferenceContainer<RT> pivot,
+ final ReferenceContainer<RT> excl) {
if (pivot == null) return null;
if (excl == null) return pivot;
if (pivot.size() == 0) return null;
@@ -405,12 +414,14 @@ public class ReferenceContainer extends RowSet { return excludeDestructiveByEnumeration(pivot, excl);
}
- private static ReferenceContainer excludeDestructiveByTest(final ReferenceContainer pivot, final ReferenceContainer excl) {
+ private static <RT extends Reference> ReferenceContainer<RT> excludeDestructiveByTest(
+ final ReferenceContainer<RT> pivot,
+ final ReferenceContainer<RT> excl) {
assert pivot.rowdef.equals(excl.rowdef) : "small = " + pivot.rowdef.toString() + "; large = " + excl.rowdef.toString();
final int keylength = pivot.rowdef.width(0);
assert (keylength == excl.rowdef.width(0));
final boolean iterate_pivot = pivot.size() < excl.size();
- final Iterator<WordReferenceRow> se = (iterate_pivot) ? pivot.entries() : excl.entries();
+ final Iterator<RT> se = (iterate_pivot) ? pivot.entries() : excl.entries();
Reference ie0, ie1;
while (se.hasNext()) {
ie0 = se.next();
@@ -424,18 +435,18 @@ public class ReferenceContainer extends RowSet { return pivot;
}
- private static ReferenceContainer excludeDestructiveByEnumeration(final ReferenceContainer pivot, final ReferenceContainer excl) {
+ private static <RT extends Reference> ReferenceContainer<RT> excludeDestructiveByEnumeration(final ReferenceContainer<RT> pivot, final ReferenceContainer<RT> excl) {
assert pivot.rowdef.equals(excl.rowdef) : "i1 = " + pivot.rowdef.toString() + "; i2 = " + excl.rowdef.toString();
final int keylength = pivot.rowdef.width(0);
assert (keylength == excl.rowdef.width(0));
if (!((pivot.rowdef.getOrdering().signature().equals(excl.rowdef.getOrdering().signature())))) return pivot; // ordering must be equal
- final Iterator<WordReferenceRow> e1 = pivot.entries();
- final Iterator<WordReferenceRow> e2 = excl.entries();
+ final Iterator<RT> e1 = pivot.entries();
+ final Iterator<RT> e2 = excl.entries();
int c;
if ((e1.hasNext()) && (e2.hasNext())) {
- WordReferenceVars ie1;
- WordReference ie2;
- ie1 = new WordReferenceVars(e1.next());
+ RT ie1;
+ RT ie2;
+ ie1 = e1.next();
ie2 = e2.next();
while (true) {
@@ -444,14 +455,14 @@ public class ReferenceContainer extends RowSet { c = pivot.rowdef.getOrdering().compare(ie1.metadataHash().getBytes(), ie2.metadataHash().getBytes());
//System.out.println("** '" + ie1.getUrlHash() + "'.compareTo('" + ie2.getUrlHash() + "')="+c);
if (c < 0) {
- if (e1.hasNext()) ie1 = new WordReferenceVars(e1.next()); else break;
+ if (e1.hasNext()) ie1 = e1.next(); else break;
} else if (c > 0) {
if (e2.hasNext()) ie2 = e2.next(); else break;
} else {
// we have found the same urls in different searches!
ie1.join(ie2);
e1.remove();
- if (e1.hasNext()) ie1 = new WordReferenceVars(e1.next()); else break;
+ if (e1.hasNext()) ie1 = e1.next(); else break;
if (e2.hasNext()) ie2 = e2.next(); else break;
}
}
@@ -468,12 +479,12 @@ public class ReferenceContainer extends RowSet { }
- public static final ByteBuffer compressIndex(final ReferenceContainer inputContainer, final ReferenceContainer excludeContainer, final long maxtime) {
+ public static final <RT extends Reference> ByteBuffer compressIndex(final ReferenceContainer<RT> inputContainer, final ReferenceContainer<RT> excludeContainer, final long maxtime) {
// collect references according to domains
final long timeout = (maxtime < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + maxtime;
final TreeMap<String, String> doms = new TreeMap<String, String>();
synchronized (inputContainer) {
- final Iterator<WordReferenceRow> i = inputContainer.entries();
+ final Iterator<RT> i = inputContainer.entries();
Reference iEntry;
String dom, paths;
while (i.hasNext()) {
@@ -508,7 +519,7 @@ public class ReferenceContainer extends RowSet { return bb;
}
- public static final void decompressIndex(final TreeMap<String, String> target, ByteBuffer ci, final String peerhash) {
+ public static final <RT extends Reference> void decompressIndex(final TreeMap<String, String> target, ByteBuffer ci, final String peerhash) {
// target is a mapping from url-hashes to a string of peer-hashes
if ((ci.byteAt(0) == '{') && (ci.byteAt(ci.length() - 1) == '}')) {
//System.out.println("DEBUG-DECOMPRESS: input is " + ci.toString());
diff --git a/source/de/anomic/kelondro/text/ReferenceContainerArray.java b/source/de/anomic/kelondro/text/ReferenceContainerArray.java index 9271f65f9..439d33db8 100644 --- a/source/de/anomic/kelondro/text/ReferenceContainerArray.java +++ b/source/de/anomic/kelondro/text/ReferenceContainerArray.java @@ -38,11 +38,12 @@ import de.anomic.kelondro.order.ByteOrder; import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.util.Log;
-public final class ReferenceContainerArray {
+public final class ReferenceContainerArray<ReferenceType extends Reference> {
+ private final ReferenceFactory<ReferenceType> factory;
private final Row payloadrow;
private final BLOBArray array;
- private final IODispatcher merger;
+ private final IODispatcher<ReferenceType> merger;
/**
* open a index container based on a BLOB dump. The content of the BLOB will not be read
@@ -56,9 +57,11 @@ public final class ReferenceContainerArray { */
public ReferenceContainerArray(
final File heapLocation,
+ final ReferenceFactory<ReferenceType> factory,
final ByteOrder termOrder,
final Row payloadrow,
- IODispatcher merger) throws IOException {
+ IODispatcher<ReferenceType> merger) throws IOException {
+ this.factory = factory;
this.payloadrow = payloadrow;
this.array = new BLOBArray(
heapLocation,
@@ -104,7 +107,7 @@ public final class ReferenceContainerArray { * objects in the cache.
* @throws IOException
*/
- public synchronized CloneableIterator<ReferenceContainer> wordContainerIterator(final String startWordHash, final boolean rot, final boolean ram) {
+ public synchronized CloneableIterator<ReferenceContainer<ReferenceType>> wordContainerIterator(final String startWordHash, final boolean rot, final boolean ram) {
try {
return new heapCacheIterator(startWordHash, rot);
} catch (IOException e) {
@@ -117,7 +120,7 @@ public final class ReferenceContainerArray { * cache iterator: iterates objects within the heap cache. This can only be used
* for write-enabled heaps, read-only heaps do not have a heap cache
*/
- public class heapCacheIterator implements CloneableIterator<ReferenceContainer>, Iterable<ReferenceContainer> {
+ public class heapCacheIterator implements CloneableIterator<ReferenceContainer<ReferenceType>>, Iterable<ReferenceContainer<ReferenceType>> {
// this class exists, because the wCache cannot be iterated with rotation
// and because every indexContainer Object that is iterated must be returned as top-level-clone
@@ -148,7 +151,7 @@ public final class ReferenceContainerArray { return iterator.hasNext();
}
- public ReferenceContainer next() {
+ public ReferenceContainer<ReferenceType> next() {
try {
if (iterator.hasNext()) {
return get(new String(iterator.next()));
@@ -169,7 +172,7 @@ public final class ReferenceContainerArray { iterator.remove();
}
- public Iterator<ReferenceContainer> iterator() {
+ public Iterator<ReferenceContainer<ReferenceType>> iterator() {
return this;
}
@@ -192,13 +195,13 @@ public final class ReferenceContainerArray { * @return the indexContainer if one exist, null otherwise
* @throws IOException
*/
- public synchronized ReferenceContainer get(final String termHash) throws IOException {
+ public synchronized ReferenceContainer<ReferenceType> get(final String termHash) throws IOException {
List<byte[]> entries = this.array.getAll(termHash.getBytes());
if (entries == null || entries.size() == 0) return null;
byte[] a = entries.remove(0);
- ReferenceContainer c = new ReferenceContainer(termHash, RowSet.importRowSet(a, payloadrow));
+ ReferenceContainer<ReferenceType> c = new ReferenceContainer<ReferenceType>(this.factory, termHash, RowSet.importRowSet(a, payloadrow));
while (entries.size() > 0) {
- c = c.merge(new ReferenceContainer(termHash, RowSet.importRowSet(entries.remove(0), payloadrow)));
+ c = c.merge(new ReferenceContainer<ReferenceType>(this.factory, termHash, RowSet.importRowSet(entries.remove(0), payloadrow)));
}
return c;
}
@@ -214,31 +217,31 @@ public final class ReferenceContainerArray { array.remove(termHash.getBytes());
}
- public synchronized int replace(final String termHash, ContainerRewriter rewriter) throws IOException {
+ public synchronized int replace(final String termHash, ContainerRewriter<ReferenceType> rewriter) throws IOException {
return array.replace(termHash.getBytes(), new BLOBRewriter(termHash, rewriter));
}
public class BLOBRewriter implements BLOB.Rewriter {
- ContainerRewriter rewriter;
+ ContainerRewriter<ReferenceType> rewriter;
String wordHash;
- public BLOBRewriter(String wordHash, ContainerRewriter rewriter) {
+ public BLOBRewriter(String wordHash, ContainerRewriter<ReferenceType> rewriter) {
this.rewriter = rewriter;
this.wordHash = wordHash;
}
public byte[] rewrite(byte[] b) {
if (b == null) return null;
- ReferenceContainer c = rewriter.rewrite(new ReferenceContainer(this.wordHash, RowSet.importRowSet(b, payloadrow)));
+ ReferenceContainer<ReferenceType> c = rewriter.rewrite(new ReferenceContainer<ReferenceType>(factory, this.wordHash, RowSet.importRowSet(b, payloadrow)));
if (c == null) return null;
return c.exportCollection();
}
}
- public interface ContainerRewriter {
+ public interface ContainerRewriter<RT extends Reference> {
- public ReferenceContainer rewrite(ReferenceContainer container);
+ public ReferenceContainer<RT> rewrite(ReferenceContainer<RT> container);
}
diff --git a/source/de/anomic/kelondro/text/ReferenceContainerCache.java b/source/de/anomic/kelondro/text/ReferenceContainerCache.java index 09f16d129..9c938f439 100644 --- a/source/de/anomic/kelondro/text/ReferenceContainerCache.java +++ b/source/de/anomic/kelondro/text/ReferenceContainerCache.java @@ -41,17 +41,16 @@ import de.anomic.kelondro.blob.HeapWriter; import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.ByteOrder;
-import de.anomic.kelondro.text.referencePrototype.WordReferenceRow;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.Log;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.index.RowSet;
-public final class ReferenceContainerCache extends AbstractIndex implements Index, IndexReader, Iterable<ReferenceContainer> {
+public final class ReferenceContainerCache<ReferenceType extends Reference> extends AbstractIndex<ReferenceType> implements Index<ReferenceType>, IndexReader<ReferenceType>, Iterable<ReferenceContainer<ReferenceType>> {
private final Row payloadrow;
private final ByteOrder termOrder;
- private SortedMap<String, ReferenceContainer> cache;
+ private SortedMap<String, ReferenceContainer<ReferenceType>> cache;
/**
* opens an existing heap file in undefined mode
@@ -60,7 +59,8 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde * @param payloadrow
* @param log
*/
- public ReferenceContainerCache(final Row payloadrow, ByteOrder termOrder) {
+ public ReferenceContainerCache(final ReferenceFactory<ReferenceType> factory, final Row payloadrow, ByteOrder termOrder) {
+ super(factory);
this.payloadrow = payloadrow;
this.termOrder = termOrder;
this.cache = null;
@@ -84,7 +84,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde * another dump reading afterwards is not possible
*/
public void initWriteMode() {
- this.cache = Collections.synchronizedSortedMap(new TreeMap<String, ReferenceContainer>(new ByteOrder.StringOrder(this.termOrder)));
+ this.cache = Collections.synchronizedSortedMap(new TreeMap<String, ReferenceContainer<ReferenceType>>(new ByteOrder.StringOrder(this.termOrder)));
}
/**
@@ -95,10 +95,10 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde public void initWriteModeFromBLOB(final File blobFile) throws IOException {
Log.logInfo("indexContainerRAMHeap", "restoring rwi blob dump '" + blobFile.getName() + "'");
final long start = System.currentTimeMillis();
- this.cache = Collections.synchronizedSortedMap(new TreeMap<String, ReferenceContainer>(new ByteOrder.StringOrder(this.termOrder)));
+ this.cache = Collections.synchronizedSortedMap(new TreeMap<String, ReferenceContainer<ReferenceType>>(new ByteOrder.StringOrder(this.termOrder)));
int urlCount = 0;
synchronized (cache) {
- for (final ReferenceContainer container : new blobFileEntries(blobFile, this.payloadrow)) {
+ for (final ReferenceContainer<ReferenceType> container : new blobFileEntries<ReferenceType>(blobFile, factory, this.payloadrow)) {
// TODO: in this loop a lot of memory may be allocated. A check if the memory gets low is necessary. But what do when the memory is low?
if (container == null) break;
//System.out.println("***DEBUG indexContainerHeap.initwriteModeFromBLOB*** container.size = " + container.size() + ", container.sorted = " + container.sorted());
@@ -126,11 +126,11 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde final long startTime = System.currentTimeMillis();
long wordcount = 0, urlcount = 0;
String wordHash = null, lwh;
- ReferenceContainer container;
+ ReferenceContainer<ReferenceType> container;
// write wCache
synchronized (cache) {
- for (final Map.Entry<String, ReferenceContainer> entry: cache.entrySet()) {
+ for (final Map.Entry<String, ReferenceContainer<ReferenceType>> entry: cache.entrySet()) {
// get entries
lwh = wordHash;
wordHash = entry.getKey();
@@ -170,15 +170,17 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde /**
* static iterator of BLOBHeap files: is used to import heap dumps into a write-enabled index heap
*/
- public static class blobFileEntries implements CloneableIterator<ReferenceContainer>, Iterable<ReferenceContainer> {
+ public static class blobFileEntries <RT extends Reference> implements CloneableIterator<ReferenceContainer<RT>>, Iterable<ReferenceContainer<RT>> {
HeapReader.entries blobs;
Row payloadrow;
File blobFile;
+ ReferenceFactory<RT> factory;
- public blobFileEntries(final File blobFile, final Row payloadrow) throws IOException {
+ public blobFileEntries(final File blobFile, ReferenceFactory<RT> factory, final Row payloadrow) throws IOException {
this.blobs = new HeapReader.entries(blobFile, payloadrow.primaryKeyLength);
this.payloadrow = payloadrow;
this.blobFile = blobFile;
+ this.factory = factory;
}
public boolean hasNext() {
@@ -192,17 +194,17 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde * return an index container
* because they may get very large, it is wise to deallocate some memory before calling next()
*/
- public ReferenceContainer next() {
+ public ReferenceContainer<RT> next() {
Map.Entry<String, byte[]> entry = blobs.next();
byte[] payload = entry.getValue();
- return new ReferenceContainer(entry.getKey(), RowSet.importRowSet(payload, payloadrow));
+ return new ReferenceContainer<RT>(factory, entry.getKey(), RowSet.importRowSet(payload, payloadrow));
}
public void remove() {
throw new UnsupportedOperationException("heap dumps are read-only");
}
- public Iterator<ReferenceContainer> iterator() {
+ public Iterator<ReferenceContainer<RT>> iterator() {
return this;
}
@@ -215,11 +217,11 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde this.close();
}
- public CloneableIterator<ReferenceContainer> clone(Object modifier) {
+ public CloneableIterator<ReferenceContainer<RT>> clone(Object modifier) {
if (blobs != null) this.blobs.close();
blobs = null;
try {
- return new blobFileEntries(this.blobFile, this.payloadrow);
+ return new blobFileEntries<RT>(this.blobFile, factory, this.payloadrow);
} catch (IOException e) {
e.printStackTrace();
return null;
@@ -230,7 +232,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde public synchronized int maxReferences() {
// iterate to find the max score
int max = 0;
- for (ReferenceContainer container : cache.values()) {
+ for (ReferenceContainer<ReferenceType> container : cache.values()) {
if (container.size() > max) max = container.size();
}
return max;
@@ -240,7 +242,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde // iterate to find the max score
int max = 0;
String hash = null;
- for (ReferenceContainer container : cache.values()) {
+ for (ReferenceContainer<ReferenceType> container : cache.values()) {
if (container.size() > max) {
max = container.size();
hash = container.getTermHash();
@@ -252,7 +254,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde public synchronized ArrayList<String> maxReferencesHash(int bound) {
// iterate to find the max score
ArrayList<String> hashes = new ArrayList<String>();
- for (ReferenceContainer container : cache.values()) {
+ for (ReferenceContainer<ReferenceType> container : cache.values()) {
if (container.size() >= bound) {
hashes.add(container.getTermHash());
}
@@ -260,18 +262,18 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde return hashes;
}
- public synchronized ReferenceContainer latest() {
- ReferenceContainer c = null;
- for (ReferenceContainer container : cache.values()) {
+ public synchronized ReferenceContainer<ReferenceType> latest() {
+ ReferenceContainer<ReferenceType> c = null;
+ for (ReferenceContainer<ReferenceType> container : cache.values()) {
if (c == null) {c = container; continue;}
if (container.lastWrote() > c.lastWrote()) {c = container; continue;}
}
return c;
}
- public synchronized ReferenceContainer first() {
- ReferenceContainer c = null;
- for (ReferenceContainer container : cache.values()) {
+ public synchronized ReferenceContainer<ReferenceType> first() {
+ ReferenceContainer<ReferenceType> c = null;
+ for (ReferenceContainer<ReferenceType> container : cache.values()) {
if (c == null) {c = container; continue;}
if (container.lastWrote() < c.lastWrote()) {c = container; continue;}
}
@@ -281,7 +283,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde public synchronized ArrayList<String> overAge(long maxage) {
ArrayList<String> hashes = new ArrayList<String>();
long limit = System.currentTimeMillis() - maxage;
- for (ReferenceContainer container : cache.values()) {
+ for (ReferenceContainer<ReferenceType> container : cache.values()) {
if (container.lastWrote() < limit) hashes.add(container.getTermHash());
}
return hashes;
@@ -292,12 +294,12 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde * in the cache, so that manipulations of the iterated objects do not change
* objects in the cache.
*/
- public synchronized CloneableIterator<ReferenceContainer> references(final String startWordHash, final boolean rot) {
+ public synchronized CloneableIterator<ReferenceContainer<ReferenceType>> references(final String startWordHash, final boolean rot) {
return new heapCacheIterator(startWordHash, rot);
}
- public Iterator<ReferenceContainer> iterator() {
+ public Iterator<ReferenceContainer<ReferenceType>> iterator() {
return references(null, false);
}
@@ -306,7 +308,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde * cache iterator: iterates objects within the heap cache. This can only be used
* for write-enabled heaps, read-only heaps do not have a heap cache
*/
- public class heapCacheIterator implements CloneableIterator<ReferenceContainer>, Iterable<ReferenceContainer> {
+ public class heapCacheIterator implements CloneableIterator<ReferenceContainer<ReferenceType>>, Iterable<ReferenceContainer<ReferenceType>> {
// this class exists, because the wCache cannot be iterated with rotation
// and because every indexContainer Object that is iterated must be returned as top-level-clone
@@ -314,7 +316,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde // plus the mentioned features
private final boolean rot;
- private Iterator<ReferenceContainer> iterator;
+ private Iterator<ReferenceContainer<ReferenceType>> iterator;
public heapCacheIterator(final String startWordHash, final boolean rot) {
this.rot = rot;
@@ -331,7 +333,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde return iterator.hasNext();
}
- public ReferenceContainer next() {
+ public ReferenceContainer<ReferenceType> next() {
if (iterator.hasNext()) {
return (iterator.next()).topLevelClone();
}
@@ -347,7 +349,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde iterator.remove();
}
- public Iterator<ReferenceContainer> iterator() {
+ public Iterator<ReferenceContainer<ReferenceType>> iterator() {
return this;
}
@@ -368,14 +370,14 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde * @param key
* @return the indexContainer if one exist, null otherwise
*/
- public ReferenceContainer get(final String key, Set<String> urlselection) {
+ public ReferenceContainer<ReferenceType> get(final String key, Set<String> urlselection) {
if (urlselection == null) return this.cache.get(key);
- ReferenceContainer c = this.cache.get(key);
+ ReferenceContainer<ReferenceType> c = this.cache.get(key);
if (c == null) return null;
// because this is all in RAM, we must clone the entries (flat)
- ReferenceContainer c1 = new ReferenceContainer(c.getTermHash(), c.row(), c.size());
- Iterator<WordReferenceRow> e = c.entries();
- WordReferenceRow ee;
+ ReferenceContainer<ReferenceType> c1 = new ReferenceContainer<ReferenceType>(factory, c.getTermHash(), c.row(), c.size());
+ Iterator<ReferenceType> e = c.entries();
+ ReferenceType ee;
while (e.hasNext()) {
ee = e.next();
if (urlselection.contains(ee.metadataHash())) c1.add(ee);
@@ -389,7 +391,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde * @return
*/
public int count(final String key) {
- ReferenceContainer c = this.cache.get(key);
+ ReferenceContainer<ReferenceType> c = this.cache.get(key);
if (c == null) return 0;
return c.size();
}
@@ -399,7 +401,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde * @param wordHash
* @return the indexContainer if the cache contained the container, null othervise
*/
- public synchronized ReferenceContainer delete(final String wordHash) {
+ public synchronized ReferenceContainer<ReferenceType> delete(final String wordHash) {
// returns the index that had been deleted
assert this.cache != null;
return cache.remove(wordHash);
@@ -407,7 +409,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde public synchronized boolean remove(final String wordHash, final String urlHash) {
assert this.cache != null;
- final ReferenceContainer c = cache.get(wordHash);
+ final ReferenceContainer<ReferenceType> c = cache.get(wordHash);
if ((c != null) && (c.remove(urlHash) != null)) {
// removal successful
if (c.size() == 0) {
@@ -423,7 +425,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde public synchronized int remove(final String wordHash, final Set<String> urlHashes) {
assert this.cache != null;
if (urlHashes.size() == 0) return 0;
- final ReferenceContainer c = cache.get(wordHash);
+ final ReferenceContainer<ReferenceType> c = cache.get(wordHash);
int count;
if ((c != null) && ((count = c.removeEntries(urlHashes)) > 0)) {
// removal successful
@@ -437,13 +439,13 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde return 0;
}
- public synchronized void add(final ReferenceContainer container) {
+ public synchronized void add(final ReferenceContainer<ReferenceType> container) {
// this puts the entries into the cache
if (this.cache == null || container == null || container.size() == 0) return;
// put new words into cache
final String wordHash = container.getTermHash();
- ReferenceContainer entries = cache.get(wordHash); // null pointer exception? wordhash != null! must be cache==null
+ ReferenceContainer<ReferenceType> entries = cache.get(wordHash); // null pointer exception? wordhash != null! must be cache==null
int added = 0;
if (entries == null) {
entries = container.topLevelClone();
@@ -458,10 +460,10 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde return;
}
- public synchronized void add(final String wordHash, final WordReferenceRow newEntry) {
+ public synchronized void add(final String wordHash, final ReferenceType newEntry) {
assert this.cache != null;
- ReferenceContainer container = cache.get(wordHash);
- if (container == null) container = new ReferenceContainer(wordHash, this.payloadrow, 1);
+ ReferenceContainer<ReferenceType> container = cache.get(wordHash);
+ if (container == null) container = new ReferenceContainer<ReferenceType>(factory, wordHash, this.payloadrow, 1);
container.put(newEntry);
cache.put(wordHash, container);
}
diff --git a/source/de/anomic/kelondro/text/ReferenceContainerOrder.java b/source/de/anomic/kelondro/text/ReferenceContainerOrder.java index fa565c662..8ad49b7f6 100644 --- a/source/de/anomic/kelondro/text/ReferenceContainerOrder.java +++ b/source/de/anomic/kelondro/text/ReferenceContainerOrder.java @@ -29,15 +29,17 @@ package de.anomic.kelondro.text; import de.anomic.kelondro.order.AbstractOrder;
import de.anomic.kelondro.order.Order;
-public class ReferenceContainerOrder extends AbstractOrder<ReferenceContainer> implements Order<ReferenceContainer>, Cloneable {
+public class ReferenceContainerOrder<ReferenceType extends Reference> extends AbstractOrder<ReferenceContainer<ReferenceType>> implements Order<ReferenceContainer<ReferenceType>>, Cloneable {
+ private final ReferenceFactory<ReferenceType> factory;
private final Order<byte[]> embeddedOrder;
- public ReferenceContainerOrder(final Order<byte[]> embedOrder) {
+ public ReferenceContainerOrder(ReferenceFactory<ReferenceType> factory, final Order<byte[]> embedOrder) {
this.embeddedOrder = embedOrder;
+ this.factory = factory;
}
- public boolean wellformed(final ReferenceContainer a) {
+ public boolean wellformed(final ReferenceContainer<ReferenceType> a) {
return embeddedOrder.wellformed(a.getTermHash().getBytes());
}
@@ -49,21 +51,21 @@ public class ReferenceContainerOrder extends AbstractOrder<ReferenceContainer> i return this.embeddedOrder.partition(key, forks);
}
- public int compare(final ReferenceContainer a, final ReferenceContainer b) {
+ public int compare(final ReferenceContainer<ReferenceType> a, final ReferenceContainer<ReferenceType> b) {
return this.embeddedOrder.compare(a.getTermHash().getBytes(), b.getTermHash().getBytes());
}
- public boolean equal(ReferenceContainer a, ReferenceContainer b) {
+ public boolean equal(ReferenceContainer<ReferenceType> a, ReferenceContainer<ReferenceType> b) {
return this.embeddedOrder.equal(a.getTermHash().getBytes(), b.getTermHash().getBytes());
}
- public void rotate(final ReferenceContainer zero) {
+ public void rotate(final ReferenceContainer<ReferenceType> zero) {
this.embeddedOrder.rotate(zero.getTermHash().getBytes());
- this.zero = new ReferenceContainer(new String(this.embeddedOrder.zero()), zero);
+ this.zero = new ReferenceContainer<ReferenceType>(this.factory, new String(this.embeddedOrder.zero()), zero);
}
- public Order<ReferenceContainer> clone() {
- return new ReferenceContainerOrder(this.embeddedOrder.clone());
+ public Order<ReferenceContainer<ReferenceType>> clone() {
+ return new ReferenceContainerOrder<ReferenceType>(this.factory, this.embeddedOrder.clone());
}
public String signature() {
@@ -74,12 +76,12 @@ public class ReferenceContainerOrder extends AbstractOrder<ReferenceContainer> i return this.embeddedOrder.cardinal(key);
}
- public boolean equals(final Order<ReferenceContainer> otherOrder) {
+ public boolean equals(final Order<ReferenceContainer<ReferenceType>> otherOrder) {
if (!(otherOrder instanceof ReferenceContainerOrder)) return false;
- return this.embeddedOrder.equals(((ReferenceContainerOrder) otherOrder).embeddedOrder);
+ return this.embeddedOrder.equals(((ReferenceContainerOrder<ReferenceType>) otherOrder).embeddedOrder);
}
- public long cardinal(final ReferenceContainer key) {
+ public long cardinal(final ReferenceContainer<ReferenceType> key) {
return this.embeddedOrder.cardinal(key.getTermHash().getBytes());
}
diff --git a/source/de/anomic/kelondro/text/ReferenceFactory.java b/source/de/anomic/kelondro/text/ReferenceFactory.java index 85daec1de..5317390eb 100644 --- a/source/de/anomic/kelondro/text/ReferenceFactory.java +++ b/source/de/anomic/kelondro/text/ReferenceFactory.java @@ -30,7 +30,6 @@ import de.anomic.kelondro.index.Row; public interface ReferenceFactory<ReferenceType extends Reference> { - public ReferenceType produce(Row.Entry e); + public ReferenceType produce(Row.Entry e, boolean fast); - public Row.Entry recycle(ReferenceType r); } diff --git a/source/de/anomic/kelondro/text/ReferenceOrder.java b/source/de/anomic/kelondro/text/ReferenceOrder.java index 52c69b6ec..4319f2a0d 100644 --- a/source/de/anomic/kelondro/text/ReferenceOrder.java +++ b/source/de/anomic/kelondro/text/ReferenceOrder.java @@ -32,6 +32,7 @@ import java.util.Iterator; import java.util.Map;
import de.anomic.kelondro.order.Bitfield;
+import de.anomic.kelondro.text.referencePrototype.WordReference;
import de.anomic.kelondro.text.referencePrototype.WordReferenceRow;
import de.anomic.kelondro.text.referencePrototype.WordReferenceVars;
import de.anomic.kelondro.util.ScoreCluster;
@@ -57,7 +58,7 @@ public class ReferenceOrder { this.language = language;
}
- public ArrayList<WordReferenceVars> normalizeWith(final ReferenceContainer container) {
+ public ArrayList<WordReferenceVars> normalizeWith(final ReferenceContainer<WordReference> container) {
// normalize ranking: find minimum and maxiumum of separate ranking criteria
assert (container != null);
ArrayList<WordReferenceVars> result = null;
@@ -120,16 +121,18 @@ public class ReferenceOrder { final Bitfield flags = t.flags();
final long tf = ((max.termFrequency() == min.termFrequency()) ? 0 : (((int)(((t.termFrequency()-min.termFrequency())*256.0)/(max.termFrequency() - min.termFrequency())))) << ranking.coeff_termfrequency);
//System.out.println("tf(" + t.urlHash + ") = " + Math.floor(1000 * t.termFrequency()) + ", min = " + Math.floor(1000 * min.termFrequency()) + ", max = " + Math.floor(1000 * max.termFrequency()) + ", tf-normed = " + tf);
+ int maxmaxpos = max.maxposition();
+ int minminpos = min.minposition();
final long r =
((256 - yacyURL.domLengthNormalized(t.metadataHash())) << ranking.coeff_domlength)
+ ((ranking.coeff_ybr > 12) ? ((256 - (plasmaSearchRankingProcess.ybr(t.metadataHash()) << 4)) << ranking.coeff_ybr) : 0)
+ ((max.urlcomps() == min.urlcomps() ) ? 0 : (256 - (((t.urlcomps() - min.urlcomps() ) << 8) / (max.urlcomps() - min.urlcomps()) )) << ranking.coeff_urlcomps)
+ ((max.urllength() == min.urllength() ) ? 0 : (256 - (((t.urllength() - min.urllength() ) << 8) / (max.urllength() - min.urllength()) )) << ranking.coeff_urllength)
- + ((max.posintext() == min.posintext() ) ? 0 : (256 - (((t.posintext() - min.posintext() ) << 8) / (max.posintext() - min.posintext()) )) << ranking.coeff_posintext)
+ + ((maxmaxpos == minminpos ) ? 0 : (256 - (((t.minposition() - minminpos ) << 8) / (maxmaxpos - minminpos) )) << ranking.coeff_posintext)
+ ((max.posofphrase() == min.posofphrase()) ? 0 : (256 - (((t.posofphrase() - min.posofphrase() ) << 8) / (max.posofphrase() - min.posofphrase()) )) << ranking.coeff_posofphrase)
+ ((max.posinphrase() == min.posinphrase()) ? 0 : (256 - (((t.posinphrase() - min.posinphrase() ) << 8) / (max.posinphrase() - min.posinphrase()) )) << ranking.coeff_posinphrase)
- + ((max.worddistance() == min.worddistance()) ? 0 : (256 - (((t.worddistance() - min.worddistance() ) << 8) / (max.worddistance() - min.worddistance()) )) << ranking.coeff_worddistance)
- + ((max.virtualAge() == min.virtualAge()) ? 0 : (((t.virtualAge() - min.virtualAge() ) << 8) / (max.virtualAge() - min.virtualAge()) ) << ranking.coeff_date)
+ + ((max.distance() == min.distance() ) ? 0 : (256 - (((t.distance() - min.distance() ) << 8) / (max.distance() - min.distance()) )) << ranking.coeff_worddistance)
+ + ((max.virtualAge() == min.virtualAge()) ? 0 : (((t.virtualAge() - min.virtualAge() ) << 8) / (max.virtualAge() - min.virtualAge()) ) << ranking.coeff_date)
+ ((max.wordsintitle() == min.wordsintitle()) ? 0 : (((t.wordsintitle() - min.wordsintitle() ) << 8) / (max.wordsintitle() - min.wordsintitle()) ) << ranking.coeff_wordsintitle)
+ ((max.wordsintext() == min.wordsintext()) ? 0 : (((t.wordsintext() - min.wordsintext() ) << 8) / (max.wordsintext() - min.wordsintext()) ) << ranking.coeff_wordsintext)
+ ((max.phrasesintext() == min.phrasesintext()) ? 0 : (((t.phrasesintext()- min.phrasesintext() ) << 8) / (max.phrasesintext()- min.phrasesintext()) ) << ranking.coeff_phrasesintext)
@@ -165,13 +168,13 @@ public class ReferenceOrder { WordReferenceVars entryMin;
WordReferenceVars entryMax;
- private final ReferenceContainer container;
+ private final ReferenceContainer<WordReference> container;
private final int start, end;
private final HashMap<String, Integer> doms;
private final Integer int1;
ArrayList<WordReferenceVars> decodedEntries;
- public minmaxfinder(final ReferenceContainer container, final int start /*including*/, final int end /*excluding*/) {
+ public minmaxfinder(final ReferenceContainer<WordReference> container, final int start /*including*/, final int end /*excluding*/) {
this.container = container;
this.start = start;
this.end = end;
diff --git a/source/de/anomic/kelondro/text/referencePrototype/WordReference.java b/source/de/anomic/kelondro/text/referencePrototype/WordReference.java index 992475d66..e330d890c 100644 --- a/source/de/anomic/kelondro/text/referencePrototype/WordReference.java +++ b/source/de/anomic/kelondro/text/referencePrototype/WordReference.java @@ -35,8 +35,6 @@ public interface WordReference extends Reference { public int hitcount(); - public int posintext(); - public int posinphrase(); public int posofphrase(); diff --git a/source/de/anomic/kelondro/text/referencePrototype/WordReferenceFactory.java b/source/de/anomic/kelondro/text/referencePrototype/WordReferenceFactory.java new file mode 100644 index 000000000..b88b697a5 --- /dev/null +++ b/source/de/anomic/kelondro/text/referencePrototype/WordReferenceFactory.java @@ -0,0 +1,41 @@ +// WordReferenceFactory.java +// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 09.04.2009 on http://yacy.net +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2009-04-03 15:23:45 +0200 (Fr, 03 Apr 2009) $ +// $LastChangedRevision: 5777 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.kelondro.text.referencePrototype; + +import de.anomic.kelondro.index.Row.Entry; +import de.anomic.kelondro.text.ReferenceFactory; + +public class WordReferenceFactory implements ReferenceFactory<WordReference> { + + public WordReference produce(Entry e, boolean fast) { + if (fast) + return new WordReferenceVars(new WordReferenceRow(e)); + else + return new WordReferenceRow(e); + } + +} diff --git a/source/de/anomic/kelondro/text/referencePrototype/WordReferenceRow.java b/source/de/anomic/kelondro/text/referencePrototype/WordReferenceRow.java index 2a4ea290a..a573c611b 100644 --- a/source/de/anomic/kelondro/text/referencePrototype/WordReferenceRow.java +++ b/source/de/anomic/kelondro/text/referencePrototype/WordReferenceRow.java @@ -32,10 +32,11 @@ import de.anomic.kelondro.index.Row.Entry; import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.Bitfield;
import de.anomic.kelondro.order.MicroDate;
+import de.anomic.kelondro.text.AbstractReference;
import de.anomic.kelondro.text.Reference;
import de.anomic.yacy.yacySeedDB;
-public final class WordReferenceRow implements WordReference, Cloneable {
+public final class WordReferenceRow extends AbstractReference implements WordReference, Cloneable {
// this object stores attributes to URL references inside RWI collections
@@ -202,10 +203,20 @@ public final class WordReferenceRow implements WordReference, Cloneable { return (int) this.entry.getColLong(col_hitcount);
}
- public int posintext() {
+ public int positions() {
+ return 1;
+ }
+
+ public int position(int p) {
+ assert p == 0 : "p = " + p;
return (int) this.entry.getColLong(col_posintext);
}
+ public void addPosition(int position) {
+ throw new UnsupportedOperationException("");
+
+ }
+
public int posinphrase() {
return (int) this.entry.getColLong(col_posinphrase);
}
@@ -277,4 +288,9 @@ public final class WordReferenceRow implements WordReference, Cloneable { public int hashCode() {
return this.metadataHash().hashCode();
}
+
+ public void join(Reference oe) {
+ throw new UnsupportedOperationException("");
+
+ }
}
diff --git a/source/de/anomic/kelondro/text/referencePrototype/WordReferenceVars.java b/source/de/anomic/kelondro/text/referencePrototype/WordReferenceVars.java index fe60bb835..328f016f0 100644 --- a/source/de/anomic/kelondro/text/referencePrototype/WordReferenceVars.java +++ b/source/de/anomic/kelondro/text/referencePrototype/WordReferenceVars.java @@ -26,21 +26,25 @@ package de.anomic.kelondro.text.referencePrototype;
+import java.util.ArrayList;
+
import de.anomic.kelondro.index.Row.Entry;
import de.anomic.kelondro.order.Bitfield;
import de.anomic.kelondro.order.MicroDate;
+import de.anomic.kelondro.text.AbstractReference;
import de.anomic.kelondro.text.Reference;
-public class WordReferenceVars implements WordReference, Cloneable {
+public class WordReferenceVars extends AbstractReference implements WordReference, Reference, Cloneable {
public Bitfield flags;
public long lastModified;
public String language, urlHash;
public char type;
- public int hitcount, llocal, lother, phrasesintext, posintext,
+ public int hitcount, llocal, lother, phrasesintext,
posinphrase, posofphrase,
urlcomps, urllength, virtualAge,
- worddistance, wordsintext, wordsintitle;
+ wordsintext, wordsintitle;
+ ArrayList<Integer> positions;
public double termFrequency;
public WordReferenceVars(final String urlHash,
@@ -50,7 +54,7 @@ public class WordReferenceVars implements WordReference, Cloneable { final int hitcount, // how often appears this word in the text
final int wordcount, // total number of words
final int phrasecount, // total number of phrases
- final int posintext, // position of word in all words
+ final ArrayList<Integer> ps, // positions of words that are joined into the reference
final int posinphrase, // position of word in its phrase
final int posofphrase, // number of the phrase where word appears
final long lastmodified, // last-modified time of the document where word appears
@@ -60,7 +64,6 @@ public class WordReferenceVars implements WordReference, Cloneable { final int outlinksSame, // outlinks to same domain
final int outlinksOther, // outlinks to other domain
final Bitfield flags, // attributes to the url and to the word according the url
- final int worddistance,
final double termfrequency
) {
if ((language == null) || (language.length() != 2)) language = "uk";
@@ -76,19 +79,19 @@ public class WordReferenceVars implements WordReference, Cloneable { this.llocal = outlinksSame;
this.lother = outlinksOther;
this.phrasesintext = outlinksOther;
- this.posintext = posintext;
+ this.positions = new ArrayList<Integer>(ps.size());
+ for (int i = 0; i < ps.size(); i++) this.positions.add(ps.get(i));
this.posinphrase = posinphrase;
this.posofphrase = posofphrase;
this.urlcomps = urlComps;
this.urllength = urlLength;
this.virtualAge = mddlm;
- this.worddistance = worddistance;
this.wordsintext = wordcount;
this.wordsintitle = titleLength;
this.termFrequency = termfrequency;
}
- public WordReferenceVars(final WordReferenceRow e) {
+ public WordReferenceVars(final WordReference e) {
this.flags = e.flags();
//this.freshUntil = e.freshUntil();
this.lastModified = e.lastModified();
@@ -99,13 +102,13 @@ public class WordReferenceVars implements WordReference, Cloneable { this.llocal = e.llocal();
this.lother = e.lother();
this.phrasesintext = e.phrasesintext();
- this.posintext = e.posintext();
+ this.positions = new ArrayList<Integer>(e.positions());
+ for (int i = 0; i < e.positions(); i++) this.positions.add(e.position(i));
this.posinphrase = e.posinphrase();
this.posofphrase = e.posofphrase();
this.urlcomps = e.urlcomps();
this.urllength = e.urllength();
this.virtualAge = e.virtualAge();
- this.worddistance = 0;
this.wordsintext = e.wordsintext();
this.wordsintitle = e.wordsintitle();
this.termFrequency = e.termFrequency();
@@ -120,7 +123,7 @@ public class WordReferenceVars implements WordReference, Cloneable { this.hitcount,
this.wordsintext,
this.phrasesintext,
- this.posintext,
+ this.positions,
this.posinphrase,
this.posofphrase,
this.lastModified,
@@ -130,15 +133,13 @@ public class WordReferenceVars implements WordReference, Cloneable { this.llocal,
this.lother,
this.flags,
- this.worddistance,
this.termFrequency);
return c;
}
public void join(final WordReferenceVars v) {
// combine the distance
- this.worddistance = this.worddistance + v.worddistance() + Math.abs(this.posintext - v.posintext);
- this.posintext = Math.min(this.posintext, v.posintext);
+ this.positions.addAll(v.positions);
this.posinphrase = (this.posofphrase == v.posofphrase) ? Math.min(this.posinphrase, v.posinphrase) : 0;
this.posofphrase = Math.min(this.posofphrase, v.posofphrase);
@@ -197,8 +198,12 @@ public class WordReferenceVars implements WordReference, Cloneable { return posinphrase;
}
- public int posintext() {
- return posintext;
+ public int positions() {
+ return this.positions.size();
+ }
+
+ public int position(int p) {
+ return this.positions.get(p);
}
public int posofphrase() {
@@ -214,7 +219,7 @@ public class WordReferenceVars implements WordReference, Cloneable { hitcount, // how often appears this word in the text
wordsintext, // total number of words
phrasesintext, // total number of phrases
- posintext, // position of word in all words
+ positions.get(0), // position of word in all words
posinphrase, // position of word in its phrase
posofphrase, // number of the phrase where word appears
lastModified, // last-modified time of the document where word appears
@@ -251,10 +256,6 @@ public class WordReferenceVars implements WordReference, Cloneable { return virtualAge;
}
- public int worddistance() {
- return worddistance;
- }
-
public int wordsintext() {
return wordsintext;
}
@@ -278,10 +279,9 @@ public class WordReferenceVars implements WordReference, Cloneable { if (this.virtualAge > (v = other.virtualAge)) this.virtualAge = v;
if (this.wordsintext > (v = other.wordsintext)) this.wordsintext = v;
if (this.phrasesintext > (v = other.phrasesintext)) this.phrasesintext = v;
- if (this.posintext > (v = other.posintext)) this.posintext = v;
+ this.positions = a(Math.min(min(this.positions), min(other.positions)));
if (this.posinphrase > (v = other.posinphrase)) this.posinphrase = v;
if (this.posofphrase > (v = other.posofphrase)) this.posofphrase = v;
- if (this.worddistance > (v = other.worddistance)) this.worddistance = v;
if (this.lastModified > (w = other.lastModified)) this.lastModified = w;
//if (this.freshUntil > (w = other.freshUntil)) this.freshUntil = w;
if (this.urllength > (v = other.urllength)) this.urllength = v;
@@ -300,10 +300,9 @@ public class WordReferenceVars implements WordReference, Cloneable { if (this.virtualAge < (v = other.virtualAge)) this.virtualAge = v;
if (this.wordsintext < (v = other.wordsintext)) this.wordsintext = v;
if (this.phrasesintext < (v = other.phrasesintext)) this.phrasesintext = v;
- if (this.posintext < (v = other.posintext)) this.posintext = v;
+ this.positions = a(Math.max(max(this.positions), max(other.positions)));
if (this.posinphrase < (v = other.posinphrase)) this.posinphrase = v;
if (this.posofphrase < (v = other.posofphrase)) this.posofphrase = v;
- if (this.worddistance < (v = other.worddistance)) this.worddistance = v;
if (this.lastModified < (w = other.lastModified)) this.lastModified = w;
//if (this.freshUntil < (w = other.freshUntil)) this.freshUntil = w;
if (this.urllength < (v = other.urllength)) this.urllength = v;
@@ -312,12 +311,12 @@ public class WordReferenceVars implements WordReference, Cloneable { if (this.termFrequency < (d = other.termFrequency)) this.termFrequency = d;
}
- public void join(final WordReference oe) {
+ public void join(final Reference r) {
// joins two entries into one entry
// combine the distance
- this.worddistance = Math.abs(this.posintext() - oe.posintext());
- this.posintext = Math.min(this.posintext, oe.posintext());
+ WordReference oe = (WordReference) r;
+ for (int i = 0; i < r.positions(); i++) this.positions.add(r.position(i));
this.posinphrase = (this.posofphrase == oe.posofphrase()) ? Math.min(this.posinphrase, oe.posinphrase()) : 0;
this.posofphrase = Math.min(this.posofphrase, oe.posofphrase());
@@ -330,5 +329,9 @@ public class WordReferenceVars implements WordReference, Cloneable { return this.urlHash.hashCode();
}
+ public void addPosition(int position) {
+ this.positions.add(position);
+ }
+
}
diff --git a/source/de/anomic/plasma/plasmaDbImporter.java b/source/de/anomic/plasma/plasmaDbImporter.java index 3f4e1b923..02f7ba1d2 100644 --- a/source/de/anomic/plasma/plasmaDbImporter.java +++ b/source/de/anomic/plasma/plasmaDbImporter.java @@ -9,7 +9,7 @@ import de.anomic.crawler.Importer; import de.anomic.kelondro.text.Reference;
import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.text.metadataPrototype.URLMetadataRow;
-import de.anomic.kelondro.text.referencePrototype.WordReferenceRow;
+import de.anomic.kelondro.text.referencePrototype.WordReference;
import de.anomic.kelondro.util.DateFormatter;
public class plasmaDbImporter extends AbstractImporter implements Importer {
@@ -101,11 +101,11 @@ public class plasmaDbImporter extends AbstractImporter implements Importer { // iterate over all words from import db
//Iterator importWordHashIterator = this.importWordIndex.wordHashes(this.wordChunkStartHash, plasmaWordIndex.RL_WORDFILES, false);
- Iterator<ReferenceContainer> indexContainerIterator = this.importWordIndex.index().references(this.wordChunkStartHash, false, 100, false).iterator();
+ Iterator<ReferenceContainer<WordReference>> indexContainerIterator = this.importWordIndex.index().references(this.wordChunkStartHash, false, 100, false).iterator();
while (!isAborted() && indexContainerIterator.hasNext()) {
final TreeSet<String> entityUrls = new TreeSet<String>();
- ReferenceContainer newContainer = null;
+ ReferenceContainer<WordReference> newContainer = null;
try {
this.wordCounter++;
newContainer = indexContainerIterator.next();
@@ -113,7 +113,7 @@ public class plasmaDbImporter extends AbstractImporter implements Importer { // loop throug the entities of the container and get the
// urlhash
- final Iterator<WordReferenceRow> importWordIdxEntries = newContainer.entries();
+ final Iterator<WordReference> importWordIdxEntries = newContainer.entries();
Reference importWordIdxEntry;
while (importWordIdxEntries.hasNext()) {
// testing if import process was aborted
@@ -203,7 +203,7 @@ public class plasmaDbImporter extends AbstractImporter implements Importer { if (!indexContainerIterator.hasNext()) {
// We may not be finished yet, try to get the next chunk of wordHashes
- final TreeSet<ReferenceContainer> containers = this.importWordIndex.index().references(this.wordHash, false, 100, false);
+ final TreeSet<ReferenceContainer<WordReference>> containers = this.importWordIndex.index().references(this.wordHash, false, 100, false);
indexContainerIterator = containers.iterator();
// Make sure we don't get the same wordhash twice, but don't skip a word
if ((indexContainerIterator.hasNext())&&(!this.wordHash.equals((indexContainerIterator.next()).getTermHash()))) {
diff --git a/source/de/anomic/plasma/plasmaRankingCRProcess.java b/source/de/anomic/plasma/plasmaRankingCRProcess.java index 9b9a0562f..e92f390c0 100644 --- a/source/de/anomic/plasma/plasmaRankingCRProcess.java +++ b/source/de/anomic/plasma/plasmaRankingCRProcess.java @@ -40,6 +40,7 @@ import de.anomic.kelondro.order.MicroDate; import de.anomic.kelondro.table.EcoTable;
import de.anomic.kelondro.text.IndexCollection;
import de.anomic.kelondro.text.ReferenceContainer;
+import de.anomic.kelondro.text.referencePrototype.WordReference;
import de.anomic.kelondro.util.DateFormatter;
import de.anomic.kelondro.util.MemoryControl;
import de.anomic.kelondro.util.AttrSeq;
@@ -141,7 +142,7 @@ public class plasmaRankingCRProcess { return true;
}
- private static boolean accumulate_upd(final File f, final ObjectIndex acc, final IndexCollection seq) throws IOException {
+ private static boolean accumulate_upd(final File f, final ObjectIndex acc, final IndexCollection<WordReference> seq) throws IOException {
// open file
AttrSeq source_cr = null;
try {
@@ -241,11 +242,11 @@ public class plasmaRankingCRProcess { // open target file
AttrSeq acc = null;
ObjectIndex newacc = null;
- IndexCollection newseq = null;
+ IndexCollection<WordReference> newseq = null;
if (newdb) {
final File path = to_file.getParentFile(); // path to storage place
newacc = new EcoTable(new File(path, CRG_accname), CRG_accrow, EcoTable.tailCacheUsageAuto, 0, 0);
- newseq = new IndexCollection(path, CRG_seqname, 12, Base64Order.enhancedCoder, 9, CRG_colrow, false);
+ newseq = new IndexCollection<WordReference>(path, CRG_seqname, plasmaWordIndex.wordReferenceFactory, 12, Base64Order.enhancedCoder, 9, CRG_colrow, false);
} else {
if (!(to_file.exists())) {
acc = new AttrSeq("Global Ranking Accumulator File",
@@ -373,16 +374,16 @@ public class plasmaRankingCRProcess { public static int genrcix(final File cr_path_in, final File rci_path_out) throws IOException {
//kelondroFlexTable acc = new kelondroFlexTable(cr_path_in, CRG_accname, kelondroBase64Order.enhancedCoder, 128 * 1024 * 1024, -1, CRG_accrow, true);
- final IndexCollection seq = new IndexCollection(cr_path_in, CRG_seqname, 12, Base64Order.enhancedCoder, 9, CRG_colrow, false);
- final IndexCollection rci = new IndexCollection(rci_path_out, RCI_colname, 6, Base64Order.enhancedCoder, 9, RCI_coli, false);
+ final IndexCollection<WordReference> seq = new IndexCollection<WordReference>(cr_path_in, CRG_seqname, plasmaWordIndex.wordReferenceFactory, 12, Base64Order.enhancedCoder, 9, CRG_colrow, false);
+ final IndexCollection<WordReference> rci = new IndexCollection<WordReference>(rci_path_out, RCI_colname, plasmaWordIndex.wordReferenceFactory, 6, Base64Order.enhancedCoder, 9, RCI_coli, false);
// loop over all referees
int count = 0;
final int size = seq.size();
final long start = System.currentTimeMillis();
long l;
- final CloneableIterator<ReferenceContainer> i = seq.references(null, false);
- ReferenceContainer keycollection;
+ final CloneableIterator<ReferenceContainer<WordReference>> i = seq.references(null, false);
+ ReferenceContainer<WordReference> keycollection;
String referee, refereeDom, anchor, anchorDom;
RowSet rci_entry;
CloneableIterator<Row.Entry> cr_entry;
diff --git a/source/de/anomic/plasma/plasmaSearchAPI.java b/source/de/anomic/plasma/plasmaSearchAPI.java index 58d1dff58..42b71ff0f 100644 --- a/source/de/anomic/plasma/plasmaSearchAPI.java +++ b/source/de/anomic/plasma/plasmaSearchAPI.java @@ -156,7 +156,7 @@ public class plasmaSearchAPI { prop.putNum("genUrlList_urlList_"+i+"_urlExists_lother", entry.word().lother()); prop.putNum("genUrlList_urlList_"+i+"_urlExists_hitcount", entry.word().hitcount()); prop.putNum("genUrlList_urlList_"+i+"_urlExists_worddistance", 0); - prop.putNum("genUrlList_urlList_"+i+"_urlExists_pos", entry.word().posintext()); + prop.putNum("genUrlList_urlList_"+i+"_urlExists_pos", entry.word().minposition()); prop.putNum("genUrlList_urlList_"+i+"_urlExists_phrase", entry.word().posofphrase()); prop.putNum("genUrlList_urlList_"+i+"_urlExists_posinphrase", entry.word().posinphrase()); prop.putNum("genUrlList_urlList_"+i+"_urlExists_urlcomps", entry.word().urlcomps()); diff --git a/source/de/anomic/plasma/plasmaSearchEvent.java b/source/de/anomic/plasma/plasmaSearchEvent.java index dac2539cc..adadd745f 100644 --- a/source/de/anomic/plasma/plasmaSearchEvent.java +++ b/source/de/anomic/plasma/plasmaSearchEvent.java @@ -42,6 +42,7 @@ import de.anomic.kelondro.order.Bitfield; import de.anomic.kelondro.text.Reference;
import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.text.metadataPrototype.URLMetadataRow;
+import de.anomic.kelondro.text.referencePrototype.WordReference;
import de.anomic.kelondro.text.referencePrototype.WordReferenceVars;
import de.anomic.kelondro.util.MemoryControl;
import de.anomic.kelondro.util.SetTools;
@@ -177,7 +178,7 @@ public final class plasmaSearchEvent { int maxcount = -1;
long mindhtdistance = Long.MAX_VALUE, l;
String wordhash;
- for (Map.Entry<String, ReferenceContainer> entry : this.rankedCache.searchContainerMaps()[0].entrySet()) {
+ for (Map.Entry<String, ReferenceContainer<WordReference>> entry : this.rankedCache.searchContainerMaps()[0].entrySet()) {
wordhash = entry.getKey();
final ReferenceContainer container = entry.getValue();
assert (container.getTermHash().equals(wordhash));
diff --git a/source/de/anomic/plasma/plasmaSearchRankingProcess.java b/source/de/anomic/plasma/plasmaSearchRankingProcess.java index f9cf6df11..baf5d87ba 100644 --- a/source/de/anomic/plasma/plasmaSearchRankingProcess.java +++ b/source/de/anomic/plasma/plasmaSearchRankingProcess.java @@ -72,7 +72,7 @@ public final class plasmaSearchRankingProcess { private final int[] flagcount; // flag counter
private final TreeSet<String> misses; // contains url-hashes that could not been found in the LURL-DB
private final plasmaWordIndex wordIndex;
- private HashMap<String, ReferenceContainer>[] localSearchContainerMaps;
+ private HashMap<String, ReferenceContainer<WordReference>>[] localSearchContainerMaps;
private final int[] domZones;
public plasmaSearchRankingProcess(
@@ -120,8 +120,9 @@ public final class plasmaSearchRankingProcess { // join and exclude the local result
timer = System.currentTimeMillis();
- final ReferenceContainer index =
+ final ReferenceContainer<WordReference> index =
ReferenceContainer.joinExcludeContainers(
+ plasmaWordIndex.wordReferenceFactory,
this.localSearchContainerMaps[0].values(),
this.localSearchContainerMaps[1].values(),
query.maxDistance);
@@ -133,7 +134,7 @@ public final class plasmaSearchRankingProcess { insertRanked(index, true, index.size());
}
- public void insertRanked(final ReferenceContainer index, final boolean local, final int fullResource) {
+ public void insertRanked(final ReferenceContainer<WordReference> index, final boolean local, final int fullResource) {
// we collect the urlhashes and construct a list with urlEntry objects
// attention: if minEntries is too high, this method will not terminate within the maxTime
@@ -241,7 +242,7 @@ public final class plasmaSearchRankingProcess { return false;
}
- public Map<String, ReferenceContainer>[] searchContainerMaps() {
+ public Map<String, ReferenceContainer<WordReference>>[] searchContainerMaps() {
// direct access to the result maps is needed for abstract generation
// this is only available if execQuery() was called before
return localSearchContainerMaps;
diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index df2b3e17a..184127b50 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -51,7 +51,10 @@ import de.anomic.kelondro.text.IndexCollectionMigration; import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.text.IODispatcher;
import de.anomic.kelondro.text.MetadataRepository;
+import de.anomic.kelondro.text.ReferenceFactory;
import de.anomic.kelondro.text.metadataPrototype.URLMetadataRow;
+import de.anomic.kelondro.text.referencePrototype.WordReference;
+import de.anomic.kelondro.text.referencePrototype.WordReferenceFactory;
import de.anomic.kelondro.text.referencePrototype.WordReferenceRow;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.kelondroException;
@@ -74,6 +77,9 @@ public final class plasmaWordIndex { public static final long targetFileSize = 100 * 1024 * 1024; // 100 MB
public static final long maxFileSize = BLOBArray.oneGigabyte; // 1GB
+ // the reference factory
+ public static final ReferenceFactory<WordReference> wordReferenceFactory = new WordReferenceFactory();
+
public static final String CRAWL_PROFILE_PROXY = "proxy";
public static final String CRAWL_PROFILE_REMOTE = "remote";
public static final String CRAWL_PROFILE_SNIPPET_LOCAL_TEXT = "snippetLocalText";
@@ -91,7 +97,7 @@ public final class plasmaWordIndex { public static final ByteOrder wordOrder = Base64Order.enhancedCoder;
- private final BufferedIndex index;
+ private final BufferedIndex<WordReference> index;
private final Log log;
private MetadataRepository metadata;
private final yacySeedDB peers;
@@ -103,7 +109,7 @@ public final class plasmaWordIndex { public CrawlProfile.entry defaultTextSnippetLocalProfile, defaultTextSnippetGlobalProfile;
public CrawlProfile.entry defaultMediaSnippetLocalProfile, defaultMediaSnippetGlobalProfile;
private final File queuesRoot;
- private IODispatcher merger;
+ private IODispatcher<WordReference> merger;
public plasmaWordIndex(
final String networkName,
@@ -141,11 +147,12 @@ public final class plasmaWordIndex { // check if the peer has migrated the index
if (new File(indexPrimaryTextLocation, "RICOLLECTION").exists()) {
- this.merger = (useCell) ? new IODispatcher(1, 1) : null;
+ this.merger = (useCell) ? new IODispatcher<WordReference>(plasmaWordIndex.wordReferenceFactory, 1, 1) : null;
if (this.merger != null) this.merger.start();
this.index = (useCell) ?
- new IndexCollectionMigration(
+ new IndexCollectionMigration<WordReference>(
indexPrimaryTextLocation,
+ wordReferenceFactory,
wordOrder,
WordReferenceRow.urlEntryRow,
entityCacheMaxSize,
@@ -154,8 +161,9 @@ public final class plasmaWordIndex { this.merger,
log)
:
- new BufferedIndexCollection(
+ new BufferedIndexCollection<WordReference>(
indexPrimaryTextLocation,
+ wordReferenceFactory,
wordOrder,
WordReferenceRow.urlEntryRow,
entityCacheMaxSize,
@@ -163,10 +171,11 @@ public final class plasmaWordIndex { redundancy,
log);
} else {
- this.merger = new IODispatcher(1, 1);
+ this.merger = new IODispatcher<WordReference>(plasmaWordIndex.wordReferenceFactory, 1, 1);
this.merger.start();
- this.index = new IndexCell(
+ this.index = new IndexCell<WordReference>(
new File(indexPrimaryTextLocation, "RICELL"),
+ wordReferenceFactory,
wordOrder,
WordReferenceRow.urlEntryRow,
entityCacheMaxSize,
@@ -267,7 +276,7 @@ public final class plasmaWordIndex { return this.peers;
}
- public BufferedIndex index() {
+ public BufferedIndex<WordReference> index() {
return this.index;
}
@@ -574,21 +583,21 @@ public final class plasmaWordIndex { }
@SuppressWarnings("unchecked")
- public HashMap<String, ReferenceContainer>[] localSearchContainers(
+ public HashMap<String, ReferenceContainer<WordReference>>[] localSearchContainers(
final TreeSet<String> queryHashes,
final TreeSet<String> excludeHashes,
final Set<String> urlselection) {
// search for the set of hashes and return a map of of wordhash:indexContainer containing the seach result
// retrieve entities that belong to the hashes
- HashMap<String, ReferenceContainer> inclusionContainers =
+ HashMap<String, ReferenceContainer<WordReference>> inclusionContainers =
(queryHashes.size() == 0) ?
- new HashMap<String, ReferenceContainer>(0) :
+ new HashMap<String, ReferenceContainer<WordReference>>(0) :
getContainers(queryHashes, urlselection);
- if ((inclusionContainers.size() != 0) && (inclusionContainers.size() < queryHashes.size())) inclusionContainers = new HashMap<String, ReferenceContainer>(0); // prevent that only a subset is returned
- final HashMap<String, ReferenceContainer> exclusionContainers =
+ if ((inclusionContainers.size() != 0) && (inclusionContainers.size() < queryHashes.size())) inclusionContainers = new HashMap<String, ReferenceContainer<WordReference>>(0); // prevent that only a subset is returned
+ final HashMap<String, ReferenceContainer<WordReference>> exclusionContainers =
(inclusionContainers.size() == 0) ?
- new HashMap<String, ReferenceContainer>(0) :
+ new HashMap<String, ReferenceContainer<WordReference>>(0) :
getContainers(excludeHashes, urlselection);
return new HashMap[]{inclusionContainers, exclusionContainers};
}
@@ -600,11 +609,11 @@ public final class plasmaWordIndex { * @param urlselection
* @return map of wordhash:indexContainer
*/
- private HashMap<String, ReferenceContainer> getContainers(final Set<String> wordHashes, final Set<String> urlselection) {
+ private HashMap<String, ReferenceContainer<WordReference>> getContainers(final Set<String> wordHashes, final Set<String> urlselection) {
// retrieve entities that belong to the hashes
- final HashMap<String, ReferenceContainer> containers = new HashMap<String, ReferenceContainer>(wordHashes.size());
+ final HashMap<String, ReferenceContainer<WordReference>> containers = new HashMap<String, ReferenceContainer<WordReference>>(wordHashes.size());
String singleHash;
- ReferenceContainer singleContainer;
+ ReferenceContainer<WordReference> singleContainer;
final Iterator<String> i = wordHashes.iterator();
while (i.hasNext()) {
@@ -620,7 +629,7 @@ public final class plasmaWordIndex { }
// check result
- if ((singleContainer == null || singleContainer.size() == 0)) return new HashMap<String, ReferenceContainer>(0);
+ if ((singleContainer == null || singleContainer.size() == 0)) return new HashMap<String, ReferenceContainer<WordReference>>(0);
containers.put(singleHash, singleContainer);
}
@@ -649,16 +658,16 @@ public final class plasmaWordIndex { public void run() {
Log.logInfo("INDEXCLEANER", "IndexCleaner-Thread started");
- ReferenceContainer container = null;
- WordReferenceRow entry = null;
+ ReferenceContainer<WordReference> container = null;
+ WordReference entry = null;
yacyURL url = null;
final HashSet<String> urlHashs = new HashSet<String>();
try {
- Iterator<ReferenceContainer> indexContainerIterator = index.references(startHash, false, 100, false).iterator();
+ Iterator<ReferenceContainer<WordReference>> indexContainerIterator = index.references(startHash, false, 100, false).iterator();
while (indexContainerIterator.hasNext() && run) {
waiter();
container = indexContainerIterator.next();
- final Iterator<WordReferenceRow> containerIterator = container.entries();
+ final Iterator<WordReference> containerIterator = container.entries();
wordHashNow = container.getTermHash();
while (containerIterator.hasNext() && run) {
waiter();
@@ -687,7 +696,7 @@ public final class plasmaWordIndex { if (!containerIterator.hasNext()) {
// We may not be finished yet, try to get the next chunk of wordHashes
- final TreeSet<ReferenceContainer> containers = index.references(container.getTermHash(), false, 100, false);
+ final TreeSet<ReferenceContainer<WordReference>> containers = index.references(container.getTermHash(), false, 100, false);
indexContainerIterator = containers.iterator();
// Make sure we don't get the same wordhash twice, but don't skip a word
if ((indexContainerIterator.hasNext()) && (!container.getTermHash().equals(indexContainerIterator.next().getTermHash()))) {
diff --git a/source/de/anomic/yacy/dht/Dispatcher.java b/source/de/anomic/yacy/dht/Dispatcher.java index 154d38fdb..0c32f73cf 100755 --- a/source/de/anomic/yacy/dht/Dispatcher.java +++ b/source/de/anomic/yacy/dht/Dispatcher.java @@ -36,8 +36,10 @@ import de.anomic.kelondro.order.Base64Order; import de.anomic.kelondro.text.BufferedIndex; import de.anomic.kelondro.text.ReferenceContainer; import de.anomic.kelondro.text.MetadataRepository; +import de.anomic.kelondro.text.referencePrototype.WordReference; import de.anomic.kelondro.text.referencePrototype.WordReferenceRow; import de.anomic.kelondro.util.Log; +import de.anomic.plasma.plasmaWordIndex; import de.anomic.server.serverProcessor; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacySeedDB; @@ -81,7 +83,7 @@ public class Dispatcher { private HashMap<String, Transmission.Chunk> transmissionCloud; // the backend is used to store the remaining indexContainers in case that the object is closed - private BufferedIndex backend; + private BufferedIndex<WordReference> backend; // the seed database private yacySeedDB seeds; @@ -96,7 +98,7 @@ public class Dispatcher { private Transmission transmission; public Dispatcher( - final BufferedIndex backend, + final BufferedIndex<WordReference> backend, final MetadataRepository repository, final yacySeedDB seeds, final boolean gzipBody, @@ -143,7 +145,7 @@ public class Dispatcher { * @return * @throws IOException */ - private ArrayList<ReferenceContainer> selectContainers( + private ArrayList<ReferenceContainer<WordReference>> selectContainers( final String hash, final String limitHash, final int maxContainerCount, @@ -151,14 +153,14 @@ public class Dispatcher { final int maxtime) throws IOException { // prefer file - ArrayList<ReferenceContainer> containers = selectContainers(hash, limitHash, maxContainerCount, maxReferenceCount, maxtime, false); + ArrayList<ReferenceContainer<WordReference>> containers = selectContainers(hash, limitHash, maxContainerCount, maxReferenceCount, maxtime, false); // if ram does not provide any result, take from file //if (containers.size() == 0) containers = selectContainers(hash, limitHash, maxContainerCount, maxtime, false); return containers; } - private ArrayList<ReferenceContainer> selectContainers( + private ArrayList<ReferenceContainer<WordReference>> selectContainers( final String hash, final String limitHash, final int maxContainerCount, @@ -166,10 +168,10 @@ public class Dispatcher { final int maxtime, final boolean ram) throws IOException { - final ArrayList<ReferenceContainer> containers = new ArrayList<ReferenceContainer>(maxContainerCount); + final ArrayList<ReferenceContainer<WordReference>> containers = new ArrayList<ReferenceContainer<WordReference>>(maxContainerCount); - final Iterator<ReferenceContainer> indexContainerIterator = this.backend.references(hash, true, ram); - ReferenceContainer container; + final Iterator<ReferenceContainer<WordReference>> indexContainerIterator = this.backend.references(hash, true, ram); + ReferenceContainer<WordReference> container; int refcount = 0; // first select the container @@ -190,8 +192,8 @@ public class Dispatcher { } // then remove the container from the backend HashSet<String> urlHashes = new HashSet<String>(); - Iterator<WordReferenceRow> it; - for (ReferenceContainer c: containers) { + Iterator<WordReference> it; + for (ReferenceContainer<WordReference> c: containers) { urlHashes.clear(); it = c.entries(); while (it.hasNext()) { @@ -213,20 +215,20 @@ public class Dispatcher { * @return */ @SuppressWarnings("unchecked") - private ArrayList<ReferenceContainer>[] splitContainers(ArrayList<ReferenceContainer> containers) { + private ArrayList<ReferenceContainer<WordReference>>[] splitContainers(ArrayList<ReferenceContainer<WordReference>> containers) { // init the result vector int partitionCount = this.seeds.scheme.verticalPartitions(); - ArrayList<ReferenceContainer>[] partitions = (ArrayList<ReferenceContainer>[]) new ArrayList[partitionCount]; - for (int i = 0; i < partitions.length; i++) partitions[i] = new ArrayList<ReferenceContainer>(); + ArrayList<ReferenceContainer<WordReference>>[] partitions = (ArrayList<ReferenceContainer<WordReference>>[]) new ArrayList[partitionCount]; + for (int i = 0; i < partitions.length; i++) partitions[i] = new ArrayList<ReferenceContainer<WordReference>>(); // check all entries and split them to the partitions - ReferenceContainer[] partitionBuffer = new ReferenceContainer[partitionCount]; + ReferenceContainer<WordReference>[] partitionBuffer = new ReferenceContainer[partitionCount]; WordReferenceRow re; for (ReferenceContainer container: containers) { // init the new partitions for (int j = 0; j < partitionBuffer.length; j++) { - partitionBuffer[j] = new ReferenceContainer(container.getTermHash(), container.row(), container.size() / partitionCount); + partitionBuffer[j] = new ReferenceContainer(plasmaWordIndex.wordReferenceFactory, container.getTermHash(), container.row(), container.size() / partitionCount); } // split the container @@ -255,9 +257,9 @@ public class Dispatcher { * stored in a cache of the Entry for later transmission to the targets, which means that * then no additional IO is necessary. */ - private void enqueueContainersToCloud(final ArrayList<ReferenceContainer>[] containers) { + private void enqueueContainersToCloud(final ArrayList<ReferenceContainer<WordReference>>[] containers) { if (transmissionCloud == null) return; - ReferenceContainer lastContainer; + ReferenceContainer<WordReference> lastContainer; String primaryTarget; Transmission.Chunk entry; for (int vertical = 0; vertical < containers.length; vertical++) { @@ -276,7 +278,7 @@ public class Dispatcher { if (entry == null) entry = transmission.newChunk(primaryTarget, targets, lastContainer.row()); // fill the entry with the containers - for (ReferenceContainer c: containers[vertical]) { + for (ReferenceContainer<WordReference> c: containers[vertical]) { entry.add(c); } @@ -293,7 +295,7 @@ public class Dispatcher { final int maxtime) throws IOException { if (this.transmissionCloud == null) return false; - ArrayList<ReferenceContainer> selectedContainerCache = selectContainers(hash, limitHash, maxContainerCount, maxReferenceCount, maxtime); + ArrayList<ReferenceContainer<WordReference>> selectedContainerCache = selectContainers(hash, limitHash, maxContainerCount, maxReferenceCount, maxtime); this.log.logInfo("selectContainersToCache: selectedContainerCache was filled with " + selectedContainerCache.size() + " entries"); if (selectedContainerCache == null || selectedContainerCache.size() == 0) { @@ -301,7 +303,7 @@ public class Dispatcher { return false; } - ArrayList<ReferenceContainer>[] splittedContainerCache = splitContainers(selectedContainerCache); + ArrayList<ReferenceContainer<WordReference>>[] splittedContainerCache = splitContainers(selectedContainerCache); selectedContainerCache = null; if (splittedContainerCache == null) { this.log.logInfo("enqueueContainersFromCache: splittedContainerCache is empty, cannot do anything here."); @@ -377,7 +379,7 @@ public class Dispatcher { if (indexingTransmissionProcessor != null) this.indexingTransmissionProcessor.announceShutdown(); if (this.transmissionCloud != null) { for (Map.Entry<String, Transmission.Chunk> e : this.transmissionCloud.entrySet()) { - for (ReferenceContainer i : e.getValue()) try {this.backend.add(i);} catch (IOException e1) {} + for (ReferenceContainer<WordReference> i : e.getValue()) try {this.backend.add(i);} catch (IOException e1) {} } this.transmissionCloud.clear(); } diff --git a/source/de/anomic/yacy/dht/Transmission.java b/source/de/anomic/yacy/dht/Transmission.java index b611279fd..44b8b9538 100644 --- a/source/de/anomic/yacy/dht/Transmission.java +++ b/source/de/anomic/yacy/dht/Transmission.java @@ -36,7 +36,7 @@ import de.anomic.kelondro.text.ReferenceContainer; import de.anomic.kelondro.text.ReferenceContainerCache; import de.anomic.kelondro.text.MetadataRepository; import de.anomic.kelondro.text.metadataPrototype.URLMetadataRow; -import de.anomic.kelondro.text.referencePrototype.WordReferenceRow; +import de.anomic.kelondro.text.referencePrototype.WordReference; import de.anomic.kelondro.util.Log; import de.anomic.plasma.plasmaWordIndex; import de.anomic.server.serverProcessorJob; @@ -44,12 +44,12 @@ import de.anomic.yacy.yacyClient; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacySeedDB; -public class Transmission { +public class Transmission { private Log log; private MetadataRepository repository; private yacySeedDB seeds; - private Index backend; + private Index<WordReference> backend; private boolean gzipBody4Transfer; private int timeout4Transfer; @@ -57,7 +57,7 @@ public class Transmission { Log log, MetadataRepository repository, yacySeedDB seeds, - Index backend, + Index<WordReference> backend, boolean gzipBody4Transfer, int timeout4Transfer) { this.log = log; @@ -75,7 +75,7 @@ public class Transmission { return new Chunk(primaryTarget, targets, payloadrow); } - public class Chunk extends serverProcessorJob implements Iterable<ReferenceContainer> { + public class Chunk extends serverProcessorJob implements Iterable<ReferenceContainer<WordReference>> { /** * a dispatcher entry contains * - the primary target, which is a word hash, as marker for the entry @@ -86,12 +86,12 @@ public class Transmission { * - a set of yacy seeds which will shrink as the containers are transmitted to them * - a counter that gives the number of sucessful and unsuccessful transmissions so far */ - private String primaryTarget; - private ReferenceContainerCache containers; + private String primaryTarget; + private ReferenceContainerCache<WordReference> containers; private HashMap<String, URLMetadataRow> references; - private HashSet<String> badReferences; - private ArrayList<yacySeed> targets; - private int hit, miss; + private HashSet<String> badReferences; + private ArrayList<yacySeed> targets; + private int hit, miss; /** * generate a new dispatcher target. such a target is defined with a primary target and @@ -107,7 +107,7 @@ public class Transmission { final Row payloadrow) { super(); this.primaryTarget = primaryTarget; - this.containers = new ReferenceContainerCache(payloadrow, plasmaWordIndex.wordOrder); + this.containers = new ReferenceContainerCache<WordReference>(plasmaWordIndex.wordReferenceFactory, payloadrow, plasmaWordIndex.wordOrder); this.containers.initWriteMode(); this.references = new HashMap<String, URLMetadataRow>(); this.badReferences = new HashSet<String>(); @@ -121,12 +121,12 @@ public class Transmission { * all entries in the container are checked and only such are stored which have a reference entry * @param container */ - public void add(ReferenceContainer container) { + public void add(ReferenceContainer<WordReference> container) { // iterate through the entries in the container and check if the reference is in the repository - Iterator<WordReferenceRow> i = container.entries(); + Iterator<WordReference> i = container.entries(); ArrayList<String> notFound = new ArrayList<String>(); while (i.hasNext()) { - WordReferenceRow e = i.next(); + WordReference e = i.next(); if (references.containsKey(e.metadataHash()) || badReferences.contains(e.metadataHash())) continue; URLMetadataRow r = repository.load(e.metadataHash(), null, 0); if (r == null) { @@ -146,7 +146,7 @@ public class Transmission { * get all containers from the entry. This method may be used to flush remaining entries * if they had been finished transmission without success (not enough peers arrived) */ - public Iterator<ReferenceContainer> iterator() { + public Iterator<ReferenceContainer<WordReference>> iterator() { return this.containers.iterator(); } @@ -201,8 +201,8 @@ public class Transmission { if (error == null) { // words successfully transfered long transferTime = System.currentTimeMillis() - start; - Iterator<ReferenceContainer> i = this.containers.iterator(); - ReferenceContainer firstContainer = (i == null) ? null : i.next(); + Iterator<ReferenceContainer<WordReference>> i = this.containers.iterator(); + ReferenceContainer<WordReference> firstContainer = (i == null) ? null : i.next(); log.logInfo("Index transfer of " + this.containers.size() + " words [" + ((firstContainer == null) ? null : firstContainer.getTermHash()) + " .. " + this.primaryTarget + "]" + " and " + this.references.size() + " URLs" + @@ -237,7 +237,7 @@ public class Transmission { } public void restore() { - for (ReferenceContainer ic : this) try { backend.add(ic); } catch (IOException e) {} + for (ReferenceContainer<WordReference> ic : this) try { backend.add(ic); } catch (IOException e) {} } } } diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index cc6f53d56..e8a3871d7 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -74,7 +74,7 @@ import de.anomic.kelondro.text.Reference; import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.text.ReferenceContainerCache;
import de.anomic.kelondro.text.metadataPrototype.URLMetadataRow;
-import de.anomic.kelondro.text.referencePrototype.WordReferenceRow;
+import de.anomic.kelondro.text.referencePrototype.WordReference;
import de.anomic.kelondro.util.ByteBuffer;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaSearchRankingProcess;
@@ -417,6 +417,7 @@ public final class yacyClient { }
}
+ @SuppressWarnings("unchecked")
public static String[] search(
final yacySeed mySeed,
final String wordhashes,
@@ -526,9 +527,9 @@ public final class yacyClient { // create containers
final int words = wordhashes.length() / yacySeedDB.commonHashLength;
- final ReferenceContainer[] container = new ReferenceContainer[words];
+ final ReferenceContainer<WordReference>[] container = new ReferenceContainer[words];
for (int i = 0; i < words; i++) {
- container[i] = ReferenceContainer.emptyContainer(wordhashes.substring(i * yacySeedDB.commonHashLength, (i + 1) * yacySeedDB.commonHashLength), count);
+ container[i] = ReferenceContainer.emptyContainer(plasmaWordIndex.wordReferenceFactory, wordhashes.substring(i * yacySeedDB.commonHashLength, (i + 1) * yacySeedDB.commonHashLength), count);
}
// insert results to containers
@@ -585,7 +586,7 @@ public final class yacyClient { // add the url entry to the word indexes
for (int m = 0; m < words; m++) {
- container[m].add(entry, System.currentTimeMillis());
+ container[m].add(entry);
}
// store url hash for statistics
@@ -857,7 +858,7 @@ public final class yacyClient { */
public static String transferIndex(
final yacySeed targetSeed,
- final ReferenceContainerCache indexes,
+ final ReferenceContainerCache<WordReference> indexes,
final HashMap<String, URLMetadataRow> urlCache,
final boolean gzipBody,
final int timeout) {
@@ -867,9 +868,9 @@ public final class yacyClient { try {
// check if we got all necessary urls in the urlCache (only for debugging)
- Iterator<WordReferenceRow> eenum;
+ Iterator<WordReference> eenum;
Reference entry;
- for (ReferenceContainer ic: indexes) {
+ for (ReferenceContainer<WordReference> ic: indexes) {
eenum = ic.entries();
while (eenum.hasNext()) {
entry = eenum.next();
@@ -944,7 +945,7 @@ public final class yacyClient { private static HashMap<String, String> transferRWI(
final yacySeed targetSeed,
- final ReferenceContainerCache indexes,
+ final ReferenceContainerCache<WordReference> indexes,
boolean gzipBody,
final int timeout) {
final String address = targetSeed.getPublicAddress();
@@ -962,9 +963,9 @@ public final class yacyClient { int indexcount = 0;
final StringBuilder entrypost = new StringBuilder(indexes.size() * 73);
- Iterator<WordReferenceRow> eenum;
+ Iterator<WordReference> eenum;
Reference entry;
- for (ReferenceContainer ic: indexes) {
+ for (ReferenceContainer<WordReference> ic: indexes) {
eenum = ic.entries();
while (eenum.hasNext()) {
entry = eenum.next();
diff --git a/source/yacy.java b/source/yacy.java index 46d325c95..e7f81916c 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -60,7 +60,7 @@ import de.anomic.kelondro.text.Reference; import de.anomic.kelondro.text.ReferenceContainer;
import de.anomic.kelondro.text.MetadataRepository;
import de.anomic.kelondro.text.metadataPrototype.URLMetadataRow;
-import de.anomic.kelondro.text.referencePrototype.WordReferenceRow;
+import de.anomic.kelondro.text.referencePrototype.WordReference;
import de.anomic.kelondro.util.DateFormatter;
import de.anomic.kelondro.util.MemoryControl;
import de.anomic.kelondro.util.ScoreCluster;
@@ -676,20 +676,20 @@ public final class yacy { if (cacheMem < 2048000) throw new OutOfMemoryError("Not enough memory available to start clean up.");
final plasmaWordIndex wordIndex = new plasmaWordIndex(networkName, log, indexPrimaryRoot, indexSecondaryRoot, 10000, false, 1, 0, false);
- final Iterator<ReferenceContainer> indexContainerIterator = wordIndex.index().references("AAAAAAAAAAAA", false, false);
+ final Iterator<ReferenceContainer<WordReference>> indexContainerIterator = wordIndex.index().references("AAAAAAAAAAAA", false, false);
long urlCounter = 0, wordCounter = 0;
long wordChunkStart = System.currentTimeMillis(), wordChunkEnd = 0;
String wordChunkStartHash = "AAAAAAAAAAAA", wordChunkEndHash;
while (indexContainerIterator.hasNext()) {
- ReferenceContainer wordIdxContainer = null;
+ ReferenceContainer<WordReference> wordIdxContainer = null;
try {
wordCounter++;
wordIdxContainer = indexContainerIterator.next();
// the combined container will fit, read the container
- final Iterator<WordReferenceRow> wordIdxEntries = wordIdxContainer.entries();
+ final Iterator<WordReference> wordIdxEntries = wordIdxContainer.entries();
Reference iEntry;
while (wordIdxEntries.hasNext()) {
iEntry = wordIdxEntries.next();
@@ -864,13 +864,13 @@ public final class yacy { log.logInfo("STARTING CREATION OF RWI-HASHLIST");
final File root = homePath;
try {
- Iterator<ReferenceContainer> indexContainerIterator = null;
+ Iterator<ReferenceContainer<WordReference>> indexContainerIterator = null;
if (resource.equals("all")) {
WordIndex = new plasmaWordIndex("freeworld", log, indexPrimaryRoot, indexSecondaryRoot, 10000, false, 1, 0, false);
indexContainerIterator = WordIndex.index().references(wordChunkStartHash, false, false);
}
int counter = 0;
- ReferenceContainer container = null;
+ ReferenceContainer<WordReference> container = null;
if (format.equals("zip")) {
log.logInfo("Writing Hashlist to ZIP-file: " + targetName + ".zip");
final ZipEntry zipEntry = new ZipEntry(targetName + ".txt");
|