Added a zim parser to the surrogate import option.

You can now import zim files into YaCy by simply moving them
to the DATA/SURROGATE/IN folder. They will be fetched and after
parsing moved to DATA/SURROGATE/OUT.
There are exceptions where the parser is not able to identify the
original URL of the documents in the zim file. In that case the file
is simply ignored.
This commit also carries an important fix to the pdf parser and an
increase of the maximum parsing speed to 60000 PPM which should make it
possible to index up to 1000 files in one second.

1 files changed, 18 insertions, 14 deletions

diff --git a/source/org/openzim/ZIMFile.java b/source/org/openzim/ZIMFile.java
index 906bf30a9..a241507ab 100644
--- a/source/org/openzim/ZIMFile.java
+++ b/source/org/openzim/ZIMFile.java
@@ -113,20 +113,24 @@ public class ZIMFile extends File {
         }

         this.mimeTypeList = mList.toArray(new String[mList.size()]);

 

-        // Initialize the Url Pointer List

-        this.urlPtrListBlob = new byte[this.header_entryCount * 8];

-        mReader.seek(this.header_urlPtrPos);

-        RandomAccessFileZIMInputStream.readFully(mReader, this.urlPtrListBlob);

-

-        // Initialize the Title Pointer List

-        this.titlePtrListBlob = new byte[this.header_entryCount * 4];

-        mReader.seek(this.header_titlePtrPos);

-        RandomAccessFileZIMInputStream.readFully(mReader, this.titlePtrListBlob);

-

-        // Initialize the Cluster Pointer List

-        this.clusterPtrListBlob = new byte[this.header_clusterCount * 8];

-        mReader.seek(this.header_clusterPtrPos);

-        RandomAccessFileZIMInputStream.readFully(mReader, this.clusterPtrListBlob);

+        try {

+            // Initialize the Url Pointer List

+            this.urlPtrListBlob = new byte[this.header_entryCount * 8];

+            mReader.seek(this.header_urlPtrPos);

+            RandomAccessFileZIMInputStream.readFully(mReader, this.urlPtrListBlob);

+

+            // Initialize the Title Pointer List

+            this.titlePtrListBlob = new byte[this.header_entryCount * 4];

+            mReader.seek(this.header_titlePtrPos);

+            RandomAccessFileZIMInputStream.readFully(mReader, this.titlePtrListBlob);

+

+            // Initialize the Cluster Pointer List

+            this.clusterPtrListBlob = new byte[this.header_clusterCount * 8];

+            mReader.seek(this.header_clusterPtrPos);

+            RandomAccessFileZIMInputStream.readFully(mReader, this.clusterPtrListBlob);

+        } catch (IndexOutOfBoundsException e) {

+            throw new IOException(e.getMessage());

+        }

     }

 

     public final String getMimeType(int idx) {