*) Migration of optional Content Parser integration

- each additional parser must be in a subpackage 
  of plasma.parser
- each parser must have its own ant build file (which will 
  be called automatically from the main build file)
- Calling the main build file results in building a separate 
  zip file for each optional parser. This zip file includes:
  + sources of the Parser.java
  + compiled classes of the Parser.java
  + needed additional libs (libx)
- To install an additional parser the user simply needs to
  extract the zip file listed above into his/her yacy directory.
- The configuration (enabling/disabling) of a parser can be done
  via the webinterface (currently the settings dialoge) and is
  done "on-the-fly". The installation can not be done "on-the-fly"
  at the moment because of classpath issues.
- The classpath of the linux startup/stop scripts is generated 
  automatically now (including all libraries from lib and libx).

*) Bugfix: File Extension was not calculated correctly by the crawler
   e.g.: file extension was accidentally: .php?param=value
   Corrected.

*) Adding additional parser for parsing of rss/atom feeds
- added needed libs to do this.

TODO:
- automatic building classpath for windows startup scripts


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@78 6c8d7289-2bf4-0310-a012-ef5d649a1542

1 files changed, 609 insertions, 406 deletions

diff --git a/source/de/anomic/plasma/plasmaParser.java b/source/de/anomic/plasma/plasmaParser.java
index ba197bf87..27d5ab5bb 100644
--- a/source/de/anomic/plasma/plasmaParser.java
+++ b/source/de/anomic/plasma/plasmaParser.java
@@ -1,406 +1,609 @@
-// plasmaParser.java 
-// ------------------------
-// part of YaCy
-// (C) by Michael Peter Christen; mc@anomic.de
-// first published on http://www.anomic.de
-// Frankfurt, Germany, 2005
-// last major change: 12.04.2005
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program; if not, write to the Free Software
-// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-//
-// Using this software in any meaning (reading, learning, copying, compiling,
-// running) means that you agree that the Author(s) is (are) not responsible
-// for cost, loss of data or any harm that may be caused directly or indirectly
-// by usage of this softare or this documentation. The usage of this software
-// is on your own risk. The installation and usage (starting/running) of this
-// software may allow other people or application to access your computer and
-// any attached devices and is highly dependent on the configuration of the
-// software which must be done by the user of the software; the author(s) is
-// (are) also not responsible for proper configuration and usage of the
-// software, even if provoked by documentation provided together with
-// the software.
-//
-// Any changes to this file according to the GPL as documented in the file
-// gpl.txt aside this file in the shipment you received can be done to the
-// lines that follows this copyright notice here, but changes must not be
-// done inside the copyright notive above. A re-distribution must contain
-// the intact and unchanged copyright notice.
-// Contributions and changes to the program code must be marked as such.
-
-// compile: javac -classpath lib/commons-collections.jar:lib/commons-pool-1.2.jar -sourcepath source source/de/anomic/plasma/plasmaParser.java
-
-
-package de.anomic.plasma;
-
-import java.io.*;
-import java.net.*;
-import java.util.*;
-
-import org.apache.commons.pool.KeyedPoolableObjectFactory;
-import org.apache.commons.pool.impl.GenericKeyedObjectPool;
-import org.apache.commons.pool.impl.GenericObjectPool;
-
-import de.anomic.plasma.parser.Parser;
-import de.anomic.server.serverFileUtils;
-import de.anomic.htmlFilter.*;
-
-public final class plasmaParser {
-
-    private final Properties parserList;
-    private final plasmaParserPool theParserPool;
-
-    public static HashSet mediaExtSet = new HashSet();
-    public static void initMediaExt(String mediaExtString) {
-	String[] xs = mediaExtString.split(",");
-	for (int i = 0; i < xs.length; i++) mediaExtSet.add(xs[i]);
-    }
-    static {
-	initMediaExt("swf,wmv,jpg,jpeg,jpe,rm,mov,mpg,mpeg,mp3,asf,gif,png,avi,zip,rar," +
-		     "sit,hqx,img,dmg,tar,gz,ps,xls,ppt,ram,bz2,arj");
-    }
-
-    public plasmaParser(File parserDispatcherPropertyFile) {
-        
-        /* ===================================================
-         * loading a list of availabe parser from file
-         * =================================================== */ 
-    	Properties prop = new Properties();
-    	try {
-    	    prop.load(new FileInputStream(parserDispatcherPropertyFile));
-    	} catch (IOException e) {
-    	    System.err.println("ERROR: " + parserDispatcherPropertyFile.toString() + " not found in settings path");
-    	}    	
-        this.parserList = prop;
-        
-        /* ===================================================
-         * initializing the parser object pool
-         * =================================================== */
-        GenericKeyedObjectPool.Config config = new GenericKeyedObjectPool.Config();
-        
-        // The maximum number of active connections that can be allocated from pool at the same time,
-        // 0 for no limit
-        config.maxActive = 0;
-        
-        // The maximum number of idle connections connections in the pool
-        // 0 = no limit.        
-        config.maxIdle = 10;    
-        
-        config.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK; 
-        config.minEvictableIdleTimeMillis = 30000; 
-        
-        this.theParserPool = new plasmaParserPool(new plasmaParserFactory(),config);           
-        
-        /* ===================================================
-         * testing if all parsers could be loaded properly.
-         * This is done now to avoid surprises at runtime. 
-         * =================================================== */
-        if (this.parserList.size() > 0) {
-			Iterator parserIterator = this.parserList.values().iterator();
-            while (parserIterator.hasNext()) {
-				String className = (String) parserIterator.next();
-                try {
-					Class.forName(className);
-                } catch (Exception e) {
-                    // if we could not load the parser we remove it from the parser list ...
-                    this.parserList.remove(className);
-                }
-            }
-        }        
-    }
-    
-    public void close() {
-        // release resources 
-        try {        
-	        // clearing the parser list
-	        this.parserList.clear();
-	        
-	        // closing the parser object pool
-	        this.theParserPool.close();
-        } catch (Exception e) {
-            //
-        }
-    }    
-    
-    public plasmaParserDocument parseSource(URL location, String mimeType, byte[] source) {
-        
-        Parser theParser = null;
-        try {
-            
-            if ((mimeType != null) && (mimeType.indexOf(";") != -1)) {
-                mimeType = mimeType.substring(0,mimeType.indexOf(";"));
-            }                        
-            
-            // getting the correct parser for the given mimeType
-            theParser = this.getParser(mimeType);
-            
-            // if a parser was found we use it ...
-            if (theParser != null) {
-                return theParser.parse(location, mimeType,source);
-            }
-        
-            // ...otherwise we make a html scraper and transformer
-            htmlFilterContentScraper scraper = new htmlFilterContentScraper(location);
-            OutputStream hfos = new htmlFilterOutputStream(null, scraper, null, false);
-
-            hfos.write(source);
-            return transformScraper(location, mimeType, scraper);
-        } catch (Exception e) {
-            return null;
-        } finally {
-            if (theParser != null) {
-                try {
-                    this.theParserPool.returnObject(mimeType, theParser);
-                } catch (Exception e) {
-                }
-            }
-        }
-    }
-
-    public plasmaParserDocument parseSource(URL location, String mimeType, File sourceFile) {
-
-        Parser theParser = null;
-        try {
-            if ((mimeType != null) && (mimeType.indexOf(";") != -1)) {
-                mimeType = mimeType.substring(0,mimeType.indexOf(";"));
-            }            
-            
-            // getting the correct parser for the given mimeType
-            theParser = this.getParser(mimeType);
-            
-            // if a parser was found we use it ...
-            if (theParser != null) {
-                return theParser.parse(location, mimeType,sourceFile);
-            }    
-            
-            // ...otherwise we make a scraper and transformer
-            htmlFilterContentScraper scraper = new htmlFilterContentScraper(location);
-            OutputStream hfos = new htmlFilterOutputStream(null, scraper, null, false);            
-            
-			serverFileUtils.copy(sourceFile, hfos);
-            return transformScraper(location, mimeType, scraper);
-        } catch (Exception e) {
-            return null;
-        } finally {
-            if (theParser != null) {
-                try {
-                    this.theParserPool.returnObject(mimeType, theParser);
-                } catch (Exception e) {
-                }
-            }
-        }
-    }
-    
-    public plasmaParserDocument transformScraper(URL location, String mimeType, htmlFilterContentScraper scraper) {
-        try {
-            return new plasmaParserDocument(new URL(urlNormalform(location)),
-                                mimeType, null, null, scraper.getHeadline(),
-                                null, null,
-                                scraper.getText(), scraper.getAnchors(), scraper.getImages());
-        } catch (MalformedURLException e) {
-            return null;
-        }
-    }
-    
-    /**
-     * This function is used to determine the parser class that should be used for a given
-     * mimetype ...
-     * @param mimeType
-     * @return
-     */
-    public Parser getParser(String mimeType) {
-        
-        if (mimeType == null) {            
-            // TODO: do automatic mimetype detection
-            return null;
-        }
-        
-        try {
-	        if (this.parserList.containsKey(mimeType)) {
-	            String parserClassName = (String)this.parserList.get(mimeType);
-	            
-                // fetching a new parser object from pool  
-				Parser theParser = (Parser) this.theParserPool.borrowObject(parserClassName);
-                
-                // checking if the created parser really supports the given mimetype 
-                HashSet supportedMimeTypes = theParser.getSupportedMimeTypes();
-                if ((supportedMimeTypes != null) && (supportedMimeTypes.contains(mimeType))) {
-					return theParser;
-                }
-                this.theParserPool.returnObject(parserClassName,theParser);
-	        }
-        } catch (Exception e) {
-            System.err.println("ERROR: Unable to load the correct parser for type " + mimeType);
-        }
-        
-        return null;
-        
-    }
-    
-    public static String urlNormalform(URL url) {
-        if (url == null) return null;
-        return urlNormalform(url.toString());
-    }
-    
-    public static String urlNormalform(String us) {
-        if (us == null) return null;
-        if (us.length() == 0) return null;
-        int p;
-        if ((p = us.indexOf("#")) >= 0) us = us.substring(0, p);
-        if (us.endsWith(":80")) us = us.substring(0, us.length() - 3);
-        if (((us.endsWith("/")) && (us.lastIndexOf('/', us.length() - 2) < 8))) us = us.substring(0, us.length() - 1);
-        return us;
-    }   
-    
-    static Map allReflinks(Map links) {
-        // we find all links that are part of a reference inside a url
-        HashMap v = new HashMap();
-        Iterator i = links.keySet().iterator();
-        String s;
-        int pos;
-        loop: while (i.hasNext()) {
-            s = (String) i.next();
-            if ((pos = s.toLowerCase().indexOf("http://",7)) > 0) {
-                i.remove();
-                s = s.substring(pos);
-                while ((pos = s.toLowerCase().indexOf("http://",7)) > 0) s = s.substring(pos);
-                if (!(v.containsKey(s))) v.put(s, "ref");
-                continue loop;
-            }
-            if ((pos = s.toLowerCase().indexOf("/www.",7)) > 0) {
-                i.remove();
-                s = "http:/" + s.substring(pos);
-                while ((pos = s.toLowerCase().indexOf("/www.",7)) > 0) s = "http:/" + s.substring(pos);
-                if (!(v.containsKey(s))) v.put(s, "ref");
-                continue loop;
-            }
-        }
-        return v;
-    }
-    
-    static Map allSubpaths(Map links) {
-        HashMap v = new HashMap();
-        Iterator i = links.keySet().iterator();
-        String s;
-        int pos;
-        while (i.hasNext()) {
-            s = (String) i.next();
-            if (s.endsWith("/")) s = s.substring(0, s.length() - 1);
-            pos = s.lastIndexOf("/");
-            while (pos > 8) {
-                s = s.substring(0, pos + 1);
-                if (!(v.containsKey(s))) v.put(s, "sub");
-                s = s.substring(0, pos);
-                pos = s.lastIndexOf("/");
-            }
-        }
-        return v;
-    }
-    
-    public static void main(String[] args) {
-	//javac -classpath lib/commons-collections.jar:lib/commons-pool-1.2.jar -sourcepath source source/de/anomic/plasma/plasmaParser.java
-	//java -cp source:lib/commons-collections.jar:lib/commons-pool-1.2.jar de.anomic.plasma.plasmaParser bug.html bug.out
-	try {
-	    File in = new File(args[0]);
-	    File out = new File(args[1]);
-	    plasmaParser theParser = new plasmaParser(new File("yacy.parser"));
-            FileInputStream theInput = new FileInputStream(in);
-	    ByteArrayOutputStream theOutput = new ByteArrayOutputStream();
-            serverFileUtils.copy(theInput, theOutput);
-            plasmaParserDocument document = theParser.parseSource(new URL("http://brain.yacy"), "text/html", theOutput.toByteArray());
-            //plasmaParserDocument document = theParser.parseSource(new URL("http://brain.yacy"), "application/pdf", theOutput.toByteArray());
-	    byte[] theText = document.getText();
-	    serverFileUtils.write(theText, out);
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-    }
-    
-}
-
-final class plasmaParserFactory implements KeyedPoolableObjectFactory {
-    
-    public plasmaParserFactory() {
-        super();  
-    }
-    
-    /**
-     * @see org.apache.commons.pool.PoolableObjectFactory#makeObject()
-     */
-    public Object makeObject(Object key) throws Exception {
-        
-        if (!(key instanceof String))
-            throw new IllegalArgumentException("The object key must be of type string.");
-        
-        Class moduleClass = Class.forName((String)key);
-        return moduleClass.newInstance();
-    }          
-    
-     /**
-     * @see org.apache.commons.pool.PoolableObjectFactory#destroyObject(java.lang.Object)
-     */
-    public void destroyObject(Object key, Object obj) {
-        if (obj instanceof Parser) {
-            Parser theParser = (Parser) obj;
-        }
-    }
-    
-    /**
-     * @see org.apache.commons.pool.PoolableObjectFactory#validateObject(java.lang.Object)
-     */
-    public boolean validateObject(Object key, Object obj) {
-        if (obj instanceof Parser) {
-            Parser theParser = (Parser) obj;
-            return true;
-        }
-        return true;
-    }
-    
-    /**
-     * @param obj 
-     * 
-     */
-    public void activateObject(Object key, Object obj)  {
-        //log.debug(" activateObject...");
-    }
-
-    /**
-     * @param obj 
-     * 
-     */
-    public void passivateObject(Object key, Object obj) { 
-        //log.debug(" passivateObject..." + obj);
-        if (obj instanceof Parser)  {
-            Parser theParser = (Parser) obj;
-            theParser.reset();
-        }
-    }
-}    
-
-final class plasmaParserPool extends GenericKeyedObjectPool {
-
-    public plasmaParserPool(plasmaParserFactory objFactory,
-            GenericKeyedObjectPool.Config config) {
-        super(objFactory, config);
-    }
-    
-
-    public Object borrowObject(Object key) throws Exception  {
-       return super.borrowObject(key);
-    }
-
-    public void returnObject(Object key, Object borrowed) throws Exception  {
-        super.returnObject(key,borrowed);
-    }        
-}   
+// plasmaParser.java 

+// ------------------------

+// part of YaCy

+// (C) by Michael Peter Christen; mc@anomic.de

+// first published on http://www.anomic.de

+// Frankfurt, Germany, 2005

+//

+// last major change: 02.05.2005 by Martin Thelian

+//

+// This program is free software; you can redistribute it and/or modify

+// it under the terms of the GNU General Public License as published by

+// the Free Software Foundation; either version 2 of the License, or

+// (at your option) any later version.

+//

+// This program is distributed in the hope that it will be useful,

+// but WITHOUT ANY WARRANTY; without even the implied warranty of

+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+// GNU General Public License for more details.

+//

+// You should have received a copy of the GNU General Public License

+// along with this program; if not, write to the Free Software

+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

+//

+// Using this software in any meaning (reading, learning, copying, compiling,

+// running) means that you agree that the Author(s) is (are) not responsible

+// for cost, loss of data or any harm that may be caused directly or indirectly

+// by usage of this softare or this documentation. The usage of this software

+// is on your own risk. The installation and usage (starting/running) of this

+// software may allow other people or application to access your computer and

+// any attached devices and is highly dependent on the configuration of the

+// software which must be done by the user of the software; the author(s) is

+// (are) also not responsible for proper configuration and usage of the

+// software, even if provoked by documentation provided together with

+// the software.

+//

+// Any changes to this file according to the GPL as documented in the file

+// gpl.txt aside this file in the shipment you received can be done to the

+// lines that follows this copyright notice here, but changes must not be

+// done inside the copyright notive above. A re-distribution must contain

+// the intact and unchanged copyright notice.

+// Contributions and changes to the program code must be marked as such.

+

+// compile: javac -classpath lib/commons-collections.jar:lib/commons-pool-1.2.jar -sourcepath source source/de/anomic/plasma/plasmaParser.java

+

+

+package de.anomic.plasma;

+

+import java.io.*;

+import java.net.*;

+import java.util.*;

+

+import org.apache.commons.pool.KeyedPoolableObjectFactory;

+import org.apache.commons.pool.impl.GenericKeyedObjectPool;

+import org.apache.commons.pool.impl.GenericObjectPool;

+

+import de.anomic.plasma.parser.Parser;

+import de.anomic.server.serverFileUtils;

+import de.anomic.htmlFilter.*;

+

+public final class plasmaParser {

+

+    /**

+     * A list containing all installed parsers and the mimeType that they support

+     * @see #loadAvailableParserList()

+     */

+    private final Properties availableParserList = new Properties();

+    

+    /**

+     * A list containing all enabled parsers and the mimeType that they can handle

+     * @see #loadEnabledParserList()

+     * @see #setEnabledParserList(Enumeration)

+     */

+    private final Properties enabledParserList = new Properties();    

+    

+    /**

+     * A list of file extensions that are supported by all enabled parsers

+     */

+    private static final HashSet supportedFileExt = new HashSet();

+    

+    /**

+     * A pool of parsers.

+     * @see plasmaParserPool

+     * @see plasmaParserFactory

+     */

+    private final plasmaParserPool theParserPool;

+

+    /**

+     * The configuration file containing a list of enabled parsers

+     * @see plasmaParser#plasmaParser(File)

+     */

+    private final File parserDispatcherPropertyFile;

+    

+    /**

+     * A list of media extensions that should not be handled by the plasmaParser

+     */

+    private static final HashSet mediaExtSet = new HashSet(28);

+    

+    /**

+     * This {@link FilenameFilter} is used to find all classes based on there filenames 

+     * which seems to be additional content parsers.

+     * Currently the filenames of all content parser classes must end with <code>Parser.class</code> 

+     */

+    private final FilenameFilter parserFileNameFilter = new FilenameFilter() {

+        public boolean accept(File dir, String name) {

+            return name.endsWith("Parser.class");

+        }

+    };

+    

+    /**

+     * This {@link FileFilter} is used to get all subpackages

+     * of the parser package.

+     */

+    private final FileFilter parserDirectoryFilter = new FileFilter() {

+        public boolean accept(File file) {

+            return file.isDirectory();

+        }

+    };    

+    

+    /**

+     * Initializing the 

+     * @see #initMediaExt(String)

+     */

+    static {

+		initMediaExt("swf,wmv,jpg,jpeg,jpe,rm,mov,mpg,mpeg,mp3,asf,gif,png,avi,zip,rar," +

+			"sit,hqx,img,dmg,tar,gz,ps,xls,ppt,ram,bz2,arj");

+    }

+    

+    public static void initMediaExt(String mediaExtString) {

+		String[] xs = mediaExtString.split(",");

+        initMediaExt(Arrays.asList(xs));

+    }

+    

+    public static void initMediaExt(List mediaExtList) {

+        synchronized (mediaExtSet) {

+            mediaExtSet.clear();

+    		mediaExtSet.addAll(mediaExtList);

+		}

+    }

+    

+    public static boolean mediaExtContains(String mediaExt) {

+        

+        synchronized (supportedFileExt) {

+			if (supportedFileExt.contains(mediaExt)) return false;

+		}

+        

+        synchronized (mediaExtSet) {

+			return mediaExtSet.contains(mediaExtSet);

+		}

+    }

+

+

+    public plasmaParser(File parserDispatcherPropertyFile) {

+        

+        this.parserDispatcherPropertyFile = parserDispatcherPropertyFile;

+

+        /* ===================================================

+         * initializing the parser object pool

+         * =================================================== */

+        GenericKeyedObjectPool.Config config = new GenericKeyedObjectPool.Config();

+        

+        // The maximum number of active connections that can be allocated from pool at the same time,

+        // 0 for no limit

+        config.maxActive = 0;

+        

+        // The maximum number of idle connections connections in the pool

+        // 0 = no limit.        

+        config.maxIdle = 10;    

+        

+        config.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK; 

+        config.minEvictableIdleTimeMillis = 30000; 

+        

+        this.theParserPool = new plasmaParserPool(new plasmaParserFactory(),config);               

+        

+        /* ===================================================

+         * loading a list of available parsers

+         * =================================================== */        

+		loadAvailableParserList();

+        

+        /* ===================================================

+         * loading a list of activated parsers

+         * =================================================== */         

+		loadEnabledParserList();

+        

+    }

+    

+    public boolean setEnabledParserList(Enumeration mimeTypes) {

+        

+        Properties newEnabledParsers = new Properties();

+        HashSet newSupportedFileExt = new HashSet();

+        

+        if (mimeTypes != null) {

+	        while (mimeTypes.hasMoreElements()) {

+	            String mimeType = (String) mimeTypes.nextElement();

+				if (this.availableParserList.containsKey(mimeType)) {

+                    Parser theParser = null;

+                    try {

+                        // getting the parser

+                        theParser = (Parser) this.theParserPool.borrowObject(this.availableParserList.get(mimeType));

+                        

+                        // getting a list of mimeTypes that the parser supports

+                        Hashtable parserSupportsMimeTypes = theParser.getSupportedMimeTypes();                        

+                        if (parserSupportsMimeTypes != null) {

+                            Object supportedExtensions = parserSupportsMimeTypes.get(mimeType);                        

+                            if ((supportedExtensions != null) && (supportedExtensions instanceof String)) {

+                        		String[] extArray = ((String)supportedExtensions).split(",");

+                                newSupportedFileExt.addAll(Arrays.asList(extArray));

+                            }

+                        }

+						newEnabledParsers.put(mimeType,this.availableParserList.get(mimeType));

+                        

+                    } catch (Exception e) { 

+                        e.printStackTrace();

+                    } finally {

+                        if (theParser != null) 

+                            try { this.theParserPool.returnObject(mimeType,theParser); } catch (Exception e) {}

+                    }

+				}

+	        }

+        }

+        

+        synchronized (this.enabledParserList) {

+			this.enabledParserList.clear();

+            this.enabledParserList.putAll(newEnabledParsers);

+		}

+        

+        

+        synchronized (supportedFileExt) {

+			supportedFileExt.clear();

+            supportedFileExt.addAll(supportedFileExt);

+		}

+        

+        return true;

+    }

+    

+    public Hashtable getEnabledParserList() {

+        synchronized (this.enabledParserList) {

+            return (Hashtable) this.enabledParserList.clone();

+		}        

+    }

+    

+    public Hashtable getAvailableParserList() {

+        return this.availableParserList;

+    }

+    

+    private void loadEnabledParserList() {

+        // loading a list of availabe parser from file

+    	Properties prop = new Properties();

+    	try {

+    	    prop.load(new FileInputStream(this.parserDispatcherPropertyFile));

+    	} catch (IOException e) {

+    	    System.err.println("ERROR: " + this.parserDispatcherPropertyFile.toString() + " not found in settings path");

+    	}    	

+

+        // enable them ...

+        this.setEnabledParserList(prop.keys());

+	}

+

+	private void loadAvailableParserList() {

+        try {

+            this.availableParserList.clear();

+            

+            // getting the current package name

+			String plasmaParserPkgName = this.getClass().getPackage().getName() + ".parser";

+			System.out.println("INFO: Searching for additional content parsers in package " + plasmaParserPkgName);

+ 

+            // getting an uri to the parser subpackage

+	        String packageURI = this.getClass().getResource("/"+plasmaParserPkgName.replace('.','/')).toString();

+			System.out.println("INFO: Parser directory is " + packageURI);           

+ 

+            // open the parser directory

+	        File parserDir = new File(new URI(packageURI));

+            if ((parserDir == null) || (!parserDir.exists()) || (!parserDir.isDirectory())) return;

+            

+            /* 

+             * loop through all subdirectories and test if we can 

+             * find an additional parser class

+             */

+            File[] parserDirectories = parserDir.listFiles(this.parserDirectoryFilter);

+            if (parserDirectories == null) return;

+			for (int parserDirNr=0; parserDirNr< parserDirectories.length; parserDirNr++) {

+                File currentDir = parserDirectories[parserDirNr];

+				System.out.println("INFO: Searching in directory " + currentDir.toString());

+                String[] parserClasses = currentDir.list(this.parserFileNameFilter);

+                if (parserClasses == null) continue;

+                

+                for (int parserNr=0; parserNr<parserClasses.length; parserNr++) {

+                	System.out.println("INFO: Testing parser class " + parserClasses[parserNr]);

+                    String className = parserClasses[parserNr].substring(0,parserClasses[parserNr].indexOf(".class"));

+                    String fullClassName = plasmaParserPkgName + "." + currentDir.getName() + "." + className;

+	                try {

+						Class parserClass = Class.forName(fullClassName);

+                        Object theParser = parserClass.newInstance();

+                        if (!(theParser instanceof Parser)) continue;

+                        Hashtable supportedMimeTypes = ((Parser)theParser).getSupportedMimeTypes();

+                        Iterator mimeTypeIterator = supportedMimeTypes.keySet().iterator();

+                        while (mimeTypeIterator.hasNext()) {

+                            String mimeType = (String) mimeTypeIterator.next();

+                            this.availableParserList.put(mimeType,fullClassName);

+							System.out.println("INFO: Found parser for mimeType " + mimeType);

+                        }

+                            

+	                } catch (Exception e) { /* we can ignore this for the moment */ }

+                }

+			}

+            

+        } catch (Exception e) {

+            System.err.println("ERROR: while trying to determine all installed parsers. " + e.getMessage());

+        }		

+	}

+

+	public void close() {        

+        // clearing the parser list

+        synchronized (this.enabledParserList) {

+        	try {

+        	    this.enabledParserList.store(new FileOutputStream(this.parserDispatcherPropertyFile),"plasmaParser configuration file");

+        	} catch (IOException e) {

+        	    System.err.println("ERROR: " + this.parserDispatcherPropertyFile.toString() + " can not be stored.");

+        	} 

+            

+	        this.enabledParserList.clear();

+		}

+        

+        // closing the parser object pool

+        try {        

+	        this.theParserPool.close();

+        } catch (Exception e) { }        

+    }    

+    

+    public plasmaParserDocument parseSource(URL location, String mimeType, byte[] source) {

+        

+        Parser theParser = null;

+        try {

+            

+            if ((mimeType != null) && (mimeType.indexOf(";") != -1)) {

+                mimeType = mimeType.substring(0,mimeType.indexOf(";"));

+            }                        

+            

+            // getting the correct parser for the given mimeType

+            theParser = this.getParser(mimeType);

+            

+            // if a parser was found we use it ...

+            if (theParser != null) {

+                return theParser.parse(location, mimeType,source);

+            }

+        

+            // ...otherwise we make a html scraper and transformer

+            htmlFilterContentScraper scraper = new htmlFilterContentScraper(location);

+            OutputStream hfos = new htmlFilterOutputStream(null, scraper, null, false);

+

+            hfos.write(source);

+            return transformScraper(location, mimeType, scraper);

+        } catch (Exception e) {

+            return null;

+        } finally {

+            if (theParser != null) {

+                try {

+                    this.theParserPool.returnObject(mimeType, theParser);

+                } catch (Exception e) {

+                }

+            }

+        }

+    }

+

+    public plasmaParserDocument parseSource(URL location, String mimeType, File sourceFile) {

+

+        Parser theParser = null;

+        try {

+            if ((mimeType != null) && (mimeType.indexOf(";") != -1)) {

+                mimeType = mimeType.substring(0,mimeType.indexOf(";"));

+            }            

+            

+            // getting the correct parser for the given mimeType

+            theParser = this.getParser(mimeType);

+            

+            // if a parser was found we use it ...

+            if (theParser != null) {

+                return theParser.parse(location, mimeType,sourceFile);

+            }    

+            

+            // ...otherwise we make a scraper and transformer

+            htmlFilterContentScraper scraper = new htmlFilterContentScraper(location);

+            OutputStream hfos = new htmlFilterOutputStream(null, scraper, null, false);            

+            

+			serverFileUtils.copy(sourceFile, hfos);

+            return transformScraper(location, mimeType, scraper);

+        } catch (Exception e) {

+            return null;

+        } finally {

+            if (theParser != null) {

+                try {

+                    this.theParserPool.returnObject(mimeType, theParser);

+                } catch (Exception e) {

+                }

+            }

+        }

+    }

+    

+    public plasmaParserDocument transformScraper(URL location, String mimeType, htmlFilterContentScraper scraper) {

+        try {

+            return new plasmaParserDocument(new URL(urlNormalform(location)),

+                                mimeType, null, null, scraper.getHeadline(),

+                                null, null,

+                                scraper.getText(), scraper.getAnchors(), scraper.getImages());

+        } catch (MalformedURLException e) {

+            return null;

+        }

+    }

+    

+    /**

+     * This function is used to determine the parser class that should be used for a given

+     * mimetype ...

+     * @param mimeType

+     * @return

+     */

+    public Parser getParser(String mimeType) {

+        

+        if (mimeType == null) {            

+            // TODO: do automatic mimetype detection

+            return null;

+        }

+        

+        try {

+            

+            // determining the proper parser class name for the mimeType

+            String parserClassName = null;

+            synchronized (this.enabledParserList) {

+    	        if (this.enabledParserList.containsKey(mimeType)) {

+    	            parserClassName = (String)this.enabledParserList.get(mimeType);

+    	        } else {

+                    return null;

+    	        }

+			}

+            

+            // fetching a new parser object from pool  

+			Parser theParser = (Parser) this.theParserPool.borrowObject(parserClassName);

+            

+            // checking if the created parser really supports the given mimetype 

+            Hashtable supportedMimeTypes = theParser.getSupportedMimeTypes();

+            if ((supportedMimeTypes != null) && (supportedMimeTypes.contains(mimeType))) {

+				return theParser;

+            }

+            this.theParserPool.returnObject(parserClassName,theParser);

+            

+        } catch (Exception e) {

+            System.err.println("ERROR: Unable to load the correct parser for type " + mimeType);

+        }

+        

+        return null;

+        

+    }

+    

+    public static String urlNormalform(URL url) {

+        if (url == null) return null;

+        return urlNormalform(url.toString());

+    }

+    

+    public static String urlNormalform(String us) {

+        if (us == null) return null;

+        if (us.length() == 0) return null;

+        int p;

+        if ((p = us.indexOf("#")) >= 0) us = us.substring(0, p);

+        if (us.endsWith(":80")) us = us.substring(0, us.length() - 3);

+        if (((us.endsWith("/")) && (us.lastIndexOf('/', us.length() - 2) < 8))) us = us.substring(0, us.length() - 1);

+        return us;

+    }   

+    

+    static Map allReflinks(Map links) {

+        // we find all links that are part of a reference inside a url

+        HashMap v = new HashMap();

+        Iterator i = links.keySet().iterator();

+        String s;

+        int pos;

+        loop: while (i.hasNext()) {

+            s = (String) i.next();

+            if ((pos = s.toLowerCase().indexOf("http://",7)) > 0) {

+                i.remove();

+                s = s.substring(pos);

+                while ((pos = s.toLowerCase().indexOf("http://",7)) > 0) s = s.substring(pos);

+                if (!(v.containsKey(s))) v.put(s, "ref");

+                continue loop;

+            }

+            if ((pos = s.toLowerCase().indexOf("/www.",7)) > 0) {

+                i.remove();

+                s = "http:/" + s.substring(pos);

+                while ((pos = s.toLowerCase().indexOf("/www.",7)) > 0) s = "http:/" + s.substring(pos);

+                if (!(v.containsKey(s))) v.put(s, "ref");

+                continue loop;

+            }

+        }

+        return v;

+    }

+    

+    static Map allSubpaths(Map links) {

+        HashMap v = new HashMap();

+        Iterator i = links.keySet().iterator();

+        String s;

+        int pos;

+        while (i.hasNext()) {

+            s = (String) i.next();

+            if (s.endsWith("/")) s = s.substring(0, s.length() - 1);

+            pos = s.lastIndexOf("/");

+            while (pos > 8) {

+                s = s.substring(0, pos + 1);

+                if (!(v.containsKey(s))) v.put(s, "sub");

+                s = s.substring(0, pos);

+                pos = s.lastIndexOf("/");

+            }

+        }

+        return v;

+    }

+    

+    public static void main(String[] args) {		

+	//javac -classpath lib/commons-collections.jar:lib/commons-pool-1.2.jar -sourcepath source source/de/anomic/plasma/plasmaParser.java

+	//java -cp source:lib/commons-collections.jar:lib/commons-pool-1.2.jar de.anomic.plasma.plasmaParser bug.html bug.out

+		try {

+			File in = new File(args[0]);

+			File out = new File(args[1]);

+			plasmaParser theParser = new plasmaParser(new File("yacy.parser"));

+			FileInputStream theInput = new FileInputStream(in);

+			ByteArrayOutputStream theOutput = new ByteArrayOutputStream();

+			serverFileUtils.copy(theInput, theOutput);

+			plasmaParserDocument document = theParser.parseSource(new URL("http://brain.yacy"), "text/html", theOutput.toByteArray());

+			//plasmaParserDocument document = theParser.parseSource(new URL("http://brain.yacy"), "application/pdf", theOutput.toByteArray());

+			byte[] theText = document.getText();

+			serverFileUtils.write(theText, out);

+        } catch (Exception e) {

+            e.printStackTrace();

+        }

+    }

+    

+}

+

+final class plasmaParserFactory implements KeyedPoolableObjectFactory {

+    

+    public plasmaParserFactory() {

+        super();  

+    }

+    

+    /**

+     * @see org.apache.commons.pool.PoolableObjectFactory#makeObject()

+     */

+    public Object makeObject(Object key) throws Exception {

+        

+        if (!(key instanceof String))

+            throw new IllegalArgumentException("The object key must be of type string.");

+        

+        Class moduleClass = Class.forName((String)key);

+        return moduleClass.newInstance();

+    }          

+    

+     /**

+     * @see org.apache.commons.pool.PoolableObjectFactory#destroyObject(java.lang.Object)

+     */

+    public void destroyObject(Object key, Object obj) {

+        if (obj instanceof Parser) {

+            Parser theParser = (Parser) obj;

+        }

+    }

+    

+    /**

+     * @see org.apache.commons.pool.PoolableObjectFactory#validateObject(java.lang.Object)

+     */

+    public boolean validateObject(Object key, Object obj) {

+        if (obj instanceof Parser) {

+            Parser theParser = (Parser) obj;

+            return true;

+        }

+        return true;

+    }

+    

+    /**

+     * @param obj 

+     * 

+     */

+    public void activateObject(Object key, Object obj)  {

+        //log.debug(" activateObject...");

+    }

+

+    /**

+     * @param obj 

+     * 

+     */

+    public void passivateObject(Object key, Object obj) { 

+        //log.debug(" passivateObject..." + obj);

+        if (obj instanceof Parser)  {

+            Parser theParser = (Parser) obj;

+            theParser.reset();

+        }

+    }

+}    

+

+final class plasmaParserPool extends GenericKeyedObjectPool {

+

+    public plasmaParserPool(plasmaParserFactory objFactory,

+            GenericKeyedObjectPool.Config config) {

+        super(objFactory, config);

+    }

+    

+

+    public Object borrowObject(Object key) throws Exception  {

+       return super.borrowObject(key);

+    }

+

+    public void returnObject(Object key, Object borrowed) throws Exception  {

+        super.returnObject(key,borrowed);

+    }        

+}   

+

+