From 35a1d85f8b7f34e3bcc45d30333a93383cb13493 Mon Sep 17 00:00:00 2001
From: Michael Peter Christen <mc@yacy.net>
Date: Sun, 5 Jul 2026 12:04:42 +0200
Subject: removed the snapshot feature and all dependencies from non-java code

---
 .../SnapshotImagesReponseWriter.java               | 205 -----
 source/net/yacy/cora/util/Html2Image.java          | 581 --------------
 source/net/yacy/crawler/CrawlSwitchboard.java      |  13 -
 source/net/yacy/crawler/RecrawlBusyThread.java     | 858 ++++++++++-----------
 source/net/yacy/crawler/data/CrawlProfile.java     |  56 --
 source/net/yacy/crawler/data/Snapshots.java        | 516 -------------
 source/net/yacy/crawler/data/Transactions.java     | 403 ----------
 source/net/yacy/document/parser/psParser.java      |  72 +-
 source/net/yacy/htroot/ConfigSearchPage_p.java     |   3 -
 source/net/yacy/htroot/CrawlStartExpert.java       |  13 -
 source/net/yacy/htroot/Crawler_p.java              |  10 -
 source/net/yacy/htroot/QuickCrawlLink_p.java       | 405 +++++-----
 source/net/yacy/htroot/api/snapshot.java           | 382 ---------
 source/net/yacy/htroot/yacysearchitem.java         |  29 -
 .../net/yacy/http/servlets/SolrSelectServlet.java  |   2 -
 source/net/yacy/search/Switchboard.java            |   8 +-
 source/net/yacy/search/SwitchboardConstants.java   |   6 -
 source/net/yacy/search/index/DocumentIndex.java    |   4 +-
 source/net/yacy/search/index/Segment.java          |  27 +-
 19 files changed, 637 insertions(+), 2956 deletions(-)
 delete mode 100644 source/net/yacy/cora/federate/solr/responsewriter/SnapshotImagesReponseWriter.java
 delete mode 100644 source/net/yacy/cora/util/Html2Image.java
 delete mode 100644 source/net/yacy/crawler/data/Snapshots.java
 delete mode 100644 source/net/yacy/crawler/data/Transactions.java
 delete mode 100644 source/net/yacy/htroot/api/snapshot.java

(limited to 'source')

diff --git a/source/net/yacy/cora/federate/solr/responsewriter/SnapshotImagesReponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/SnapshotImagesReponseWriter.java
deleted file mode 100644
index f68c6ca48..000000000
--- a/source/net/yacy/cora/federate/solr/responsewriter/SnapshotImagesReponseWriter.java
+++ /dev/null
@@ -1,205 +0,0 @@
-package net.yacy.cora.federate.solr.responsewriter;
-
-import java.io.IOException;
-import java.io.Writer;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.IndexableField;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrDocumentList;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.response.QueryResponseWriter;
-import org.apache.solr.response.ResultContext;
-import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.search.DocIterator;
-import org.apache.solr.search.DocList;
-import org.apache.solr.search.SolrIndexSearcher;
-
-import net.yacy.cora.util.CommonPattern;
-import net.yacy.search.schema.CollectionSchema;
-
-/**
- * this writer is supposed to be used to generate iframes. It generates links for the /api/snapshot.jpg servlet.
- */
-public class SnapshotImagesReponseWriter implements QueryResponseWriter, SolrjResponseWriter  {
-
-    private static final Set<String> DEFAULT_FIELD_LIST = new HashSet<>();
-    
-    static {
-        DEFAULT_FIELD_LIST.add(CollectionSchema.id.getSolrFieldName());
-        DEFAULT_FIELD_LIST.add(CollectionSchema.sku.getSolrFieldName());
-    }
-    
-    /** Default width for each snapshot image */
-    private static final int DEFAULT_WIDTH = 256;
-    
-    /** Default height for each snapshot image */
-    private static final int DEFAULT_HEIGTH = 256;
-    
-    public SnapshotImagesReponseWriter() {
-        super();
-    }
-    
-    @Override
-    public String getContentType(SolrQueryRequest arg0, SolrQueryResponse arg1) {
-        return "text/html";
-    }
-
-    @Override
-    public void init(@SuppressWarnings("rawtypes") NamedList n) {
-    }
-    
-	/**
-	 * Compute the root path as a relative URL prefix from the original servlet
-	 * request URI if provided in the Solr request context, otherwise return a
-	 * regular "/". Using relative URLs when possible makes deployment behind a
-	 * reverse proxy more reliable and convenient as no URL rewriting is needed.
-	 * 
-	 * @param request the Solr request.
-	 * @return the root context path to use as a prefix for resources URLs such as
-	 *         stylesheets
-	 */
-	private String getRootPath(final SolrQueryRequest request) {
-		String rootPath = "/";
-		if (request != null) {
-			final Map<Object, Object> context = request.getContext();
-			if (context != null) {
-				final Object requestUriObj = context.get("requestURI");
-				if (requestUriObj instanceof String) {
-					String servletRequestUri = (String) requestUriObj;
-					if (servletRequestUri.startsWith("/")) {
-						servletRequestUri = servletRequestUri.substring(1);
-
-						final String[] pathParts = CommonPattern.SLASH.split(servletRequestUri);
-						if (pathParts.length > 1) {
-							final StringBuilder sb = new StringBuilder();
-							for (int i = 1; i < pathParts.length; i++) {
-								sb.append("../");
-							}
-							rootPath = sb.toString();
-						}
-					}
-				}
-			}
-		}
-		return rootPath;
-	}
-    
-    /**
-     * Append the response HTML head to the writer.
-     * @param writer an open output writer. Must not be null.
-     * @param request the Solr request
-     * @throws IOException when a write error occurred
-     */
-	private void writeHtmlHead(final Writer writer, final SolrQueryRequest request) throws IOException {
-		final String rootPath = getRootPath(request);
-		
-		writer.write("<!DOCTYPE html>\n");
-		writer.write("<html lang=\"en\">");
-		writer.write("<head>\n");
-		writer.write("<meta charset=\"UTF-8\">");
-		writer.write("<title>Documents snapshots</title>\n");
-        writer.write("<link rel=\"stylesheet\" type=\"text/css\" media=\"all\" href=\"" + rootPath + "env/base.css\" />\n");
-        writer.write("<link rel=\"stylesheet\" type=\"text/css\" media=\"screen\" href=\"" + rootPath + "env/style.css\" />\n");
-		writer.write("</head>\n");
-	}
-
-    @Override
-    public void write(final Writer writer, final SolrQueryRequest request, final SolrQueryResponse rsp) throws IOException {
-            final NamedList<?> values = rsp.getValues();
-            assert values.get("responseHeader") != null;
-            assert values.get("response") != null;
-
-            writeHtmlHead(writer, request);
-            writer.write("<body id=\"SnapshotImagesReponseWriter\">\n");
-            final SolrParams originalParams = request.getOriginalParams();
-            
-            final int width = originalParams != null ? originalParams.getInt("width", DEFAULT_WIDTH) : DEFAULT_WIDTH;
-            final int height = originalParams != null ? originalParams.getInt("height", DEFAULT_HEIGTH) : DEFAULT_HEIGTH;
-            
-            final DocList response = ((ResultContext) values.get("response")).getDocList();
-            final int sz = response.size();
-            if (sz > 0) {
-                final SolrIndexSearcher searcher = request.getSearcher();
-                final DocIterator iterator = response.iterator();
-                while (iterator.hasNext()) {
-                    final int id = iterator.nextDoc();
-                    final Document doc = searcher.doc(id, DEFAULT_FIELD_LIST);
-                    final IndexableField docId = doc.getField(CollectionSchema.id.getSolrFieldName());
-                    final IndexableField docSku = doc.getField(CollectionSchema.sku.getSolrFieldName());
-                    if(docId != null && docSku != null) {
-                    	writeDoc(writer, width, height, docId.stringValue(), docSku.stringValue());
-                    }
-                }
-            }
-            
-            writer.write("</body></html>\n");
-        
-    }
-
-    /**
-     * Process a document and append its representation to the output writer.
-     * @param writer an open ouput writer. Must not be null.
-     * @param width the width of the snapshot image to render
-     * @param height the height of the snapshot image to render
-     * @param docId the document id (URL hash).
-	 * @param docUrl the document URL.
-     * @throws IOException when a write error occurred
-     */
-	private void writeDoc(final Writer writer, final int width, final int height, final String docId, final String docUrl)
-			throws IOException {
-		if(docId != null && docUrl != null) {
-			writer.write("<a href=\"");
-			writer.write(docUrl);
-			writer.write("\" class=\"forceNoExternalIcon\"><img width=\"");
-			writer.write(String.valueOf(width));
-			writer.write("\" height=\"");
-			writer.write(String.valueOf(height));
-			writer.write("\" src=\"/api/snapshot.jpg?urlhash=");
-			writer.write(docId);
-			writer.write("&amp;width=");
-			writer.write(String.valueOf(width));
-			writer.write("&amp;height=");
-			writer.write(String.valueOf(height));
-			writer.write("\" alt=\"");
-			writer.write(docUrl);
-			writer.write("\"></a>\n");
-		}
-	}
-    
-	@Override
-	public void write(final Writer writer, final SolrQueryRequest request, final String coreName,
-			final QueryResponse rsp) throws IOException {
-
-		writeHtmlHead(writer, request);
-		writer.write("<body id=\"SnapshotImagesReponseWriter\">\n");
-		final SolrParams originalParams = request.getOriginalParams();
-
-        final int width = originalParams != null ? originalParams.getInt("width", DEFAULT_WIDTH) : DEFAULT_WIDTH;
-        final int height = originalParams != null ? originalParams.getInt("height", DEFAULT_HEIGTH) : DEFAULT_HEIGTH;
-
-		final SolrDocumentList docList = rsp.getResults();
-		final int sz = docList.size();
-		if (sz > 0) {
-			final Iterator<SolrDocument> iterator = docList.iterator();
-			while (iterator.hasNext()) {
-				final SolrDocument doc = iterator.next();
-				final Object docId = doc.getFieldValue(CollectionSchema.id.getSolrFieldName());
-				final Object docSku = doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());
-				if (docId != null && docSku != null) {
-					writeDoc(writer, width, height, docId.toString(), docSku.toString());
-				}
-			}
-		}
-
-		writer.write("</body></html>\n");
-	}
-
-}
diff --git a/source/net/yacy/cora/util/Html2Image.java b/source/net/yacy/cora/util/Html2Image.java
deleted file mode 100644
index ccd44d31a..000000000
--- a/source/net/yacy/cora/util/Html2Image.java
+++ /dev/null
@@ -1,581 +0,0 @@
-/**
- *  Html2Image
- *  Copyright 2014 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
- *  First published 26.11.2014 on http://yacy.net
- *
- *  This library is free software; you can redistribute it and/or
- *  modify it under the terms of the GNU Lesser General Public
- *  License as published by the Free Software Foundation; either
- *  version 2.1 of the License, or (at your option) any later version.
- *
- *  This library is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  Lesser General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public License
- *  along with this program in the file lgpl21.txt
- *  If not, see <http://www.gnu.org/licenses/>.
- */
-
-package net.yacy.cora.util;
-
-import java.awt.Container;
-import java.awt.Dimension;
-import java.awt.Graphics;
-import java.awt.Image;
-import java.awt.MediaTracker;
-import java.awt.image.BufferedImage;
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import javax.imageio.ImageIO;
-import javax.swing.JEditorPane;
-import javax.swing.text.Document;
-import javax.swing.text.Element;
-import javax.swing.text.View;
-import javax.swing.text.ViewFactory;
-import javax.swing.text.html.HTMLDocument;
-import javax.swing.text.html.HTMLEditorKit;
-import javax.swing.text.html.ImageView;
-
-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.rendering.ImageType;
-import org.apache.pdfbox.rendering.PDFRenderer;
-
-import net.yacy.cora.document.id.MultiProtocolURL;
-import net.yacy.cora.protocol.ClientIdentification;
-import net.yacy.cora.protocol.Domains;
-import net.yacy.cora.protocol.http.HTTPClient;
-import net.yacy.document.ImageParser;
-import net.yacy.kelondro.util.FileUtils;
-import net.yacy.kelondro.util.OS;
-
-/**
- * Convert html to an copy on disk-image in a other file format
- * currently (pdf and/or jpg)
- */
-public class Html2Image {
-
-    // Mac
-    /**
-     * Path to wkhtmltopdf executable on Mac OS when installed using
-     * wkhtmltox-n.n.n.macos-cocoa.pkg from https://wkhtmltopdf.org/downloads.html.
-     * This can also be a path on Debian or another Gnu/Linux distribution.
-     */
-    private final static File wkhtmltopdfMac = new File("/usr/local/bin/wkhtmltopdf");
-
-    // to install imagemagick, download from http://cactuslab.com/imagemagick/assets/ImageMagick-6.8.9-9.pkg.zip
-    // the convert command from imagemagick needs ghostscript, if not present on older macs, download a version of gs from http://pages.uoregon.edu/koch/
-
-    private final static File convertMac1 = new File("/opt/local/bin/convert");
-    private final static File convertMac2 = new File("/opt/ImageMagick/bin/convert");
-
-    /* Debian packages to install: apt-get install wkhtmltopdf imagemagick xvfb ghostscript
-     The imagemagick policy at /etc should also be checked :
-     if it contains a line such as <policy domain="coder" rights="none" pattern="PDF" /> it must be edited with rights="read" at minimum
-     */
-    private final static File wkhtmltopdfDebian = new File("/usr/bin/wkhtmltopdf"); // there is no wkhtmltoimage, use convert to create images
-    private final static File convertDebian = new File("/usr/bin/convert");
-
-    /**
-     * Path to wkhtmltopdf executable on Windows, when installed with default
-     * settings using wkhtmltox-n.n.n.msvc2015-win64.exe from
-     * https://wkhtmltopdf.org/downloads.html
-     */
-    private static final File WKHTMLTOPDF_WINDOWS = new File("C:\\Program Files\\wkhtmltopdf\\bin\\wkhtmltopdf.exe");
-
-    /**
-     * Path to wkhtmltopdf executable on Windows, when installed with default
-     * settings using wkhtmltox-n.n.n.msvc2015-win32.exe from
-     * https://wkhtmltopdf.org/downloads.html
-     */
-    private static final File WKHTMLTOPDF_WINDOWS_X86 = new File(
-            "C:\\Program Files (x86)\\wkhtmltopdf\\bin\\wkhtmltopdf.exe");
-
-    /** Command to use when wkhtmltopdf is included in the system Path */
-    private static final String WKHTMLTOPDF_COMMAND = "wkhtmltopdf";
-
-    /** Command to use when imagemagick convert is included in the system Path */
-    private static final String CONVERT_COMMAND = "convert";
-
-    private static boolean usexvfb = false;
-
-    /**
-     * @return when the wkhtmltopdf command is detected as available in the system
-     */
-    public static boolean wkhtmltopdfAvailable() {
-        /* Check wkhtmltopdf common installation paths and system Path */
-        return wkhtmltopdfExecutable() != null || wkhtmltopdfAvailableInPath();
-    }
-
-    /**
-     * @return a wkhtmltopdf executable file when one can be found, null otherwise
-     */
-    private static File wkhtmltopdfExecutable() {
-        File executable = null;
-        if(OS.isWindows) {
-            if(WKHTMLTOPDF_WINDOWS.exists()) {
-                executable = WKHTMLTOPDF_WINDOWS;
-            } else if(WKHTMLTOPDF_WINDOWS_X86.exists()) {
-                executable = WKHTMLTOPDF_WINDOWS_X86;
-            }
-        } else {
-            if(wkhtmltopdfMac.exists()) {
-                executable = wkhtmltopdfMac;
-            } else if(wkhtmltopdfDebian.exists()) {
-                executable = wkhtmltopdfDebian;
-            }
-        }
-        return executable;
-    }
-
-    /**
-     * @return true when wkhtmltopdf is available in system path
-     */
-    private static boolean wkhtmltopdfAvailableInPath() {
-        boolean available = false;
-        try {
-            final Process p = Runtime.getRuntime().exec(WKHTMLTOPDF_COMMAND + " -V");
-            available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0;
-        } catch (final IOException e) {
-            ConcurrentLog.fine("Html2Image", "wkhtmltopdf is not included in system path.");
-        } catch (final InterruptedException e) {
-            Thread.currentThread().interrupt(); // preserve thread interrupted state
-        }
-        return available;
-    }
-
-    /**
-     * @return a imagemagick convert executable file when one can be found, null otherwise
-     */
-    private static File convertExecutable() {
-        File executable = null;
-        if(!OS.isWindows) {
-            if(convertMac1.exists()) {
-                executable = convertMac1;
-            } else if(convertMac2.exists()) {
-                executable = convertMac2;
-            } else if(convertDebian.exists()) {
-                executable = convertDebian;
-            }
-        }
-        return executable;
-    }
-
-    /**
-     * @return when the imagemagick convert command is detected as available in the system
-     */
-    public static boolean convertAvailable() {
-        /* Check convert common installation paths and system Path */
-        return convertExecutable() != null || convertAvailableInPath();
-    }
-
-    /**
-     * @return when imagemagick convert is available in system path
-     */
-    private static boolean convertAvailableInPath() {
-        boolean available = false;
-        if(!OS.isWindows) { // on MS Windows convert is a system tool to convert volumes from FAT to NTFS
-            try {
-                final Process p = Runtime.getRuntime().exec(CONVERT_COMMAND + " -version");
-                available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0;
-            } catch (final IOException e) {
-                ConcurrentLog.fine("Html2Image", "convert is not included in system path.");
-            } catch (final InterruptedException e) {
-                Thread.currentThread().interrupt(); // preserve thread interrupted state
-            }
-        }
-        return available;
-    }
-
-    /**
-     * Run the wkhtmltopdf external tool to fetch and render to PDF a web resource.
-     * wKhtmltopdf may be called multiple times with various parameters flavors in
-     * case of failure.
-     *
-     * @param url         the URL of a web resource to fetch, render and convert to
-     *                    a pdf file. Must not be null.
-     * @param proxy       the eventual proxy address to use. Can be null. Must be of
-     *                    the form http://host:port; use YaCy here as proxy which is
-     *                    mostly http://localhost:8090
-     * @param destination the destination PDF file that should be written. Must not
-     *                    be null.
-     * @param maxSeconds  the maximum time in seconds to wait for each wkhtmltopdf
-     *                    call termination. Beyond this limit the process is killed.
-     * @return true when the destination file was successfully written
-     */
-    public static boolean writeWkhtmltopdf(final String url, final String proxy, final String userAgent, final String acceptLanguage, final File destination, final long maxSeconds) {
-        boolean success = false;
-        for (final boolean ignoreErrors: new boolean[]{false, true}) {
-            success = writeWkhtmltopdfInternal(url, proxy, destination, userAgent, acceptLanguage, ignoreErrors, maxSeconds);
-            if (success) break;
-            if (!success && proxy != null) {
-                ConcurrentLog.warn("Html2Image", "trying to load without proxy: " + url);
-                success = writeWkhtmltopdfInternal(url, null, destination, userAgent, acceptLanguage, ignoreErrors, maxSeconds);
-                if (success) break;
-            }
-        }
-        if (success) {
-            ConcurrentLog.info("Html2Image", "wrote " + destination.toString() + " for " + url);
-        } else {
-            ConcurrentLog.warn("Html2Image", "could not generate snapshot for " + url);
-        }
-        return success;
-    }
-
-    /**
-     * Run wkhtmltopdf in a separate process to fetch and render to PDF a web
-     * resource.
-     *
-     * @param url          the URL of a web resource to fetch, render and convert to
-     *                     a pdf file. Must not be null.
-     * @param proxy        the eventual proxy address to use. Can be null.
-     * @param destination  the destination PDF file that should be written. Must not
-     *                     be null.
-     * @param userAgent    TODO: implement
-     * @param acceptLanguage TODO: implement
-     * @param ignoreErrors when true wkhtmltopdf is instructed to ignore load errors
-     * @param maxSeconds   the maximum time in seconds to wait for the wkhtmltopdf
-     *                     dedicated process termination. Beyond this limit the
-     *                     process is killed.
-     * @return true when the destination file was successfully written
-     */
-    private static boolean writeWkhtmltopdfInternal(final String url, final String proxy, final File destination,
-            final String userAgent, final String acceptLanguage, final boolean ignoreErrors, final long maxSeconds) {
-        final String wkhtmltopdfCmd;
-        final File wkhtmltopdf = wkhtmltopdfExecutable();
-        if(wkhtmltopdf != null) {
-            wkhtmltopdfCmd = wkhtmltopdf.getAbsolutePath();
-        } else if(wkhtmltopdfAvailableInPath()) {
-            wkhtmltopdfCmd = WKHTMLTOPDF_COMMAND;
-        } else {
-            ConcurrentLog.warn("Html2Pdf", "Unable to locate wkhtmltopdf executable on this system!");
-            return false;
-        }
-        String commandline =
-                wkhtmltopdfCmd + " -q --title '" + url + "' " +
-                        //acceptLanguage == null ? "" : "--custom-header 'Accept-Language' '" + acceptLanguage + "' " +
-                        //(userAgent == null ? "" : "--custom-header \"User-Agent\" \"" + userAgent + "\" --custom-header-propagation ") +
-                        (proxy == null ? "" : "--proxy " + proxy + " ") +
-                        (ignoreErrors ? (OS.isMacArchitecture ? "--load-error-handling ignore " : "--ignore-load-errors ") : "") + // some versions do not have that flag and fail if attempting to use it...
-                        //"--footer-font-name 'Courier' --footer-font-size 9 --footer-left [webpage] --footer-right [date]/[time]([page]/[topage]) " +
-                        "--footer-left [webpage] --footer-right '[date]/[time]([page]/[topage])' --footer-font-size 7 " +
-                        url + " " + destination.getAbsolutePath();
-        try {
-            ConcurrentLog.info("Html2Pdf", "creating pdf from url " + url + " with command: " + commandline);
-            if (!usexvfb && execWkhtmlToPdf(proxy, destination, commandline, maxSeconds)) {
-                return true;
-            }
-            // if this fails, we should try to wrap the X server with a virtual screen using xvfb, this works on headless servers
-            commandline = "xvfb-run -a " + commandline;
-            return execWkhtmlToPdf(proxy, destination, commandline, maxSeconds);
-        } catch (final IOException e) {
-            ConcurrentLog.warn("Html2Pdf", "exception while creation of pdf with command: " + commandline, e);
-            return false;
-        }
-    }
-
-    /**
-     * Run a wkhtmltopdf commandline in a separate process.
-     *
-     * @param proxy       the eventual proxy address to use. Can be null.
-     * @param destination the destination PDF file that should be written. Must not
-     *                    be null.
-     * @param commandline the wkhtmltopdf command line to execute. Must not be null.
-     * @param maxSeconds  the maximum time in seconds to wait for the process
-     *                    termination. Beyond this limit the process is killed.
-     * @return true when the destination file was successfully written
-     * @throws IOException when an unexpected error occurred
-     */
-    private static boolean execWkhtmlToPdf(final String proxy, final File destination, final String commandline, final long maxSeconds) throws IOException {
-        final Process p = Runtime.getRuntime().exec(commandline);
-
-        try {
-            p.waitFor(maxSeconds, TimeUnit.SECONDS);
-        } catch (final InterruptedException e) {
-            p.destroyForcibly();
-            ConcurrentLog.warn("Html2Pdf", "Interrupted creation of pdf. Killing the process started with command : " + commandline);
-            Thread.currentThread().interrupt(); // Keep the thread interrupted state
-            return false;
-        }
-        if(p.isAlive()) {
-            ConcurrentLog.warn("Html2Pdf", "Creation of pdf did not terminate within " + maxSeconds + " seconds. Killing the process started with command : " + commandline);
-            p.destroyForcibly();
-            return false;
-        }
-        if (p.exitValue() == 0 && destination.exists()) {
-            return true;
-        }
-        final List<String> messages = OS.readStreams(p);
-        ConcurrentLog.warn("Html2Image", "failed to create pdf " + (proxy == null ? "" : "using proxy " + proxy) + " with command : " + commandline);
-        for (final String message : messages) {
-            ConcurrentLog.warn("Html2Image", ">> " + message);
-        }
-        return false;
-    }
-
-    /**
-     * Convert a pdf (first page) to an image. Proper values are i.e. width = 1024, height = 1024, density = 300, quality = 75
-     * using internal pdf library or external command line tool on linux or mac
-     * @param pdf input pdf file. Must not be null.
-     * @param image output image file. Must not be null, and should end with ".jpg" or ".png".
-     * @param width output width in pixels
-     * @param height output height in pixels
-     * @param density (dpi)
-     * @param quality JPEG/PNG compression level
-     * @return true when the ouput image file was successfully written.
-     */
-    public static boolean pdf2image(final File pdf, final File image, final int width, final int height, final int density, final int quality) {
-        /* Deduce the ouput image format from the file extension */
-        String imageFormat = MultiProtocolURL.getFileExtension(image.getName());
-        if(imageFormat.isEmpty()) {
-            /* Use JPEG as a default fallback */
-            imageFormat = "jpg";
-        }
-        String convertCmd = null;
-        final File convert = convertExecutable();
-        if(convert != null) {
-            convertCmd = convert.getAbsolutePath();
-        } else if(convertAvailableInPath()) {
-            convertCmd = CONVERT_COMMAND;
-        } else {
-            ConcurrentLog.info("Html2Image", "Unable to locate convert executable on this system!");
-        }
-
-        // convert pdf to jpg using internal pdfbox capability
-        if (convertCmd == null) {
-            try (final PDDocument pdoc = Loader.loadPDF(pdf);) {
-
-                final BufferedImage bi = new PDFRenderer(pdoc).renderImageWithDPI(0, density, ImageType.RGB);
-
-                return ImageIO.write(bi, imageFormat, image);
-
-            } catch (final IOException ex) {
-                ConcurrentLog.warn("Html2Image", "Failed to create image with pdfbox"
-                        + (ex.getMessage() != null ? " : " + ex.getMessage() : ""));
-                return false;
-            }
-        }
-
-        // convert using external command line utility
-        try {
-            // i.e. convert -density 300 -trim yacy.pdf[0] -trim -resize 1024x -crop x1024+0+0 -quality 75% yacy-convert-300.jpg
-            // note: both -trim are necessary, otherwise it is trimmed only on one side. The [0] selects the first page of the pdf
-            final String command = convertCmd + " -alpha remove -density " + density + " -trim " + pdf.getAbsolutePath() + "[0] -trim -resize " + width + "x -crop x" + height + "+0+0 -quality " + quality + "% " + image.getAbsolutePath();
-            List<String> message = OS.execSynchronous(new String[] {
-                    convertCmd,
-                    "-alpha", "remove", "-density", Integer.toString(density),
-                    "-trim", pdf.getAbsolutePath() + "[0]", "-trim",
-                    "-resize", Integer.toString(width) + "x",
-                    "-crop", "x" + Integer.toString(height) + "+0+0",
-                    "-quality", Integer.toString(quality) + "%",
-                    image.getAbsolutePath()
-            });
-            if (image.exists()) return true;
-            ConcurrentLog.warn("Html2Image", "failed to create image with command: " + command);
-            for (final String m: message) ConcurrentLog.warn("Html2Image", ">> " + m);
-
-            // another try for mac: use Image Events using AppleScript in osacript commands...
-            // the following command overwrites a pdf with an png, so we must make a copy first
-            if (!OS.isMacArchitecture) return false;
-            final File pngFile = new File(pdf.getAbsolutePath() + ".tmp.pdf");
-            org.apache.commons.io.FileUtils.copyFile(pdf, pngFile);
-            final String[] commandx = {"osascript",
-                    "-e", "set ImgFile to \"" + pngFile.getAbsolutePath() + "\"",
-                    "-e", "tell application \"Image Events\"",
-                    "-e", "set Img to open file ImgFile",
-                    "-e", "save Img as PNG",
-                    "-e", "end tell"};
-            //ConcurrentLog.warn("Html2Image", "failed to create image with command: " + commandx);
-            message = OS.execSynchronous(commandx);
-            for (final String m: message) ConcurrentLog.warn("Html2Image", ">> " + m);
-            // now we must read and convert this file to the target format with the target size 1024x1024
-            try {
-                final File newPngFile = new File(pngFile.getAbsolutePath() + ".png");
-                pngFile.renameTo(newPngFile);
-                final Image img = ImageParser.parse(pngFile.getAbsolutePath(), FileUtils.read(newPngFile));
-                if(img == null) {
-                    /* Should not happen. If so, ImageParser.parse() should already have logged about the error */
-                    return false;
-                }
-                final Image scaled = img.getScaledInstance(width, height, Image.SCALE_AREA_AVERAGING);
-                final MediaTracker mediaTracker = new MediaTracker(new Container());
-                mediaTracker.addImage(scaled, 0);
-                try {mediaTracker.waitForID(0);} catch (final InterruptedException e) {}
-                // finally write the image
-                final BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
-                bi.createGraphics().drawImage(scaled, 0, 0, width, height, null);
-                ImageIO.write(bi, imageFormat, image);
-                newPngFile.delete();
-                return image.exists();
-            } catch (final IOException e) {
-                ConcurrentLog.logException(e);
-                return false;
-            }
-        } catch (final IOException e) {
-            e.printStackTrace();
-            return false;
-        }
-    }
-
-    /**
-     * render a html page with a JEditorPane, which can do html up to html v 3.2. No CSS supported!
-     * @param url
-     * @param size
-     * @throws IOException
-     */
-    public static void writeSwingImage(final String url, final Dimension size, final File destination) throws IOException {
-
-        // set up a pane for rendering
-        final JEditorPane htmlPane = new JEditorPane();
-        htmlPane.setSize(size);
-        htmlPane.setEditable(false);
-        final HTMLEditorKit kit = new HTMLEditorKit() {
-
-            private static final long serialVersionUID = 1L;
-
-            @Override
-            public Document createDefaultDocument() {
-                final HTMLDocument doc = (HTMLDocument) super.createDefaultDocument();
-                doc.setAsynchronousLoadPriority(-1);
-                return doc;
-            }
-
-            @Override
-            public ViewFactory getViewFactory() {
-                return new HTMLFactory() {
-                    @Override
-                    public View create(final Element elem) {
-                        final View view = super.create(elem);
-                        if (view instanceof ImageView) {
-                            ((ImageView) view).setLoadsSynchronously(true);
-                        }
-                        return view;
-                    }
-                };
-            }
-        };
-        htmlPane.setEditorKitForContentType("text/html", kit);
-        htmlPane.setContentType("text/html");
-        htmlPane.addPropertyChangeListener(evt -> {
-        });
-
-        // load the page
-        try {
-            htmlPane.setPage(url);
-        } catch (final IOException e) {
-            e.printStackTrace();
-        }
-
-        // render the page
-        final Dimension prefSize = htmlPane.getPreferredSize();
-        final BufferedImage img = new BufferedImage(prefSize.width, htmlPane.getPreferredSize().height, BufferedImage.TYPE_INT_ARGB);
-        final Graphics graphics = img.getGraphics();
-        htmlPane.setSize(prefSize);
-        htmlPane.paint(graphics);
-        ImageIO.write(img, destination.getName().endsWith("jpg") ? "jpg" : "png", destination);
-    }
-
-    /**
-     * Test PDF or image snapshot generation for a given URL.
-     * @param args main arguments list:
-     * <ol>
-     * 	<li>Source remote URL (required)</li>
-     * 	<li>Target local file path (required)</li>
-     * 	<li>Snapshot generation method identifier (optional) :
-     * 		<ul>
-     * 			<li>"wkhtmltopdf" (default): generate a PDF snapshot using external wkhtmltopdf tool.</li>
-     * 			<li>"swing" : use JRE provided Swing to generate a jpg or png image snapshot.</li>
-     * 		</ul>
-     * 	</li>
-     * </ol>
-     */
-    public static void main(final String[] args) {
-        final String usageMessage = "Usage : java " + Html2Image.class.getName()
-                + " <url> <target-file[.pdf|.jpg|.png]> [wkhtmltopdf|swing]";
-        int exitStatus = 0;
-        try {
-            if (args.length < 2) {
-                System.out.println("Missing required parameter(s).");
-                System.out.println(usageMessage);
-                exitStatus = 1;
-                return;
-            }
-            final String targetPath = args[1];
-            if (args.length < 3 || "wkhtmltopdf".equals(args[2])) {
-                if(Html2Image.wkhtmltopdfAvailable()) {
-                    final File targetPdfFile;
-                    if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {
-                        targetPdfFile = new File(targetPath.substring(0, targetPath.length() - 4) + ".pdf");
-                    } else if(targetPath.endsWith(".pdf")) {
-                        targetPdfFile = new File(targetPath);
-                    } else {
-                        System.out.println("Unsupported output format");
-                        System.out.println(usageMessage);
-                        exitStatus = 1;
-                        return;
-                    }
-                    if(Html2Image.writeWkhtmltopdf(args[0], null, ClientIdentification.yacyInternetCrawlerAgent.userAgent(),
-                            "en-us,en;q=0.5", targetPdfFile, 30)) {
-                        if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {
-                            if(Html2Image.pdf2image(targetPdfFile, new File(targetPath), 1024, 1024, 300, 75)) {
-                                ConcurrentLog.info("Html2Image", "wrote " + targetPath + " converted from " + targetPdfFile);
-                            } else {
-                                exitStatus = 1;
-                                return;
-                            }
-                        }
-                    } else {
-                        exitStatus = 1;
-                        return;
-                    }
-                } else {
-                    System.out.println("Unable to locate wkhtmltopdf executable on this system!");
-                    exitStatus = 1;
-                    return;
-                }
-            } else if ("swing".equals(args[2])) {
-                if(targetPath.endsWith(".pdf")) {
-                    System.out.println("Pdf output format is not supported with swing method.");
-                    exitStatus = 1;
-                    return;
-                }
-                if(!targetPath.endsWith(".jpg") && !targetPath.endsWith(".png")) {
-                    System.out.println("Unsupported output format");
-                    System.out.println(usageMessage);
-                    exitStatus = 1;
-                    return;
-                }
-
-                try {
-                    Html2Image.writeSwingImage(args[0], new Dimension(1200, 2000), new File(targetPath));
-                } catch (final IOException e) {
-                    e.printStackTrace();
-                    exitStatus = 1;
-                    return;
-                }
-            } else {
-                System.out.println("Unknown method : please specify either wkhtmltopdf or swing.");
-                exitStatus = 1;
-                return;
-            }
-        } finally {
-            /* Shutdown running threads */
-            Domains.close();
-            try {
-                HTTPClient.closeConnectionManager();
-            } catch (final InterruptedException e) {
-                Thread.currentThread().interrupt(); // restore interrupted state
-            }
-            ConcurrentLog.shutdown();
-            if(exitStatus != 0) {
-                System.exit(exitStatus);
-            }
-        }
-    }
-
-}
diff --git a/source/net/yacy/crawler/CrawlSwitchboard.java b/source/net/yacy/crawler/CrawlSwitchboard.java
index 4f63608ad..8ad8373c2 100644
--- a/source/net/yacy/crawler/CrawlSwitchboard.java
+++ b/source/net/yacy/crawler/CrawlSwitchboard.java
@@ -304,8 +304,6 @@ public final class CrawlSwitchboard {
                 sb.getConfigBool(SwitchboardConstants.AUTOCRAWL_INDEX_MEDIA, true),
                 false,
                 false,
-                -1,
-                false, true, CrawlProfile.MATCH_NEVER_STRING,
                 CacheStrategy.NOCACHE,
                 "robot_" + CRAWL_PROFILE_AUTOCRAWL_DEEP,
                 ClientIdentification.yacyInternetCrawlerAgentName,
@@ -339,8 +337,6 @@ public final class CrawlSwitchboard {
                 sb.getConfigBool(SwitchboardConstants.AUTOCRAWL_INDEX_MEDIA, true),
                 false,
                 false,
-                -1,
-                false, true, CrawlProfile.MATCH_NEVER_STRING,
                 CacheStrategy.NOCACHE,
                 "robot_" + CRAWL_PROFILE_AUTOCRAWL_SHALLOW,
                 ClientIdentification.yacyInternetCrawlerAgentName,
@@ -374,7 +370,6 @@ public final class CrawlSwitchboard {
                 sb.getConfigBool(SwitchboardConstants.PROXY_INDEXING_LOCAL_MEDIA, true),
                 true,
                 sb.getConfigBool(SwitchboardConstants.PROXY_INDEXING_REMOTE, false),
-                -1, false, true, CrawlProfile.MATCH_NEVER_STRING,
                 CacheStrategy.IFFRESH,
                 "robot_" + CRAWL_PROFILE_PROXY,
                 ClientIdentification.yacyProxyAgentName,
@@ -408,7 +403,6 @@ public final class CrawlSwitchboard {
                 true,
                 false,
                 false,
-                -1, false, true, CrawlProfile.MATCH_NEVER_STRING,
                 CacheStrategy.IFFRESH,
                 "robot_" + CRAWL_PROFILE_REMOTE,
                 ClientIdentification.yacyInternetCrawlerAgentName,
@@ -442,7 +436,6 @@ public final class CrawlSwitchboard {
                 false,
                 true,
                 false,
-                -1, false, true, CrawlProfile.MATCH_NEVER_STRING,
                 CacheStrategy.IFEXIST,
                 "robot_" + CRAWL_PROFILE_SNIPPET_LOCAL_TEXT,
                 ClientIdentification.yacyIntranetCrawlerAgentName,
@@ -476,7 +469,6 @@ public final class CrawlSwitchboard {
                 true,
                 true,
                 false,
-                -1, false, true, CrawlProfile.MATCH_NEVER_STRING,
                 CacheStrategy.IFEXIST,
                 "robot_" + CRAWL_PROFILE_SNIPPET_GLOBAL_TEXT,
                 ClientIdentification.yacyIntranetCrawlerAgentName,
@@ -518,7 +510,6 @@ public final class CrawlSwitchboard {
                 false,
                 true,
                 false,
-                -1, false, true, CrawlProfile.MATCH_NEVER_STRING,
                 CacheStrategy.IFEXIST,
                 "robot_" + CRAWL_PROFILE_GREEDY_LEARNING_TEXT,
                 ClientIdentification.browserAgentName,
@@ -552,7 +543,6 @@ public final class CrawlSwitchboard {
                 false, // indexMedia
                 true,
                 false,
-                -1, false, true, CrawlProfile.MATCH_NEVER_STRING,
                 CacheStrategy.IFEXIST,
                 "robot_" + CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA,
                 ClientIdentification.yacyIntranetCrawlerAgentName,
@@ -586,7 +576,6 @@ public final class CrawlSwitchboard {
                 true,
                 true,
                 false,
-                -1, false, true, CrawlProfile.MATCH_NEVER_STRING,
                 CacheStrategy.IFEXIST,
                 "robot_" + CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA,
                 ClientIdentification.yacyIntranetCrawlerAgentName,
@@ -620,7 +609,6 @@ public final class CrawlSwitchboard {
                 false,
                 false,
                 false,
-                -1, false, true, CrawlProfile.MATCH_NEVER_STRING,
                 CacheStrategy.NOCACHE,
                 "robot_" + CRAWL_PROFILE_PACKS,
                 ClientIdentification.yacyIntranetCrawlerAgentName,
@@ -657,7 +645,6 @@ public final class CrawlSwitchboard {
                 true,
                 false,
                 false,
-                -1, false, true, CrawlProfile.MATCH_NEVER_STRING,
                 CacheStrategy.NOCACHE,
                 collection,
                 ClientIdentification.yacyIntranetCrawlerAgentName,
diff --git a/source/net/yacy/crawler/RecrawlBusyThread.java b/source/net/yacy/crawler/RecrawlBusyThread.java
index 10867d2c0..b74586fc4 100644
--- a/source/net/yacy/crawler/RecrawlBusyThread.java
+++ b/source/net/yacy/crawler/RecrawlBusyThread.java
@@ -1,429 +1,429 @@
-/**
- * RecrawlBusyThread.java
- * SPDX-FileCopyrightText: 2015 by Burkhard Buelte
- * SPDX-License-Identifier: GPL-2.0-or-later
- * First released 15.05.2015 at https://yacy.net
- *
- * This is a part of YaCy, a peer-to-peer based web search engine
- *
- * LICENSE
- *
- * This library is free software; you can redistribute it and/or modify it under
- * the terms of the GNU Lesser General Public License as published by the Free
- * Software Foundation; either version 2.1 of the License, or (at your option)
- * any later version.
- *
- * This library is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
- * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
- * details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program in the file lgpl21.txt If not, see
- * <http://www.gnu.org/licenses/>.
- */
-package net.yacy.crawler;
-
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.time.LocalDateTime;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrDocumentList;
-
-import net.yacy.cora.document.encoding.ASCII;
-import net.yacy.cora.document.id.DigestURL;
-import net.yacy.cora.federate.solr.connector.SolrConnector;
-import net.yacy.cora.federate.yacy.CacheStrategy;
-import net.yacy.cora.protocol.ClientIdentification;
-import net.yacy.cora.util.ConcurrentLog;
-import net.yacy.crawler.data.CrawlProfile;
-import net.yacy.crawler.data.NoticedURL;
-import net.yacy.crawler.retrieval.Request;
-import net.yacy.document.parser.html.TagValency;
-import net.yacy.kelondro.workflow.AbstractBusyThread;
-import net.yacy.search.Switchboard;
-import net.yacy.search.schema.CollectionSchema;
-
-/**
- * Selects documents by a query from the local index
- * and feeds the found urls to the crawler to recrawl the documents.
- * This is intended to keep the index up-to-date
- * Currently the doucments are selected by expired fresh_date_dt field
- * an added to the crawler in smaller chunks (see chunksize) as long as no other crawl is running.
- */
-public class RecrawlBusyThread extends AbstractBusyThread {
-
-    /** The thread name */
-    public final static String THREAD_NAME = "recrawlindex";
-
-    /** The default selection query */
-    public static final String DEFAULT_QUERY = CollectionSchema.fresh_date_dt.getSolrFieldName()+":[* TO NOW/DAY-1DAY]";
-
-    /** Default value for inclusion or not of documents with a https status different from 200 (success) */
-    public static final boolean DEFAULT_INCLUDE_FAILED = false;
-
-    /** The default value whether to delete on Recrawl */
-    public static final boolean DEFAULT_DELETE_ON_RECRAWL = false;
-
-    /** The current query selecting documents to recrawl */
-    private String currentQuery;
-
-    /** flag if docs with httpstatus_i <> 200 shall be recrawled */
-    private boolean includefailed;
-
-    /** flag whether to delete on Recrawl */
-    private boolean deleteOnRecrawl;
-
-    private int chunkstart = 0;
-    private final int chunksize = 100;
-    private final Switchboard sb;
-
-    /** buffer of urls to recrawl */
-    private final Set<DigestURL> urlstack;
-
-    /** The total number of candidate URLs found for recrawl */
-    private long urlsToRecrawl = 0;
-
-    /** Total number of URLs added to the crawler queue for recrawl */
-    private long recrawledUrlsCount = 0;
-
-    /** Total number of URLs rejected for some reason by the crawl stacker or the crawler queue */
-    private long rejectedUrlsCount = 0;
-
-    /** Total number of malformed URLs found */
-    private long malformedUrlsCount = 0;
-
-    /** Total number of malformed URLs deleted from index */
-    private long malformedUrlsDeletedCount = 0;
-
-    private final String solrSortBy;
-
-    /** Set to true when more URLs are still to be processed */
-    private boolean moreToRecrawl = true;
-
-    /** True when the job terminated early because an error occurred when requesting the Solr index, or the Solr index was closed */
-    private boolean terminatedBySolrFailure = false;
-
-    /** The recrawl job start time */
-    private LocalDateTime startTime;
-
-    /** The recrawl job end time */
-    private LocalDateTime endTime;
-
-    /**
-     * @param xsb
-     *            the Switchboard instance holding server environment
-     * @param query
-     *            the Solr selection query
-     * @param includeFailed
-     *            set to true when documents with a https status different from 200
-     *            (success) must be included
-     */
-    public RecrawlBusyThread(final Switchboard xsb, final String query, final boolean includeFailed, final boolean deleteOnRecrawl) {
-        super(3000, 1000); // set lower limits of cycle delay
-        this.setName(THREAD_NAME);
-        this.setIdleSleep(10*60000); // set actual cycle delays
-        this.setBusySleep(2*60000);
-        this.setPriority(Thread.MIN_PRIORITY);
-        this.setLoadPreReqisite(1);
-        this.sb = xsb;
-        this.currentQuery = query;
-        this.includefailed = includeFailed;
-        this.deleteOnRecrawl = deleteOnRecrawl;
-        this.urlstack = new HashSet<>();
-        // workaround to prevent solr exception on existing index (not fully reindexed) since intro of schema with docvalues
-        // org.apache.solr.core.SolrCore java.lang.IllegalStateException: unexpected docvalues type NONE for field 'load_date_dt' (expected=NUMERIC). Use UninvertingReader or index with docvalues.
-        this.solrSortBy = CollectionSchema.load_date_dt.getSolrFieldName() + " asc";
-
-        final SolrConnector solrConnector = this.sb.index.fulltext().getDefaultConnector();
-        if (solrConnector != null && !solrConnector.isClosed()) {
-            /* Ensure indexed data is up-to-date before running the main job */
-            solrConnector.commit(true);
-        }
-    }
-
-    /**
-     * Set the query to select documents to recrawl
-     * and resets the counter to start a fresh query loop
-     * @param q select query
-     * @param includefailedurls true=all http status docs are recrawled, false=httpstatus=200 docs are recrawled
-     * @param deleteOnRecrawl
-     */
-    public void setQuery(String q, boolean includefailedurls, final boolean deleteOnRecrawl) {
-        this.currentQuery = q;
-        this.includefailed = includefailedurls;
-        this.deleteOnRecrawl = deleteOnRecrawl;
-        this.chunkstart = 0;
-    }
-
-    public String getQuery() {
-        return this.currentQuery;
-    }
-
-    /**
-     *
-     * @param queryBase
-     *            the base query
-     * @param includeFailed
-     *            set to true when documents with a https status different from 200
-     *            (success) must be included
-     * @return the Solr selection query for candidate URLs to recrawl
-     */
-    public static final String buildSelectionQuery(final String queryBase, final boolean includeFailed) {
-        return includeFailed ? queryBase : queryBase + " AND (" + CollectionSchema.httpstatus_i.name() + ":200)";
-    }
-
-    /**
-     * Flag to include failed urls (httpstatus_i <> 200)
-     * if true -> currentQuery is used as is,
-     * if false -> the term " AND (httpstatus_i:200)" is appended to currentQuery
-     * @param includefailedurls
-     */
-    public void setIncludeFailed(boolean includefailedurls) {
-        this.includefailed = includefailedurls;
-    }
-
-    public boolean getIncludeFailed () {
-        return this.includefailed;
-    }
-
-    public void setDeleteOnRecrawl(final boolean deleteOnRecrawl) {
-        this.deleteOnRecrawl = deleteOnRecrawl;
-    }
-
-    public boolean getDeleteOnRecrawl() {
-        return this.deleteOnRecrawl;
-    }
-
-    /**
-     * feed urls to the local crawler
-     * (Switchboard.addToCrawler() is not used here, as there existing urls are always skipped)
-     *
-     * @return true if urls were added/accepted to the crawler
-     */
-    private boolean feedToCrawler() {
-
-        int added = 0;
-
-        if (!this.urlstack.isEmpty()) {
-            final CrawlProfile profile = this.sb.crawler.defaultRecrawlJobProfile;
-
-            for (final DigestURL url : this.urlstack) {
-                final Request request = new Request(ASCII.getBytes(this.sb.peers.mySeed().hash), url, null, "",
-                        new Date(), profile.handle(), 0, profile.timezoneOffset());
-                String acceptedError = this.sb.crawlStacker.checkAcceptanceChangeable(url, profile, 0);
-                if (!this.includefailed && acceptedError == null) { // skip check if failed docs to be included
-                    acceptedError = this.sb.crawlStacker.checkAcceptanceInitially(url, profile);
-                }
-                if (acceptedError != null) {
-                    this.rejectedUrlsCount++;
-                    ConcurrentLog.info(THREAD_NAME, "addToCrawler: cannot load " + url.toNormalform(true) + ": " + acceptedError);
-                    continue;
-                }
-                final String s;
-                s = this.sb.crawlQueues.noticeURL.push(NoticedURL.StackType.LOCAL, request, profile, this.sb.robots);
-
-                if (s != null) {
-                    this.rejectedUrlsCount++;
-                    ConcurrentLog.info(THREAD_NAME, "addToCrawler: failed to add " + url.toNormalform(true) + ": " + s);
-                } else {
-                    added++;
-                    this.recrawledUrlsCount++;
-                }
-            }
-            this.urlstack.clear();
-        }
-        return (added > 0);
-    }
-
-    /**
-     * Process query and hand over urls to the crawler
-     *
-     * @return true if something processed
-     */
-    @Override
-    public boolean job() {
-        // more than chunksize crawls are running, do nothing
-        if (this.sb.crawlQueues.coreCrawlJobSize() > this.chunksize) {
-            return false;
-        }
-
-        boolean didSomething = false;
-        if (this.urlstack.isEmpty()) {
-            if(!this.moreToRecrawl) {
-                /* We do not remove the thread from the Switchboard worker threads using serverSwitch.terminateThread(String,boolean),
-                 * because we want to be able to provide a report after its termination */
-                this.terminate(false);
-            } else {
-                this.moreToRecrawl = this.processSingleQuery();
-                /* Even if no more URLs are to recrawl, the job has done something by searching the Solr index */
-                didSomething = true;
-            }
-        } else {
-            didSomething = this.feedToCrawler();
-        }
-        return didSomething;
-    }
-
-    @Override
-    public synchronized void start() {
-        this.startTime = LocalDateTime.now();
-        super.start();
-    }
-
-    @Override
-    public void terminate(boolean waitFor) {
-        super.terminate(waitFor);
-        this.endTime = LocalDateTime.now();
-    }
-
-    /**
-     * Selects documents to recrawl the urls
-     * @return true if query has more results
-     */
-    private boolean processSingleQuery() {
-        if (!this.urlstack.isEmpty()) {
-            return true;
-        }
-        SolrDocumentList docList = null;
-        final SolrConnector solrConnector = this.sb.index.fulltext().getDefaultConnector();
-        if (solrConnector == null || solrConnector.isClosed()) {
-            this.urlsToRecrawl = 0;
-            this.terminatedBySolrFailure = true;
-            return false;
-        }
-
-        try {
-            // query all or only httpstatus=200 depending on includefailed flag
-            docList = solrConnector.getDocumentListByQuery(RecrawlBusyThread.buildSelectionQuery(this.currentQuery, this.includefailed),
-                this.solrSortBy, this.chunkstart, this.chunksize, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName());
-            this.urlsToRecrawl = docList.getNumFound();
-        } catch (final Throwable e) {
-            this.urlsToRecrawl = 0;
-            this.terminatedBySolrFailure = true;
-        }
-
-        if (docList != null) {
-            final Set<String> tobedeletedIDs = new HashSet<>();
-            for (final SolrDocument doc : docList) {
-                try {
-                    this.urlstack.add(new DigestURL((String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName())));
-                    if (this.deleteOnRecrawl) tobedeletedIDs.add((String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName()));
-                } catch (final MalformedURLException ex) {
-                    this.malformedUrlsCount++;
-                    // if index entry hasn't a valid url (useless), delete it
-                    tobedeletedIDs.add((String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName()));
-                    this.malformedUrlsDeletedCount++;
-                    ConcurrentLog.severe(THREAD_NAME, "deleted index document with invalid url " + (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName()));
-                }
-            }
-
-            if (!tobedeletedIDs.isEmpty()) try {
-                solrConnector.deleteByIds(tobedeletedIDs);
-                solrConnector.commit(false);
-            } catch (final IOException e) {
-                ConcurrentLog.severe(THREAD_NAME, "error deleting IDs ", e);
-            }
-
-            this.chunkstart = this.deleteOnRecrawl? 0 : this.chunkstart + this.chunksize;
-        }
-
-        if (docList == null || docList.size() < this.chunksize) {
-            return false;
-        }
-        return true;
-    }
-
-    /**
-     * @return a new default CrawlProfile instance to be used for recrawl jobs.
-     */
-    public static CrawlProfile buildDefaultCrawlProfile() {
-        final CrawlProfile profile = new CrawlProfile(CrawlSwitchboard.CRAWL_PROFILE_RECRAWL_JOB, CrawlProfile.MATCH_ALL_STRING, // crawlerUrlMustMatch
-                CrawlProfile.MATCH_NEVER_STRING, // crawlerUrlMustNotMatch
-                CrawlProfile.MATCH_ALL_STRING, // crawlerIpMustMatch
-                CrawlProfile.MATCH_NEVER_STRING, // crawlerIpMustNotMatch
-                CrawlProfile.MATCH_NEVER_STRING, // crawlerCountryMustMatch
-                CrawlProfile.MATCH_NEVER_STRING, // crawlerNoDepthLimitMatch
-                CrawlProfile.MATCH_ALL_STRING, // indexUrlMustMatch
-                CrawlProfile.MATCH_NEVER_STRING, // indexUrlMustNotMatch
-                CrawlProfile.MATCH_ALL_STRING, // indexContentMustMatch
-                CrawlProfile.MATCH_NEVER_STRING, // indexContentMustNotMatch
-                false, //noindexWhenCanonicalUnequalURL
-                0, false, CrawlProfile.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_RECRAWL_JOB_RECRAWL_CYCLE), -1,
-                true, true, true, false, // crawlingQ, followFrames, obeyHtmlRobotsNoindex, obeyHtmlRobotsNofollow,
-                true, true, true, false, -1, false, true, CrawlProfile.MATCH_NEVER_STRING, CacheStrategy.IFFRESH,
-                "robot_" + CrawlSwitchboard.CRAWL_PROFILE_RECRAWL_JOB,
-                ClientIdentification.yacyInternetCrawlerAgentName,
-                TagValency.EVAL, null, null, 0);
-        return profile;
-    }
-
-    @Override
-    public int getJobCount() {
-        return this.urlstack.size();
-    }
-
-    /**
-     * @return The total number of candidate URLs found for recrawl
-     */
-    public long getUrlsToRecrawl() {
-        return this.urlsToRecrawl;
-    }
-
-    /**
-     * @return The total number of URLs added to the crawler queue for recrawl
-     */
-    public long getRecrawledUrlsCount() {
-        return this.recrawledUrlsCount;
-    }
-
-    /**
-     * @return The total number of URLs rejected for some reason by the crawl
-     *         stacker or the crawler queue
-     */
-    public long getRejectedUrlsCount() {
-        return this.rejectedUrlsCount;
-    }
-
-    /**
-     * @return The total number of malformed URLs found
-     */
-    public long getMalformedUrlsCount() {
-        return this.malformedUrlsCount;
-    }
-
-    /**
-     * @return The total number of malformed URLs deleted from index
-     */
-    public long getMalformedUrlsDeletedCount() {
-        return this.malformedUrlsDeletedCount;
-    }
-
-    /**
-     * @return true when the job terminated early because an error occurred when
-     *         requesting the Solr index, or the Solr index was closed
-     */
-    public boolean isTerminatedBySolrFailure() {
-        return this.terminatedBySolrFailure;
-    }
-
-    /** @return The recrawl job start time */
-    public LocalDateTime getStartTime() {
-        return this.startTime;
-    }
-
-    /** @return The recrawl job end time */
-    public LocalDateTime getEndTime() {
-        return this.endTime;
-    }
-
-    @Override
-    public void freemem() {
-        this.urlstack.clear();
-    }
-
-}
+/**
+ * RecrawlBusyThread.java
+ * SPDX-FileCopyrightText: 2015 by Burkhard Buelte
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ * First released 15.05.2015 at https://yacy.net
+ *
+ * This is a part of YaCy, a peer-to-peer based web search engine
+ *
+ * LICENSE
+ *
+ * This library is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation; either version 2.1 of the License, or (at your option)
+ * any later version.
+ *
+ * This library is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt If not, see
+ * <http://www.gnu.org/licenses/>.
+ */
+package net.yacy.crawler;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.time.LocalDateTime;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+
+import net.yacy.cora.document.encoding.ASCII;
+import net.yacy.cora.document.id.DigestURL;
+import net.yacy.cora.federate.solr.connector.SolrConnector;
+import net.yacy.cora.federate.yacy.CacheStrategy;
+import net.yacy.cora.protocol.ClientIdentification;
+import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.crawler.data.CrawlProfile;
+import net.yacy.crawler.data.NoticedURL;
+import net.yacy.crawler.retrieval.Request;
+import net.yacy.document.parser.html.TagValency;
+import net.yacy.kelondro.workflow.AbstractBusyThread;
+import net.yacy.search.Switchboard;
+import net.yacy.search.schema.CollectionSchema;
+
+/**
+ * Selects documents by a query from the local index
+ * and feeds the found urls to the crawler to recrawl the documents.
+ * This is intended to keep the index up-to-date
+ * Currently the doucments are selected by expired fresh_date_dt field
+ * an added to the crawler in smaller chunks (see chunksize) as long as no other crawl is running.
+ */
+public class RecrawlBusyThread extends AbstractBusyThread {
+
+    /** The thread name */
+    public final static String THREAD_NAME = "recrawlindex";
+
+    /** The default selection query */
+    public static final String DEFAULT_QUERY = CollectionSchema.fresh_date_dt.getSolrFieldName()+":[* TO NOW/DAY-1DAY]";
+
+    /** Default value for inclusion or not of documents with a https status different from 200 (success) */
+    public static final boolean DEFAULT_INCLUDE_FAILED = false;
+
+    /** The default value whether to delete on Recrawl */
+    public static final boolean DEFAULT_DELETE_ON_RECRAWL = false;
+
+    /** The current query selecting documents to recrawl */
+    private String currentQuery;
+
+    /** flag if docs with httpstatus_i <> 200 shall be recrawled */
+    private boolean includefailed;
+
+    /** flag whether to delete on Recrawl */
+    private boolean deleteOnRecrawl;
+
+    private int chunkstart = 0;
+    private final int chunksize = 100;
+    private final Switchboard sb;
+
+    /** buffer of urls to recrawl */
+    private final Set<DigestURL> urlstack;
+
+    /** The total number of candidate URLs found for recrawl */
+    private long urlsToRecrawl = 0;
+
+    /** Total number of URLs added to the crawler queue for recrawl */
+    private long recrawledUrlsCount = 0;
+
+    /** Total number of URLs rejected for some reason by the crawl stacker or the crawler queue */
+    private long rejectedUrlsCount = 0;
+
+    /** Total number of malformed URLs found */
+    private long malformedUrlsCount = 0;
+
+    /** Total number of malformed URLs deleted from index */
+    private long malformedUrlsDeletedCount = 0;
+
+    private final String solrSortBy;
+
+    /** Set to true when more URLs are still to be processed */
+    private boolean moreToRecrawl = true;
+
+    /** True when the job terminated early because an error occurred when requesting the Solr index, or the Solr index was closed */
+    private boolean terminatedBySolrFailure = false;
+
+    /** The recrawl job start time */
+    private LocalDateTime startTime;
+
+    /** The recrawl job end time */
+    private LocalDateTime endTime;
+
+    /**
+     * @param xsb
+     *            the Switchboard instance holding server environment
+     * @param query
+     *            the Solr selection query
+     * @param includeFailed
+     *            set to true when documents with a https status different from 200
+     *            (success) must be included
+     */
+    public RecrawlBusyThread(final Switchboard xsb, final String query, final boolean includeFailed, final boolean deleteOnRecrawl) {
+        super(3000, 1000); // set lower limits of cycle delay
+        this.setName(THREAD_NAME);
+        this.setIdleSleep(10*60000); // set actual cycle delays
+        this.setBusySleep(2*60000);
+        this.setPriority(Thread.MIN_PRIORITY);
+        this.setLoadPreReqisite(1);
+        this.sb = xsb;
+        this.currentQuery = query;
+        this.includefailed = includeFailed;
+        this.deleteOnRecrawl = deleteOnRecrawl;
+        this.urlstack = new HashSet<>();
+        // workaround to prevent solr exception on existing index (not fully reindexed) since intro of schema with docvalues
+        // org.apache.solr.core.SolrCore java.lang.IllegalStateException: unexpected docvalues type NONE for field 'load_date_dt' (expected=NUMERIC). Use UninvertingReader or index with docvalues.
+        this.solrSortBy = CollectionSchema.load_date_dt.getSolrFieldName() + " asc";
+
+        final SolrConnector solrConnector = this.sb.index.fulltext().getDefaultConnector();
+        if (solrConnector != null && !solrConnector.isClosed()) {
+            /* Ensure indexed data is up-to-date before running the main job */
+            solrConnector.commit(true);
+        }
+    }
+
+    /**
+     * Set the query to select documents to recrawl
+     * and resets the counter to start a fresh query loop
+     * @param q select query
+     * @param includefailedurls true=all http status docs are recrawled, false=httpstatus=200 docs are recrawled
+     * @param deleteOnRecrawl
+     */
+    public void setQuery(String q, boolean includefailedurls, final boolean deleteOnRecrawl) {
+        this.currentQuery = q;
+        this.includefailed = includefailedurls;
+        this.deleteOnRecrawl = deleteOnRecrawl;
+        this.chunkstart = 0;
+    }
+
+    public String getQuery() {
+        return this.currentQuery;
+    }
+
+    /**
+     *
+     * @param queryBase
+     *            the base query
+     * @param includeFailed
+     *            set to true when documents with a https status different from 200
+     *            (success) must be included
+     * @return the Solr selection query for candidate URLs to recrawl
+     */
+    public static final String buildSelectionQuery(final String queryBase, final boolean includeFailed) {
+        return includeFailed ? queryBase : queryBase + " AND (" + CollectionSchema.httpstatus_i.name() + ":200)";
+    }
+
+    /**
+     * Flag to include failed urls (httpstatus_i <> 200)
+     * if true -> currentQuery is used as is,
+     * if false -> the term " AND (httpstatus_i:200)" is appended to currentQuery
+     * @param includefailedurls
+     */
+    public void setIncludeFailed(boolean includefailedurls) {
+        this.includefailed = includefailedurls;
+    }
+
+    public boolean getIncludeFailed () {
+        return this.includefailed;
+    }
+
+    public void setDeleteOnRecrawl(final boolean deleteOnRecrawl) {
+        this.deleteOnRecrawl = deleteOnRecrawl;
+    }
+
+    public boolean getDeleteOnRecrawl() {
+        return this.deleteOnRecrawl;
+    }
+
+    /**
+     * feed urls to the local crawler
+     * (Switchboard.addToCrawler() is not used here, as there existing urls are always skipped)
+     *
+     * @return true if urls were added/accepted to the crawler
+     */
+    private boolean feedToCrawler() {
+
+        int added = 0;
+
+        if (!this.urlstack.isEmpty()) {
+            final CrawlProfile profile = this.sb.crawler.defaultRecrawlJobProfile;
+
+            for (final DigestURL url : this.urlstack) {
+                final Request request = new Request(ASCII.getBytes(this.sb.peers.mySeed().hash), url, null, "",
+                        new Date(), profile.handle(), 0, profile.timezoneOffset());
+                String acceptedError = this.sb.crawlStacker.checkAcceptanceChangeable(url, profile, 0);
+                if (!this.includefailed && acceptedError == null) { // skip check if failed docs to be included
+                    acceptedError = this.sb.crawlStacker.checkAcceptanceInitially(url, profile);
+                }
+                if (acceptedError != null) {
+                    this.rejectedUrlsCount++;
+                    ConcurrentLog.info(THREAD_NAME, "addToCrawler: cannot load " + url.toNormalform(true) + ": " + acceptedError);
+                    continue;
+                }
+                final String s;
+                s = this.sb.crawlQueues.noticeURL.push(NoticedURL.StackType.LOCAL, request, profile, this.sb.robots);
+
+                if (s != null) {
+                    this.rejectedUrlsCount++;
+                    ConcurrentLog.info(THREAD_NAME, "addToCrawler: failed to add " + url.toNormalform(true) + ": " + s);
+                } else {
+                    added++;
+                    this.recrawledUrlsCount++;
+                }
+            }
+            this.urlstack.clear();
+        }
+        return (added > 0);
+    }
+
+    /**
+     * Process query and hand over urls to the crawler
+     *
+     * @return true if something processed
+     */
+    @Override
+    public boolean job() {
+        // more than chunksize crawls are running, do nothing
+        if (this.sb.crawlQueues.coreCrawlJobSize() > this.chunksize) {
+            return false;
+        }
+
+        boolean didSomething = false;
+        if (this.urlstack.isEmpty()) {
+            if(!this.moreToRecrawl) {
+                /* We do not remove the thread from the Switchboard worker threads using serverSwitch.terminateThread(String,boolean),
+                 * because we want to be able to provide a report after its termination */
+                this.terminate(false);
+            } else {
+                this.moreToRecrawl = this.processSingleQuery();
+                /* Even if no more URLs are to recrawl, the job has done something by searching the Solr index */
+                didSomething = true;
+            }
+        } else {
+            didSomething = this.feedToCrawler();
+        }
+        return didSomething;
+    }
+
+    @Override
+    public synchronized void start() {
+        this.startTime = LocalDateTime.now();
+        super.start();
+    }
+
+    @Override
+    public void terminate(boolean waitFor) {
+        super.terminate(waitFor);
+        this.endTime = LocalDateTime.now();
+    }
+
+    /**
+     * Selects documents to recrawl the urls
+     * @return true if query has more results
+     */
+    private boolean processSingleQuery() {
+        if (!this.urlstack.isEmpty()) {
+            return true;
+        }
+        SolrDocumentList docList = null;
+        final SolrConnector solrConnector = this.sb.index.fulltext().getDefaultConnector();
+        if (solrConnector == null || solrConnector.isClosed()) {
+            this.urlsToRecrawl = 0;
+            this.terminatedBySolrFailure = true;
+            return false;
+        }
+
+        try {
+            // query all or only httpstatus=200 depending on includefailed flag
+            docList = solrConnector.getDocumentListByQuery(RecrawlBusyThread.buildSelectionQuery(this.currentQuery, this.includefailed),
+                this.solrSortBy, this.chunkstart, this.chunksize, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName());
+            this.urlsToRecrawl = docList.getNumFound();
+        } catch (final Throwable e) {
+            this.urlsToRecrawl = 0;
+            this.terminatedBySolrFailure = true;
+        }
+
+        if (docList != null) {
+            final Set<String> tobedeletedIDs = new HashSet<>();
+            for (final SolrDocument doc : docList) {
+                try {
+                    this.urlstack.add(new DigestURL((String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName())));
+                    if (this.deleteOnRecrawl) tobedeletedIDs.add((String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName()));
+                } catch (final MalformedURLException ex) {
+                    this.malformedUrlsCount++;
+                    // if index entry hasn't a valid url (useless), delete it
+                    tobedeletedIDs.add((String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName()));
+                    this.malformedUrlsDeletedCount++;
+                    ConcurrentLog.severe(THREAD_NAME, "deleted index document with invalid url " + (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName()));
+                }
+            }
+
+            if (!tobedeletedIDs.isEmpty()) try {
+                solrConnector.deleteByIds(tobedeletedIDs);
+                solrConnector.commit(false);
+            } catch (final IOException e) {
+                ConcurrentLog.severe(THREAD_NAME, "error deleting IDs ", e);
+            }
+
+            this.chunkstart = this.deleteOnRecrawl? 0 : this.chunkstart + this.chunksize;
+        }
+
+        if (docList == null || docList.size() < this.chunksize) {
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * @return a new default CrawlProfile instance to be used for recrawl jobs.
+     */
+    public static CrawlProfile buildDefaultCrawlProfile() {
+        final CrawlProfile profile = new CrawlProfile(CrawlSwitchboard.CRAWL_PROFILE_RECRAWL_JOB, CrawlProfile.MATCH_ALL_STRING, // crawlerUrlMustMatch
+                CrawlProfile.MATCH_NEVER_STRING, // crawlerUrlMustNotMatch
+                CrawlProfile.MATCH_ALL_STRING, // crawlerIpMustMatch
+                CrawlProfile.MATCH_NEVER_STRING, // crawlerIpMustNotMatch
+                CrawlProfile.MATCH_NEVER_STRING, // crawlerCountryMustMatch
+                CrawlProfile.MATCH_NEVER_STRING, // crawlerNoDepthLimitMatch
+                CrawlProfile.MATCH_ALL_STRING, // indexUrlMustMatch
+                CrawlProfile.MATCH_NEVER_STRING, // indexUrlMustNotMatch
+                CrawlProfile.MATCH_ALL_STRING, // indexContentMustMatch
+                CrawlProfile.MATCH_NEVER_STRING, // indexContentMustNotMatch
+                false, //noindexWhenCanonicalUnequalURL
+                0, false, CrawlProfile.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_RECRAWL_JOB_RECRAWL_CYCLE), -1,
+                true, true, true, false, // crawlingQ, followFrames, obeyHtmlRobotsNoindex, obeyHtmlRobotsNofollow,
+                true, true, true, false, CacheStrategy.IFFRESH,
+                "robot_" + CrawlSwitchboard.CRAWL_PROFILE_RECRAWL_JOB,
+                ClientIdentification.yacyInternetCrawlerAgentName,
+                TagValency.EVAL, null, null, 0);
+        return profile;
+    }
+
+    @Override
+    public int getJobCount() {
+        return this.urlstack.size();
+    }
+
+    /**
+     * @return The total number of candidate URLs found for recrawl
+     */
+    public long getUrlsToRecrawl() {
+        return this.urlsToRecrawl;
+    }
+
+    /**
+     * @return The total number of URLs added to the crawler queue for recrawl
+     */
+    public long getRecrawledUrlsCount() {
+        return this.recrawledUrlsCount;
+    }
+
+    /**
+     * @return The total number of URLs rejected for some reason by the crawl
+     *         stacker or the crawler queue
+     */
+    public long getRejectedUrlsCount() {
+        return this.rejectedUrlsCount;
+    }
+
+    /**
+     * @return The total number of malformed URLs found
+     */
+    public long getMalformedUrlsCount() {
+        return this.malformedUrlsCount;
+    }
+
+    /**
+     * @return The total number of malformed URLs deleted from index
+     */
+    public long getMalformedUrlsDeletedCount() {
+        return this.malformedUrlsDeletedCount;
+    }
+
+    /**
+     * @return true when the job terminated early because an error occurred when
+     *         requesting the Solr index, or the Solr index was closed
+     */
+    public boolean isTerminatedBySolrFailure() {
+        return this.terminatedBySolrFailure;
+    }
+
+    /** @return The recrawl job start time */
+    public LocalDateTime getStartTime() {
+        return this.startTime;
+    }
+
+    /** @return The recrawl job end time */
+    public LocalDateTime getEndTime() {
+        return this.endTime;
+    }
+
+    @Override
+    public void freemem() {
+        this.urlstack.clear();
+    }
+
+}
diff --git a/source/net/yacy/crawler/data/CrawlProfile.java b/source/net/yacy/crawler/data/CrawlProfile.java
index 574a164fb..af6c60926 100644
--- a/source/net/yacy/crawler/data/CrawlProfile.java
+++ b/source/net/yacy/crawler/data/CrawlProfile.java
@@ -120,10 +120,6 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
         STORE_HTCACHE                ("storeHTCache",               false, CrawlAttribute.BOOLEAN, "Store in HTCache"),
         CACHE_STRAGEGY               ("cacheStrategy",              false, CrawlAttribute.STRING,  "Cache Strategy (NOCACHE,IFFRESH,IFEXIST,CACHEONLY)"),
         AGENT_NAME                   ("agentName",                  false, CrawlAttribute.STRING,  "User Agent Profile Name"),
-        SNAPSHOTS_MAXDEPTH           ("snapshotsMaxDepth",          false, CrawlAttribute.INTEGER, "Max Depth for Snapshots"),
-        SNAPSHOTS_REPLACEOLD         ("snapshotsReplaceOld",        false, CrawlAttribute.BOOLEAN, "Multiple Snapshot Versions - replace old with new"),
-        SNAPSHOTS_MUSTNOTMATCH       ("snapshotsMustnotmatch",      false, CrawlAttribute.STRING,  "must-not-match filter for snapshot generation"),
-        SNAPSHOTS_LOADIMAGE          ("snapshotsLoadImage",         false, CrawlAttribute.BOOLEAN, "Flag for Snapshot image generation"),
         REMOTE_INDEXING              ("remoteIndexing",             false, CrawlAttribute.BOOLEAN, "Remote Indexing (only for p2p networks)"),
         INDEX_TEXT                   ("indexText",                  false, CrawlAttribute.BOOLEAN, "Index Text"),
         INDEX_MEDIA                  ("indexMedia",                 false, CrawlAttribute.BOOLEAN, "Index Media"),
@@ -174,7 +170,6 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
      * @see CollectionSchema#content_type  */
     private Pattern indexMediaTypeMustNotMatch = null;
 
-    private Pattern snapshotsMustnotmatch = null;
 
     private final Map<String, AtomicInteger> doms;
     private final TagValency defaultValency;
@@ -204,10 +199,6 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
      * @param indexMedia true if media content of URL shall be indexed
      * @param storeHTCache true if content chall be kept in cache after indexing
      * @param remoteIndexing true if part of the crawl job shall be distributed
-     * @param snapshotsMaxDepth if the current crawl depth is equal or below that given depth, a snapshot is generated
-     * @param snapshotsLoadImage true if graphical (== pdf) shapshots shall be made
-     * @param snapshotsReplaceOld true if snapshots shall not be historized
-     * @param snapshotsMustnotmatch a regular expression; if it matches on the url, the snapshot is not generated
      * @param xsstopw true if static stop words shall be ignored
      * @param xdstopw true if dynamic stop words shall be ignored
      * @param xpstopw true if parent stop words shall be ignored
@@ -235,10 +226,6 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
                  final boolean indexMedia,
                  final boolean storeHTCache,
                  final boolean remoteIndexing,
-                 final int snapshotsMaxDepth,
-                 final boolean snapshotsLoadImage,
-                 final boolean snapshotsReplaceOld,
-                 final String snapshotsMustnotmatch,
                  final CacheStrategy cacheStrategy,
                  final String collections,
                  final String userAgentName,
@@ -281,10 +268,6 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
         this.put(CrawlAttribute.INDEX_MEDIA.key,               indexMedia);
         this.put(CrawlAttribute.STORE_HTCACHE.key,             storeHTCache);
         this.put(CrawlAttribute.REMOTE_INDEXING.key,           remoteIndexing);
-        this.put(CrawlAttribute.SNAPSHOTS_MAXDEPTH.key,        snapshotsMaxDepth);
-        this.put(CrawlAttribute.SNAPSHOTS_LOADIMAGE.key,       snapshotsLoadImage);
-        this.put(CrawlAttribute.SNAPSHOTS_REPLACEOLD.key,      snapshotsReplaceOld);
-        this.put(CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key,    snapshotsMustnotmatch);
         this.put(CrawlAttribute.CACHE_STRAGEGY.key,            cacheStrategy.toString());
         this.put(CrawlAttribute.COLLECTIONS.key,               CommonPattern.SPACE.matcher(collections.trim()).replaceAll(""));
         // we transform the ignore_class_name and scraper information into a JSON Array
@@ -895,41 +878,6 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
         return (r.equals(Boolean.TRUE.toString()));
     }
 
-    public int snapshotMaxdepth() {
-        final String r = this.get(CrawlAttribute.SNAPSHOTS_MAXDEPTH.key);
-        if (r == null) return -1;
-        try {
-            final int i = Integer.parseInt(r);
-            if (i < 0) return -1;
-            return i;
-        } catch (final NumberFormatException e) {
-            ConcurrentLog.logException(e);
-            return -1;
-        }
-    }
-
-    public boolean snapshotLoadImage() {
-        final String r = this.get(CrawlAttribute.SNAPSHOTS_LOADIMAGE.key);
-        if (r == null) return false;
-        return (r.equals(Boolean.TRUE.toString()));
-    }
-
-    public boolean snapshotReplaceold() {
-        final String r = this.get(CrawlAttribute.SNAPSHOTS_REPLACEOLD.key);
-        if (r == null) return false;
-        return (r.equals(Boolean.TRUE.toString()));
-    }
-
-    public Pattern snapshotsMustnotmatch() {
-        if (this.snapshotsMustnotmatch == null) {
-            final String r = this.get(CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key);
-            try {
-                this.snapshotsMustnotmatch = (r == null || r.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
-            } catch (final PatternSyntaxException e) { this.snapshotsMustnotmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
-        }
-        return this.snapshotsMustnotmatch;
-    }
-
     public int timezoneOffset() {
         final String timezoneOffset = this.get(CrawlAttribute.TIMEZONEOFFSET.key);
         if (timezoneOffset == null) return 0;
@@ -1038,10 +986,6 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
         prop.put(CRAWL_PROFILE_PREFIX + count + "_storeHTCache", this.storeHTCache() ? 1 : 0);
         prop.putXML(CRAWL_PROFILE_PREFIX + count + "_cacheStrategy", this.get(CrawlAttribute.CACHE_STRAGEGY.key));
         prop.putXML(CRAWL_PROFILE_PREFIX + count + "_agentName", this.get(CrawlAttribute.AGENT_NAME.key));
-        prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.SNAPSHOTS_MAXDEPTH.key, this.get(CrawlAttribute.SNAPSHOTS_MAXDEPTH.key));
-        prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.SNAPSHOTS_REPLACEOLD.key, this.get(CrawlAttribute.SNAPSHOTS_REPLACEOLD.key));
-        prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key, this.get(CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key));
-        prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.SNAPSHOTS_LOADIMAGE.key, this.get(CrawlAttribute.SNAPSHOTS_LOADIMAGE.key));
         prop.put(CRAWL_PROFILE_PREFIX + count + "_remoteIndexing", this.remoteIndexing() ? 1 : 0);
         prop.put(CRAWL_PROFILE_PREFIX + count + "_indexText", this.indexText() ? 1 : 0);
         prop.put(CRAWL_PROFILE_PREFIX + count + "_indexMedia", this.indexMedia() ? 1 : 0);
diff --git a/source/net/yacy/crawler/data/Snapshots.java b/source/net/yacy/crawler/data/Snapshots.java
deleted file mode 100644
index 0463d0d42..000000000
--- a/source/net/yacy/crawler/data/Snapshots.java
+++ /dev/null
@@ -1,516 +0,0 @@
-/**
- *  DocumentImage
- *  SPDX-FileCopyrightText: 2014 Michael Peter Christen <mc@yacy.net)> 
- *  SPDX-License-Identifier: GPL-2.0-or-later
- *  First released 29.11.2014 at https://yacy.net
- *
- *  This library is free software; you can redistribute it and/or
- *  modify it under the terms of the GNU Lesser General Public
- *  License as published by the Free Software Foundation; either
- *  version 2.1 of the License, or (at your option) any later version.
- *
- *  This library is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  Lesser General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public License
- *  along with this program in the file lgpl21.txt
- *  If not, see <http://www.gnu.org/licenses/>.
- */
-
-package net.yacy.crawler.data;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
-import java.text.ParseException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.solr.common.SolrDocument;
-
-import net.yacy.cora.date.GenericFormatter;
-import net.yacy.cora.document.encoding.ASCII;
-import net.yacy.cora.document.id.DigestURL;
-import net.yacy.cora.util.ConcurrentLog;
-import net.yacy.search.index.Fulltext;
-import net.yacy.search.schema.CollectionSchema;
-
-/**
- * This class hosts document snapshots.
- * 
- * The storage is organized in the following hierarchy:
- * - in the root path are subpaths for each host:port
- * - in the host:port path are subpaths for the crawl depth, two digits length
- * - in the crawl depth path are subpaths for the first two charaters of the url-hash, called shard
- * - in the shard path are files, named with <urlhash>'.'<date>.<ext>
- * .. where the <date> has the form "yyyyMMdd" and ext may be one of {pdf,jpg,png,xml,json}.
- * The pdf is created with wxhtmltopdf, jpg/png is created with convert
- * and the xml/json is an extract from solr.
- * 
- * The construction of the file name with the date allows to make several copies of the same document
- * for different snapshot-times. The usage of the crawl depth makes it easier to extract a specific part
- * of the domain.
- */
-public class Snapshots {
-
-    private File storageLocation;
-    
-    private Map<String, TreeMap<Integer, TreeSet<String>>> directory; // a TreeMap for each domain (host.port) where the key is the depth and the value is a Set containing a key/urlhash id to get all files into a specific order to provide a recent view on the documents
-    
-    public Snapshots(final File location) {
-        this.storageLocation = location;
-        this.storageLocation.mkdirs();
-        // scan the location to fill the directory
-        this.directory = new HashMap<>();
-        for (String hostport: location.list()) {
-            TreeMap<Integer, TreeSet<String>> domaindepth = new TreeMap<>();
-            this.directory.put(hostport, domaindepth);
-            File domaindir = new File(location, hostport);
-            if (domaindir.isDirectory()) domainscan: for (String depth: domaindir.list()) {
-                TreeSet<String> dateid = new TreeSet<>();
-                Integer depthi = -1;
-                try {
-                    depthi = Integer.parseInt(depth);
-                } catch (NumberFormatException e) {
-                    continue domainscan;
-                }
-                domaindepth.put(depthi, dateid);
-                File sharddir = new File(domaindir, depth);
-                if (sharddir.isDirectory()) for (String shard: sharddir.list()) {
-                    File snapshotdir = new File(sharddir, shard);
-                    if (snapshotdir.isDirectory()) {
-                        for (String snapshotfile: snapshotdir.list()) {
-                            if (snapshotfile.endsWith(".xml")) {
-                                String s = snapshotfile.substring(0, snapshotfile.length() - 4);
-                                int p = s.indexOf('.');
-                                assert p == 12;
-                                if (p > 0) {
-                                    String key = s.substring(p + 1) + '.' + s.substring(0, p);
-                                    dateid.add(key);
-                                }
-                            }
-                        }
-                    }
-                }
-                if (dateid.size() == 0) domaindepth.remove(depthi);
-            }
-            if (domaindepth.size() == 0) this.directory.remove(hostport);
-        }
-    }
-
-    /**
-     * get the number of entries in the snapshot directory
-     * @return the total number of different documents
-     */
-    public int size() {
-        int c = 0;
-        for (Map<Integer, TreeSet<String>> m: directory.values()) {
-            for (TreeSet<String> n: m.values()) {
-                c += n.size();
-            }
-        }
-        return c;
-    }
-
-    /**
-     * get a list of <host>.<port> names in the snapshot directory
-     * @return
-     */
-    public Set<String> listHosts() {
-        return directory.keySet();
-    }
-    
-    public final class Revisions {
-        public final int depth;
-        public final Date[] dates;
-        public final String urlhash;
-        public final String url;
-        public final File[] pathtoxml;
-        public Revisions(final String hostport, final int depth, final String datehash) {
-            this.depth = depth;
-            int p = datehash.indexOf('.');
-            this.dates = new Date[1];
-            String datestring = datehash.substring(0, p);
-            this.dates[0] = parseDate(datestring);
-            this.urlhash = datehash.substring(p + 1);
-            this.pathtoxml = new File[1];
-            this.pathtoxml[0] = new File(pathToShard(hostport, urlhash, depth), this.urlhash + "." + datestring + ".xml");
-            String u = null;
-            if (this.pathtoxml[0].exists()) {
-            	BufferedReader reader = null;
-                try {
-                    reader = new BufferedReader(new InputStreamReader(new FileInputStream(this.pathtoxml[0])));
-                    String line;
-                    while ((line = reader.readLine()) != null) {
-                        if (line.startsWith("<str name=\"sku\">")) {
-                            u = line.substring(16, line.length() - 6);
-                            break;
-                        }
-                    }
-                } catch (IOException e) {
-                	ConcurrentLog.warn("SNAPSHOTS", "Error while reading file " + this.pathtoxml[0]);
-                } finally {
-                	if(reader != null) {
-                		try {
-							reader.close();
-						} catch (IOException ignored) {
-							ConcurrentLog.warn("SNAPSHOTS", "Could not close input stream on file " + this.pathtoxml[0]);
-						}
-                	}
-                }
-            }
-            this.url = u; 
-        }
-    }
-    
-    public Revisions getRevisions(String urlhash) {
-        if (urlhash == null || urlhash.length() == 0) return null;
-        // search for the hash, we must iterate through all entries
-        for (Map.Entry<String, TreeMap<Integer, TreeSet<String>>> hostportDomaindepth: this.directory.entrySet()) {
-            String hostport = hostportDomaindepth.getKey();
-            for (Map.Entry<Integer, TreeSet<String>> depthDateHash: hostportDomaindepth.getValue().entrySet()) {
-                int depth = depthDateHash.getKey();
-                for (String dateHash: depthDateHash.getValue()) {
-                    if (dateHash.endsWith(urlhash)) {
-                        return new Revisions(hostport, depth, dateHash);
-                    }
-                }
-            }
-        }
-        return null;
-    }
-    
-    /**
-     * list the snapshots for a given host name
-     * @param hostport the <host>.<port> identifier for the domain (with the same format as applied by the Snapshots.pathToHostPortDir() function)
-     * @param depth restrict the result to the given depth or if depth == -1 do not restrict to a depth
-     * @return a map with a set for each depth in the domain of the host name
-     */
-    public TreeMap<Integer, Collection<Revisions>> listIDs(final String hostport, final int depth) {
-        TreeMap<Integer, Collection<Revisions>> result = new TreeMap<>();
-        TreeMap<Integer, TreeSet<String>> list = directory.get(hostport);
-        if (list != null) {
-            for (Map.Entry<Integer, TreeSet<String>> entry: list.entrySet()) {
-                if (depth != -1 && entry.getKey() != depth) continue;
-                Collection<Revisions> r = new ArrayList<>(entry.getValue().size());
-                for (String datehash: entry.getValue()) {
-                    r.add(new Revisions(hostport, entry.getKey(), datehash));
-                }
-                result.put(entry.getKey(), r);
-            }
-        }
-        return result;
-    }
-
-    /**
-     * get the number of snapshots for the given host name
-     * @param hostport the <host>.<port> identifier for the domain
-     * @param depth restrict the result to the given depth or if depth == -1 do not restrict to a depth
-     * @return a count, the total number of documents for the domain and depth
-     */
-    public int listIDsSize(final String hostport, final int depth) {
-        int count = 0;
-        TreeMap<Integer, TreeSet<String>> list = directory.get(hostport);
-        if (list != null) {
-            for (Map.Entry<Integer, TreeSet<String>> entry: list.entrySet()) {
-                if (depth != -1 && entry.getKey() != depth) continue;
-                count += entry.getValue().size();
-            }
-        }
-        return count;
-    }
-    
-    /**
-     * Compute the path of a snapshot. This does not create the snapshot, only gives a path.
-     * Also, the path to the storage location is not created.
-     * @param url
-     * @param ext
-     * @param depth
-     * @param date
-     * @return a file to the snapshot
-     */
-    public File definePath(final DigestURL url, final int depth, final Date date, final String ext) {
-        String id = ASCII.String(url.hash());
-        String ds = GenericFormatter.SHORT_MINUTE_FORMATTER.format(date);
-        return new File(pathToShard(url, depth), id + "." + ds + "." + ext);
-    }
-
-    /**
-     * Write information about the storage of a snapshot to the Snapshot-internal index.
-     * The actual writing of files to the target directory must be done elsewehre, this method does not store the snapshot files.
-     * @param url
-     * @param depth
-     * @param date
-     */
-    public void announceStorage(final DigestURL url, final int depth, final Date date) {
-        String id = ASCII.String(url.hash());
-        String ds = GenericFormatter.SHORT_MINUTE_FORMATTER.format(date);
-        String pathToHostPortDir = pathToHostPortDir(url.getHost(), url.getPort());
-        TreeMap<Integer, TreeSet<String>> domaindepth = this.directory.get(pathToHostPortDir);
-        if (domaindepth == null) {domaindepth = new TreeMap<Integer, TreeSet<String>>(); this.directory.put(pathToHostPortDir(url.getHost(), url.getPort()), domaindepth);}
-        TreeSet<String> dateid = domaindepth.get(depth);
-        if (dateid == null) {dateid = new TreeSet<String>(); domaindepth.put(depth, dateid);}
-        dateid.add(ds + '.' + id);
-    }
-
-    /**
-     * Delete information about the storage of a snapshot to the Snapshot-internal index.
-     * The actual deletion of files in the target directory must be done elsewhere, this method does not store the snapshot files.
-     * @param url
-     * @param depth
-     */
-    public Set<Date> announceDeletion(final DigestURL url, final int depth) {
-        HashSet<Date> dates = new HashSet<>();
-        String id = ASCII.String(url.hash());
-        String pathToHostPortDir = pathToHostPortDir(url.getHost(), url.getPort());
-        TreeMap<Integer, TreeSet<String>> domaindepth = this.directory.get(pathToHostPortDir);
-        if (domaindepth == null) return dates;
-        TreeSet<String> dateid = domaindepth.get(depth);
-        if (dateid == null) return dates;
-        Iterator<String> i = dateid.iterator();
-        while (i.hasNext()) {
-            String dis = i.next();
-            if (dis.endsWith("." + id)) {
-                String d = dis.substring(0, dis.length() - id.length() - 1);
-                Date date = parseDate(d);
-                if (date != null) dates.add(date);
-                i.remove();
-            }
-        }
-        if (dateid.size() == 0) domaindepth.remove(depth);
-        if (domaindepth.size() == 0) this.directory.remove(pathToHostPortDir);
-        return dates;
-    }
-    
-    /**
-     * Order enum class for the select method
-     */
-    public static enum Order {
-        ANY, OLDESTFIRST, LATESTFIRST;
-    }
-
-    /**
-     * select a set of urlhashes from the snapshot directory. The selection either ordered
-     * by generation date (upwards == OLDESTFIRST or downwards == LATESTFIRST) or with any
-     * order. The result set can be selected either with a given host or a depth
-     * @param host selected host or null for all hosts
-     * @param depth selected depth or null for all depths
-     * @param order Order.ANY, Order.OLDESTFIRST or Order.LATESTFIRST
-     * @param maxcount the maximum number of hosthashes. If unlimited, submit Integer.MAX_VALUE
-     * @return a map of hosthashes with the associated creation date
-     */
-    public LinkedHashMap<String, Revisions> select(final String host, final Integer depth, final Order order, int maxcount) {
-        TreeMap<String, String[]> dateIdResult = new TreeMap<>();
-        if (host == null && depth == null) {
-            loop: for (Map.Entry<String, TreeMap<Integer, TreeSet<String>>> hostportDepths: this.directory.entrySet()) {
-                for (Map.Entry<Integer, TreeSet<String>> depthIds: hostportDepths.getValue().entrySet()) {
-                    for (String id: depthIds.getValue()) {
-                        dateIdResult.put(id, new String[]{hostportDepths.getKey(), Integer.toString(depthIds.getKey())});
-                        if (order == Order.ANY && dateIdResult.size() >= maxcount) break loop;
-                    }
-                }
-            }
-        }
-        if (host == null && depth != null) {
-            loop: for (Map.Entry<String, TreeMap<Integer, TreeSet<String>>> hostportDepths: this.directory.entrySet()) {
-                TreeSet<String> ids = hostportDepths.getValue().get(depth);
-                if (ids != null) for (String id: ids) {
-                    dateIdResult.put(id, new String[]{hostportDepths.getKey(), Integer.toString(depth)});
-                    if (order == Order.ANY && dateIdResult.size() >= maxcount) break loop;
-                }
-            }
-        }
-        if (host != null && depth == null) {
-            String hostport = pathToHostPortDir(host, 80);
-            TreeMap<Integer, TreeSet<String>> depthIdsMap = this.directory.get(hostport);
-            if(depthIdsMap == null && isIpv6AddrHost(host)) {
-            	/* If the host is a raw IPV6 address, we check also if a snapshot was recorded with the old format (without percent-encoding) */
-                hostport = pathToHostPortDir(host, 80, false);
-                depthIdsMap = this.directory.get(hostport);            	
-            }
-            if (depthIdsMap != null) {
-            	loop: for (Map.Entry<Integer, TreeSet<String>> depthIds: depthIdsMap.entrySet()) {
-            		for (String id: depthIds.getValue()) {
-            			dateIdResult.put(id, new String[]{hostport, Integer.toString(depthIds.getKey())});
-            			if (order == Order.ANY && dateIdResult.size() >= maxcount) break loop;
-            		}
-            	}
-            }
-        }
-        if (host != null && depth != null) {
-            String hostport = pathToHostPortDir(host, 80);
-            TreeMap<Integer, TreeSet<String>> domaindepth = this.directory.get(hostport);
-            if(domaindepth == null && isIpv6AddrHost(host)) {
-            	/* If the host is a raw IPV6 address, we check also if a snapshot was recorded with the old format (without percent-encoding) */
-                hostport = pathToHostPortDir(host, 80, false);
-                domaindepth = this.directory.get(hostport);            	
-            }
-            if (domaindepth != null) {
-                TreeSet<String> ids = domaindepth.get(depth);
-                if (ids != null) loop: for (String id: ids) {
-                    dateIdResult.put(id, new String[]{hostport, Integer.toString(depth)});
-                    if (order == Order.ANY && dateIdResult.size() >= maxcount) break loop;
-                }
-            }
-        }
-        LinkedHashMap<String, Revisions> result = new LinkedHashMap<>();
-        Iterator<Map.Entry<String, String[]>> i = order == Order.LATESTFIRST ? dateIdResult.descendingMap().entrySet().iterator() : dateIdResult.entrySet().iterator();
-        while (i.hasNext() && result.size() < maxcount) {
-            Map.Entry<String, String[]> entry = i.next();
-            String datehash = entry.getKey();
-            int p = datehash.indexOf('.');
-            assert p >= 0;
-            Revisions r = new Revisions(entry.getValue()[0], Integer.parseInt(entry.getValue()[1]), datehash);
-            result.put(datehash.substring(p + 1), r);
-        }
-        return result;
-    }
-    
-    private static Date parseDate(String d) {
-        try {
-            return GenericFormatter.SHORT_MINUTE_FORMATTER.parse(d, 0).getTime();
-        } catch (ParseException e) {
-            try {
-                return GenericFormatter.SHORT_DAY_FORMATTER.parse(d, 0).getTime();
-            } catch (ParseException ee) {
-                return null;
-            }
-        }
-    }
-    
-    /**
-     * get the depth to a document, helper method for definePath to determine the depth value
-     * @param url
-     * @param fulltext
-     * @return the crawldepth of the document
-     */
-    public int getDepth(final DigestURL url, final Fulltext fulltext) {
-        Integer depth = null;
-        if (fulltext.getDefaultConfiguration().contains(CollectionSchema.crawldepth_i)) {
-            try {
-                SolrDocument doc = fulltext.getDefaultConnector().getDocumentById(ASCII.String(url.hash()), CollectionSchema.crawldepth_i.getSolrFieldName());
-                if (doc != null) {
-                    depth = (Integer) doc.getFieldValue(CollectionSchema.crawldepth_i.getSolrFieldName());
-                }
-            } catch (IOException e) {
-            }
-        }
-        return depth == null ? 0 : depth;
-    }
-
-    /**
-     * for a given url, get all paths for storage locations.
-     * The locations are all for the single url but may represent different storage times.
-     * This method is inefficient because it tests all different depths, it would be better to use
-     * findPaths/3 with a given depth.
-     * @param url
-     * @param ext
-     * @return a set of files for snapshots of the url
-     */
-    public Collection<File> findPaths(final DigestURL url, final String ext) {
-        for (int i = 0; i < 100; i++) {
-            Collection<File> paths = findPaths(url, i, ext);
-            if (paths.size() > 0) return paths;
-        }
-        return new ArrayList<>(0);
-    }
-    
-    // pathtoxml = <storageLocation>/<host>.<port>/<depth>/<shard>/<urlhash>.<date>.xml
-    
-    /**
-     * for a given url, get all paths for storage locations.
-     * The locations are all for the single url but may represent different storage times.
-     * @param url
-     * @param ext required extension or null if the extension must not be checked
-     * @param depth
-     * @return a set of files for snapshots of the url
-     */
-    public Collection<File> findPaths(final DigestURL url, final int depth, final String ext) {
-        String id = ASCII.String(url.hash());
-        File pathToShard = pathToShard(url, depth);
-        if(!pathToShard.exists() && isIpv6AddrHost(url.getHost())) {
-        	/* If the host is a raw IPV6 address, we check also if a snapshot was recorded with the old format (without percent-encoding) */
-        	pathToShard = pathToShard(pathToHostPortDir(url.getHost(), url.getPort(), false), ASCII.String(url.hash()), depth);
-        }
-        String[] list = pathToShard.exists() && pathToShard.isDirectory() ? pathToShard.list() : null; // may be null if path does not exist
-        ArrayList<File> paths = new ArrayList<>();
-        if (list != null) {
-            for (String f: list) {
-                if (f.startsWith(id) && (ext == null || f.endsWith(ext))) paths.add(new File(pathToShard, f));
-            }
-        }
-        return paths;
-    }
-
-    private File pathToShard(final DigestURL url, final int depth) {
-        return pathToShard(pathToHostPortDir(url.getHost(), url.getPort()), ASCII.String(url.hash()), depth);
-    }
-    
-    private File pathToShard(final String hostport, final String urlhash, final int depth) {
-        File pathToHostDir = new File(storageLocation, hostport);
-        File pathToDepthDir = new File(pathToHostDir, pathToDepthDir(depth));
-        File pathToShard = new File(pathToDepthDir, pathToShard(urlhash));
-        return pathToShard;
-    }
-    
-    /**
-     * @param host a domain name or IP address
-     * @return true when the host string is a raw IPV6 address (with square brackets)
-     */
-    private boolean isIpv6AddrHost(final String host) {
-    	return (host != null && host.startsWith("[") && host.endsWith("]") && host.contains(":"));
-    }
-    
-    /**
-     * @param host a domain name or IP address
-     * @param port a port number
-     * @return a representation of the host and port encoding IPV6 addresses for better support accross file systems (notably FAT or NTFS)
-     */
-    private String pathToHostPortDir(final String host, final int port) {
-    	return pathToHostPortDir(host, port, true);
-    }
-
-    /**
-     * @param host a domain name or IP address
-     * @param port a port number
-     * @param encodeIpv6 when true, encode the host for better support accross file systems (notably FAT or NTFS)
-     * @return a representation of the host and port
-     */
-    private String pathToHostPortDir(final String host, final int port, final boolean encodeIpv6) {
-    	String encodedHost = host;
-        if(encodeIpv6 && isIpv6AddrHost(host)) {
-        	/* Percent-encode the host name when it is an IPV6 address, as the ':' character is illegal in a file name on MS Windows FAT32 and NTFS file systems */
-        	try {
-        		encodedHost = URLEncoder.encode(host, StandardCharsets.UTF_8.name());
-			} catch (final UnsupportedEncodingException e) {
-				/* This should not happen has UTF-8 encoding support is required for any JVM implementation */
-			}
-        }
-        return encodedHost + "." + port;
-    }
-    
-    private String pathToDepthDir(final int depth) {
-        return depth < 10 ? "0" + depth : Integer.toString(depth);
-    }
-    
-    private String pathToShard(final String urlhash) {
-        return urlhash.substring(0, 2);
-    }
-
-}
diff --git a/source/net/yacy/crawler/data/Transactions.java b/source/net/yacy/crawler/data/Transactions.java
deleted file mode 100644
index 016cd06c6..000000000
--- a/source/net/yacy/crawler/data/Transactions.java
+++ /dev/null
@@ -1,403 +0,0 @@
-/**
- *  Transactions
- *  SPDX-FileCopyrightText: 2014 Michael Peter Christen <mc@yacy.net)>
- *  SPDX-License-Identifier: GPL-2.0-or-later
- *  First released 08.12.2014 at https://yacy.net
- *
- *  This library is free software; you can redistribute it and/or
- *  modify it under the terms of the GNU Lesser General Public
- *  License as published by the Free Software Foundation; either
- *  version 2.1 of the License, or (at your option) any later version.
- *
- *  This library is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  Lesser General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public License
- *  along with this program in the file lgpl21.txt
- *  If not, see <http://www.gnu.org/licenses/>.
- */
-
-package net.yacy.crawler.data;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.net.MalformedURLException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.solr.common.SolrInputDocument;
-
-import net.yacy.cora.document.id.DigestURL;
-import net.yacy.cora.federate.solr.responsewriter.EnhancedXMLResponseWriter;
-import net.yacy.cora.protocol.ClientIdentification;
-import net.yacy.cora.util.ConcurrentLog;
-import net.yacy.cora.util.Html2Image;
-import net.yacy.crawler.data.Snapshots.Order;
-import net.yacy.crawler.data.Snapshots.Revisions;
-import net.yacy.search.schema.CollectionSchema;
-
-/**
- * This is a static class holding one or several Snapshot directories
- * Transacted snapshots are moved from the inventory snapshot directory to the archive snapshot directory.
- *
- */
-public class Transactions {
-
-    private final static String XML_PREFIX = "<response>\n<!--\n";
-    private final static char[] WHITESPACE = new char[132];
-    //private final static int WHITESPACE_START = XML_PREFIX.length();
-    //private final static int WHITESPACE_LENGTH = WHITESPACE.length;
-    private static File transactionDir = null, inventoryDir = null, archiveDir = null;
-    private static Snapshots inventory = null, archive = null;
-    private static ExecutorService executor = Executors.newCachedThreadPool();
-    private static AtomicInteger executorRunning = new AtomicInteger(0);
-
-    /** the maximum to wait for each wkhtmltopdf call when rendering PDF snapshots */
-    private static long wkhtmltopdfTimeout = 30;
-
-    static {
-        for (int i = 0; i < WHITESPACE.length; i++) WHITESPACE[i] = 32;
-    }
-
-    public static enum State {
-        INVENTORY("inventory"), ARCHIVE("archive"), ANY(null);
-        public String dirname;
-        State(String dirname) {
-            this.dirname = dirname;
-        }
-    }
-
-    /**
-     * @param dir the parent directory of inventory and archive snapshots.
-     * @param wkhtmltopdfSecondsTimeout the maximum to wait for each wkhtmltopdf call when rendering PDF snapshots
-     */
-    public static void init(final File dir, final long wkhtmltopdfSecondsTimeout) {
-        transactionDir = dir;
-        transactionDir.mkdirs();
-        inventoryDir = new File(transactionDir, State.INVENTORY.dirname);
-        inventory = new Snapshots(inventoryDir);
-        archiveDir = new File(transactionDir, State.ARCHIVE.dirname);
-        archive = new Snapshots(archiveDir);
-        wkhtmltopdfTimeout = wkhtmltopdfSecondsTimeout;
-    }
-
-    public static synchronized void migrateIPV6Snapshots() {
-    	executor.shutdown();
-    	try {
-			executor.awaitTermination(10, TimeUnit.SECONDS);
-		} catch (final InterruptedException e) {
-			return;
-		}
-    }
-
-    /**
-     * get the number of entries for each of the transaction states
-     * @return the total number of different documents for each transaction state
-     */
-    public static Map<String, Integer> sizes() {
-        HashMap<String, Integer> m = new HashMap<>();
-        m.put(State.INVENTORY.name(), inventory.size());
-        m.put(State.ARCHIVE.name(), archive.size());
-        return m;
-    }
-
-    public static Revisions getRevisions(final State state, final String urlhash) {
-        switch (state) {
-            case INVENTORY : return inventory.getRevisions(urlhash);
-            case ARCHIVE : return archive.getRevisions(urlhash);
-            default : Revisions a = inventory.getRevisions(urlhash); return a == null ? archive.getRevisions(urlhash) : a;
-        }
-    }
-
-    /**
-     * get a list of <host>.<port> names in the snapshot directory
-     * @return the list of the given state. if the state is ALL or unknown, all lists are combined
-     */
-    public static Set<String> listHosts(final State state) {
-        switch (state) {
-            case INVENTORY : return inventory.listHosts();
-            case ARCHIVE : return archive.listHosts();
-            default : Set<String> a = inventory.listHosts(); a.addAll(archive.listHosts()); return a;
-        }
-    }
-
-    /**
-     * list the snapshots for a given host name
-     * @param hostport the <host>.<port> identifier for the domain (with the same format as applied by the Snapshots.pathToHostPortDir() function).
-     * @param depth restrict the result to the given depth or if depth == -1 do not restrict to a depth
-     * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
-     * @return a map with a set for each depth in the domain of the host name
-     */
-    public static TreeMap<Integer, Collection<Revisions>> listIDs(final String hostport, final int depth, final State state) {
-        switch (state) {
-            case INVENTORY : return inventory.listIDs(hostport, depth);
-            case ARCHIVE : return archive.listIDs(hostport, depth);
-            default : TreeMap<Integer, Collection<Revisions>> a = inventory.listIDs(hostport, depth); a.putAll(archive.listIDs(hostport, depth)); return a;
-        }
-    }
-
-    /**
-     * get the number of snapshots for the given host name
-     * @param hostport the <host>.<port> identifier for the domain
-     * @param depth restrict the result to the given depth or if depth == -1 do not restrict to a depth
-     * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
-     * @return a count, the total number of documents for the domain and depth
-     */
-    public static int listIDsSize(final String hostport, final int depth, final State state) {
-        switch (state) {
-            case INVENTORY : return inventory.listIDsSize(hostport, depth);
-            case ARCHIVE : return archive.listIDsSize(hostport, depth);
-            default : return inventory.listIDsSize(hostport, depth) + archive.listIDsSize(hostport, depth);
-        }
-    }
-
-    public static boolean store(final SolrInputDocument doc, final boolean concurrency, final boolean loadImage, final boolean replaceOld, final String proxy, final String acceptLanguage) {
-
-        // GET METADATA FROM DOC
-        final String urls = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());
-        final Date date = (Date) doc.getFieldValue(CollectionSchema.load_date_dt.getSolrFieldName());
-        final Integer o_depth = (Integer) doc.getFieldValue(CollectionSchema.crawldepth_i.getSolrFieldName()); // may return null
-        final int depth = o_depth == null ? 0 : o_depth.intValue();
-
-        DigestURL url;
-        try {
-            url = new DigestURL(urls);
-        } catch (MalformedURLException e) {
-            ConcurrentLog.logException(e);
-            return false;
-        }
-
-        boolean success = loadImage ? store(url, date, depth, concurrency, replaceOld, proxy, acceptLanguage) : true;
-        if (success) {
-            // STORE METADATA FOR THE IMAGE
-            File metadataPath = Transactions.definePath(url, depth, date, "xml", Transactions.State.INVENTORY);
-            metadataPath.getParentFile().mkdirs();
-            if (doc != null) {
-            	try (
-            		/* Resources automatically closed by this try-with-resources statement */
-                    final FileOutputStream fos = new FileOutputStream(metadataPath);
-                    final OutputStreamWriter osw = new OutputStreamWriter(fos);
-            	) {
-                    osw.write(XML_PREFIX);
-                    osw.write(WHITESPACE); osw.write("\n-->\n"); // placeholder for transaction information properties (a hack to attach metadata to metadata)
-                    osw.write("<result name=\"response\" numFound=\"1\" start=\"0\">\n");
-                    EnhancedXMLResponseWriter.writeDoc(osw, doc);
-                    osw.write("</result>\n");
-                    osw.write("</response>\n");
-            	} catch (IOException e) {
-            		ConcurrentLog.logException(e);
-            		success = false;
-            	}
-            	if(success) {
-            		Transactions.announceStorage(url, depth, date, State.INVENTORY);
-            	}
-            }
-
-        }
-
-        return success;
-    }
-
-
-    public static boolean store(final DigestURL url, final Date date, final int depth, final boolean concurrency, final boolean replaceOld, final String proxy, final String acceptLanguage) {
-
-        // CLEAN UP OLD DATA (if wanted)
-        Collection<File> oldPaths = Transactions.findPaths(url, depth, null, Transactions.State.INVENTORY);
-        if (replaceOld && oldPaths != null) {
-            for (File oldPath: oldPaths) {
-            	oldPath.delete();
-            }
-        }
-
-        // STORE METADATA FOR THE IMAGE
-        File metadataPath = Transactions.definePath(url, depth, date, "xml", Transactions.State.INVENTORY);
-        metadataPath.getParentFile().mkdirs();
-        boolean success = true;
-
-        // STORE AN IMAGE
-        final String urls = url.toNormalform(true);
-        final File pdfPath = Transactions.definePath(url, depth, date, "pdf", Transactions.State.INVENTORY);
-        if (concurrency && executorRunning.intValue() < Runtime.getRuntime().availableProcessors()) {
-            Thread t = new Thread("Transactions.store"){
-                @Override
-                public void run() {
-                    executorRunning.incrementAndGet();
-                    try {
-                        Html2Image.writeWkhtmltopdf(urls, proxy, ClientIdentification.browserAgent.userAgent(), acceptLanguage, pdfPath, wkhtmltopdfTimeout);
-                    } catch (Throwable e) {} finally {
-                    executorRunning.decrementAndGet();
-                    }
-                }
-            };
-            executor.execute(t);
-        } else {
-            success = Html2Image.writeWkhtmltopdf(urls, proxy, ClientIdentification.browserAgent.userAgent(), acceptLanguage, pdfPath, wkhtmltopdfTimeout);
-        }
-
-        return success;
-    }
-
-    /**
-     * Announce the commit of a snapshot: this will move all data for the given urlhash from the inventory to the archive
-     * The index within the snapshot management will update also.
-     * @param urlhash
-     * @return a revision object from the moved document if the commit has succeeded, null if something went wrong
-     */
-    public static Revisions commit(String urlhash) {
-        return transact(urlhash, State.INVENTORY, State.ARCHIVE);
-    }
-
-    /**
-     * Announce the rollback of a snapshot: this will move all data for the given urlhash from the archive to the inventory
-     * The index within the snapshot management will update also.
-     * @param urlhash
-     * @return a revision object from the moved document if the commit has succeeded, null if something went wrong
-     */
-    public static Revisions rollback(String urlhash) {
-        return transact(urlhash, State.ARCHIVE, State.INVENTORY);
-    }
-
-    private static Revisions transact(final String urlhash, final State from, final State to) {
-        Revisions r = Transactions.getRevisions(from, urlhash);
-        if (r == null) return null;
-        // we take all pathtoxml and move that to archive
-        for (File f: r.pathtoxml) {
-            String name = f.getName();
-            String nameStub = name.substring(0, name.length() - 4);
-            File sourceParent = f.getParentFile();
-            File targetParent = new File(sourceParent.getAbsolutePath().replace("/" + from.dirname + "/", "/" + to.dirname + "/"));
-            targetParent.mkdirs();
-            // list all files in the parent directory
-            for (String a: sourceParent.list()) {
-                if (a.startsWith(nameStub)) {
-                    new File(sourceParent, a).renameTo(new File(targetParent, a));
-                }
-            }
-            // delete empty directories
-            while (sourceParent.list().length == 0) {
-                sourceParent.delete();
-                sourceParent = sourceParent.getParentFile();
-            }
-        }
-        // announce the movement
-        DigestURL durl;
-        try {
-            durl = new DigestURL(r.url);
-            Transactions.announceDeletion(durl, r.depth, from);
-            Transactions.announceStorage(durl, r.depth, r.dates[0], to);
-            return r;
-        } catch (MalformedURLException e) {
-            ConcurrentLog.logException(e);
-        }
-
-        return null;
-    }
-
-    /**
-     * select a set of urlhashes from the snapshot directory. The selection either ordered
-     * by generation date (upwards == OLDESTFIRST or downwards == LATESTFIRST) or with any
-     * order. The result set can be selected either with a given host or a depth
-     * @param host selected host or null for all hosts
-     * @param depth selected depth or null for all depths
-     * @param order Order.ANY, Order.OLDESTFIRST or Order.LATESTFIRST
-     * @param maxcount the maximum number of hosthashes. If unlimited, submit Integer.MAX_VALUE
-     * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
-     * @return a map of hosthashes with the associated creation date
-     */
-    public static LinkedHashMap<String, Revisions> select(String host, Integer depth, final Order order, int maxcount, State state) {
-        LinkedHashMap<String, Revisions> result = new LinkedHashMap<>();
-        if (state == State.INVENTORY || state == State.ANY) result.putAll(inventory.select(host, depth, order, maxcount));
-        if (state == State.ARCHIVE || state == State.ANY) result.putAll(archive.select(host, depth, order, maxcount));
-        return result;
-    }
-
-    /**
-     * Compute the path of a snapshot. This does not create the snapshot, only gives a path.
-     * Also, the path to the storage location is not created.
-     * @param url
-     * @param depth
-     * @param date
-     * @param ext
-     * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
-     * @return a file to the snapshot
-     */
-    public static File definePath(final DigestURL url, final int depth, final Date date, final String ext, State state) {
-        if (state == State.ANY) throw new RuntimeException("definePath must be selected with INVENTORY or ARCHIVE state");
-        if (state == State.INVENTORY) return inventory.definePath(url, depth, date, ext);
-        if (state == State.ARCHIVE) return archive.definePath(url, depth, date, ext);
-        return null;
-    }
-
-    /**
-     * Write information about the storage of a snapshot to the Snapshot-internal index.
-     * The actual writing of files to the target directory must be done elsewehre, this method does not store the snapshot files.
-     * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
-     * @param url
-     * @param depth
-     * @param date
-     */
-    public static void announceStorage(final DigestURL url, final int depth, final Date date, State state) {
-        if (state == State.INVENTORY || state == State.ANY) inventory.announceStorage(url, depth, date);
-        if (state == State.ARCHIVE || state == State.ANY) archive.announceStorage(url, depth, date);
-    }
-
-    /**
-     * Delete information about the storage of a snapshot to the Snapshot-internal index.
-     * The actual deletion of files in the target directory must be done elsewehre, this method does not store the snapshot files.
-     * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
-     * @param url
-     * @param depth
-     */
-    public static void announceDeletion(final DigestURL url, final int depth, final State state) {
-        if (state == State.INVENTORY || state == State.ANY) inventory.announceDeletion(url, depth);
-        if (state == State.ARCHIVE || state == State.ANY) archive.announceDeletion(url, depth);
-    }
-
-    /**
-     * for a given url, get all paths for storage locations.
-     * The locations are all for the single url but may represent different storage times.
-     * This method is inefficient because it tests all different depths, it would be better to use
-     * findPaths/3 with a given depth.
-     * @param url
-     * @param ext required extension or null if the extension must not be checked
-     * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
-     * @return a set of files for snapshots of the url
-     */
-    public static Collection<File> findPaths(final DigestURL url, final String ext, State state) {
-        Collection<File> result = new ArrayList<>();
-        if (state == State.INVENTORY || state == State.ANY) result.addAll(inventory.findPaths(url, ext));
-        if (state == State.ARCHIVE || state == State.ANY) result.addAll(archive.findPaths(url, ext));
-        return result;
-    }
-
-    /**
-     * for a given url, get all paths for storage locations.
-     * The locations are all for the single url but may represent different storage times.
-     * @param url
-     * @param ext required extension or null if the extension must not be checked
-     * @param depth
-     * @param state the wanted transaction state, State.INVENTORY, State.ARCHIVE or State.ANY
-     * @return a set of files for snapshots of the url
-     */
-    public static Collection<File> findPaths(final DigestURL url, final int depth, final String ext, State state) {
-        Collection<File> result = new ArrayList<>();
-        if (state == State.INVENTORY || state == State.ANY) result.addAll(inventory.findPaths(url, depth, ext));
-        if (state == State.ARCHIVE || state == State.ANY) result.addAll(archive.findPaths(url, depth, ext));
-        return result;
-    }
-
-}
diff --git a/source/net/yacy/document/parser/psParser.java b/source/net/yacy/document/parser/psParser.java
index 86b4ca4d9..9d98e7f25 100644
--- a/source/net/yacy/document/parser/psParser.java
+++ b/source/net/yacy/document/parser/psParser.java
@@ -33,7 +33,6 @@ import java.io.File;
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
 import java.util.Date;
 
@@ -47,10 +46,6 @@ import net.yacy.kelondro.util.FileUtils;
 
 public class psParser extends AbstractParser implements Parser {
 
-    private final static Object modeScan = new Object();
-    private static boolean modeScanDone = false;
-    private static String parserMode = "java";
-
     public psParser() {
         super("PostScript Document Parser");
         this.SUPPORTED_EXTENSIONS.add("ps");
@@ -59,33 +54,8 @@ public class psParser extends AbstractParser implements Parser {
         this.SUPPORTED_MIME_TYPES.add("application/x-postscript");
         this.SUPPORTED_MIME_TYPES.add("application/x-ps");
         this.SUPPORTED_MIME_TYPES.add("application/x-postscript-not-eps");
-        if (!modeScanDone) synchronized (modeScan) {
-        	if (testForPs2Ascii()) parserMode = "ps2ascii";
-        	else parserMode = "java";
-        	modeScanDone = true;
-		}
     }
 
-    private boolean testForPs2Ascii() {
-        try {
-            String procOutputLine = null;
-            final StringBuilder procOutput = new StringBuilder(80);
-
-            final Process ps2asciiProc = Runtime.getRuntime().exec(new String[]{"ps2ascii", "--version"});
-            final BufferedReader stdOut = new BufferedReader(new InputStreamReader(ps2asciiProc.getInputStream()));
-            while ((procOutputLine = stdOut.readLine()) != null) {
-                procOutput.append(procOutputLine).append(", ");
-            }
-            stdOut.close();
-            final int returnCode = ps2asciiProc.waitFor();
-            return (returnCode == 0);
-        } catch (final Exception e) {
-            if (AbstractParser.log != null) AbstractParser.log.info("ps2ascii not found. Switching to java parser mode.");
-            return false;
-        }
-    }
-
-
     private Document[] parse(final DigestURL location, final String mimeType, @SuppressWarnings("unused") final String charset, final File sourceFile) throws Parser.Failure, InterruptedException {
 
     	File outputFile = null;
@@ -93,12 +63,7 @@ public class psParser extends AbstractParser implements Parser {
         	// creating a temp file for the output
         	outputFile = FileUtils.createTempFile(this.getClass(), "ascii.txt");
 
-        	// decide with parser mode to use
-            if (parserMode.equals("ps2ascii")) {
-                parseUsingPS2ascii(sourceFile,outputFile);
-            } else {
-                parseUsingJava(sourceFile,outputFile);
-            }
+            parseUsingJava(sourceFile,outputFile);
 
             // return result
             final Document[] docs = new Document[]{new Document(
@@ -221,41 +186,6 @@ public class psParser extends AbstractParser implements Parser {
 
     }
 
-    /**
-     * This function requires the ghostscript-library
-     * @param inputFile
-     * @param outputFile
-     * @throws Exception
-     */
-    private void parseUsingPS2ascii(final File inputFile, final File outputFile) throws Exception {
-    	int execCode = 0;
-    	StringBuilder procErr = null;
-    	try {
-            String procOutputLine;
-            final StringBuilder procOut = new StringBuilder();
-            procErr = new StringBuilder();
-
-            final Process ps2asciiProc = Runtime.getRuntime().exec(new String[]{"ps2ascii", inputFile.getAbsolutePath(),outputFile.getAbsolutePath()});
-            final BufferedReader stdOut = new BufferedReader(new InputStreamReader(ps2asciiProc.getInputStream()));
-            final BufferedReader stdErr = new BufferedReader(new InputStreamReader(ps2asciiProc.getErrorStream()));
-            while ((procOutputLine = stdOut.readLine()) != null) {
-                procOut.append(procOutputLine);
-            }
-            stdOut.close();
-            while ((procOutputLine = stdErr.readLine()) != null) {
-                procErr.append(procOutputLine);
-            }
-            stdErr.close();
-            execCode = ps2asciiProc.waitFor();
-    	} catch (final Exception e) {
-            final String errorMsg = "Unable to convert ps to ascii. " + e.getMessage();
-            AbstractParser.log.severe(errorMsg);
-            throw new Exception(errorMsg);
-    	}
-
-    	if (execCode != 0) throw new Exception("Unable to convert ps to ascii. ps2ascii returned statuscode " + execCode + "\n" + procErr.toString());
-    }
-
     @Override
     public Document[] parse(
             final DigestURL location,
diff --git a/source/net/yacy/htroot/ConfigSearchPage_p.java b/source/net/yacy/htroot/ConfigSearchPage_p.java
index 9b60ae6fb..11d70cb24 100644
--- a/source/net/yacy/htroot/ConfigSearchPage_p.java
+++ b/source/net/yacy/htroot/ConfigSearchPage_p.java
@@ -100,7 +100,6 @@ public class ConfigSearchPage_p {
                 sb.setConfig("search.result.show.cache", post.getBoolean("search.result.show.cache"));
                 sb.setConfig("search.result.show.proxy", post.getBoolean("search.result.show.proxy"));
                 sb.setConfig("search.result.show.indexbrowser", post.getBoolean("search.result.show.indexbrowser"));
-                sb.setConfig("search.result.show.snapshots", post.getBoolean("search.result.show.snapshots"));
 
                 // construct navigation String
                 final Set<String> navConfigs = new HashSet<>();
@@ -190,7 +189,6 @@ public class ConfigSearchPage_p {
                 sb.setConfig("search.result.show.cache", config.getProperty("search.result.show.cache","true"));
                 sb.setConfig("search.result.show.proxy", config.getProperty("search.result.show.proxy","false"));
                 sb.setConfig("search.result.show.indexbrowser", config.getProperty("search.result.show.indexbrowser","true"));
-                sb.setConfig("search.result.show.snapshots", config.getProperty("search.result.show.snapshots","true"));
 				sb.setConfig(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,
 						config.getProperty(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,
 								String.valueOf(QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT)));
@@ -254,7 +252,6 @@ public class ConfigSearchPage_p {
         prop.put("search.result.show.cache", sb.getConfigBool("search.result.show.cache", false) ? 1 : 0);
         prop.put("search.result.show.proxy", sb.getConfigBool("search.result.show.proxy", false) ? 1 : 0);
         prop.put("search.result.show.indexbrowser", sb.getConfigBool("search.result.show.indexbrowser", false) ? 1 : 0);
-        prop.put("search.result.show.snapshots", sb.getConfigBool("search.result.show.snapshots", false) ? 1 : 0);
         prop.put("search.result.show.ranking", sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_SHOW_RANKING, SwitchboardConstants.SEARCH_RESULT_SHOW_RANKING_DEFAULT) ? 1 : 0);
 
         final Set<String> navConfigs = sb.getConfigSet("search.navigation");
diff --git a/source/net/yacy/htroot/CrawlStartExpert.java b/source/net/yacy/htroot/CrawlStartExpert.java
index c0d0f3a38..78bf9baaa 100644
--- a/source/net/yacy/htroot/CrawlStartExpert.java
+++ b/source/net/yacy/htroot/CrawlStartExpert.java
@@ -36,7 +36,6 @@ import net.yacy.cora.federate.solr.instance.EmbeddedInstance;
 import net.yacy.cora.lod.vocabulary.Tagging;
 import net.yacy.cora.protocol.ClientIdentification;
 import net.yacy.cora.protocol.RequestHeader;
-import net.yacy.cora.util.Html2Image;
 import net.yacy.crawler.data.CrawlProfile;
 import net.yacy.crawler.data.CrawlProfile.CrawlAttribute;
 import net.yacy.document.LibraryProvider;
@@ -633,18 +632,6 @@ public class CrawlStartExpert {
             prop.put("vocabularySelect_vocabularyset", count);
         }
 
-        // ---------- Snapshot generation
-        final boolean wkhtmltopdfAvailable = Html2Image.wkhtmltopdfAvailable();
-        //boolean convertAvailable = Html2Image.convertAvailable();
-        prop.put("snapshotsMaxDepth", post == null ? "-1" : post.get("snapshotsMaxDepth", "-1"));
-        prop.put("snapshotsMustnotmatch", post == null ? "" : post.get("snapshotsMustnotmatch", ""));
-        if (wkhtmltopdfAvailable) {
-            prop.put("snapshotEnableImages", 1);
-            prop.put("snapshotEnableImages_snapshotsLoadImageChecked", post == null ? 1 : post.getBoolean("snapshotsLoadImage") ? 1 : 0);
-        } else {
-            prop.put("snapshotEnableImages", 0);
-        }
-
         // ---------- Index Administration
         // Do Local Indexing
         if (post == null) {
diff --git a/source/net/yacy/htroot/Crawler_p.java b/source/net/yacy/htroot/Crawler_p.java
index fa94dbc6f..79934c786 100644
--- a/source/net/yacy/htroot/Crawler_p.java
+++ b/source/net/yacy/htroot/Crawler_p.java
@@ -480,12 +480,6 @@ public class Crawler_p {
                     }
                 }
 
-                final String snapshotsMaxDepthString = post.get("snapshotsMaxDepth", "-1");
-                final int snapshotsMaxDepth = Integer.parseInt(snapshotsMaxDepthString);
-                final boolean snapshotsLoadImage = post.getBoolean("snapshotsLoadImage");
-                final boolean snapshotsReplaceOld = post.getBoolean("snapshotsReplaceOld");
-                final String snapshotsMustnotmatch = post.get("snapshotsMustnotmatch", "");
-
                 final String valency_switch_tag_names_s = post.get("valency_switch_tag_names");
                 final Set<String> valency_switch_tag_names = new HashSet<>();
                 if (valency_switch_tag_names_s != null) {
@@ -626,10 +620,6 @@ public class Crawler_p {
                             indexMedia,
                             storeHTCache,
                             crawlOrder,
-                            snapshotsMaxDepth,
-                            snapshotsLoadImage,
-                            snapshotsReplaceOld,
-                            snapshotsMustnotmatch,
                             cachePolicy,
                             collection,
                             agentName,
diff --git a/source/net/yacy/htroot/QuickCrawlLink_p.java b/source/net/yacy/htroot/QuickCrawlLink_p.java
index e78557c98..80c84f102 100644
--- a/source/net/yacy/htroot/QuickCrawlLink_p.java
+++ b/source/net/yacy/htroot/QuickCrawlLink_p.java
@@ -1,204 +1,203 @@
-//QuickCrawlLink_p.java
-//-----------------------
-//part of the AnomicHTTPD caching proxy
-//(C) by Michael Peter Christen; mc@yacy.net
-//first published on http://www.anomic.de
-//Frankfurt, Germany, 2004
-//
-//This file was contributed by Martin Thelian
-//$LastChangedDate$
-//$LastChangedBy$
-//$LastChangedRevision$
-//
-//This program is free software; you can redistribute it and/or modify
-//it under the terms of the GNU General Public License as published by
-//the Free Software Foundation; either version 2 of the License, or
-//(at your option) any later version.
-//
-//This program is distributed in the hope that it will be useful,
-//but WITHOUT ANY WARRANTY; without even the implied warranty of
-//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-//GNU General Public License for more details.
-//
-//You should have received a copy of the GNU General Public License
-//along with this program; if not, write to the Free Software
-//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
-//You must compile this file with
-//javac -classpath .:../classes IndexCreate_p.java
-//if the shell's current path is HTROOT
-
-
-package net.yacy.htroot;
-
-import java.net.MalformedURLException;
-import java.util.Date;
-
-import net.yacy.cora.document.encoding.UTF8;
-import net.yacy.cora.document.id.DigestURL;
-import net.yacy.cora.federate.yacy.CacheStrategy;
-import net.yacy.cora.protocol.ClientIdentification;
-import net.yacy.cora.protocol.Domains;
-import net.yacy.cora.protocol.HeaderFramework;
-import net.yacy.cora.protocol.RequestHeader;
-import net.yacy.crawler.data.CrawlProfile;
-import net.yacy.crawler.retrieval.Request;
-import net.yacy.document.parser.html.TagValency;
-import net.yacy.search.Switchboard;
-import net.yacy.search.SwitchboardConstants;
-import net.yacy.search.index.Segment;
-import net.yacy.server.serverObjects;
-import net.yacy.server.serverSwitch;
-
-public class QuickCrawlLink_p {
-
-    /**
-     * Example Javascript to call this servlet:
-     * <code>javascript:w = window.open('http://user:pwd@localhost:8090/QuickCrawlLink_p.html?indexText=on&indexMedia=on&crawlingQ=on&xdstopw=on&title=' + escape(document.title) + '&url=' + location.href,'_blank','height=150,width=500,resizable=yes,scrollbar=no,directory=no,menubar=no,location=no'); w.focus();</code>
-     * @param header the complete HTTP header of the request
-     * @param post any arguments for this servlet, the request carried with (GET as well as POST)
-     * @param env the serverSwitch object holding all runtime-data
-     * @return the rewrite-properties for the template
-     */
-    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
-
-        final serverObjects prop = new serverObjects();
-        final Switchboard sb = (Switchboard) env;
-
-        int port;
-
-        // get the http host header
-        if (header.containsKey(HeaderFramework.HOST)) {
-            port = header.getServerPort();
-        } else {
-            port = sb.getConfigInt(SwitchboardConstants.SERVER_PORT, 8090);
-        }
-
-        prop.put("mode_host", Domains.LOCALHOST);
-        prop.put("mode_port", port);
-
-        if (post == null) {
-            // send back usage example
-            prop.put("mode", "0");
-            return prop;
-        }
-
-        // get the URL
-        String crawlingStart = post.get("url", "");
-
-        if (crawlingStart.length() != 0) {
-            prop.put("mode", "1");
-            crawlingStart = UTF8.decodeURL(crawlingStart);
-
-             // get segment
-            final Segment indexSegment = sb.index;
-
-            // get the browser title
-            String title = post.get("title", "");
+//QuickCrawlLink_p.java
+//-----------------------
+//part of the AnomicHTTPD caching proxy
+//(C) by Michael Peter Christen; mc@yacy.net
+//first published on http://www.anomic.de
+//Frankfurt, Germany, 2004
+//
+//This file was contributed by Martin Thelian
+//$LastChangedDate$
+//$LastChangedBy$
+//$LastChangedRevision$
+//
+//This program is free software; you can redistribute it and/or modify
+//it under the terms of the GNU General Public License as published by
+//the Free Software Foundation; either version 2 of the License, or
+//(at your option) any later version.
+//
+//This program is distributed in the hope that it will be useful,
+//but WITHOUT ANY WARRANTY; without even the implied warranty of
+//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+//GNU General Public License for more details.
+//
+//You should have received a copy of the GNU General Public License
+//along with this program; if not, write to the Free Software
+//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+//You must compile this file with
+//javac -classpath .:../classes IndexCreate_p.java
+//if the shell's current path is HTROOT
+
+
+package net.yacy.htroot;
+
+import java.net.MalformedURLException;
+import java.util.Date;
+
+import net.yacy.cora.document.encoding.UTF8;
+import net.yacy.cora.document.id.DigestURL;
+import net.yacy.cora.federate.yacy.CacheStrategy;
+import net.yacy.cora.protocol.ClientIdentification;
+import net.yacy.cora.protocol.Domains;
+import net.yacy.cora.protocol.HeaderFramework;
+import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.crawler.data.CrawlProfile;
+import net.yacy.crawler.retrieval.Request;
+import net.yacy.document.parser.html.TagValency;
+import net.yacy.search.Switchboard;
+import net.yacy.search.SwitchboardConstants;
+import net.yacy.search.index.Segment;
+import net.yacy.server.serverObjects;
+import net.yacy.server.serverSwitch;
+
+public class QuickCrawlLink_p {
+
+    /**
+     * Example Javascript to call this servlet:
+     * <code>javascript:w = window.open('http://user:pwd@localhost:8090/QuickCrawlLink_p.html?indexText=on&indexMedia=on&crawlingQ=on&xdstopw=on&title=' + escape(document.title) + '&url=' + location.href,'_blank','height=150,width=500,resizable=yes,scrollbar=no,directory=no,menubar=no,location=no'); w.focus();</code>
+     * @param header the complete HTTP header of the request
+     * @param post any arguments for this servlet, the request carried with (GET as well as POST)
+     * @param env the serverSwitch object holding all runtime-data
+     * @return the rewrite-properties for the template
+     */
+    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
+
+        final serverObjects prop = new serverObjects();
+        final Switchboard sb = (Switchboard) env;
+
+        int port;
+
+        // get the http host header
+        if (header.containsKey(HeaderFramework.HOST)) {
+            port = header.getServerPort();
+        } else {
+            port = sb.getConfigInt(SwitchboardConstants.SERVER_PORT, 8090);
+        }
+
+        prop.put("mode_host", Domains.LOCALHOST);
+        prop.put("mode_port", port);
+
+        if (post == null) {
+            // send back usage example
+            prop.put("mode", "0");
+            return prop;
+        }
+
+        // get the URL
+        String crawlingStart = post.get("url", "");
+
+        if (crawlingStart.length() != 0) {
+            prop.put("mode", "1");
+            crawlingStart = UTF8.decodeURL(crawlingStart);
+
+             // get segment
+            final Segment indexSegment = sb.index;
+
+            // get the browser title
+            String title = post.get("title", "");
             if (title.length() != 0) {
-                title = crawlingStart;
-            	/* Decode eventual special(non ASCII) characters in title */
-            	title = UTF8.decodeURL(title);
-            }
-
-            // get other parameters if set
-            final String crawlingMustMatch = post.get("mustmatch", CrawlProfile.MATCH_ALL_STRING);
-            final String crawlingMustNotMatch = post.get("mustnotmatch", CrawlProfile.MATCH_NEVER_STRING);
-            final int CrawlingDepth = post.getInt("crawlingDepth", 0);
-            final boolean crawlingQ = post.get("crawlingQ", "").equals("on");
-            final boolean followFrames = post.get("followFrames", "").equals("on");
-            final boolean obeyHtmlRobotsNoindex = post.get("obeyHtmlRobotsNoindex", "").equals("on");
-            final boolean obeyHtmlRobotsNofollow = post.get("obeyHtmlRobotsNofollow", "").equals("on");
-            final boolean indexText = post.get("indexText", "off").equals("on");
-            final boolean indexMedia = post.get("indexMedia", "off").equals("on");
-            final boolean storeHTCache = post.get("storeHTCache", "").equals("on");
-            final boolean remoteIndexing = post.get("crawlOrder", "").equals("on");
-            final String collection = post.get("collection", "user");
-
-            prop.put("mode_url", (crawlingStart == null) ? "unknown" : crawlingStart);
-            prop.putHTML("mode_title", (title == null) ? "unknown" : title);
-
-            crawlingStart = crawlingStart.trim();
-            try {crawlingStart = new DigestURL(crawlingStart).toNormalform(true);} catch (final MalformedURLException e1) {}
-
-            // check if url is proper
-            DigestURL crawlingStartURL = null;
-            try {
-                crawlingStartURL = new DigestURL(crawlingStart);
-            } catch (final MalformedURLException e) {
-                prop.put("mode_status", "1");
-                prop.put("mode_code", "1");
-                return prop;
-            }
-
-            final byte[] urlhash = crawlingStartURL.hash();
-            indexSegment.fulltext().remove(urlhash);
-            sb.crawlQueues.noticeURL.removeByURLHash(urlhash);
-            final int timezoneOffset = post.getInt("timezoneOffset", 0);
-
-            // create crawling profile
-            CrawlProfile pe = null;
-            try {
-                pe = new CrawlProfile(
-                        (crawlingStartURL.getHost() == null) ? crawlingStartURL.toNormalform(true) : crawlingStartURL.getHost(),
-                        crawlingMustMatch,               //crawlerUrlMustMatch
-                        crawlingMustNotMatch,            //crawlerUrlMustNotMatch
-                        CrawlProfile.MATCH_ALL_STRING,   //crawlerIpMustMatch
-                        CrawlProfile.MATCH_NEVER_STRING, //crawlerIpMustNotMatch
-                        CrawlProfile.MATCH_NEVER_STRING, //crawlerCountryMustMatch
-                        CrawlProfile.MATCH_NEVER_STRING, //crawlerNoDepthLimitMatch
-                        CrawlProfile.MATCH_ALL_STRING,   //indexUrlMustMatch
-                        CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatch
-                        CrawlProfile.MATCH_ALL_STRING,   //indexContentMustMatch
-                        CrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch
-                        false,
-                        CrawlingDepth,
-                        true,
-                        CrawlProfile.getRecrawlDate(60 * 24 * 30), // recrawlIfOlder (minutes); here: one month
-                        -1, // domMaxPages, if negative: no count restriction
-                        crawlingQ, followFrames,
-                        obeyHtmlRobotsNoindex, obeyHtmlRobotsNofollow,
-                        indexText, indexMedia,
-                        storeHTCache, remoteIndexing,
-                        -1, false, true, CrawlProfile.MATCH_NEVER_STRING,
-                        CacheStrategy.IFFRESH,
-                        collection,
-                        ClientIdentification.yacyIntranetCrawlerAgentName,
-                        TagValency.EVAL, null, null,
-                        timezoneOffset);
-                sb.crawler.putActive(pe.handle().getBytes(), pe);
-            } catch (final Exception e) {
-                // mist
-                prop.put("mode_status", "2");//Error with url
-                prop.put("mode_code", "2");
-                prop.putHTML("mode_status_error", e.getMessage());
-                return prop;
-            }
-
-            // stack URL
-            String reasonString;
-            reasonString = sb.crawlStacker.stackCrawl(new Request(
-                    sb.peers.mySeed().hash.getBytes(),
-                    crawlingStartURL,
-                    null,
-                    (title==null)?"CRAWLING-ROOT":title,
-                    new Date(),
-                    pe.handle(),
-                    0,
-                    pe.timezoneOffset()
-                ));
-
-            // validate rejection reason
-            if (reasonString == null) {
-                prop.put("mode_status", "0");//start msg
-                prop.put("mode_code", "0");
-            } else {
-                prop.put("mode_status", "3");//start msg
-                prop.put("mode_code","3");
-                prop.putHTML("mode_status_error", reasonString);
-            }
-        }
-
-        return prop;
-    }
-}
+                title = crawlingStart;
+            	/* Decode eventual special(non ASCII) characters in title */
+            	title = UTF8.decodeURL(title);
+            }
+
+            // get other parameters if set
+            final String crawlingMustMatch = post.get("mustmatch", CrawlProfile.MATCH_ALL_STRING);
+            final String crawlingMustNotMatch = post.get("mustnotmatch", CrawlProfile.MATCH_NEVER_STRING);
+            final int CrawlingDepth = post.getInt("crawlingDepth", 0);
+            final boolean crawlingQ = post.get("crawlingQ", "").equals("on");
+            final boolean followFrames = post.get("followFrames", "").equals("on");
+            final boolean obeyHtmlRobotsNoindex = post.get("obeyHtmlRobotsNoindex", "").equals("on");
+            final boolean obeyHtmlRobotsNofollow = post.get("obeyHtmlRobotsNofollow", "").equals("on");
+            final boolean indexText = post.get("indexText", "off").equals("on");
+            final boolean indexMedia = post.get("indexMedia", "off").equals("on");
+            final boolean storeHTCache = post.get("storeHTCache", "").equals("on");
+            final boolean remoteIndexing = post.get("crawlOrder", "").equals("on");
+            final String collection = post.get("collection", "user");
+
+            prop.put("mode_url", (crawlingStart == null) ? "unknown" : crawlingStart);
+            prop.putHTML("mode_title", (title == null) ? "unknown" : title);
+
+            crawlingStart = crawlingStart.trim();
+            try {crawlingStart = new DigestURL(crawlingStart).toNormalform(true);} catch (final MalformedURLException e1) {}
+
+            // check if url is proper
+            DigestURL crawlingStartURL = null;
+            try {
+                crawlingStartURL = new DigestURL(crawlingStart);
+            } catch (final MalformedURLException e) {
+                prop.put("mode_status", "1");
+                prop.put("mode_code", "1");
+                return prop;
+            }
+
+            final byte[] urlhash = crawlingStartURL.hash();
+            indexSegment.fulltext().remove(urlhash);
+            sb.crawlQueues.noticeURL.removeByURLHash(urlhash);
+            final int timezoneOffset = post.getInt("timezoneOffset", 0);
+
+            // create crawling profile
+            CrawlProfile pe = null;
+            try {
+                pe = new CrawlProfile(
+                        (crawlingStartURL.getHost() == null) ? crawlingStartURL.toNormalform(true) : crawlingStartURL.getHost(),
+                        crawlingMustMatch,               //crawlerUrlMustMatch
+                        crawlingMustNotMatch,            //crawlerUrlMustNotMatch
+                        CrawlProfile.MATCH_ALL_STRING,   //crawlerIpMustMatch
+                        CrawlProfile.MATCH_NEVER_STRING, //crawlerIpMustNotMatch
+                        CrawlProfile.MATCH_NEVER_STRING, //crawlerCountryMustMatch
+                        CrawlProfile.MATCH_NEVER_STRING, //crawlerNoDepthLimitMatch
+                        CrawlProfile.MATCH_ALL_STRING,   //indexUrlMustMatch
+                        CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatch
+                        CrawlProfile.MATCH_ALL_STRING,   //indexContentMustMatch
+                        CrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch
+                        false,
+                        CrawlingDepth,
+                        true,
+                        CrawlProfile.getRecrawlDate(60 * 24 * 30), // recrawlIfOlder (minutes); here: one month
+                        -1, // domMaxPages, if negative: no count restriction
+                        crawlingQ, followFrames,
+                        obeyHtmlRobotsNoindex, obeyHtmlRobotsNofollow,
+                        indexText, indexMedia,
+                        storeHTCache, remoteIndexing,
+                        CacheStrategy.IFFRESH,
+                        collection,
+                        ClientIdentification.yacyIntranetCrawlerAgentName,
+                        TagValency.EVAL, null, null,
+                        timezoneOffset);
+                sb.crawler.putActive(pe.handle().getBytes(), pe);
+            } catch (final Exception e) {
+                // mist
+                prop.put("mode_status", "2");//Error with url
+                prop.put("mode_code", "2");
+                prop.putHTML("mode_status_error", e.getMessage());
+                return prop;
+            }
+
+            // stack URL
+            String reasonString;
+            reasonString = sb.crawlStacker.stackCrawl(new Request(
+                    sb.peers.mySeed().hash.getBytes(),
+                    crawlingStartURL,
+                    null,
+                    (title==null)?"CRAWLING-ROOT":title,
+                    new Date(),
+                    pe.handle(),
+                    0,
+                    pe.timezoneOffset()
+                ));
+
+            // validate rejection reason
+            if (reasonString == null) {
+                prop.put("mode_status", "0");//start msg
+                prop.put("mode_code", "0");
+            } else {
+                prop.put("mode_status", "3");//start msg
+                prop.put("mode_code","3");
+                prop.putHTML("mode_status_error", reasonString);
+            }
+        }
+
+        return prop;
+    }
+}
diff --git a/source/net/yacy/htroot/api/snapshot.java b/source/net/yacy/htroot/api/snapshot.java
deleted file mode 100644
index 9b69e7751..000000000
--- a/source/net/yacy/htroot/api/snapshot.java
+++ /dev/null
@@ -1,382 +0,0 @@
-/**
- *  snapshot
- *  Copyright 2014 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
- *  First released 02.12.2014 at https://yacy.net
- *
- *  This library is free software; you can redistribute it and/or
- *  modify it under the terms of the GNU Lesser General Public
- *  License as published by the Free Software Foundation; either
- *  version 2.1 of the License, or (at your option) any later version.
- *
- *  This library is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  Lesser General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public License
- *  along with this program in the file lgpl21.txt
- *  If not, see <http://www.gnu.org/licenses/>.
- */
-
-package net.yacy.htroot.api;
-
-import java.awt.Container;
-import java.awt.Image;
-import java.awt.MediaTracker;
-import java.awt.image.BufferedImage;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.util.Collection;
-import java.util.Date;
-import java.util.Map;
-import java.util.TreeMap;
-
-import org.apache.http.HttpStatus;
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrInputDocument;
-import org.json.JSONException;
-import org.json.JSONObject;
-
-import net.yacy.cora.document.encoding.ASCII;
-import net.yacy.cora.document.encoding.UTF8;
-import net.yacy.cora.document.feed.RSSFeed;
-import net.yacy.cora.document.feed.RSSMessage;
-import net.yacy.cora.document.id.DigestURL;
-import net.yacy.cora.protocol.HeaderFramework;
-import net.yacy.cora.protocol.RequestHeader;
-import net.yacy.cora.util.ConcurrentLog;
-import net.yacy.cora.util.Html2Image;
-import net.yacy.crawler.data.Snapshots;
-import net.yacy.crawler.data.Snapshots.Revisions;
-import net.yacy.crawler.data.Transactions;
-import net.yacy.document.ImageParser;
-import net.yacy.http.servlets.TemplateMissingParameterException;
-import net.yacy.http.servlets.TemplateProcessingException;
-import net.yacy.kelondro.util.FileUtils;
-import net.yacy.peers.graphics.EncodedImage;
-import net.yacy.search.Switchboard;
-import net.yacy.search.SwitchboardConstants;
-import net.yacy.server.serverObjects;
-import net.yacy.server.serverSwitch;
-
-public class snapshot {
-
-    //width = 1024, height = 1024, density = 300, quality = 75
-    private final static int DEFAULT_WIDTH = 1024;
-    private final static int DEFAULT_HEIGHT = 1024;
-    private final static int DEFAULT_DENSITY = 300;
-    private final static int DEFAULT_QUALITY = 75;
-    private final static String DEFAULT_EXT = "jpg";
-
-    public static Object respond(final RequestHeader header, serverObjects post, final serverSwitch env) {
-        final Switchboard sb = (Switchboard) env;
-
-    	final serverObjects defaultResponse = new serverObjects();
-
-
-        final boolean authenticated = sb.adminAuthenticated(header) >= 2;
-        final String ext = header.get(HeaderFramework.CONNECTION_PROP_EXT, "");
-
-        if(ext.isEmpty()) {
-			throw new TemplateProcessingException("Missing extension. Try with rss, xml, json, pdf, png or jpg." + ext,
-					HttpStatus.SC_BAD_REQUEST);
-        }
-
-
-        if (ext.equals("rss")) {
-            // create a report about the content of the snapshot directory
-            if (!authenticated) {
-            	defaultResponse.authenticationRequired();
-            	return defaultResponse;
-            }
-            final int maxcount = post == null ? 10 : post.getInt("maxcount", 10);
-            final int depthx = post == null ? -1 : post.getInt("depth", -1);
-            final Integer depth = depthx == -1 ? null : depthx;
-            final String orderx = post == null ? "ANY" : post.get("order", "ANY");
-            final Snapshots.Order order = Snapshots.Order.valueOf(orderx);
-            final String statex = post == null ? Transactions.State.INVENTORY.name() : post.get("state", Transactions.State.INVENTORY.name());
-            final Transactions.State state = Transactions.State.valueOf(statex);
-            final String host = post == null ? null : post.get("host");
-            final Map<String, Revisions> iddate = Transactions.select(host, depth, order, maxcount, state);
-            // now select the URL from the index for these ids in iddate and make an RSS feed
-            final RSSFeed rssfeed = new RSSFeed(Integer.MAX_VALUE);
-            rssfeed.setChannel(new RSSMessage("Snapshot list for host = " + host + ", depth = " + depth + ", order = " + order + ", maxcount = " + maxcount, "", ""));
-            for (final Map.Entry<String, Revisions> e: iddate.entrySet()) {
-                try {
-                    final String u = e.getValue().url == null ? sb.index.fulltext().getURL(e.getKey()) : e.getValue().url;
-                    if (u == null) continue;
-                    final RSSMessage message = new RSSMessage(u, "", new DigestURL(u), e.getKey());
-                    message.setPubDate(e.getValue().dates[0]);
-                    rssfeed.addMessage(message);
-                } catch (final IOException ee) {
-                    ConcurrentLog.logException(ee);
-                }
-            }
-            final byte[] rssBinary = UTF8.getBytes(rssfeed.toString());
-            return new ByteArrayInputStream(rssBinary);
-        }
-
-        // for the following methods we (mostly) need an url or a url hash
-        if (post == null) post = new serverObjects();
-        final boolean xml = ext.equals("xml");
-        final boolean pdf = ext.equals("pdf");
-        if (pdf && !authenticated) {
-        	defaultResponse.authenticationRequired();
-        	return defaultResponse;
-        }
-        final boolean pngjpg = ext.equals("png") || ext.equals(DEFAULT_EXT);
-        String urlhash = post.get("urlhash", "");
-        final String url = post.get("url", "");
-        DigestURL durl = null;
-        if (urlhash.length() == 0 && url.length() > 0) {
-            try {
-                durl = new DigestURL(url);
-                urlhash = ASCII.String(durl.hash());
-            } catch (final MalformedURLException e) {
-            }
-        }
-        if (durl == null && urlhash.length() > 0) {
-            try {
-                final String u = sb.index.fulltext().getURL(urlhash);
-                durl = u == null ? null : new DigestURL(u);
-            } catch (final IOException e) {
-                ConcurrentLog.logException(e);
-            }
-        }
-
-        if (ext.equals("json")) {
-            // command interface: view and change a transaction state, get metadata about transactions in the past
-            final String command = post.get("command", "metadata");
-            final String statename = post.get("state");
-            final JSONObject result = new JSONObject();
-            try {
-                if (command.equals("status")) {
-                    // return a status of the transaction archive
-                    final JSONObject sizes = new JSONObject();
-                    for (final Map.Entry<String, Integer> state: Transactions.sizes().entrySet()) sizes.put(state.getKey(), state.getValue());
-                    result.put("size", sizes);
-                } else if (command.equals("list")) {
-                    if (!authenticated) {
-                    	defaultResponse.authenticationRequired();
-                    	return defaultResponse;
-                    }
-                    // return a status of the transaction archive
-                    final String host = post.get("host");
-                    final String depth = post.get("depth");
-                    final int depthi = depth == null ? -1 : Integer.parseInt(depth);
-                    for (final Transactions.State state: statename == null ?
-                            new Transactions.State[]{Transactions.State.INVENTORY, Transactions.State.ARCHIVE} :
-                            new Transactions.State[]{Transactions.State.valueOf(statename)}) {
-                        if (host == null) {
-                            final JSONObject hostCountInventory = new JSONObject();
-                            for (final String h: Transactions.listHosts(state)) {
-                                final int size = Transactions.listIDsSize(h, depthi, state);
-                                if (size > 0) hostCountInventory.put(h, size);
-                            }
-                            result.put("count." + state.name(), hostCountInventory);
-                        } else {
-                            final TreeMap<Integer, Collection<Revisions>> ids = Transactions.listIDs(host, depthi, state);
-                            if (ids == null) {
-                                result.put("result", "fail");
-                                result.put("comment", "no entries for host " + host + " found");
-                            } else {
-                                for (final Map.Entry<Integer, Collection<Revisions>> entry: ids.entrySet()) {
-                                    for (final Revisions r: entry.getValue()) {
-                                        try {
-                                            final JSONObject metadata = new JSONObject();
-                                            final String u = r.url != null ? r.url : sb.index.fulltext().getURL(r.urlhash);
-                                            metadata.put("url", u == null ? "unknown" : u);
-                                            metadata.put("dates", r.dates);
-                                            assert r.depth == entry.getKey().intValue();
-                                            metadata.put("depth", entry.getKey().intValue());
-                                            result.put(r.urlhash, metadata);
-                                        } catch (final IOException e) {}
-                                    }
-                                }
-                            }
-                        }
-                    }
-                } else if (command.equals("commit")) {
-                    if (!authenticated) {
-                    	defaultResponse.authenticationRequired();
-                    	return defaultResponse;
-                    }
-                    final Revisions r = Transactions.commit(urlhash);
-                    if (r != null) {
-                        result.put("result", "success");
-                        result.put("depth", r.depth);
-                        result.put("url", r.url);
-                        result.put("dates", r.dates);
-                    } else {
-                        result.put("result", "fail");
-                    }
-                    result.put("urlhash", urlhash);
-                } else if (command.equals("rollback")) {
-                    if (!authenticated) {
-                    	defaultResponse.authenticationRequired();
-                    	return defaultResponse;
-                    }
-                    final Revisions r = Transactions.rollback(urlhash);
-                    if (r != null) {
-                        result.put("result", "success");
-                        result.put("depth", r.depth);
-                        result.put("url", r.url);
-                        result.put("dates", r.dates);
-                    } else {
-                        result.put("result", "fail");
-                    }
-                    result.put("urlhash", urlhash);
-                } else if (command.equals("metadata")) {
-                    try {
-                        Revisions r;
-                        Transactions.State state = statename == null || statename.length() == 0 ? null : Transactions.State.valueOf(statename);
-                        if (state == null) {
-                            r = Transactions.getRevisions(Transactions.State.INVENTORY, urlhash);
-                            if (r != null) state = Transactions.State.INVENTORY;
-                            r = Transactions.getRevisions(Transactions.State.ARCHIVE, urlhash);
-                            if (r != null) state = Transactions.State.ARCHIVE;
-                        } else {
-                            r = Transactions.getRevisions(state, urlhash);
-                        }
-                        if (r != null) {
-                            final JSONObject metadata = new JSONObject();
-                            final String u = r.url != null ? r.url : sb.index.fulltext().getURL(r.urlhash);
-                            metadata.put("url", u == null ? "unknown" : u);
-                            metadata.put("dates", r.dates);
-                            metadata.put("depth", r.depth);
-                            metadata.put("state", state.name());
-                            result.put(r.urlhash, metadata);
-                        }
-                    } catch (IOException |IllegalArgumentException e) {}
-                }
-            } catch (final JSONException e) {
-                ConcurrentLog.logException(e);
-            }
-            String json = result.toString();
-            if (post.containsKey("callback")) json = post.get("callback") + "([" + json + "]);";
-            return new ByteArrayInputStream(UTF8.getBytes(json));
-        }
-
-        // for the following methods we always need the durl to fetch data
-        if (durl == null) {
-        	throw new TemplateMissingParameterException("Missing valid url or urlhash parameter");
-        }
-
-        if (xml) {
-            final Collection<File> xmlSnapshots = Transactions.findPaths(durl, "xml", Transactions.State.ANY);
-            File xmlFile = null;
-            if (xmlSnapshots.isEmpty()) {
-				throw new TemplateProcessingException("Could not find the xml snapshot file.", HttpStatus.SC_NOT_FOUND);
-            }
-            xmlFile = xmlSnapshots.iterator().next();
-            try {
-                final byte[] xmlBinary = FileUtils.read(xmlFile);
-                return new ByteArrayInputStream(xmlBinary);
-            } catch (final IOException e) {
-                ConcurrentLog.logException(e);
-                throw new TemplateProcessingException("Could not read the xml snapshot file.");
-            }
-        }
-
-        if (pdf || pngjpg) {
-            Collection<File> pdfSnapshots = Transactions.findPaths(durl, "pdf", Transactions.State.INVENTORY);
-            File pdfFile = null;
-            if (pdfSnapshots.isEmpty()) {
-                // if the client is authenticated, we create the pdf on the fly!
-                if (!authenticated) {
-					throw new TemplateProcessingException(
-							"Could not find the pdf snapshot file. You must be authenticated to generate one on the fly.",
-							HttpStatus.SC_NOT_FOUND);
-                }
-                final SolrDocument sd = sb.index.fulltext().getMetadata(durl.hash());
-                boolean success = false;
-                if (sd == null) {
-                    success = Transactions.store(durl, new Date(), 99, false, true, sb.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false) ? "http://127.0.0.1:" + sb.getConfigInt(SwitchboardConstants.SERVER_PORT, 8090) : null, sb.getConfig("crawler.http.acceptLanguage", null));
-                } else {
-                    final SolrInputDocument sid = sb.index.fulltext().getDefaultConfiguration().toSolrInputDocument(sd);
-                    success = Transactions.store(sid, false, true, true, sb.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false) ? "http://127.0.0.1:" + sb.getConfigInt(SwitchboardConstants.SERVER_PORT, 8090) : null, sb.getConfig("crawler.http.acceptLanguage", null));
-                }
-                if (success) {
-                    pdfSnapshots = Transactions.findPaths(durl, "pdf", Transactions.State.ANY);
-                    if (!pdfSnapshots.isEmpty()) {
-                    	pdfFile = pdfSnapshots.iterator().next();
-                    }
-                }
-            } else {
-                pdfFile = pdfSnapshots.iterator().next();
-            }
-            if (pdfFile == null) {
-				throw new TemplateProcessingException(
-						"Could not find the pdf snapshot file and could not generate one on the fly.",
-						HttpStatus.SC_NOT_FOUND);
-            }
-            if (pdf) {
-                try {
-                    final byte[] pdfBinary = FileUtils.read(pdfFile);
-                    return new ByteArrayInputStream(pdfBinary);
-                } catch (final IOException e) {
-                    ConcurrentLog.logException(e);
-					throw new TemplateProcessingException("Could not read the pdf snapshot file.");
-                }
-            }
-
-            if (pngjpg) {
-                final int width = Math.min(post.getInt("width", DEFAULT_WIDTH), DEFAULT_WIDTH);
-                final int height = Math.min(post.getInt("height", DEFAULT_HEIGHT), DEFAULT_HEIGHT);
-                String imageFileStub = pdfFile.getAbsolutePath(); imageFileStub = imageFileStub.substring(0, imageFileStub.length() - 3); // cut off extension
-                final File imageFile = new File(imageFileStub + DEFAULT_WIDTH + "." + DEFAULT_HEIGHT + "." + ext);
-                if (!imageFile.exists() && authenticated) {
-                    if(!Html2Image.pdf2image(pdfFile, imageFile, DEFAULT_WIDTH, DEFAULT_HEIGHT, DEFAULT_DENSITY, DEFAULT_QUALITY)) {
-						throw new TemplateProcessingException(
-								"Could not generate the " + ext + " image snapshot file.");
-                    }
-                }
-                if (!imageFile.exists()) {
-					throw new TemplateProcessingException(
-							"Could not find the " + ext
-									+ " image snapshot file. You must be authenticated to generate one on the fly.",
-							HttpStatus.SC_NOT_FOUND);
-                }
-                if (width == DEFAULT_WIDTH && height == DEFAULT_HEIGHT) {
-                    try {
-                        final byte[] imageBinary = FileUtils.read(imageFile);
-                        return new ByteArrayInputStream(imageBinary);
-                    } catch (final IOException e) {
-                        ConcurrentLog.logException(e);
-						throw new TemplateProcessingException("Could not read the " + ext + " image snapshot file.");
-                    }
-                }
-                // lets read the file and scale
-                Image image;
-                try {
-                    image = ImageParser.parse(imageFile.getAbsolutePath(), FileUtils.read(imageFile));
-                    if(image == null) {
-                    	throw new TemplateProcessingException("Could not parse the " + ext + " image snapshot file.");
-                    }
-                    final Image scaled = image.getScaledInstance(width, height, Image.SCALE_AREA_AVERAGING);
-                    final MediaTracker mediaTracker = new MediaTracker(new Container());
-                    mediaTracker.addImage(scaled, 0);
-                    try {mediaTracker.waitForID(0);} catch (final InterruptedException e) {}
-
-					/*
-					 * Ensure there is no alpha component on the ouput image, as it is pointless
-					 * here and it is not well supported by the JPEGImageWriter from OpenJDK
-					 */
-					final BufferedImage scaledBufferedImg = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
-                    scaledBufferedImg.createGraphics().drawImage(scaled, 0, 0, width, height, null);
-                    return new EncodedImage(scaledBufferedImg, ext, true);
-                } catch (final IOException e) {
-                    ConcurrentLog.logException(e);
-					throw new TemplateProcessingException("Could not scale the " + ext + " image snapshot file.");
-                }
-
-            }
-        }
-
-		throw new TemplateProcessingException(
-				"Unsupported extension : " + ext + ". Try with rss, xml, json, pdf, png or jpg.",
-				HttpStatus.SC_BAD_REQUEST);
-    }
-}
diff --git a/source/net/yacy/htroot/yacysearchitem.java b/source/net/yacy/htroot/yacysearchitem.java
index 28a7ecac9..a85679f0a 100644
--- a/source/net/yacy/htroot/yacysearchitem.java
+++ b/source/net/yacy/htroot/yacysearchitem.java
@@ -27,7 +27,6 @@
 package net.yacy.htroot;
 
 import java.awt.Dimension;
-import java.io.File;
 import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URLEncoder;
@@ -36,7 +35,6 @@ import java.util.Collection;
 import java.util.Comparator;
 import java.util.Date;
 import java.util.Iterator;
-import java.util.Locale;
 import java.util.Set;
 import java.util.StringTokenizer;
 import java.util.TreeSet;
@@ -55,8 +53,6 @@ import net.yacy.cora.protocol.RequestHeader.FileType;
 import net.yacy.cora.util.ConcurrentLog;
 import net.yacy.cora.util.Memory;
 import net.yacy.crawler.data.Cache;
-import net.yacy.crawler.data.Transactions;
-import net.yacy.crawler.data.Transactions.State;
 import net.yacy.crawler.retrieval.Response;
 import net.yacy.data.URLLicense;
 import net.yacy.data.UserDB;
@@ -291,7 +287,6 @@ public class yacysearchitem {
             final Date[] events = result.events();
             final boolean showEvent = events != null && events.length > 0 && sb.getConfig("search.navigation", "").indexOf("date",0) >= 0;
             prop.put("content_showEvent", showEvent ? 1 : 0);
-            final Collection<File> snapshotPaths = sb.getConfigBool("search.result.show.snapshots", true) ? Transactions.findPaths(result.url(), null, State.ANY) : null;
             if (fileType == FileType.HTML) { // html template specific settings
 				final boolean showKeywords = (sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS,
 						SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS_DEFAULT) && !result.dc_subject().isEmpty());
@@ -305,7 +300,6 @@ public class yacysearchitem {
                 prop.put("content_showCache", sb.getConfigBool("search.result.show.cache", true) && Cache.has(resultURL.hash()) ? 1 : 0);
                 prop.put("content_showProxy", sb.getConfigBool("search.result.show.proxy", true) && sb.getConfigBool("proxyURL", false) ? 1 : 0);
                 prop.put("content_showIndexBrowser", sb.getConfigBool("search.result.show.indexbrowser", true) ? 1 : 0);
-                prop.put("content_showSnapshots", snapshotPaths != null && snapshotPaths.size() > 0 && sb.getConfigBool("search.result.show.snapshots", true) ? 1 : 0);
                 prop.put("content_showVocabulary", sb.getConfigBool("search.result.show.vocabulary", true) ? 1 : 0);
                 prop.put("content_showRanking", sb.getConfigBool("search.result.show.ranking", false) ? 1 : 0);
 
@@ -376,29 +370,6 @@ public class yacysearchitem {
                     prop.put("content_showVocabulary_vocabulary", 0);
                     prop.put("content_showVocabulary", 0);
                 }
-                if (snapshotPaths != null && snapshotPaths.size() > 0) {
-            		/* Only add a link to the eventual snapshot file in the format it is stored (no resource fetching and conversion here) */
-                	String selectedExt = null, ext;
-                	for(final File snapshot : snapshotPaths) {
-                		ext = MultiProtocolURL.getFileExtension(snapshot.getName());
-                		if("jpg".equals(ext) || "png".equals(ext)) {
-                			/* Prefer snapshots in jpeg or png format */
-                			selectedExt = ext;
-                			break;
-                		} else if("pdf".equals(ext)) {
-                			selectedExt = ext;
-                		} else if("xml".equals(ext) && selectedExt == null) {
-                			/* Use the XML metadata snapshot in last resort */
-                			selectedExt = ext;
-                		}
-                	}
-                	if(selectedExt != null) {
-                		prop.putHTML("content_showSnapshots_extension", selectedExt.toUpperCase(Locale.ROOT));
-                		prop.putHTML("content_showSnapshots_link", "api/snapshot." + selectedExt + "?url=" + resultURL);
-                	} else {
-                		prop.put("content_showSnapshots", 0);
-                	}
-                }
                 prop.put("content_showRanking_ranking", Float.toString(result.score()));
                 prop.put("content_ranking", Float.toString(result.score()));
             }
diff --git a/source/net/yacy/http/servlets/SolrSelectServlet.java b/source/net/yacy/http/servlets/SolrSelectServlet.java
index 50eb41fc6..5b29b9adc 100644
--- a/source/net/yacy/http/servlets/SolrSelectServlet.java
+++ b/source/net/yacy/http/servlets/SolrSelectServlet.java
@@ -45,7 +45,6 @@ import net.yacy.cora.federate.solr.responsewriter.EnhancedXMLResponseWriter;
 import net.yacy.cora.federate.solr.responsewriter.GrepHTMLResponseWriter;
 import net.yacy.cora.federate.solr.responsewriter.HTMLResponseWriter;
 import net.yacy.cora.federate.solr.responsewriter.OpensearchResponseWriter;
-import net.yacy.cora.federate.solr.responsewriter.SnapshotImagesReponseWriter;
 import net.yacy.cora.federate.solr.responsewriter.SolrjResponseWriter;
 import net.yacy.cora.federate.solr.responsewriter.YJsonResponseWriter;
 import net.yacy.cora.protocol.RequestHeader;
@@ -109,7 +108,6 @@ public class SolrSelectServlet extends HttpServlet {
         RESPONSE_WRITER.put("xslt", xsltWriter); // try i.e. http://localhost:8090/solr/select?q=*:*&start=0&rows=10&wt=xslt&tr=json.xsl
         RESPONSE_WRITER.put("exml", new EnhancedXMLResponseWriter());
         RESPONSE_WRITER.put("html", new HTMLResponseWriter());
-        RESPONSE_WRITER.put("snapshots", new SnapshotImagesReponseWriter());
         RESPONSE_WRITER.put("grephtml", new GrepHTMLResponseWriter());
         RESPONSE_WRITER.put("rss", opensearchResponseWriter); //try http://localhost:8090/solr/select?wt=rss&q=olympia&hl=true&hl.fl=text_t,h1,h2
         RESPONSE_WRITER.put("opensearch", opensearchResponseWriter); //try http://localhost:8090/solr/select?wt=rss&q=olympia&hl=true&hl.fl=text_t,h1,h2
diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java
index 4e813c782..1d0b8d42a 100644
--- a/source/net/yacy/search/Switchboard.java
+++ b/source/net/yacy/search/Switchboard.java
@@ -148,7 +148,6 @@ import net.yacy.crawler.data.NoticedURL.StackType;
 import net.yacy.crawler.data.ResultImages;
 import net.yacy.crawler.data.ResultURLs;
 import net.yacy.crawler.data.ResultURLs.EventOrigin;
-import net.yacy.crawler.data.Transactions;
 import net.yacy.crawler.retrieval.Request;
 import net.yacy.crawler.retrieval.Response;
 import net.yacy.crawler.robots.RobotsTxt;
@@ -789,9 +788,6 @@ public final class Switchboard extends serverSwitch {
                         SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT_DEFAULT),
                 this.getConfigInt(SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL,
                         SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL_DEFAULT));
-        final File transactiondir = new File(this.htCachePath, "snapshots");
-        Transactions.init(transactiondir, this.getConfigLong(SwitchboardConstants.SNAPSHOTS_WKHTMLTOPDF_TIMEOUT,
-                SwitchboardConstants.SNAPSHOTS_WKHTMLTOPDF_TIMEOUT_DEFAULT));
 
         // create the packs directories
         this.packsHoldPath = this.getDataPath(SwitchboardConstants.PACKS_HOLD_PATH, SwitchboardConstants.PACKS_HOLD_PATH_DEFAULT);
@@ -3455,9 +3451,7 @@ public final class Switchboard extends serverSwitch {
                         condenser,
                         searchEvent,
                         sourceName,
-                        this.getConfigBool(SwitchboardConstants.NETWORK_UNIT_DHT, false),
-                        this.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false) ? "http://127.0.0.1:" + sb.getConfigInt(SwitchboardConstants.SERVER_PORT, 8090) : null,
-                                this.getConfig("crawler.http.acceptLanguage", null));
+                        this.getConfigBool(SwitchboardConstants.NETWORK_UNIT_DHT, false));
         final RSSFeed feed =
                 EventChannel.channels(queueEntry.initiator() == null
                 ? EventChannel.PROXY
diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java
index 0e391e8ac..c846d88ec 100644
--- a/source/net/yacy/search/SwitchboardConstants.java
+++ b/source/net/yacy/search/SwitchboardConstants.java
@@ -360,12 +360,6 @@ public final class SwitchboardConstants {
     public static final String CRAWLER_USER_AGENT_MINIMUMDELTA  = "crawler.userAgent.minimumdelta";
     public static final String CRAWLER_USER_AGENT_CLIENTTIMEOUT = "crawler.userAgent.clienttimeout";
 
-    /** Key of the setting controlling the maximum time to wait for each wkhtmltopdf call when rendering PDF snapshots */
-    public static final String SNAPSHOTS_WKHTMLTOPDF_TIMEOUT          = "snapshots.wkhtmltopdf.timeout";
-
-    /** Default maximum time in seconds to wait for each wkhtmltopdf call when rendering PDF snapshots*/
-    public static final long SNAPSHOTS_WKHTMLTOPDF_TIMEOUT_DEFAULT   = 30;
-
     /* --- debug flags ---  */
 
     /** when set to true : do not use the local dht/rwi index (which is not done if we do remote searches) */
diff --git a/source/net/yacy/search/index/DocumentIndex.java b/source/net/yacy/search/index/DocumentIndex.java
index 1d374c72d..e5da7a2ca 100644
--- a/source/net/yacy/search/index/DocumentIndex.java
+++ b/source/net/yacy/search/index/DocumentIndex.java
@@ -193,9 +193,7 @@ public class DocumentIndex extends Segment {
                     condenser,
                     null,
                     DocumentIndex.class.getName() + ".add",
-                    false,
-                    null,
-                    null);
+                    false);
         }
         return rows;
     }
diff --git a/source/net/yacy/search/index/Segment.java b/source/net/yacy/search/index/Segment.java
index 7333f463a..0a5a22fba 100644
--- a/source/net/yacy/search/index/Segment.java
+++ b/source/net/yacy/search/index/Segment.java
@@ -63,12 +63,10 @@ import net.yacy.cora.util.ConcurrentLog;
 import net.yacy.cora.util.LookAheadIterator;
 import net.yacy.cora.util.SpaceExceededException;
 import net.yacy.crawler.data.CrawlProfile;
-import net.yacy.crawler.data.Transactions;
 import net.yacy.crawler.retrieval.Response;
 import net.yacy.document.Condenser;
 import net.yacy.document.Document;
 import net.yacy.document.Parser;
-import net.yacy.document.parser.htmlParser;
 import net.yacy.kelondro.data.citation.CitationReference;
 import net.yacy.kelondro.data.citation.CitationReferenceFactory;
 import net.yacy.kelondro.data.word.Word;
@@ -596,9 +594,7 @@ public class Segment {
             final Condenser condenser,
             final SearchEvent searchEvent,
             final String sourceName, // contains the crawl profile hash if this comes from a web crawl
-            final boolean storeToRWI,
-            final String proxy,
-            final String acceptLanguage
+            final boolean storeToRWI
             ) {
         final CollectionConfiguration collectionConfig = this.fulltext.getDefaultConfiguration();
         final String language = votedLanguage(url, url.toNormalform(true), document, condenser); // identification of the language
@@ -608,7 +604,7 @@ public class Segment {
 				this.fulltext().useWebgraph() ? this.fulltext.getWebgraphConfiguration() : null, sourceName);
 
 		return storeDocument(url, crawlProfile, responseHeader, document, vector, language, condenser,
-				searchEvent, sourceName, storeToRWI, proxy, acceptLanguage);
+				searchEvent, sourceName, storeToRWI);
     }
 
     public SolrInputDocument storeDocument(
@@ -621,9 +617,7 @@ public class Segment {
             final Condenser condenser,
             final SearchEvent searchEvent,
             final String sourceName, // contains the crawl profile hash if this comes from a web crawl
-            final boolean storeToRWI,
-            final String proxy,
-            final String acceptLanguage
+            final boolean storeToRWI
             ) {
         final long startTime = System.currentTimeMillis();
 
@@ -647,21 +641,6 @@ public class Segment {
         // ENRICH DOCUMENT WITH RANKING INFORMATION
         this.fulltext.getDefaultConfiguration().postprocessing_references(this.getReferenceReportCache(), vector, url, null);
 
-        // CREATE SNAPSHOT
-        if ((url.getProtocol().equals("http") || url.getProtocol().equals("https")) &&
-                crawlProfile != null && document.getDepth() <= crawlProfile.snapshotMaxdepth() &&
-                !crawlProfile.snapshotsMustnotmatch().matcher(urlNormalform).matches()) {
-            // load pdf in case that is wanted. This can later be used to compute a web page preview in the search results
-            Parser p = document.getParserObject();
-            boolean mimesupported = false;
-            if (p instanceof htmlParser)
-                    mimesupported = ((htmlParser)p).supportedMimeTypes().contains(document.dc_format());
-
-            if (mimesupported)
-                // STORE IMAGE AND METADATA
-                Transactions.store(vector, true, crawlProfile.snapshotLoadImage(), crawlProfile.snapshotReplaceold(), proxy, acceptLanguage);
-        }
-
         // STORE TO SOLR
         this.putDocument(vector);
         List<SolrInputDocument> webgraph = vector.getWebgraphDocuments();
-- 
cgit v1.2.3