diff options
| author | sixcooler <sgaebel@sixcooler.de> | 2011-11-24 19:21:58 +0100 |
|---|---|---|
| committer | sixcooler <sgaebel@sixcooler.de> | 2011-11-24 19:21:58 +0100 |
| commit | 69570fda245dfc6f67591bf8acf115ef9f938ab8 (patch) | |
| tree | 57dca7784f1627566bcaf821ac664fb83adc380b | |
| parent | 8f19fbc46d0059d7f64ddd0cc7dad8f3c7444666 (diff) | |
bring my master to stuff from remote
25 files changed, 48 insertions, 595 deletions
diff --git a/.gitignore b/.gitignore index 9fa982899..20e054dc6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,6 @@ *.class lib/yacycore.jar libbuild/svnRevNr.jar -libbuild/GitRevTask.jar gen/** DATA/ classes/ -RELEASE/ -/yacy.pid diff --git a/build.properties b/build.properties index 9325ef6f9..678872126 100644 --- a/build.properties +++ b/build.properties @@ -7,7 +7,7 @@ releaseVersion=0.99 stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
sourceReleaseFile=yacy_src_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
releaseFileParentDir=yacy
-releaseNr=$Revision: 8091 $
+releaseNr=$Revision$
privateKeyFile=private.key
# defining some file/directory access rights
@@ -57,7 +57,6 @@ <property name="release_windows" location="${release}/WINDOWS"/> <property name="release_mac" location="${release}/MAC"/> <property name="svnEntriesFile" location=".svn/entries"/> - <property name="git" location=".git"/> <property name="defaults" location="defaults"/> <!-- defining all needed directory names for packing search widget--> @@ -70,47 +69,6 @@ <property name="PKGMANAGER" value="false"/> <property name="RESTARTCMD" value="/etc/init.d/yacy restart"/> - <!-- determining if the .git directory exists --> - <condition property="isGit"> - <available file="${git}" /> - </condition> - - <target name="buildGitRevTask"> - <delete file="${libbuild}/GitRevTask.jar" failonerror="false" /> - <javac srcdir="${libbuild}/GitRevTask"> - <classpath> - <pathelement location="${libbuild}/org.eclipse.jgit-1.1.0.201109151100-r.jar" /> - </classpath> - </javac> - <jar destfile="${libbuild}/GitRevTask.jar" basedir="${libbuild}/GitRevTask"> - <manifest> - <attribute name="Main-Class" value="GitRevTask"/> - </manifest> - </jar> - </target> - - <target name="determineGitRevision" if="isGit" depends="buildGitRevTask"> - <taskdef resource="GitRevTask.properties"> - <classpath> - <pathelement location="${libbuild}/GitRevTask.jar" /> - <pathelement location="${libbuild}/org.eclipse.jgit-1.1.0.201109151100-r.jar" /> - </classpath> - </taskdef> - <gitRev repoPath="${yacyroot}" property="baseRevisionNr" /> - - <!-- replacing the old with the new revision number --> - <copy file="build.properties" tofile="build.properties.new"> - <filterchain> - <tokenfilter> - <replaceregex pattern="^releaseNr=(.*)" - replace="releaseNr=$Revision: ${baseRevisionNr} $" /> - </tokenfilter> - </filterchain> - </copy> - <delete file="build.properties"/> - <move file="build.properties.new" tofile="build.properties"/> - </target> - <!-- determining if the .svn directory exists --> <condition property="svnEntriesFileExists"> <available file="${svnEntriesFile}" /> @@ -153,7 +111,7 @@ </target> <!-- reading the build properties from file --> - <target name="readBuildProperties" depends="determineGitRevision,determineRevisionNr"> + <target name="readBuildProperties" depends="determineRevisionNr"> <!-- loading some property values from file --> <loadproperties srcFile="build.properties"> <filterchain> @@ -352,8 +310,6 @@ <!-- copy Unix wrappers --> <copy todir="${release_main}"> <fileset dir="."> - <include name="yacy.blue"/> - <include name="cleanUp.sh"/> <include name="startYACY.sh"/> <include name="stopYACY.sh"/> <include name="killYACY.sh"/> @@ -552,14 +508,12 @@ <delete file="${lib}/yacycore.jar" failonerror="false"/> <delete file="${lib}/svnRevNr.jar" failonerror="false"/> <delete file="${libbuild}/svnRevNr.jar" failonerror="false"/> - <delete file="${libbuild}/GitRevTask.jar" failonerror="false"/> <delete failonerror="false"> <fileset dir="${src}" includes="**/*.class" /> <fileset dir="${build}" includes="**/*.class" /> <fileset dir="${htroot}" includes="**/*.class" /> <fileset dir="test/" includes="**/*.class" /> <fileset dir="${libbuild}/svnRevNr" includes="**/*.class" /> - <fileset dir="${libbuild}/GitRevTask" includes="**/*.class" /> <fileset dir="." includes="TEST-*" /> </delete> </target> diff --git a/cleanUp.sh b/cleanUp.sh deleted file mode 100755 index e11a5b3b9..000000000 --- a/cleanUp.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -JAVA_ARGS="-Xmx1024m -cp lib/yacycore.jar:lib/jcifs-1.3.15.jar:lib/httpcore-4.0.1.jar de.anomic.data.URLAnalysis" -nice -n 19 java $JAVA_ARGS -incell DATA/INDEX/freeworld/SEGMENTS/default used.dump -nice -n 19 java $JAVA_ARGS -diffurlcol DATA/INDEX/freeworld/SEGMENTS/default used.dump diffurlcol.dump -rm used.dump -nice -n 19 java $JAVA_ARGS -copy DATA/INDEX/freeworld/SEGMENTS/default diffurlcol.dump -rm diffurlcol.dump diff --git a/defaults/yacy.network.freeworld.unit b/defaults/yacy.network.freeworld.unit index 6218695bf..456dafd1d 100644 --- a/defaults/yacy.network.freeworld.unit +++ b/defaults/yacy.network.freeworld.unit @@ -75,7 +75,6 @@ network.unit.bootstrap.seedlist0 = http://www.yacy.net/seed.txt network.unit.bootstrap.seedlist1 = http://home.arcor.de/hermens/yacy/seed.txt network.unit.bootstrap.seedlist2 = http://low.audioattack.de/yacy/seed.txt network.unit.bootstrap.seedlist3 = http://www.lulabad.de/seed.txt -network.unit.bootstrap.seedlist4 = http://sixcooler.de/yacy/seed.txt # each network may use different yacy distributions. # the auto-updater can access network-specific update locations diff --git a/htroot/Status.html b/htroot/Status.html index a535eb501..a1091e970 100644 --- a/htroot/Status.html +++ b/htroot/Status.html @@ -15,7 +15,7 @@ }
}
function reloadGraph() {
- document.images["graph"].src="PerformanceGraph.png?time="+(new Date()).getTime();
+ document.images["graph"].src="PerformanceGraph.png?nomem=&time="+(new Date()).getTime();
window.status="";
}
function init() {
@@ -48,7 +48,7 @@ <h2><img src="/env/grafics/kaskelix.png" width="128" height="64" alt="YaCy Websearch" style="margin-right:5px; vertical-align:middle;" />Welcome to YaCy!</h2>
<iframe src="rssTerminal.html?set=PEERNEWS,REMOTESEARCH,LOCALSEARCH,REMOTEINDEXING,LOCALINDEXING,DHTRECEIVE,DHTSEND,PROXY&width=600px&height=180px&maxlines=20&maxwidth=120"
style="width:600px;height:180px;margin:0px;border:1px solid black;" scrolling="no" name="newsframe"></iframe><br />
- <img src="PerformanceGraph.png" id="graph" alt="PerformanceGraph" width="660" height="240"/><br />
+ <img src="PerformanceGraph.png?nomem=" id="graph" alt="PerformanceGraph" width="660" height="240"/><br />
<img src="Banner.png?textcolor=000000&bgcolor=e7effc&bordercolor=5090d0" alt="banner" width="468" height="60"/>
</div>
@@ -99,7 +99,7 @@ #(warningMemoryLow)#::
<dt class="hintIcon"><img src="env/grafics/bad.png" width="32" height="32" alt="bad"/></dt>
- <dd class="hint">Low memory was detected. DHT has been disabled. Please fix
+ <dd class="hint">Free memory is lower than #[minSpace]#. DHT has been disabled. Please fix
it as soon as possible and restart YaCy.
</dd>
#(/warningMemoryLow)#
diff --git a/libbuild/GitRevTask/GitRevTask.java b/libbuild/GitRevTask/GitRevTask.java deleted file mode 100644 index 3d9302d98..000000000 --- a/libbuild/GitRevTask/GitRevTask.java +++ /dev/null @@ -1,96 +0,0 @@ -import java.io.File; -import java.io.IOException; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.tools.ant.BuildException; -import org.apache.tools.ant.Project; - -import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevTag; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.storage.file.FileRepositoryBuilder; - - -public class GitRevTask extends org.apache.tools.ant.Task { - - private String repoPath; - private String property; - - public void setRepoPath(final String repoPath) { - this.repoPath = repoPath; - } - - public void setProperty(String property) { - this.property = property; - } - - public void execute() { - if (this.property==null || this.property.length() == 0) { - log("svn entries file name property was not set properly",Project.MSG_ERR); - return; - } - - String revision = null; - String lastTag = null; - try { - final File src = new File(repoPath); - final Repository repo = new FileRepositoryBuilder().readEnvironment() - .findGitDir(src).build(); - final ObjectId head = repo.resolve("HEAD"); - final String gitrev = head.getName().substring(0, 8); - - final Git git = new Git(repo); - final List<RevTag> tags = git.tagList().call(); - - final RevWalk walk = new RevWalk(repo); - walk.markStart(walk.parseCommit(head)); - int distance = 0; - for (final RevCommit commit : walk) { - for (final RevTag tag : tags) { - if (commit.equals(tag.getObject())) { - lastTag = tag.getShortMessage(); - break; - } - } - if (lastTag == null) lastTag = findRev(commit.getFullMessage()); - if (lastTag != null || distance++ > 99) break; - } - walk.dispose(); - if (lastTag == null) { - revision = "dev" + "-" + gitrev; - } else { - revision = lastTag + "-" + distance + "-" + gitrev; - } - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - Project theProject = getProject(); - if (theProject != null) { - theProject.setProperty(this.property, lastTag); - log("Property '" + this.property + "' set to '" + revision + "'", Project.MSG_VERBOSE); - } - } - - private String findRev(final String message) { - final Pattern pattern = Pattern.compile("trunk@(\\d{4})\\s+"); - final Matcher matcher = pattern.matcher(message); - if (matcher.find()) { - return matcher.group(1); - } - return null; - } - - public static void main(String[] args) { - GitRevTask gitRevTask = new GitRevTask(); - gitRevTask.setRepoPath("/home/sgaebel/git/yacy.rc1"); - - gitRevTask.execute(); - } -} diff --git a/libbuild/GitRevTask/GitRevTask.properties b/libbuild/GitRevTask/GitRevTask.properties deleted file mode 100644 index 13effd982..000000000 --- a/libbuild/GitRevTask/GitRevTask.properties +++ /dev/null @@ -1 +0,0 @@ -gitRev=GitRevTask
\ No newline at end of file diff --git a/libbuild/org.eclipse.jgit-1.1.0.201109151100-r.jar b/libbuild/org.eclipse.jgit-1.1.0.201109151100-r.jar Binary files differdeleted file mode 100644 index 40d5c0c50..000000000 --- a/libbuild/org.eclipse.jgit-1.1.0.201109151100-r.jar +++ /dev/null diff --git a/libbuild/org.eclipse.jgit-1.1.0.201109151100-r.license b/libbuild/org.eclipse.jgit-1.1.0.201109151100-r.license deleted file mode 100644 index 1b85c6466..000000000 --- a/libbuild/org.eclipse.jgit-1.1.0.201109151100-r.license +++ /dev/null @@ -1,37 +0,0 @@ -This program and the accompanying materials are made available -under the terms of the Eclipse Distribution License v1.0 which -accompanies this distribution, is reproduced below, and is -available at http://www.eclipse.org/org/documents/edl-v10.php - -All rights reserved. - -Redistribution and use in source and binary forms, with or -without modification, are permitted provided that the following -conditions are met: - -- Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -- Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - -- Neither the name of the Eclipse Foundation, Inc. nor the - names of its contributors may be used to endorse or promote - products derived from this software without specific prior - written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND -CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/source/de/anomic/data/URLAnalysis.java b/source/de/anomic/data/URLAnalysis.java index 1a02c92d9..f9d09ce58 100644 --- a/source/de/anomic/data/URLAnalysis.java +++ b/source/de/anomic/data/URLAnalysis.java @@ -59,7 +59,6 @@ import net.yacy.kelondro.index.HandleSet; import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
-import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.rwi.ReferenceContainerArray;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.search.index.MetadataRepository;
@@ -144,7 +143,7 @@ public class URLAnalysis { i.remove();
}
}
- // Runtime.getRuntime().gc();
+ Runtime.getRuntime().gc();
}
public static void genstat(final String urlfile) {
@@ -385,7 +384,7 @@ public class URLAnalysis { writeSet(trunk + "." + filecount, gz, urls);
filecount++;
urls.clear();
- // Runtime.getRuntime().gc();
+ Runtime.getRuntime().gc();
}
}
reader.close();
@@ -472,40 +471,6 @@ public class URLAnalysis { System.out.println("URL DELETE finished deletions, " + mr.size() + " entries left in URL database");
}
- public static void cleanCopy(final String metadataPath, final String diffFile) throws IOException, RowSpaceExceededException {
- System.out.println("URL COPY startup");
- final HandleSet hs = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, new File(diffFile));
- if (hs.size() < 1000){
- System.out.println("to few Items to delete\n");
- return;
- }
- final MetadataRepository mrold = new MetadataRepository(new File(metadataPath), "text.urlmd", false, false);
- final MetadataRepository mrnew = new MetadataRepository(new File(metadataPath), "urlmd", false, false);
- final CloneableIterator<URIMetadataRow> mIter = mrold.entries();
-
- System.out.println("\n\nURL COPY loaded dump, starting clean copy from " + mrold.size() + "\n");
- URIMetadataRow row;
- while (mIter.hasNext()) {
- row = mIter.next();
- if(!hs.has(row.hash()) && !DigestURI.isLocal(row.hash())) mrnew.store(row);
- }
- System.out.println("URL COPY finished copy of " + mrnew.size() + " entries in new URL database");
- mrold.close();
- mrnew.close();
- final String[] tablefile = new File(metadataPath).list();
- File f;
- for (int i = 0; i < tablefile.length; i++) {
- if (tablefile[i].startsWith("text.urlmd")) {
- f = new File(metadataPath, tablefile[i]);
- f.renameTo(new File(metadataPath, "old." + tablefile[i]));
- }
- if (tablefile[i].startsWith("urlmd")) {
- f = new File(metadataPath, tablefile[i]);
- f.renameTo(new File(metadataPath, "text." + tablefile[i]));
- }
- }
- }
-
public static void main(final String[] args) {
if (args[0].equals("-stat") && args.length >= 2) {
// generate a statistics about common words in file, store to <file>.stat
@@ -553,15 +518,6 @@ public class URLAnalysis { } catch (final Exception e) {
Log.logException(e);
}
- } else if (args[0].equals("-copy") && args.length >= 3) {
- // copy from URLS as given by urlreference diff dump
- // example:
- // java -Xmx1024m -cp lib/yacycore.jar de.anomic.data.URLAnalysis -copy DATA/INDEX/freeworld/TEXT/METADATA diffurlcol.dump
- try {
- cleanCopy(args[1], args[2]);
- } catch (final Exception e) {
- Log.logException(e);
- }
} else {
System.out.println("usage:");
System.out.println();
@@ -586,15 +542,12 @@ public class URLAnalysis { System.out.println("-delete <path-to-URL-DB> <diff-dump>");
System.out.println(" delete all urls that are listed in the diff-dump from the url-db");
System.out.println();
- System.out.println("-copy <path-to-URL-DB> <diff-dump>");
- System.out.println(" copy all urls that are listed in the diff-dump from the url-db");
- System.out.println();
System.out.println("to do a complete clean-up of the url database, start the following:");
System.out.println();
- System.out.println("java -Xmx1000m -cp lib/yacycore.jar de.anomic.data.URLAnalysis -incollection DATA/INDEX/freeworld/SEGMENTS/default used.dump");
- System.out.println("java -Xmx1000m -cp lib/yacycore.jar de.anomic.data.URLAnalysis -diffurlcol DATA/INDEX/freeworld/SEGMENTS/default used.dump diffurlcol.dump");
- System.out.println("java -Xmx1000m -cp lib/yacycore.jar de.anomic.data.URLAnalysis -export DATA/INDEX/freeworld/SEGMENTS/default xml urls.xml diffurlcol.dump");
- System.out.println("java -Xmx1000m -cp lib/yacycore.jar de.anomic.data.URLAnalysis -delete DATA/INDEX/freeworld/SEGMENTS/default diffurlcol.dump");
+ System.out.println("java -Xmx1000m -cp classes de.anomic.data.URLAnalysis -incollection DATA/INDEX/freeworld/TEXT/RICOLLECTION used.dump");
+ System.out.println("java -Xmx1000m -cp classes de.anomic.data.URLAnalysis -diffurlcol DATA/INDEX/freeworld/TEXT used.dump diffurlcol.dump");
+ System.out.println("java -Xmx1000m -cp classes de.anomic.data.URLAnalysis -export DATA/INDEX/freeworld/TEXT xml urls.xml diffurlcol.dump");
+ System.out.println("java -Xmx1000m -cp classes de.anomic.data.URLAnalysis -delete DATA/INDEX/freeworld/TEXT diffurlcol.dump");
System.out.println();
}
System.exit(0); // kill remaining threads
diff --git a/source/net/yacy/cora/document/MultiProtocolURI.java b/source/net/yacy/cora/document/MultiProtocolURI.java index cd888c281..57cc12cbc 100644 --- a/source/net/yacy/cora/document/MultiProtocolURI.java +++ b/source/net/yacy/cora/document/MultiProtocolURI.java @@ -487,9 +487,7 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU || ch == '.' || ch == '!' || ch == '~' || ch == '*' || ch == '\'' || ch == '(' - || ch == ')' || ch == ';' - || ch == ',' || ch == '=' - || ch == '+') { + || ch == ')' || ch == ';') { sbuf.append((char)ch); } else if (ch == '/') { // reserved, but may appear in post part where it should not be replaced sbuf.append((char)ch); diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java index 6becfc1c5..fc1819c45 100644 --- a/source/net/yacy/cora/protocol/http/HTTPClient.java +++ b/source/net/yacy/cora/protocol/http/HTTPClient.java @@ -49,9 +49,8 @@ import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.cora.protocol.HeaderFramework; +import org.apache.commons.httpclient.cookie.CookiePolicy; import org.apache.http.Header; -import org.apache.http.HeaderElement; -import org.apache.http.HeaderElementIterator; import org.apache.http.HttpEntity; import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpHost; @@ -65,11 +64,9 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.client.params.CookiePolicy; import org.apache.http.client.params.HttpClientParams; import org.apache.http.client.protocol.ClientContext; import org.apache.http.conn.ClientConnectionManager; -import org.apache.http.conn.ConnectionKeepAliveStrategy; import org.apache.http.conn.params.ConnRouteParams; import org.apache.http.conn.routing.HttpRoute; import org.apache.http.conn.scheme.PlainSocketFactory; @@ -80,10 +77,10 @@ import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.mime.MultipartEntity; import org.apache.http.entity.mime.content.ContentBody; import org.apache.http.entity.mime.content.StringBody; +import org.apache.http.impl.client.AbstractHttpClient; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; -import org.apache.http.message.BasicHeaderElementIterator; import org.apache.http.params.BasicHttpParams; import org.apache.http.params.HttpConnectionParams; import org.apache.http.params.HttpParams; @@ -191,12 +188,10 @@ public class HTTPClient { httpClient = new DefaultHttpClient(clientConnectionManager, httpParams); // disable the cookiestore, cause this may cause segfaults and is not needed ((DefaultHttpClient) httpClient).setCookieStore(null); - // add cutom keep alive strategy - addCustomKeepAliveStrategy((DefaultHttpClient) httpClient); // ask for gzip - ((DefaultHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor()); + ((AbstractHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor()); // uncompress gzip - ((DefaultHttpClient) httpClient).addResponseInterceptor(new GzipResponseInterceptor()); + ((AbstractHttpClient) httpClient).addResponseInterceptor(new GzipResponseInterceptor()); if (idledConnectionEvictor == null) { idledConnectionEvictor = new IdledConnectionEvictor(clientConnectionManager); @@ -660,7 +655,7 @@ public class HTTPClient { if (ProxySettings.use) ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost()); // TODO find a better way for this - ProxySettings.setProxyCreds((DefaultHttpClient) httpClient); + ProxySettings.setProxyCreds((AbstractHttpClient) httpClient); } private void storeConnectionInfo(final HttpUriRequest httpUriRequest) { @@ -705,38 +700,6 @@ public class HTTPClient { final SSLSocketFactory sslSF = new SSLSocketFactory(sslContext, SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); return sslSF; } - - /** - * If the Keep-Alive header is not present in the response, - * HttpClient assumes the connection can be kept alive indefinitely. - * Here we limit this to 5 seconds. - * - * @param defaultHttpClient - */ - private static void addCustomKeepAliveStrategy(final DefaultHttpClient defaultHttpClient) { - defaultHttpClient.setKeepAliveStrategy(new ConnectionKeepAliveStrategy() { - public long getKeepAliveDuration(HttpResponse response, HttpContext context) { - // Honor 'keep-alive' header - String param, value; - HeaderElement element; - HeaderElementIterator it = new BasicHeaderElementIterator( - response.headerIterator(HTTP.CONN_KEEP_ALIVE)); - while (it.hasNext()) { - element = it.nextElement(); - param = element.getName(); - value = element.getValue(); - if (value != null && param.equalsIgnoreCase("timeout")) { - try { - return Long.parseLong(value) * 1000; - } catch(final NumberFormatException e) { - } - } - } - // Keep alive for 5 seconds only - return 5 * 1000; - } - }); - } /** * testing diff --git a/source/net/yacy/document/ImageParser.java b/source/net/yacy/document/ImageParser.java index 35aa8aa54..0e135712b 100644 --- a/source/net/yacy/document/ImageParser.java +++ b/source/net/yacy/document/ImageParser.java @@ -23,10 +23,7 @@ package net.yacy.document; import java.awt.Container; import java.awt.Image; import java.awt.MediaTracker; -import java.io.ByteArrayInputStream; -import java.io.IOException; - -import javax.imageio.ImageIO; +import java.awt.Toolkit; import net.yacy.document.parser.images.bmpParser; import net.yacy.document.parser.images.icoParser; @@ -35,34 +32,36 @@ public class ImageParser { public static final Image parse(final String filename, final byte[] source) { final MediaTracker mediaTracker = new MediaTracker(new Container()); - Image image = null; + Image image; if (((filename.endsWith(".ico")) || (filename.endsWith(".bmp"))) && (bmpParser.isBMP(source))) { // parse image with BMP parser image = bmpParser.parse(source).getImage(); + if (image == null) return null; } else if ((filename.endsWith(".ico")) && (icoParser.isICO(source))) { // parse image with ICO parser - final icoParser icoparser = new icoParser(source); + icoParser icoparser; + icoparser = new icoParser(source); image = icoparser.getImage(0); + if (image == null) return null; } else { // awt can handle jpg, png and gif formats, try it - // image = Toolkit.getDefaultToolkit().createImage(source); - + image = Toolkit.getDefaultToolkit().createImage(source); + /* try { ImageIO.setUseCache(false); // do not write a cache to disc; keep in RAM image = ImageIO.read(new ByteArrayInputStream(source)); } catch (IOException e) { -// Image i = Toolkit.getDefaultToolkit().createImage(source); -// mediaTracker.addImage(i, 0); -// try {mediaTracker.waitForID(0);} catch (final InterruptedException ee) {} -// -// int width = i.getWidth(null); if (width < 0) width = 96; // bad hack -// int height = i.getHeight(null); if (height < 0) height = 96; // bad hack -// image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); -// image.createGraphics().drawImage(i, 0, 0, width, height, null); + Image i = Toolkit.getDefaultToolkit().createImage(source); + mediaTracker.addImage(i, 0); + try {mediaTracker.waitForID(0);} catch (final InterruptedException ee) {} + + int width = i.getWidth(null); if (width < 0) width = 96; // bad hack + int height = i.getHeight(null); if (height < 0) height = 96; // bad hack + image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); + image.createGraphics().drawImage(i, 0, 0, width, height, null); } - + */ } - if (image == null) return null; final int handle = image.hashCode(); mediaTracker.addImage(image, handle); diff --git a/source/net/yacy/document/parser/pdfParser.java b/source/net/yacy/document/parser/pdfParser.java index 21f801dbd..2d5dd75c7 100644 --- a/source/net/yacy/document/parser/pdfParser.java +++ b/source/net/yacy/document/parser/pdfParser.java @@ -70,7 +70,7 @@ public class pdfParser extends AbstractParser implements Parser { public Document[] parse(final MultiProtocolURI location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException {
// check memory for parser
- if (!MemoryControl.request(200 * 1024 * 1024, false))
+ if (!MemoryControl.request(200 * 1024 * 1024, true))
throw new Parser.Failure("Not enough Memory available for pdf parser: " + MemoryControl.available(), location);
// create a pdf parser
diff --git a/source/net/yacy/document/parser/zipParser.java b/source/net/yacy/document/parser/zipParser.java index 841c2cac0..b216fe099 100644 --- a/source/net/yacy/document/parser/zipParser.java +++ b/source/net/yacy/document/parser/zipParser.java @@ -63,7 +63,7 @@ public class zipParser extends AbstractParser implements Parser { final String charset, final InputStream source)
throws Parser.Failure, InterruptedException {
// check memory for parser
- if (!MemoryControl.request(200 * 1024 * 1024, false))
+ if (!MemoryControl.request(200 * 1024 * 1024, true))
throw new Parser.Failure("Not enough Memory available for zip parser: " + MemoryControl.available(), url);
Document[] docs = null;
diff --git a/source/net/yacy/kelondro/blob/ArrayStack.java b/source/net/yacy/kelondro/blob/ArrayStack.java index d4aa56f2b..bf2eb3301 100755 --- a/source/net/yacy/kelondro/blob/ArrayStack.java +++ b/source/net/yacy/kelondro/blob/ArrayStack.java @@ -82,7 +82,6 @@ public class ArrayStack implements BLOB { public static final long maxFileSize = Integer.MAX_VALUE; public static final long oneMonth = 1000L * 60L * 60L * 24L * 365L / 12L; - private static final long oneWeek = 1000L * 60L * 60L * 24L * 7L; protected int keylength; protected ByteOrder ordering; @@ -115,7 +114,7 @@ public class ArrayStack implements BLOB { this.ordering = ordering; this.buffersize = buffersize; this.heapLocation = heapLocation; - this.fileAgeLimit = oneWeek; + this.fileAgeLimit = oneMonth; this.fileSizeLimit = maxFileSize; this.repositoryAgeMax = Long.MAX_VALUE; this.repositorySizeMax = Long.MAX_VALUE; @@ -381,7 +380,7 @@ public class ArrayStack implements BLOB { public void setMaxAge(final long maxAge) { this.repositoryAgeMax = maxAge; - this.fileAgeLimit = Math.min(oneWeek, maxAge / 10); + this.fileAgeLimit = Math.min(oneMonth, maxAge / 10); } public void setMaxSize(final long maxSize) { diff --git a/source/net/yacy/kelondro/util/G1MemoryStrategy.java b/source/net/yacy/kelondro/util/G1MemoryStrategy.java deleted file mode 100644 index 2f8cd7b85..000000000 --- a/source/net/yacy/kelondro/util/G1MemoryStrategy.java +++ /dev/null @@ -1,186 +0,0 @@ -// MemoryControl.java -// ------------------------------------------- -// (C) 2011 by Sebastian Gaebel -// first published 22.08.2011 on http://yacy.net -// -// $LastChangedDate: 2011-08-18 00:24:17 +0200 (Do, 18. Aug 2011) $ -// $LastChangedRevision: 7883 $ -// $LastChangedBy: orbiter $ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package net.yacy.kelondro.util; - -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryMXBean; -import java.lang.management.MemoryPoolMXBean; -import java.lang.management.MemoryUsage; - -public class G1MemoryStrategy extends MemoryStrategy { - - private MemoryPoolMXBean eden, survivor, old; - private MemoryMXBean heap; - - public G1MemoryStrategy() { - name = "G1 Memory Strategy"; - error = initPoolBeans(); - heap = ManagementFactory.getMemoryMXBean(); - if (lastGC == 0l) gc(10000, "initial gc - to get proper results"); // this is necessary on some GCs / vm - if (error) log.logWarning(name + ": not a generational heap"); - } - - /** - * memory that is free without increasing of total memory taken from os - * @return bytes - */ - protected final long free() { - return oldUsage(false).getCommitted() - oldUsage(false).getUsed(); - } - - /** - * memory that is available including increasing total memory up to maximum - * Smallest of both old and young - * @return bytes - */ - protected final long available() { - return oldAvailable(); - } - - /** - * memory that is currently bound in objects - * @return used bytes - */ - protected final long used() { - return heap.getHeapMemoryUsage().getUsed(); - } - - /** - * currently allocated memory in the Java virtual machine; may vary over time - * @return bytes - */ - protected final long total() { - return heap.getHeapMemoryUsage().getCommitted(); - } - - /** - * maximum memory the Java virtual will allocate machine; may vary over time in some cases - * @return bytes - */ - protected final long maxMemory() { - return heap.getHeapMemoryUsage().getMax(); - } - - /** - * checks if a specified amount of bytes are available - * after the jvm recycled unused objects - * - * @param size the requested amount of free memory in bytes - * @param force specifies whether ignoring prefered size - * @return whether enough memory could be freed (or is free) or not - */ - protected final boolean request(final long size, final boolean force, boolean shortStatus) { - debugOut(); - if (size == 0l) return true; // does not make sense to check - returning true without setting shortStatus (which also doesn't make sense to me) - final boolean unknown = size < 0l; // size < 0 indicate an unknown size - maybe from gziped streams - final boolean r = unknown? properState() : size < oldAvailable(); - shortStatus = !r; - return r; - } - - /** - * @return bytes available to allocate in Tenured Space (Old Generation) - */ - private final long oldAvailable() { - return oldUsage(true).getCommitted() - oldUsage(true).getUsed(); - } - - /** - * G1 works different than traditional generation-stukture. - * The heap iss segmented which can be young, survivior or old generation. - * For proper running young and survior need to fit into old space - * - * @return Memory is in proper state - */ - protected boolean properState() { - return (oldUsage(true).getUsed() + survivorUsage(false).getCommitted() + youngUsage(false).getCommitted()) < oldUsage(false).getCommitted(); - } - - /** - * @param collected specifies whether trying to get the memory usage after the jvm recycled unused objects - * @return MemoryUsage of Eden Space aka Young Gen - */ - private MemoryUsage youngUsage(final boolean collected) { - if (collected) { - final MemoryUsage usage = eden.getCollectionUsage(); - if (usage != null) return usage; - error = true; - log.logWarning(name + ": no young colletion usage available"); - } - return eden.getUsage(); - } - - /** - * @param collected specifies whether trying to get the memory usage after the jvm recycled unused objects - * @return MemoryUsage of Survivor - */ - private MemoryUsage survivorUsage(final boolean collected) { - if (collected) { - final MemoryUsage usage = survivor.getCollectionUsage(); - if (usage != null) return usage; - error = true; - log.logWarning(name + ": no survivior colletion usage available"); - } - return survivor.getUsage(); - } - - /** - * @param collected specifies whether trying to get the memory usage after the jvm recycled unused objects - * @return MemoryUsage of Old Gen - */ - private MemoryUsage oldUsage(final boolean collected) { - if (collected) { - final MemoryUsage usage = old.getCollectionUsage(); - if (usage != null) return usage; - error = true; - log.logWarning(name + ": no old colletion usage available"); - } - return old.getUsage(); - } - - private boolean initPoolBeans() { - for (final MemoryPoolMXBean bean : ManagementFactory.getMemoryPoolMXBeans()) { - if (!bean.getName().startsWith("G1")) continue; //this strategy is G1 only - if (bean.getName().contains("Eden")) { - eden = bean; - } else if (bean.getName().contains("Survivor")) { - survivor = bean; - } else if (bean.getName().contains("Old") || bean.getName().contains("Tenured")) { - old = bean; - } - } - return eden == null || survivor == null || old == null; - } - - private void debugOut() { -// System.out.println("young: " + youngUsage(false).getUsed() + " " + youngUsage(false).getCommitted() + " " + youngUsage(false).getMax()); -// System.out.println("->: " + youngUsage(true).getUsed() + " " + youngUsage(true).getCommitted() + " " + youngUsage(true).getMax()); -// System.out.println("survivor: " + survivorUsage(false).getUsed() + " " + survivorUsage(false).getCommitted() + " " + survivorUsage(false).getMax()); -// System.out.println("->: " + survivorUsage(true).getUsed() + " " + survivorUsage(true).getCommitted() + " " + survivorUsage(true).getMax()); - System.out.println("old: " + oldUsage(false).getUsed() + " " + oldUsage(false).getCommitted() + " " + oldUsage(false).getMax()); - System.out.println("->: " + oldUsage(true).getUsed() + " " + oldUsage(true).getCommitted() + " " + oldUsage(true).getMax()); - } -} diff --git a/source/net/yacy/kelondro/util/MemoryControl.java b/source/net/yacy/kelondro/util/MemoryControl.java index d1b46a8ba..55f2e4a8c 100644 --- a/source/net/yacy/kelondro/util/MemoryControl.java +++ b/source/net/yacy/kelondro/util/MemoryControl.java @@ -37,9 +37,6 @@ public class MemoryControl { if (strategy == null || strategy.hasError()) {
if (!usingStandardStrategy) {
strategy = new GenerationMemoryStrategy();
-// if (strategy.hasError()) { // perhaps we do have a G1
-// strategy = new G1MemoryStrategy();
-// }
// fall back if error detected
if (strategy.hasError()) {
usingStandardStrategy = true;
@@ -190,20 +187,17 @@ public class MemoryControl { System.out.println("computed max = " + (maxMemory() / mb) + " mb");
System.out.println("using " + getStrategyName());
final byte[][] x = new byte[100000][];
- byte[] y;
+
for (int i = 0; i < 100000; i++) {
- if (i== 10) System.gc();
if (request(mb, force))
{
- y = new byte[mb];
- x[i] = y;
+ x[i] = new byte[mb];
System.out.println("used = " + i + " / " + (used() /mb) +
", total = " + (total() / mb) +
", free = " + (free() / mb) +
", max = " + (maxMemory() / mb) +
", avail = " + (available() / mb) +
- (usingStandardStrategy? ", averageGC = " + ((StandardMemoryStrategy)getStrategy()).getAverageGCFree() :
- ", proper = " + properState()));
+ (usingStandardStrategy? ", averageGC = " + ((StandardMemoryStrategy)getStrategy()).getAverageGCFree() : ""));
} else System.exit(0);
}
diff --git a/source/net/yacy/kelondro/util/MemoryTracker.java b/source/net/yacy/kelondro/util/MemoryTracker.java index 3da30b35d..7d6e87b33 100644 --- a/source/net/yacy/kelondro/util/MemoryTracker.java +++ b/source/net/yacy/kelondro/util/MemoryTracker.java @@ -33,7 +33,7 @@ public class MemoryTracker extends Thread { private static MemoryTracker systemProfiler = null; public static void startSystemProfiling() { - systemProfiler = new MemoryTracker(3000); + systemProfiler = new MemoryTracker(1500); systemProfiler.start(); } @@ -52,7 +52,7 @@ public class MemoryTracker extends Thread { public void run() { try { while (running) { - EventTracker.update(EventTracker.EClass.MEMORY, Long.valueOf(MemoryControl.available()), true); + EventTracker.update(EventTracker.EClass.MEMORY, Long.valueOf(MemoryControl.used()), true); try { Thread.sleep(this.delaytime); } catch (final InterruptedException e) { diff --git a/source/net/yacy/peers/graphics/ProfilingGraph.java b/source/net/yacy/peers/graphics/ProfilingGraph.java index 1ebe95ab0..da0bc9c8a 100644 --- a/source/net/yacy/peers/graphics/ProfilingGraph.java +++ b/source/net/yacy/peers/graphics/ProfilingGraph.java @@ -121,8 +121,8 @@ public class ProfilingGraph { bytes = ((Long) event.payload).longValue(); x1 = (int) (time/1000); y1 = (int) (bytes / 1024 / 1024); - // chart.setColor("AAAAFF"); - // chart.chartDot(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_RIGHT, x1, y1, 2, null, 0); + chart.setColor("AAAAFF"); + chart.chartDot(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_RIGHT, x1, y1, 2, null, 0); chart.setColor("0000FF"); if (x0 < 0) chart.chartLine(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_RIGHT, x0, y0, x1, y1); x0 = x1; y0 = y1; diff --git a/startYACY.sh b/startYACY.sh index 67894fb40..481618c83 100755 --- a/startYACY.sh +++ b/startYACY.sh @@ -6,10 +6,7 @@ PIDFILE="yacy.pid" OS="`uname`" #get javastart args -JAVA_ARGS="-server -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8"; -#CMS: JAVA_ARGS="-server -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:NewRatio=2 -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8"; -#G1: JAVA_ARGS="-server -XX:+UnlockExperimentalVMOptions -XX:+UseG1GC -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8"; -#org: JAVA_ARGS="-server -Xss256k -XX:MaxPermSize=256m -XX:ReservedCodeCacheSize=1024m -XX:-UseGCOverheadLimit -XX:+UseAdaptiveSizePolicy -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8"; +JAVA_ARGS="-server -Xss256k -XX:MaxPermSize=256m -XX:ReservedCodeCacheSize=1024m -XX:-UseGCOverheadLimit -XX:+UseAdaptiveSizePolicy -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8"; #JAVA_ARGS="-verbose:gc -XX:+PrintGCTimeStamps -XX:+PrintGCDetails $JAVA_ARGS"; #check if OS is Sun Solaris or one of the OpenSolaris distributions and use different version of id if necessary diff --git a/yacy.blue b/yacy.blue deleted file mode 100644 index 3abbd1f04..000000000 --- a/yacy.blue +++ /dev/null @@ -1,12 +0,0 @@ -anal -fuck -fucked -incest -lolita -naked -porn -preteen -pre-teen -pussy -teen -teens diff --git a/yacy.logging b/yacy.logging index 26d7bbd0d..ea200b078 100644 --- a/yacy.logging +++ b/yacy.logging @@ -38,7 +38,7 @@ handlers = java.util.logging.FileHandler,\ # Properties for the FileHandler
java.util.logging.FileHandler.limit = 1048576
-java.util.logging.FileHandler.count = 9
+java.util.logging.FileHandler.count = 20
java.util.logging.FileHandler.pattern = DATA/LOG/yacy%u%g.log
java.util.logging.FileHandler.level = ALL
java.util.logging.FileHandler.formatter=net.yacy.kelondro.logging.SimpleLogFormatter
@@ -60,7 +60,7 @@ net.yacy.kelondro.logging.ConsoleOutErrHandler.formatter=net.yacy.kelondro.loggi # Properties for the proxy access logger
de.anomic.http.server.HTTPDProxyHandler.logging.enabled = TRUE
de.anomic.http.server.HTTPDProxyHandler.logging.FileHandler.limit = 1048576
-de.anomic.http.server.HTTPDProxyHandler.logging.FileHandler.count = 5
+de.anomic.http.server.HTTPDProxyHandler.logging.FileHandler.count = 20
de.anomic.http.server.HTTPDProxyHandler.logging.FileHandler.pattern = DATA/LOG/proxyAccess%u%g.log
# Properties for the LogalizerHandler
diff --git a/yacy.stopwords b/yacy.stopwords index 79413771b..e69de29bb 100644 --- a/yacy.stopwords +++ b/yacy.stopwords @@ -1,61 +0,0 @@ -aber -als -an -an -auch -auf -aus -aus -bei -da -das -dass -daß -dem -den -denn -der -des -die -du -ein -eine -er -es -es -euch -für -hat -hat -ich -ihm -ihn -ihnen -ihr -im -in -in -ist -mehr -mir -mit -nach -nicht -oder -sein -seine -sich -sie -sind -so -und -von -vor -was -werden -werden -wie -wird -www -zu -zum |
