summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rwxr-xr-xbin/localsearch.sh4
-rw-r--r--htroot/Blacklist_p.java11
-rw-r--r--source/de/anomic/data/ListManager.java1
-rw-r--r--source/net/yacy/repository/Blacklist.java59
4 files changed, 42 insertions, 33 deletions
diff --git a/bin/localsearch.sh b/bin/localsearch.sh
index 50b737616..8acf586ba 100755
--- a/bin/localsearch.sh
+++ b/bin/localsearch.sh
@@ -2,9 +2,9 @@
cd "`dirname $0`"
port=$(grep ^port= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
if which curl &>/dev/null; then
- curl -s "http://localhost:$port/yacysearch.rss?resource=local&verify=ifexist&query=$1" | awk '/^<link>/{ gsub("<link>","" );gsub("<\/link>","" ); print $0 }'
+ curl -s "http://localhost:$port/yacysearch.rss?resource=local&verify=false&query=$1" | awk '/^<link>/{ gsub("<link>","" );gsub("<\/link>","" ); print $0 }'
elif which wget &>/dev/null; then
- wget -q -O - "http://localhost:$port/yacysearch.rss?resource=local&verify=ifexist&query=$1" | awk '/^<link>/{ gsub("<link>","" );gsub("<\/link>","" ); print $0 }'
+ wget -q -O - "http://localhost:$port/yacysearch.rss?resource=local&verify=false&query=$1" | awk '/^<link>/{ gsub("<link>","" );gsub("<\/link>","" ); print $0 }'
else
echo "Neither curl nor wget installed!"
exit 1
diff --git a/htroot/Blacklist_p.java b/htroot/Blacklist_p.java
index 5d0ada5fc..8d2d130ad 100644
--- a/htroot/Blacklist_p.java
+++ b/htroot/Blacklist_p.java
@@ -216,7 +216,8 @@ public class Blacklist_p {
ListManager.removeFromListSet(supportedBlacklistType + ".BlackLists",blacklistToUse);
}
}
-
+
+ Switchboard.urlBlacklist.clear();
ListManager.reloadBlacklists();
} else if (post.containsKey("shareList")) {
@@ -256,6 +257,8 @@ public class Blacklist_p {
}
}
}
+
+ Switchboard.urlBlacklist.clear();
ListManager.reloadBlacklists();
} else if (post.containsKey("addBlacklistEntry")) {
@@ -275,6 +278,8 @@ public class Blacklist_p {
prop.put("LOCATION", temp);
return prop;
}
+
+ Switchboard.urlBlacklist.clear();
ListManager.reloadBlacklists();
} else if ("moveBlacklistEntry".equals(action)) {
@@ -307,6 +312,8 @@ public class Blacklist_p {
}
}
}
+
+ Switchboard.urlBlacklist.clear();
ListManager.reloadBlacklists();
} else if ("editBlacklistEntry".equals(action)) {
@@ -346,6 +353,8 @@ public class Blacklist_p {
}
}
}
+
+ Switchboard.urlBlacklist.clear();
ListManager.reloadBlacklists();
prop.putHTML(DISABLED + EDIT + "currentBlacklist", blacklistToUse);
diff --git a/source/de/anomic/data/ListManager.java b/source/de/anomic/data/ListManager.java
index 051580a8c..46a9e7a8b 100644
--- a/source/de/anomic/data/ListManager.java
+++ b/source/de/anomic/data/ListManager.java
@@ -207,7 +207,6 @@ public class ListManager {
blacklistFiles.add(blFile);
}
- Switchboard.urlBlacklist.clear();
Switchboard.urlBlacklist.loadList(
blacklistFiles.toArray(new BlacklistFile[blacklistFiles.size()]),
"/");
diff --git a/source/net/yacy/repository/Blacklist.java b/source/net/yacy/repository/Blacklist.java
index b4934e030..750ab6abf 100644
--- a/source/net/yacy/repository/Blacklist.java
+++ b/source/net/yacy/repository/Blacklist.java
@@ -51,11 +51,10 @@ import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.SetTools;
+import net.yacy.search.Switchboard;
public class Blacklist {
-
- private static final File BLACKLIST_DHT_CACHEFILE = new File("DATA/WORK/blacklistCache_DHT.ser");
-
+
public enum BlacklistType {
DHT, CRAWLER, PROXY, SEARCH, SURFTIPS, NEWS;
@@ -94,28 +93,22 @@ public class Blacklist {
private File blacklistRootPath = null;
private final ConcurrentMap<BlacklistType, HandleSet> cachedUrlHashs;
- private final ConcurrentMap<BlacklistType, ConcurrentMap<String, List<Pattern>>> hostpaths_matchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here
- private final ConcurrentMap<BlacklistType, ConcurrentMap<String, List<Pattern>>> hostpaths_notmatchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here
-
+ private final ConcurrentMap<BlacklistType, Map<String, List<Pattern>>> hostpaths_matchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here
+ private final ConcurrentMap<BlacklistType, Map<String, List<Pattern>>> hostpaths_notmatchable; // key=host, value=path; mapped url is http://host/path; path does not start with '/' here
public Blacklist(final File rootPath) {
setRootPath(rootPath);
// prepare the data structure
- this.hostpaths_matchable = new ConcurrentHashMap<BlacklistType, ConcurrentMap<String, List<Pattern>>>();
- this.hostpaths_notmatchable = new ConcurrentHashMap<BlacklistType, ConcurrentMap<String, List<Pattern>>>();
+ this.hostpaths_matchable = new ConcurrentHashMap<BlacklistType, Map<String, List<Pattern>>>();
+ this.hostpaths_notmatchable = new ConcurrentHashMap<BlacklistType, Map<String, List<Pattern>>>();
this.cachedUrlHashs = new ConcurrentHashMap<BlacklistType, HandleSet>();
for (final BlacklistType blacklistType : BlacklistType.values()) {
this.hostpaths_matchable.put(blacklistType, new ConcurrentHashMap<String, List<Pattern>>());
this.hostpaths_notmatchable.put(blacklistType, new ConcurrentHashMap<String, List<Pattern>>());
- this.hostpaths_notmatchable.put(blacklistType, new ConcurrentHashMap<String, List<Pattern>>());
- if (blacklistType.equals(BlacklistType.DHT)) {
- loadDHTCache();
- } else {
- this.cachedUrlHashs.put(blacklistType, new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0));
- }
+ loadDHTCache(blacklistType);
}
}
@@ -127,8 +120,10 @@ public class Blacklist {
public synchronized void close() {
Log.logFine("Blacklist", "Shutting down blacklists ...");
- // Save DHT cache
- saveDHTCache();
+ // Save cache
+ for (final BlacklistType blacklistType : BlacklistType.values()) {
+ saveDHTCache(blacklistType);
+ }
Log.logFine("Blacklist", "All blacklists has been shutdown.");
}
@@ -147,7 +142,7 @@ public class Blacklist {
this.blacklistRootPath = rootPath;
}
- protected ConcurrentMap<String, List<Pattern>> getBlacklistMap(final BlacklistType blacklistType, final boolean matchable) {
+ protected Map<String, List<Pattern>> getBlacklistMap(final BlacklistType blacklistType, final boolean matchable) {
return (matchable) ? this.hostpaths_matchable.get(blacklistType) : this.hostpaths_notmatchable.get(blacklistType);
}
@@ -156,7 +151,7 @@ public class Blacklist {
}
public void clear() {
- for (final ConcurrentMap<String, List<Pattern>> entry : this.hostpaths_matchable.values()) {
+ for (final Map<String, List<Pattern>> entry : this.hostpaths_matchable.values()) {
entry.clear();
}
for (final Map<String, List<Pattern>> entry : this.hostpaths_notmatchable.values()) {
@@ -195,8 +190,8 @@ public class Blacklist {
* @param sep
*/
private void loadList(final BlacklistFile blFile, final String sep) {
- final ConcurrentMap<String, List<Pattern>> blacklistMapMatch = getBlacklistMap(blFile.getType(), true);
- final ConcurrentMap<String, List<Pattern>> blacklistMapNotMatch = getBlacklistMap(blFile.getType(), false);
+ final Map<String, List<Pattern>> blacklistMapMatch = getBlacklistMap(blFile.getType(), true);
+ final Map<String, List<Pattern>> blacklistMapNotMatch = getBlacklistMap(blFile.getType(), false);
Set<Map.Entry<String, List<String>>> loadedBlacklist;
Map.Entry<String, List<String>> loadedEntry;
List<Pattern> paths;
@@ -528,10 +523,15 @@ public class Blacklist {
return blacklist != null && blacklist.contains(newEntry);
}
- public final void saveDHTCache() {
+ private static File DHTCacheFile(BlacklistType type) {
+ String BLACKLIST_DHT_CACHEFILE_NAME = "DATA/LISTS/blacklist_" + type.name() + "_Cache.ser";
+ return new File(Switchboard.getSwitchboard().dataPath, BLACKLIST_DHT_CACHEFILE_NAME);
+ }
+
+ private final void saveDHTCache(BlacklistType type) {
try {
- final ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(BLACKLIST_DHT_CACHEFILE));
- out.writeObject(getCacheUrlHashsSet(BlacklistType.DHT));
+ final ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(DHTCacheFile(type)));
+ out.writeObject(getCacheUrlHashsSet(type));
out.close();
} catch (final IOException e) {
@@ -539,13 +539,15 @@ public class Blacklist {
}
}
- public final void loadDHTCache() {
+ private final void loadDHTCache(BlacklistType type) {
try {
- if (BLACKLIST_DHT_CACHEFILE.exists()) {
- final ObjectInputStream in = new ObjectInputStream(new FileInputStream(BLACKLIST_DHT_CACHEFILE));
- this.cachedUrlHashs.put(BlacklistType.DHT, (HandleSet) in.readObject());
+ File cachefile = DHTCacheFile(type);
+ if (cachefile.exists()) {
+ final ObjectInputStream in = new ObjectInputStream(new FileInputStream(cachefile));
+ this.cachedUrlHashs.put(type, (HandleSet) in.readObject());
in.close();
- return;
+ } else {
+ this.cachedUrlHashs.put(type, new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0));
}
} catch (final ClassNotFoundException e) {
Log.logException(e);
@@ -554,6 +556,5 @@ public class Blacklist {
} catch (final IOException e) {
Log.logException(e);
}
- this.cachedUrlHashs.put(BlacklistType.DHT, new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0));
}
}
generated by cgit v1.2.3 (git 2.46.0) at 2026-08-08 09:29:56 +0000