diff options
| -rw-r--r-- | htroot/AccessTracker_p.java | 4 | ||||
| -rw-r--r-- | htroot/CrawlStartExpert.html | 4 | ||||
| -rw-r--r-- | source/net/yacy/cora/protocol/Scanner.java | 1 | ||||
| -rw-r--r-- | source/net/yacy/document/parser/html/ContentScraper.java | 6 |
4 files changed, 8 insertions, 7 deletions
diff --git a/htroot/AccessTracker_p.java b/htroot/AccessTracker_p.java index 6dcb80c71..07f9e422d 100644 --- a/htroot/AccessTracker_p.java +++ b/htroot/AccessTracker_p.java @@ -24,12 +24,10 @@ // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import java.util.AbstractMap; import java.util.Collection; import java.util.ConcurrentModificationException; import java.util.Date; import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -39,9 +37,7 @@ import java.util.concurrent.LinkedBlockingQueue; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.sorting.ConcurrentScoreMap; import net.yacy.cora.sorting.OrderedScoreMap; -import net.yacy.cora.sorting.ScoreMap; import net.yacy.cora.util.CommonPattern; import net.yacy.cora.util.ConcurrentLog; import net.yacy.peers.Seed; diff --git a/htroot/CrawlStartExpert.html b/htroot/CrawlStartExpert.html index b07b26aed..0681684ba 100644 --- a/htroot/CrawlStartExpert.html +++ b/htroot/CrawlStartExpert.html @@ -370,10 +370,10 @@ <legend>Content Filter</legend> <p>These are limitations on parts of a document. The filter will be applied after a web page was loaded.</p> <dl> - <dt>Filter div class names</dt> + <dt>Filter div or nav class names</dt> <dd> <table style="border-width: 0px"> - <tr><td style="width:110px">set of CSS class names</td><td><input name="ignoreclassname" id="ignoreclassname" type="text" size="55" maxlength="100000" value="#[ignoreclassname]#" onblur="if (this.value=='') this.value='';"/></td><td>comma-separated list of <div> element class names which should be filtered out</td></tr> + <tr><td style="width:110px">set of CSS class names</td><td><input name="ignoreclassname" id="ignoreclassname" type="text" size="55" maxlength="100000" value="#[ignoreclassname]#" onblur="if (this.value=='') this.value='';"/></td><td>comma-separated list of <div> or <nav> element class names which should be filtered out</td></tr> </table> </dd> </dl> diff --git a/source/net/yacy/cora/protocol/Scanner.java b/source/net/yacy/cora/protocol/Scanner.java index 6d11388aa..8c84931a9 100644 --- a/source/net/yacy/cora/protocol/Scanner.java +++ b/source/net/yacy/cora/protocol/Scanner.java @@ -152,6 +152,7 @@ public class Scanner { if (access != Access.unknown) Scanner.this.services.put(this, access); } } catch (final OutOfMemoryError e) { + e.printStackTrace(); } } public long age() { diff --git a/source/net/yacy/document/parser/html/ContentScraper.java b/source/net/yacy/document/parser/html/ContentScraper.java index 1a4d46bab..36fb1e34c 100644 --- a/source/net/yacy/document/parser/html/ContentScraper.java +++ b/source/net/yacy/document/parser/html/ContentScraper.java @@ -129,6 +129,7 @@ public class ContentScraper extends AbstractScraper implements Scraper { script(TagType.pair),
span(TagType.pair),
div(TagType.pair),
+ nav(TagType.pair),
article(TagType.pair), // html5
time(TagType.pair), // html5 <time datetime>
// tags used to capture tag content
@@ -1020,7 +1021,10 @@ public class ContentScraper extends AbstractScraper implements Scraper { }
/* Parent is not marked as ignored : let's check the current tag */
- if (!ignore && this.ignoreDivClassNames != null && tag != null && TagName.div.name().equals(tag.name)) {
+ if (!ignore &&
+ this.ignoreDivClassNames != null &&
+ tag != null &&
+ (TagName.div.name().equals(tag.name) || TagName.nav.name().equals(tag.name))) {
final String classAttr = tag.opts.getProperty("class", EMPTY_STRING);
final Set<String> classes = ContentScraper.parseSpaceSeparatedTokens(classAttr);
ignore = !Collections.disjoint(this.ignoreDivClassNames, classes);
|