diff options
| -rw-r--r-- | htroot/ConfigAccountList_p.java | 7 | ||||
| -rw-r--r-- | htroot/IndexImportMediawiki_p.java | 9 | ||||
| -rw-r--r-- | htroot/Table_API_p.java | 11 | ||||
| -rw-r--r-- | source/net/yacy/cora/date/GenericFormatter.java | 232 | ||||
| -rw-r--r-- | source/net/yacy/cora/protocol/HeaderFramework.java | 2 | ||||
| -rw-r--r-- | source/net/yacy/document/DateDetection.java | 4 | ||||
| -rw-r--r-- | source/net/yacy/kelondro/blob/ArrayStack.java | 2 | ||||
| -rw-r--r-- | source/net/yacy/kelondro/blob/MapHeap.java | 15 | ||||
| -rw-r--r-- | source/net/yacy/kelondro/blob/Tables.java | 2 | ||||
| -rw-r--r-- | source/net/yacy/kelondro/data/meta/URIMetadataNode.java | 83 | ||||
| -rw-r--r-- | source/net/yacy/peers/Network.java | 17 | ||||
| -rw-r--r-- | source/net/yacy/peers/NewsDB.java | 2 | ||||
| -rw-r--r-- | source/net/yacy/peers/Protocol.java | 18 | ||||
| -rw-r--r-- | source/net/yacy/peers/Seed.java | 54 | ||||
| -rw-r--r-- | source/net/yacy/search/index/Fulltext.java | 8 | ||||
| -rw-r--r-- | test/java/net/yacy/cora/date/GenericFormatterTest.java | 87 |
16 files changed, 449 insertions, 104 deletions
diff --git a/htroot/ConfigAccountList_p.java b/htroot/ConfigAccountList_p.java index e6ee44f63..ea772af4b 100644 --- a/htroot/ConfigAccountList_p.java +++ b/htroot/ConfigAccountList_p.java @@ -18,8 +18,8 @@ // along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-import java.util.Date;
import java.util.Iterator;
+
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.data.UserDB;
@@ -30,7 +30,7 @@ import net.yacy.server.serverSwitch; public class ConfigAccountList_p {
- public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
+ public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, final serverSwitch env) {
final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard) env;
@@ -54,7 +54,8 @@ public class ConfigAccountList_p { prop.putHTML("userlist_" + numUsers + "_firstname", entry.getFirstName());
prop.putHTML("userlist_" + numUsers + "_address", entry.getAddress());
if (entry.getLastAccess() != null) {
- prop.put("userlist_" + numUsers + "_lastaccess", GenericFormatter.FORMAT_SIMPLE.format(new Date(entry.getLastAccess())));
+ prop.put("userlist_" + numUsers + "_lastaccess",
+ GenericFormatter.formatSafely(entry.getLastAccess(), GenericFormatter.FORMAT_SIMPLE));
} else {
prop.put("userlist_" + numUsers + "_lastaccess", "never");
}
diff --git a/htroot/IndexImportMediawiki_p.java b/htroot/IndexImportMediawiki_p.java index c2279255e..0b95382d7 100644 --- a/htroot/IndexImportMediawiki_p.java +++ b/htroot/IndexImportMediawiki_p.java @@ -25,6 +25,7 @@ import java.io.File; import java.io.IOException; import java.net.MalformedURLException; +import java.time.Instant; import java.util.Date; import org.apache.http.Header; @@ -131,11 +132,11 @@ public class IndexImportMediawiki_p { if (status == 0 && post.getBoolean("iffresh")) { long lastModified = getLastModified(sourceURL); - if (lastExecutionDate != null && lastModified != 0L - && lastModified <= lastExecutionDate.getTime()) { + if (lastExecutionDate != null && lastModified != 0L && Instant.ofEpochMilli(lastModified) + .isBefore(lastExecutionDate.toInstant())) { status = 5; - prop.put("import_status_lastImportDate", - GenericFormatter.FORMAT_SIMPLE.format(lastExecutionDate)); + prop.put("import_status_lastImportDate", GenericFormatter + .formatSafely(lastExecutionDate.toInstant(), GenericFormatter.FORMAT_SIMPLE)); /* the import is not performed, but we increase here the api call count */ if(sb.tables != null) { diff --git a/htroot/Table_API_p.java b/htroot/Table_API_p.java index 9e73163f6..1b36adaba 100644 --- a/htroot/Table_API_p.java +++ b/htroot/Table_API_p.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.text.DateFormat; import java.text.ParseException; import java.util.ArrayList; import java.util.Date; @@ -58,6 +59,8 @@ public class Table_API_p { final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); + final DateFormat dateFormat = GenericFormatter.newSimpleDateFormat(); + prop.put("showexec", 0); prop.put("showtable", 0); @@ -182,7 +185,7 @@ public class Table_API_p { final int time = row.get(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, 0); final String dateNextExecStr = entry.getValue().trim(); try { - final Date dateNextExec = GenericFormatter.FORMAT_SIMPLE.parse(dateNextExecStr); + final Date dateNextExec = dateFormat.parse(dateNextExecStr); if(time != 0) { // Check there is effectively a schedule period on this row if(dateNextExec.before(now)) { @@ -390,8 +393,8 @@ public class Table_API_p { prop.put("showtable_list_" + count + "_pk", UTF8.String(row.getPK())); prop.put("showtable_list_" + count + "_count", count); prop.put("showtable_list_" + count + "_callcount", callcount); - prop.put("showtable_list_" + count + "_dateRecording", date_recording == null ? "-" : GenericFormatter.FORMAT_SIMPLE.format(date_recording)); - prop.put("showtable_list_" + count + "_dateLastExec", date_last_exec == null ? "-" : GenericFormatter.FORMAT_SIMPLE.format(date_last_exec)); + prop.put("showtable_list_" + count + "_dateRecording", date_recording == null ? "-" : dateFormat.format(date_recording)); + prop.put("showtable_list_" + count + "_dateLastExec", date_last_exec == null ? "-" : dateFormat.format(date_last_exec)); prop.put("showtable_list_" + count + "_editableDateNext", time != 0); final String enteredDateBeforeNow = nextExecDatesBeforeNow.get(rowPKStr); @@ -407,7 +410,7 @@ public class Table_API_p { } prop.put("showtable_list_" + count + "_editableDateNext_dateLastExecPattern", GenericFormatter.PATTERN_SIMPLE_REGEX); - prop.put("showtable_list_" + count + "_editableDateNext_dateNextExec", date_next_exec == null ? "-" : GenericFormatter.FORMAT_SIMPLE.format(date_next_exec)); + prop.put("showtable_list_" + count + "_editableDateNext_dateNextExec", date_next_exec == null ? "-" : dateFormat.format(date_next_exec)); prop.put("showtable_list_" + count + "_editableDateNext_pk", rowPKStr); prop.put("showtable_list_" + count + "_type", row.get(WorkTables.TABLE_API_COL_TYPE)); diff --git a/source/net/yacy/cora/date/GenericFormatter.java b/source/net/yacy/cora/date/GenericFormatter.java index 5190bc726..31a5ac67d 100644 --- a/source/net/yacy/cora/date/GenericFormatter.java +++ b/source/net/yacy/cora/date/GenericFormatter.java @@ -27,6 +27,13 @@ package net.yacy.cora.date; import java.text.DecimalFormat; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.DateTimeException; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; import java.util.Calendar; import java.util.Date; import java.util.Locale; @@ -44,37 +51,159 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter public static final String PATTERN_ANSIC = "EEE MMM d HH:mm:ss yyyy"; public static final String PATTERN_SIMPLE = "yyyy/MM/dd HH:mm:ss"; - /** A regular expression matching the PATTERN_SIMPLE pattern (does not control last day of month (30/31 or 28/29 for february). - * Can be used as a HTML5 input field validation pattern */ - public static final String PATTERN_SIMPLE_REGEX ="[0-9]{4}/(0[1-9]|1[012])/(0[1-9]|1[0-9]|2[0-9]|3[01]) (0[0-9]|1[0-9]|2[0-3])(:[0-5][0-9]){2}"; - - public static final SimpleDateFormat FORMAT_SHORT_DAY = new SimpleDateFormat(PATTERN_SHORT_DAY, Locale.US); - public static final SimpleDateFormat FORMAT_SHORT_MINUTE = new SimpleDateFormat(PATTERN_SHORT_MINUTE, Locale.US); - public static final SimpleDateFormat FORMAT_SHORT_SECOND = new SimpleDateFormat(PATTERN_SHORT_SECOND, Locale.US); - public static final SimpleDateFormat FORMAT_SHORT_MILSEC = new SimpleDateFormat(PATTERN_SHORT_MILSEC, Locale.US); - public static final SimpleDateFormat FORMAT_RFC1123_SHORT = new SimpleDateFormat(PATTERN_RFC1123_SHORT, Locale.US); - public static final SimpleDateFormat FORMAT_ANSIC = new SimpleDateFormat(PATTERN_ANSIC, Locale.US); - public static final SimpleDateFormat FORMAT_SIMPLE = new SimpleDateFormat(PATTERN_SIMPLE, Locale.US); - - static { - // we want GMT times on the formats as well as they don't support any timezone - FORMAT_SHORT_DAY.setTimeZone(UTCtimeZone); - FORMAT_SHORT_SECOND.setTimeZone(UTCtimeZone); - FORMAT_SHORT_MILSEC.setTimeZone(UTCtimeZone); - } + /** + * A regular expression matching the PATTERN_SIMPLE pattern (does not control + * last day of month (30/31 or 28/29 for february). Can be used as a HTML5 input + * field validation pattern + */ + public static final String PATTERN_SIMPLE_REGEX = "[0-9]{4}/(0[1-9]|1[012])/(0[1-9]|1[0-9]|2[0-9]|3[01]) (0[0-9]|1[0-9]|2[0-3])(:[0-5][0-9]){2}"; + + /** + * A thread-safe date formatter using the + * {@link GenericFormatter#PATTERN_SHORT_DAY} pattern with the US locale on the + * UTC time zone. + */ + public static final DateTimeFormatter FORMAT_SHORT_DAY = DateTimeFormatter + .ofPattern(PATTERN_SHORT_DAY.replace("yyyy", "uuuu"), Locale.US).withZone(ZoneOffset.UTC); + + /** + * A thread-safe date formatter using the + * {@link GenericFormatter#PATTERN_SHORT_MINUTE} pattern with the US locale on + * the system time zone. + */ + public static final DateTimeFormatter FORMAT_SHORT_MINUTE = DateTimeFormatter + .ofPattern(PATTERN_SHORT_MINUTE.replace("yyyy", "uuuu"), Locale.US).withZone(ZoneId.systemDefault()); + + + /** + * A thread-safe date formatter using the + * {@link GenericFormatter#PATTERN_SHORT_SECOND} pattern with the US locale on + * the UTC time zone. + */ + public static final DateTimeFormatter FORMAT_SHORT_SECOND = DateTimeFormatter + .ofPattern(PATTERN_SHORT_SECOND.replace("yyyy", "uuuu"), Locale.US).withZone(ZoneOffset.UTC); + + /** + * A thread-safe date formatter using the + * {@link GenericFormatter#PATTERN_SHORT_MILSEC} pattern with the US locale on + * the UTC time zone. + */ + public static final DateTimeFormatter FORMAT_SHORT_MILSEC = new DateTimeFormatterBuilder() + .appendPattern(PATTERN_SHORT_MILSEC.replace("yyyy", "uuuu").replaceAll("SSS", "")) + .appendValue(ChronoField.MILLI_OF_SECOND, 3).toFormatter().withLocale(Locale.US) + .withZone(ZoneOffset.UTC);/* we can not use here the 'SSS' pattern for milliseconds on JDK 8 (see https://bugs.openjdk.java.net/browse/JDK-8031085) */ + + /** + * A thread-safe date formatter using the + * {@link GenericFormatter#PATTERN_RFC1123_SHORT} pattern with the US locale on + * the system time zone. + */ + public static final DateTimeFormatter FORMAT_RFC1123_SHORT = DateTimeFormatter + .ofPattern(PATTERN_RFC1123_SHORT.replace("yyyy", "uuuu"), Locale.US).withZone(ZoneId.systemDefault()); + + /** + * A thread-safe date formatter using the {@link GenericFormatter#PATTERN_ANSIC} + * pattern with the US locale on the system time zone. + */ + public static final DateTimeFormatter FORMAT_ANSIC = DateTimeFormatter.ofPattern(PATTERN_ANSIC.replace("yyyy", "uuuu"), Locale.US) + .withZone(ZoneId.systemDefault()); + + /** + * A thread-safe date formatter using the + * {@link GenericFormatter#PATTERN_SIMPLE} pattern (adapted for the DateTimeFormatter class) with the US locale on the + * system time zone. + */ + public static final DateTimeFormatter FORMAT_SIMPLE = DateTimeFormatter.ofPattern(PATTERN_SIMPLE.replace("yyyy", "uuuu"), Locale.US) + .withZone(ZoneId.systemDefault()); + + /** + * @return a new SimpleDateFormat instance using the + * {@link GenericFormatter#PATTERN_SHORT_DAY} pattern with the US + * locale. + */ + public static SimpleDateFormat newShortDayFormat() { + final SimpleDateFormat dateFormat = new SimpleDateFormat(GenericFormatter.PATTERN_SHORT_DAY, Locale.US); + + // we want GMT times on the formats as well as they don't support any timezone + dateFormat.setTimeZone(UTCtimeZone); + + return dateFormat; + } + + /** + * @return a new SimpleDateFormat instance using the + * {@link GenericFormatter#PATTERN_SHORT_MINUTE} pattern with the US + * locale. + */ + public static SimpleDateFormat newShortMinuteFormat() { + return new SimpleDateFormat(GenericFormatter.PATTERN_SHORT_MINUTE, Locale.US); + } + + /** + * @return a new SimpleDateFormat instance using the + * {@link GenericFormatter#PATTERN_SHORT_SECOND} pattern with the US + * locale. + */ + public static SimpleDateFormat newShortSecondFormat() { + final SimpleDateFormat dateFormat = new SimpleDateFormat(GenericFormatter.PATTERN_SHORT_SECOND, Locale.US); + + // we want GMT times on the formats as well as they don't support any timezone + dateFormat.setTimeZone(UTCtimeZone); + + return dateFormat; + } + + /** + * @return a new SimpleDateFormat instance using the + * {@link GenericFormatter#PATTERN_SHORT_MILSEC} pattern with the US + * locale. + */ + public static SimpleDateFormat newShortMilsecFormat() { + final SimpleDateFormat dateFormat = new SimpleDateFormat(GenericFormatter.PATTERN_SHORT_MILSEC, Locale.US); + + // we want GMT times on the formats as well as they don't support any timezone + dateFormat.setTimeZone(UTCtimeZone); + + return dateFormat; + } + + /** + * @return a new SimpleDateFormat instance using the + * {@link GenericFormatter#PATTERN_RFC1123_SHORT} pattern with the US + * locale. + */ + public static SimpleDateFormat newRfc1123ShortFormat() { + return new SimpleDateFormat(GenericFormatter.PATTERN_RFC1123_SHORT, Locale.US); + } + + /** + * @return a new SimpleDateFormat instance using the + * {@link GenericFormatter#PATTERN_ANSIC} pattern with the US locale. + */ + public static SimpleDateFormat newAnsicFormat() { + return new SimpleDateFormat(GenericFormatter.PATTERN_ANSIC, Locale.US); + } + + /** + * @return a new SimpleDateFormat instance using the + * {@link GenericFormatter#PATTERN_SIMPLE} pattern with the US locale. + */ + public static SimpleDateFormat newSimpleDateFormat() { + return new SimpleDateFormat(GenericFormatter.PATTERN_SIMPLE, Locale.US); + } public static final long time_second = 1000L; public static final long time_minute = 60000L; public static final long time_hour = 60 * time_minute; public static final long time_day = 24 * time_hour; - public static final GenericFormatter SHORT_DAY_FORMATTER = new GenericFormatter(FORMAT_SHORT_DAY, time_minute); - public static final GenericFormatter SHORT_MINUTE_FORMATTER = new GenericFormatter(FORMAT_SHORT_MINUTE, time_second); - public static final GenericFormatter SHORT_SECOND_FORMATTER = new GenericFormatter(FORMAT_SHORT_SECOND, time_second); - public static final GenericFormatter SHORT_MILSEC_FORMATTER = new GenericFormatter(FORMAT_SHORT_MILSEC, 1); - public static final GenericFormatter RFC1123_SHORT_FORMATTER = new GenericFormatter(FORMAT_RFC1123_SHORT, time_minute); - public static final GenericFormatter ANSIC_FORMATTER = new GenericFormatter(FORMAT_ANSIC, time_second); - public static final GenericFormatter SIMPLE_FORMATTER = new GenericFormatter(FORMAT_SIMPLE, time_second); + public static final GenericFormatter SHORT_DAY_FORMATTER = new GenericFormatter(newShortDayFormat(), time_minute); + public static final GenericFormatter SHORT_MINUTE_FORMATTER = new GenericFormatter(newShortMinuteFormat(), time_second); + public static final GenericFormatter SHORT_SECOND_FORMATTER = new GenericFormatter(newShortSecondFormat(), time_second); + public static final GenericFormatter SHORT_MILSEC_FORMATTER = new GenericFormatter(newShortMilsecFormat(), 1); + public static final GenericFormatter RFC1123_SHORT_FORMATTER = new GenericFormatter(newRfc1123ShortFormat(), time_minute); + public static final GenericFormatter ANSIC_FORMATTER = new GenericFormatter(newAnsicFormat(), time_second); + public static final GenericFormatter SIMPLE_FORMATTER = new GenericFormatter(newSimpleDateFormat(), time_second); private final SimpleDateFormat dateFormat; private final long maxCacheDiff; @@ -201,6 +330,59 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter } private final static DecimalFormat D2 = new DecimalFormat("00"); + + /** + * Safely format the given time value using the given formatter. Fallback to + * ISO-8601 representation or to raw time value without exception when the + * format can not be applied. + * + * @param time + * a time value as millisecnods from Epoch (1970-01-01T00:00:00Z) + * @param formatter + * the formatter to use + * @return a String representation of the time value + */ + public static String formatSafely(final long time, final DateTimeFormatter formatter) { + String res; + try { + res = formatSafely(Instant.ofEpochMilli(time), formatter); + } catch (final DateTimeException e) { + /* + * Can occur on Instant.ofEpochMilli when the time value is greater than + * Instant.MAX.toEpochMilli() or lower than Instant.MIN.toEpochMilli() + */ + res = String.valueOf(time); + } + return res; + } + + /** + * Safely format the given instant using the given formatter. Fallback to + * ISO-8601 representation without exception when the format can not be applied. + * + * @param instant + * the instant to format + * @param formatter + * the formatter to use + * @return a String representation of the time value + */ + public static String formatSafely(final Instant instant, final DateTimeFormatter formatter) { + String res; + if (instant == null) { + res = ""; + } else { + try { + if (formatter != null) { + res = formatter.format(instant); + } else { + res = instant.toString(); + } + } catch (final DateTimeException e) { + res = instant.toString(); + } + } + return res; + } public static void main(String[] args) { System.out.println(UTCDiffString()); diff --git a/source/net/yacy/cora/protocol/HeaderFramework.java b/source/net/yacy/cora/protocol/HeaderFramework.java index cf7f073e2..3595cc933 100644 --- a/source/net/yacy/cora/protocol/HeaderFramework.java +++ b/source/net/yacy/cora/protocol/HeaderFramework.java @@ -245,7 +245,7 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri // RFC 1036/850 (old) "Monday, 12-Nov-07 10:11:12 GMT"
FORMAT_RFC1036,
// ANSI C asctime() "Mon Nov 12 10:11:12 2007"
- GenericFormatter.FORMAT_ANSIC,
+ GenericFormatter.newAnsicFormat(),
};
diff --git a/source/net/yacy/document/DateDetection.java b/source/net/yacy/document/DateDetection.java index 56b2e2ecf..b7ec754ad 100644 --- a/source/net/yacy/document/DateDetection.java +++ b/source/net/yacy/document/DateDetection.java @@ -674,8 +674,8 @@ public class DateDetection { // check standard date formats try {d = CONFORM.parse(text);} catch (ParseException e) {} //if (d == null) try {d = GenericFormatter.FORMAT_SHORT_DAY.parse(text);} catch (ParseException e) {} // did not work well and fired for wrong formats; do not use - if (d == null) try {d = GenericFormatter.FORMAT_RFC1123_SHORT.parse(text);} catch (ParseException e) {} - if (d == null) try {d = GenericFormatter.FORMAT_ANSIC.parse(text);} catch (ParseException e) {} + if (d == null) try {d = GenericFormatter.newRfc1123ShortFormat().parse(text);} catch (ParseException e) {} + if (d == null) try {d = GenericFormatter.newAnsicFormat().parse(text);} catch (ParseException e) {} if (d == null) { // check other date formats diff --git a/source/net/yacy/kelondro/blob/ArrayStack.java b/source/net/yacy/kelondro/blob/ArrayStack.java index 46da81835..91f029d61 100644 --- a/source/net/yacy/kelondro/blob/ArrayStack.java +++ b/source/net/yacy/kelondro/blob/ArrayStack.java @@ -103,7 +103,7 @@ public class ArrayStack implements BLOB { private final ExecutorService executor; // use our own formatter to prevent concurrency locks with other processes - private final static GenericFormatter my_SHORT_MILSEC_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_MILSEC, 1); + private final static GenericFormatter my_SHORT_MILSEC_FORMATTER = new GenericFormatter(GenericFormatter.newShortMilsecFormat(), 1); public ArrayStack( diff --git a/source/net/yacy/kelondro/blob/MapHeap.java b/source/net/yacy/kelondro/blob/MapHeap.java index f8d34cc38..743a2e7a0 100644 --- a/source/net/yacy/kelondro/blob/MapHeap.java +++ b/source/net/yacy/kelondro/blob/MapHeap.java @@ -32,6 +32,8 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; +import java.time.DateTimeException; +import java.time.Instant; import java.util.AbstractMap; import java.util.Collection; import java.util.HashMap; @@ -144,9 +146,6 @@ public class MapHeap implements Map<byte[], Map<String, String>> { } - // use our own formatter to prevent concurrency locks with other processes - private final static GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); - /** * write a whole byte array as Map to the table * @param key the primary key @@ -159,7 +158,15 @@ public class MapHeap implements Map<byte[], Map<String, String>> { assert key.length > 0; assert newMap != null; key = normalizeKey(key); - final String s = map2string(newMap, "W" + my_SHORT_SECOND_FORMATTER.format() + " "); + String formattedTime; + try { + /* Prefer using first the shared and thread-safe DateTimeFormatter instance */ + formattedTime = GenericFormatter.FORMAT_SHORT_SECOND.format(Instant.now()); + } catch (final DateTimeException e) { + /* This should not happen, but rather than failing we fallback to the old formatter wich uses synchronization locks */ + formattedTime = GenericFormatter.SHORT_SECOND_FORMATTER.format(); + } + final String s = map2string(newMap, "W" + formattedTime + " "); assert s != null; final byte[] sb = UTF8.getBytes(s); if (this.cache == null) { diff --git a/source/net/yacy/kelondro/blob/Tables.java b/source/net/yacy/kelondro/blob/Tables.java index ec4cb3205..af3519a2e 100644 --- a/source/net/yacy/kelondro/blob/Tables.java +++ b/source/net/yacy/kelondro/blob/Tables.java @@ -75,7 +75,7 @@ public class Tables implements Iterable<String> { private int keymaxlen; // use our own formatter to prevent concurrency locks with other processes - private final static GenericFormatter my_SHORT_MILSEC_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_MILSEC, 1); + private final static GenericFormatter my_SHORT_MILSEC_FORMATTER = new GenericFormatter(GenericFormatter.newShortMilsecFormat(), 1); public Tables(final File location, final int keymaxlen) { this.location = new File(location.getAbsolutePath()); diff --git a/source/net/yacy/kelondro/data/meta/URIMetadataNode.java b/source/net/yacy/kelondro/data/meta/URIMetadataNode.java index 35d0e1c0d..1313f15b4 100644 --- a/source/net/yacy/kelondro/data/meta/URIMetadataNode.java +++ b/source/net/yacy/kelondro/data/meta/URIMetadataNode.java @@ -26,6 +26,9 @@ import java.awt.Dimension; import java.io.IOException; import java.net.MalformedURLException; import java.text.ParseException; +import java.time.DateTimeException; +import java.time.LocalDate; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -125,24 +128,12 @@ public class URIMetadataNode extends SolrDocument /* implements Comparable<URIMe this.lon = (lons == null) ? 0.0d : Double.parseDouble(lons); this.lat = (lats == null) ? 0.0d : Double.parseDouble(lats); - // create new formatters to make concurrency possible - final GenericFormatter formatter = new GenericFormatter(GenericFormatter.FORMAT_SHORT_DAY, GenericFormatter.time_minute); - - try { - this.setField(CollectionSchema.last_modified.name(), formatter.parse(prop.getProperty("mod", "20000101"), 0).getTime()); - } catch (final ParseException e) { - this.setField(CollectionSchema.last_modified.name(), new Date()); - } - try { - this.setField(CollectionSchema.load_date_dt.name(), formatter.parse(prop.getProperty("load", "20000101"), 0).getTime()); - } catch (final ParseException e) { - this.setField(CollectionSchema.load_date_dt.name(), new Date()); - } - try { - this.setField(CollectionSchema.fresh_date_dt.name(), formatter.parse(prop.getProperty("fresh", "20000101"), 0).getTime()); - } catch (final ParseException e) { - this.setField(CollectionSchema.fresh_date_dt.name(), new Date()); - } + this.setField(CollectionSchema.last_modified.name(), parseShortDayDate(prop.getProperty("mod", "20000101"))); + + this.setField(CollectionSchema.load_date_dt.name(), parseShortDayDate(prop.getProperty("load", "20000101"))); + + this.setField(CollectionSchema.fresh_date_dt.name(), parseShortDayDate(prop.getProperty("fresh", "20000101"))); + this.setField(CollectionSchema.referrer_id_s.name(), prop.getProperty("referrer", "")); // this.setField(CollectionSchema.md5_s.name(), prop.getProperty("md5", "")); // always 0 (not used / calculated) this.setField(CollectionSchema.size_i.name(), Integer.parseInt(prop.getProperty("size", "0"))); @@ -725,14 +716,56 @@ public class URIMetadataNode extends SolrDocument /* implements Comparable<URIMe return null; } } - + + /** + * Format a date using the short day format. + * @param date the date to format. Must not be null. + * @return the formatted date + * @throws NullPointerException when date is null. + */ + private String formatShortDayDate(final Date date) { + String formattedDate; + try { + /* Prefer using first the thread-safe shared instance of DateTimeFormatter */ + formattedDate = GenericFormatter.FORMAT_SHORT_DAY.format(date.toInstant()); + } catch (final DateTimeException e) { + /* + * Should not happen, but rather than failing it is preferable to use the old + * formatter which uses synchronization locks + */ + formattedDate = GenericFormatter.SHORT_DAY_FORMATTER.format(date); + } + return formattedDate; + } + + /** + * Parse a date string with the short day format. + * + * @param dateStr + * a date representation as a String. Must not be null. + * @return the parsed Date or the current date when an parsing error occurred. + */ + private Date parseShortDayDate(final String dateStr) { + Date parsed; + try { + /* Prefer using first the thread-safe shared instance of DateTimeFormatter */ + parsed = Date.from(LocalDate.parse(dateStr, GenericFormatter.FORMAT_SHORT_DAY).atStartOfDay() + .toInstant(ZoneOffset.UTC)); + } catch (final RuntimeException e) { + /* Retry with the old formatter which uses synchronization locks */ + try { + parsed = GenericFormatter.SHORT_DAY_FORMATTER.parse(dateStr, 0).getTime(); + } catch (final ParseException pe) { + parsed = new Date(); + } + } + return parsed; + } + protected StringBuilder corePropList() { // generate a parseable string; this is a simple property-list final StringBuilder s = new StringBuilder(300); - // create new formatters to make concurrency possible - final GenericFormatter formatter = new GenericFormatter(GenericFormatter.FORMAT_SHORT_DAY, GenericFormatter.time_minute); - try { s.append("hash=").append(ASCII.String(this.hash())); s.append(",url=").append(crypt.simpleEncode(this.url().toNormalform(true))); @@ -742,9 +775,9 @@ public class URIMetadataNode extends SolrDocument /* implements Comparable<URIMe s.append(",publisher=").append(crypt.simpleEncode(this.dc_publisher())); s.append(",lat=").append(this.lat()); s.append(",lon=").append(this.lon()); - s.append(",mod=").append(formatter.format(this.moddate())); - s.append(",load=").append(formatter.format(this.loaddate())); - s.append(",fresh=").append(formatter.format(this.freshdate())); + s.append(",mod=").append(formatShortDayDate(this.moddate())); + s.append(",load=").append(formatShortDayDate(this.loaddate())); + s.append(",fresh=").append(formatShortDayDate(this.freshdate())); s.append(",referrer=").append(this.referrerHash() == null ? "" : ASCII.String(this.referrerHash())); //s.append(",md5=").append(this.md5()); // md5 never calculated / not used, also removed from this(prop) 2015-11-27 s.append(",size=").append(this.filesize()); diff --git a/source/net/yacy/peers/Network.java b/source/net/yacy/peers/Network.java index ec2c6aadd..1a1bbb669 100644 --- a/source/net/yacy/peers/Network.java +++ b/source/net/yacy/peers/Network.java @@ -40,7 +40,6 @@ package net.yacy.peers; import java.net.MalformedURLException; import java.util.Collections; -import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; @@ -49,6 +48,7 @@ import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; + import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.feed.RSSFeed; @@ -188,11 +188,6 @@ public class Network publishMySeed(); } - // use our own formatter to prevent concurrency locks with other processes - private final static GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter( - GenericFormatter.FORMAT_SHORT_SECOND, - GenericFormatter.time_second); - protected class publishThread extends Thread { private final Seed seed; @@ -256,13 +251,19 @@ public class Network // update last seed date if ( newSeed.getLastSeenUTC() >= this.seed.getLastSeenUTC() ) { if ( log.isFine() ) { - log.fine("publish: recently handshaked " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getIPs() + " with old LastSeen: '" + my_SHORT_SECOND_FORMATTER.format(new Date(newSeed.getLastSeenUTC())) + "'"); + final String newSeedLastSeenStr = GenericFormatter.formatSafely( + newSeed.getLastSeenUTC(), GenericFormatter.FORMAT_SHORT_SECOND); + log.fine("publish: recently handshaked " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getIPs() + " with old LastSeen: '" + newSeedLastSeenStr + "'"); } newSeed.setLastSeenUTC(); Network.this.sb.peers.peerActions.peerArrival(newSeed, true); } else { if ( log.isFine() ) { - log.fine("publish: recently handshaked " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getIPs() + " with old LastSeen: '" + my_SHORT_SECOND_FORMATTER.format(new Date(newSeed.getLastSeenUTC())) + "', this is more recent: '" + my_SHORT_SECOND_FORMATTER.format(new Date(this.seed.getLastSeenUTC())) + "'"); + final String newSeedLastSeenStr = GenericFormatter.formatSafely( + newSeed.getLastSeenUTC(), GenericFormatter.FORMAT_SHORT_SECOND); + final String thisSeedLastSeenStr = GenericFormatter.formatSafely( + newSeed.getLastSeenUTC(), GenericFormatter.FORMAT_SHORT_SECOND); + log.fine("publish: recently handshaked " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getIPs() + " with old LastSeen: '" + newSeedLastSeenStr + "', this is more recent: '" + thisSeedLastSeenStr + "'"); } this.seed.setLastSeenUTC(); Network.this.sb.peers.peerActions.peerArrival(this.seed, true); diff --git a/source/net/yacy/peers/NewsDB.java b/source/net/yacy/peers/NewsDB.java index a984e3dec..044530903 100644 --- a/source/net/yacy/peers/NewsDB.java +++ b/source/net/yacy/peers/NewsDB.java @@ -162,7 +162,7 @@ public class NewsDB { }
// use our own formatter to prevent concurrency locks with other processes
- private final static GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second);
+ private final static GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.newShortSecondFormat(), GenericFormatter.time_second);
private Record b2r(final Row.Entry b) {
if (b == null) return null;
diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index e3cfca1ca..537bda1de 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -49,8 +49,11 @@ import java.io.IOException; import java.io.InputStreamReader;
import java.net.InetAddress;
import java.net.MalformedURLException;
+import java.time.DateTimeException;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -2148,10 +2151,17 @@ public final class Protocol { if ( targetHash != null ) parts.put("youare", UTF8.StringBody(targetHash));
// time information for synchronization
- // use our own formatter to prevent concurrency locks with other processes
- final GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second);
- parts.put("mytime", UTF8.StringBody(my_SHORT_SECOND_FORMATTER.format()));
- parts.put("myUTC", UTF8.StringBody(Long.toString(System.currentTimeMillis())));
+ final long myTime = System.currentTimeMillis();
+ String formattedTime;
+ try {
+ /* Prefer using first the shared and thread-safe DateTimeFormatter instance */
+ formattedTime = GenericFormatter.FORMAT_SHORT_SECOND.format(Instant.ofEpochMilli(myTime));
+ } catch(final DateTimeException e) {
+ /* This should not happen, but rather than failing we fallback to the old formatter wich uses synchronization locks */
+ formattedTime = GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(myTime));
+ }
+ parts.put("mytime", UTF8.StringBody(formattedTime));
+ parts.put("myUTC", UTF8.StringBody(Long.toString(myTime)));
// network identification
parts.put(SwitchboardConstants.NETWORK_NAME, UTF8.StringBody(Switchboard.getSwitchboard().getConfig(
diff --git a/source/net/yacy/peers/Seed.java b/source/net/yacy/peers/Seed.java index 3ea735780..7d27b92d2 100644 --- a/source/net/yacy/peers/Seed.java +++ b/source/net/yacy/peers/Seed.java @@ -49,6 +49,10 @@ import java.net.InetAddress; import java.net.MalformedURLException;
import java.net.URL;
import java.text.ParseException;
+import java.time.DateTimeException;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
@@ -855,14 +859,15 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed> return Integer.parseInt(port);
}
- /** puts the current time into the lastseen field and cares about the time differential to UTC */
+ /** puts the current UTC time into the lastseen field */
public final void setLastSeenUTC() {
- // because java thinks it must apply the UTC offset to the current time,
- // to create a string that looks like our current time, it adds the local UTC offset to the
- // time. To create a corrected UTC Date string, we first subtract the local UTC offset.
- final GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes
- final String ls = my_SHORT_SECOND_FORMATTER.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/));
- this.dna.put(Seed.LASTSEEN, ls);
+ try {
+ /* Prefer using first the shared and thread-safe DateTimeFormatter instance */
+ this.dna.put(Seed.LASTSEEN, GenericFormatter.FORMAT_SHORT_SECOND.format(Instant.now()));
+ } catch(final DateTimeException e) {
+ /* This should not happen, but rather than failing we fallback to the old formatter wich uses synchronization locks */
+ this.dna.put(Seed.LASTSEEN, GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date()));
+ }
}
/**
@@ -870,9 +875,17 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed> */
public final long getLastSeenUTC() {
try {
- final GenericFormatter my_SHORT_SECOND_FORMATTER =
- new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes
- final long t = my_SHORT_SECOND_FORMATTER.parse(get(Seed.LASTSEEN, "20040101000000"), 0).getTime().getTime();
+ final String lastSeenStr = get(Seed.LASTSEEN, "20040101000000");
+ long t;
+ try {
+ /* Prefer using first the shared and thread-safe DateTimeFormatter instance */
+ t = LocalDateTime.parse(lastSeenStr, GenericFormatter.FORMAT_SHORT_SECOND).toInstant(ZoneOffset.UTC).toEpochMilli();
+ } catch(final RuntimeException e) {
+ /* Retry with the old date API parser */
+ final GenericFormatter my_SHORT_SECOND_FORMATTER =
+ new GenericFormatter(GenericFormatter.newShortSecondFormat(), GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes
+ t = my_SHORT_SECOND_FORMATTER.parse(lastSeenStr, 0).getTime().getTime();
+ }
// getTime creates a UTC time number. But in this case java thinks, that the given
// time string is a local time, which has a local UTC offset applied.
// Therefore java subtracts the local UTC offset, to get a UTC number.
@@ -903,13 +916,20 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed> return this.birthdate;
}
long b;
- try {
- final GenericFormatter my_SHORT_SECOND_FORMATTER =
- new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes
- b = my_SHORT_SECOND_FORMATTER.parse(get(Seed.BDATE, "20040101000000"), 0).getTime().getTime();
- } catch (final ParseException e ) {
- b = System.currentTimeMillis();
- }
+ final String bdateStr = get(Seed.BDATE, "20040101000000");
+ try {
+ /* Prefer using first the shared and thread-safe DateTimeFormatter instance */
+ b = LocalDateTime.parse(bdateStr, GenericFormatter.FORMAT_SHORT_SECOND).toInstant(ZoneOffset.UTC).toEpochMilli();
+ } catch(final RuntimeException e) {
+ /* Retry with the old date API parser */
+ try {
+ final GenericFormatter my_SHORT_SECOND_FORMATTER =
+ new GenericFormatter(GenericFormatter.newShortSecondFormat(), GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes
+ b = my_SHORT_SECOND_FORMATTER.parse(bdateStr, 0).getTime().getTime();
+ } catch (final ParseException pe) {
+ b = System.currentTimeMillis();
+ }
+ }
this.birthdate = b;
return this.birthdate;
}
diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java index 6620831d1..a4938cb0f 100644 --- a/source/net/yacy/search/index/Fulltext.java +++ b/source/net/yacy/search/index/Fulltext.java @@ -741,9 +741,9 @@ public final class Fulltext { String s = new File(path, yacy_dump_prefix + - "f" + GenericFormatter.FORMAT_SHORT_MINUTE.format(firstdate) + "_" + - "l" + GenericFormatter.FORMAT_SHORT_MINUTE.format(lastdate) + "_" + - "n" + GenericFormatter.FORMAT_SHORT_MINUTE.format(new Date(now)) + "_" + + "f" + GenericFormatter.SHORT_MINUTE_FORMATTER.format(firstdate) + "_" + + "l" + GenericFormatter.SHORT_MINUTE_FORMATTER.format(lastdate) + "_" + + "n" + GenericFormatter.SHORT_MINUTE_FORMATTER.format(new Date(now)) + "_" + "c" + String.format("%1$012d", doccount)).getAbsolutePath() + "_tc"; // the name ends with the transaction token ('c' = 'created') // create export file name @@ -767,7 +767,7 @@ public final class Fulltext { public static void main(String args[]) { Date firstdate = null; - System.out.println(GenericFormatter.FORMAT_SHORT_MINUTE.format(firstdate)); + System.out.println(GenericFormatter.SHORT_MINUTE_FORMATTER.format(firstdate)); } public Export export() { diff --git a/test/java/net/yacy/cora/date/GenericFormatterTest.java b/test/java/net/yacy/cora/date/GenericFormatterTest.java new file mode 100644 index 000000000..6ddbd00dc --- /dev/null +++ b/test/java/net/yacy/cora/date/GenericFormatterTest.java @@ -0,0 +1,87 @@ +// GenericFormatterTest.java +// Copyright 2018 by luccioman; https://github.com/luccioman +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package net.yacy.cora.date; + +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.util.Date; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Unit tests for the {@link GenericFormatter} class. + */ +public class GenericFormatterTest { + + /** + * Check that the date patterns are properly written : no error should occur + * when using them for formatting and parsing. + */ + @Test + public void testFormats() { + final Instant time = Instant.parse("2018-06-28T10:49:35.726Z"); + + String formatted = GenericFormatter.FORMAT_SHORT_DAY.format(time); + System.out.println("GenericFormatter.FORMAT_SHORT_DAY : " + formatted); + SimpleDateFormat oldApiFormat = GenericFormatter.newShortDayFormat(); + Assert.assertEquals(oldApiFormat.format(Date.from(time)), formatted); + GenericFormatter.FORMAT_SHORT_DAY.parse("20180628"); + + formatted = GenericFormatter.FORMAT_SHORT_MINUTE.format(time); + System.out.println("GenericFormatter.FORMAT_SHORT_MINUTE : " + formatted); + oldApiFormat = GenericFormatter.newShortMinuteFormat(); + Assert.assertEquals(oldApiFormat.format(Date.from(time)), formatted); + GenericFormatter.FORMAT_SHORT_MINUTE.parse("201806281407"); + + formatted = GenericFormatter.FORMAT_SHORT_SECOND.format(time); + System.out.println("GenericFormatter.FORMAT_SHORT_SECOND : " + formatted); + oldApiFormat = GenericFormatter.newShortSecondFormat(); + Assert.assertEquals(oldApiFormat.format(Date.from(time)), formatted); + GenericFormatter.FORMAT_SHORT_SECOND.parse("20180628140713"); + + formatted = GenericFormatter.FORMAT_SHORT_MILSEC.format(time); + System.out.println("GenericFormatter.FORMAT_SHORT_MILSEC : " + formatted); + oldApiFormat = GenericFormatter.newShortMilsecFormat(); + Assert.assertEquals(oldApiFormat.format(Date.from(time)), formatted); + GenericFormatter.FORMAT_SHORT_MILSEC.parse("20180628140713921"); + + formatted = GenericFormatter.FORMAT_RFC1123_SHORT.format(time); + System.out.println("GenericFormatter.FORMAT_RFC1123_SHORT : " + formatted); + oldApiFormat = GenericFormatter.newRfc1123ShortFormat(); + Assert.assertEquals(oldApiFormat.format(Date.from(time)), formatted); + GenericFormatter.FORMAT_RFC1123_SHORT.parse("Thu, 28 Jun 2018"); + + formatted = GenericFormatter.FORMAT_ANSIC.format(time); + System.out.println("GenericFormatter.FORMAT_ANSIC : " + formatted); + oldApiFormat = GenericFormatter.newAnsicFormat(); + Assert.assertEquals(oldApiFormat.format(Date.from(time)), formatted); + GenericFormatter.FORMAT_ANSIC.parse("Thu Jun 28 15:48:17 2018"); + + formatted = GenericFormatter.FORMAT_SIMPLE.format(time); + System.out.println("GenericFormatter.FORMAT_SIMPLE : " + formatted); + oldApiFormat = GenericFormatter.newSimpleDateFormat(); + Assert.assertEquals(oldApiFormat.format(Date.from(time)), formatted); + GenericFormatter.FORMAT_SIMPLE.parse("2018/06/28 15:27:45"); + } + +} |
