From 5d60d6464b5731cd940293d4ba63ce4f195f5e4e Mon Sep 17 00:00:00 2001 From: Sami Salonen Date: Sun, 29 Nov 2020 12:24:16 +0200 Subject: [PATCH] [dynamodb] Fixed persistence issues with zoneddatetime (#9157) * [dynamodb] fix serialization of ZonedDateTime timestamps Resolves #9156. AWS SDK does not know how to serialize ZonedDateTime. We resolve this by providing explicit serializer/deserializer. In addition, integration tests had issue with timezones. The persistence converts all timestamps from the DynamoDB to system timezone for display purposes. This was not taken into account in the tests and failure was expected with non-UTC system timezone. Signed-off-by: Sami Salonen --- .../internal/AbstractDynamoDBItem.java | 37 ++++++++++++++++--- .../internal/DynamoDBBigDecimalItem.java | 2 + .../internal/DynamoDBHistoricItem.java | 9 ++++- .../dynamodb/internal/DynamoDBStringItem.java | 2 + .../internal/DateTimeItemIntegrationTest.java | 22 +++++++++-- 5 files changed, 61 insertions(+), 11 deletions(-) diff --git a/bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/AbstractDynamoDBItem.java b/bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/AbstractDynamoDBItem.java index a12d3c580..b1108ca32 100644 --- a/bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/AbstractDynamoDBItem.java +++ b/bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/AbstractDynamoDBItem.java @@ -13,7 +13,6 @@ package org.openhab.persistence.dynamodb.internal; import java.math.BigDecimal; -import java.text.DateFormat; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; @@ -51,6 +50,8 @@ import org.openhab.core.types.UnDefType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverter; + /** * Base class for all DynamoDBItem. Represents openHAB Item serialized in a suitable format for the database * @@ -60,8 +61,8 @@ import org.slf4j.LoggerFactory; */ public abstract class AbstractDynamoDBItem implements DynamoDBItem { - public static final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern(DATE_FORMAT) - .withZone(ZoneId.of("UTC")); + private static final ZoneId UTC = ZoneId.of("UTC"); + public static final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern(DATE_FORMAT).withZone(UTC); private static final String UNDEFINED_PLACEHOLDER = ""; @@ -91,6 +92,29 @@ public abstract class AbstractDynamoDBItem implements DynamoDBItem { return dtoclass; } + /** + * Custom converter for serialization/deserialization of ZonedDateTime. + * + * Serialization: ZonedDateTime is first converted to UTC and then stored with format yyyy-MM-dd'T'HH:mm:ss.SSS'Z' + * This allows easy sorting of values since all timestamps are in UTC and string ordering can be used. + * + * @author Sami Salonen - Initial contribution + * + */ + public static final class ZonedDateTimeConverter implements DynamoDBTypeConverter { + + @Override + public String convert(ZonedDateTime time) { + return DATEFORMATTER.format(time.withZoneSameInstant(UTC)); + } + + @Override + public ZonedDateTime unconvert(String serialized) { + return ZonedDateTime.parse(serialized, DATEFORMATTER); + } + } + + private static final ZonedDateTimeConverter zonedDateTimeConverter = new ZonedDateTimeConverter(); private final Logger logger = LoggerFactory.getLogger(AbstractDynamoDBItem.class); protected String name; @@ -117,7 +141,8 @@ public abstract class AbstractDynamoDBItem implements DynamoDBItem { return new DynamoDBBigDecimalItem(name, ((UpDownType) state) == UpDownType.UP ? BigDecimal.ONE : BigDecimal.ZERO, time); } else if (state instanceof DateTimeType) { - return new DynamoDBStringItem(name, ((DateTimeType) state).getZonedDateTime().format(DATEFORMATTER), time); + return new DynamoDBStringItem(name, + zonedDateTimeConverter.convert(((DateTimeType) state).getZonedDateTime()), time); } else if (state instanceof UnDefType) { return new DynamoDBStringItem(name, UNDEFINED_PLACEHOLDER, time); } else if (state instanceof StringListType) { @@ -151,7 +176,7 @@ public abstract class AbstractDynamoDBItem implements DynamoDBItem { // Parse ZoneDateTime from string. DATEFORMATTER assumes UTC in case it is not clear // from the string (should be). // We convert to default/local timezone for user convenience (e.g. display) - state[0] = new DateTimeType(ZonedDateTime.parse(dynamoStringItem.getState(), DATEFORMATTER) + state[0] = new DateTimeType(zonedDateTimeConverter.unconvert(dynamoStringItem.getState()) .withZoneSameInstant(ZoneId.systemDefault())); } catch (DateTimeParseException e) { logger.warn("Failed to parse {} as date. Outputting UNDEF instead", @@ -211,6 +236,6 @@ public abstract class AbstractDynamoDBItem implements DynamoDBItem { @Override public String toString() { - return DateFormat.getDateTimeInstance().format(time) + ": " + name + " -> " + state.toString(); + return DATEFORMATTER.format(time) + ": " + name + " -> " + state.toString(); } } diff --git a/bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/DynamoDBBigDecimalItem.java b/bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/DynamoDBBigDecimalItem.java index dd4b08495..69d247934 100644 --- a/bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/DynamoDBBigDecimalItem.java +++ b/bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/DynamoDBBigDecimalItem.java @@ -20,6 +20,7 @@ import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBAttribute; import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBDocument; import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBHashKey; import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBRangeKey; +import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverted; /** * DynamoDBItem for items that can be serialized as DynamoDB number @@ -62,6 +63,7 @@ public class DynamoDBBigDecimalItem extends AbstractDynamoDBItem { @Override @DynamoDBRangeKey(attributeName = ATTRIBUTE_NAME_TIMEUTC) + @DynamoDBTypeConverted(converter = ZonedDateTimeConverter.class) public ZonedDateTime getTime() { return time; } diff --git a/bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/DynamoDBHistoricItem.java b/bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/DynamoDBHistoricItem.java index c7eaecf43..26083ffe7 100644 --- a/bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/DynamoDBHistoricItem.java +++ b/bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/DynamoDBHistoricItem.java @@ -12,8 +12,9 @@ */ package org.openhab.persistence.dynamodb.internal; -import java.text.DateFormat; +import java.time.ZoneId; import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import org.eclipse.jdt.annotation.NonNullByDefault; import org.openhab.core.persistence.HistoricItem; @@ -26,6 +27,10 @@ import org.openhab.core.types.State; */ @NonNullByDefault public class DynamoDBHistoricItem implements HistoricItem { + private static final ZoneId UTC = ZoneId.of("UTC"); + private static final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern(DynamoDBItem.DATE_FORMAT) + .withZone(UTC); + private final String name; private final State state; private final ZonedDateTime timestamp; @@ -53,6 +58,6 @@ public class DynamoDBHistoricItem implements HistoricItem { @Override public String toString() { - return DateFormat.getDateTimeInstance().format(timestamp) + ": " + name + " -> " + state.toString(); + return name + ": " + DATEFORMATTER.format(timestamp) + ": " + state.toString(); } } diff --git a/bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/DynamoDBStringItem.java b/bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/DynamoDBStringItem.java index 0c164b5ad..2448d33ad 100644 --- a/bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/DynamoDBStringItem.java +++ b/bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/DynamoDBStringItem.java @@ -18,6 +18,7 @@ import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBAttribute; import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBDocument; import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBHashKey; import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBRangeKey; +import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverted; /** * DynamoDBItem for items that can be serialized as DynamoDB string @@ -49,6 +50,7 @@ public class DynamoDBStringItem extends AbstractDynamoDBItem { @Override @DynamoDBRangeKey(attributeName = ATTRIBUTE_NAME_TIMEUTC) + @DynamoDBTypeConverted(converter = ZonedDateTimeConverter.class) public ZonedDateTime getTime() { return time; } diff --git a/bundles/org.openhab.persistence.dynamodb/src/test/java/org/openhab/persistence/dynamodb/internal/DateTimeItemIntegrationTest.java b/bundles/org.openhab.persistence.dynamodb/src/test/java/org/openhab/persistence/dynamodb/internal/DateTimeItemIntegrationTest.java index 22460fcd7..8b9265a42 100644 --- a/bundles/org.openhab.persistence.dynamodb/src/test/java/org/openhab/persistence/dynamodb/internal/DateTimeItemIntegrationTest.java +++ b/bundles/org.openhab.persistence.dynamodb/src/test/java/org/openhab/persistence/dynamodb/internal/DateTimeItemIntegrationTest.java @@ -12,6 +12,8 @@ */ package org.openhab.persistence.dynamodb.internal; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.time.ZonedDateTime; import org.eclipse.jdt.annotation.NonNullByDefault; @@ -34,8 +36,10 @@ public class DateTimeItemIntegrationTest extends AbstractTwoItemIntegrationTest private static final ZonedDateTime ZDT2 = ZonedDateTime.parse("2016-06-15T16:00:00.123Z"); private static final ZonedDateTime ZDT_BETWEEN = ZonedDateTime.parse("2016-06-15T14:00:00Z"); + // State1 stored as DateTimeType wrapping ZonedDateTime specified in UTC private static final DateTimeType STATE1 = new DateTimeType(ZDT1); - private static final DateTimeType STATE2 = new DateTimeType(ZDT2); + // State2 stored as DateTimeType wrapping ZonedDateTime specified in UTC+5 + private static final DateTimeType STATE2 = new DateTimeType(ZDT2.withZoneSameInstant(ZoneOffset.ofHours(5))); private static final DateTimeType STATE_BETWEEN = new DateTimeType(ZDT_BETWEEN); @BeforeAll @@ -65,12 +69,24 @@ public class DateTimeItemIntegrationTest extends AbstractTwoItemIntegrationTest @Override protected State getFirstItemState() { - return STATE1; + // The persistence converts to system default timezone + // Thus we need to convert here as well for comparison + // In the logs: + // [main] TRACE org.openhab.persistence.dynamodb.internal.DynamoDBPersistenceService - Dynamo item datetime + // (Type=DateTimeItem, State=2016-06-15T16:00:00.123+0000, Label=null, Category=null) converted to historic + // item: datetime: 2020-11-28T11:29:54.326Z: 2016-06-15T19:00:00.123+0300 + return STATE1.toZone(ZoneId.systemDefault()); } @Override protected State getSecondItemState() { - return STATE2; + // The persistence converts to system default timezone + // Thus we need to convert here as well for comparison + // In the logs: + // [main] TRACE org.openhab.persistence.dynamodb.internal.DynamoDBPersistenceService - Dynamo item datetime + // (Type=DateTimeItem, State=2016-06-15T16:00:00.123+0000, Label=null, Category=null) converted to historic + // item: datetime: 2020-11-28T11:29:54.326Z: 2016-06-15T19:00:00.123+0300 + return STATE2.toZone(ZoneId.systemDefault()); } @Override