[dynamodb] Fixed persistence issues with zoneddatetime (#9157)

* [dynamodb] fix serialization of ZonedDateTime timestamps

Resolves #9156.

AWS SDK does not know how to serialize ZonedDateTime. We resolve this
by providing explicit serializer/deserializer.

In addition, integration tests had issue with timezones. The persistence
converts all timestamps from the DynamoDB to system timezone for display
purposes. This was not taken into account in the tests and failure was
expected with non-UTC system timezone.

Signed-off-by: Sami Salonen <ssalonen@gmail.com>

bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/AbstractDynamoDBItem.java

@@ -13,7 +13,6 @@
 package org.openhab.persistence.dynamodb.internal;
 
 import java.math.BigDecimal;
-import java.text.DateFormat;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
@@ -51,6 +50,8 @@ import org.openhab.core.types.UnDefType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverter;
+
 /**
  * Base class for all DynamoDBItem. Represents openHAB Item serialized in a suitable format for the database
  *
@@ -60,8 +61,8 @@ import org.slf4j.LoggerFactory;
  */
 public abstract class AbstractDynamoDBItem<T> implements DynamoDBItem<T> {
 
-    public static final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern(DATE_FORMAT)
+    private static final ZoneId UTC = ZoneId.of("UTC");
-            .withZone(ZoneId.of("UTC"));
+    public static final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern(DATE_FORMAT).withZone(UTC);
 
     private static final String UNDEFINED_PLACEHOLDER = "<org.openhab.core.types.UnDefType.UNDEF>";
 
@@ -91,6 +92,29 @@ public abstract class AbstractDynamoDBItem<T> implements DynamoDBItem<T> {
         return dtoclass;
     }
 
+    /**
+     * Custom converter for serialization/deserialization of ZonedDateTime.
+     *
+     * Serialization: ZonedDateTime is first converted to UTC and then stored with format yyyy-MM-dd'T'HH:mm:ss.SSS'Z'
+     * This allows easy sorting of values since all timestamps are in UTC and string ordering can be used.
+     *
+     * @author Sami Salonen - Initial contribution
+     *
+     */
+    public static final class ZonedDateTimeConverter implements DynamoDBTypeConverter<String, ZonedDateTime> {
+
+        @Override
+        public String convert(ZonedDateTime time) {
+            return DATEFORMATTER.format(time.withZoneSameInstant(UTC));
+        }
+
+        @Override
+        public ZonedDateTime unconvert(String serialized) {
+            return ZonedDateTime.parse(serialized, DATEFORMATTER);
+        }
+    }
+
+    private static final ZonedDateTimeConverter zonedDateTimeConverter = new ZonedDateTimeConverter();
     private final Logger logger = LoggerFactory.getLogger(AbstractDynamoDBItem.class);
 
     protected String name;
@@ -117,7 +141,8 @@ public abstract class AbstractDynamoDBItem<T> implements DynamoDBItem<T> {
             return new DynamoDBBigDecimalItem(name,
                     ((UpDownType) state) == UpDownType.UP ? BigDecimal.ONE : BigDecimal.ZERO, time);
         } else if (state instanceof DateTimeType) {
-            return new DynamoDBStringItem(name, ((DateTimeType) state).getZonedDateTime().format(DATEFORMATTER), time);
+            return new DynamoDBStringItem(name,
+                    zonedDateTimeConverter.convert(((DateTimeType) state).getZonedDateTime()), time);
         } else if (state instanceof UnDefType) {
             return new DynamoDBStringItem(name, UNDEFINED_PLACEHOLDER, time);
         } else if (state instanceof StringListType) {
@@ -151,7 +176,7 @@ public abstract class AbstractDynamoDBItem<T> implements DynamoDBItem<T> {
                         // Parse ZoneDateTime from string. DATEFORMATTER assumes UTC in case it is not clear
                         // from the string (should be).
                         // We convert to default/local timezone for user convenience (e.g. display)
-                        state[0] = new DateTimeType(ZonedDateTime.parse(dynamoStringItem.getState(), DATEFORMATTER)
+                        state[0] = new DateTimeType(zonedDateTimeConverter.unconvert(dynamoStringItem.getState())
                                 .withZoneSameInstant(ZoneId.systemDefault()));
                     } catch (DateTimeParseException e) {
                         logger.warn("Failed to parse {} as date. Outputting UNDEF instead",
@@ -211,6 +236,6 @@ public abstract class AbstractDynamoDBItem<T> implements DynamoDBItem<T> {
 
     @Override
     public String toString() {
-        return DateFormat.getDateTimeInstance().format(time) + ": " + name + " -> " + state.toString();
+        return DATEFORMATTER.format(time) + ": " + name + " -> " + state.toString();
     }
 }

bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/DynamoDBBigDecimalItem.java

@@ -20,6 +20,7 @@ import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBAttribute;
 import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBDocument;
 import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBHashKey;
 import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBRangeKey;
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverted;
 
 /**
  * DynamoDBItem for items that can be serialized as DynamoDB number
@@ -62,6 +63,7 @@ public class DynamoDBBigDecimalItem extends AbstractDynamoDBItem<BigDecimal> {
 
     @Override
     @DynamoDBRangeKey(attributeName = ATTRIBUTE_NAME_TIMEUTC)
+    @DynamoDBTypeConverted(converter = ZonedDateTimeConverter.class)
     public ZonedDateTime getTime() {
         return time;
     }

bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/DynamoDBHistoricItem.java

@@ -12,8 +12,9 @@
  */
 package org.openhab.persistence.dynamodb.internal;
 
-import java.text.DateFormat;
+import java.time.ZoneId;
 import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
 
 import org.eclipse.jdt.annotation.NonNullByDefault;
 import org.openhab.core.persistence.HistoricItem;
@@ -26,6 +27,10 @@ import org.openhab.core.types.State;
  */
 @NonNullByDefault
 public class DynamoDBHistoricItem implements HistoricItem {
+    private static final ZoneId UTC = ZoneId.of("UTC");
+    private static final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern(DynamoDBItem.DATE_FORMAT)
+            .withZone(UTC);
+
     private final String name;
     private final State state;
     private final ZonedDateTime timestamp;
@@ -53,6 +58,6 @@ public class DynamoDBHistoricItem implements HistoricItem {
 
     @Override
     public String toString() {
-        return DateFormat.getDateTimeInstance().format(timestamp) + ": " + name + " -> " + state.toString();
+        return name + ": " + DATEFORMATTER.format(timestamp) + ": " + state.toString();
     }
 }

bundles/org.openhab.persistence.dynamodb/src/main/java/org/openhab/persistence/dynamodb/internal/DynamoDBStringItem.java

@@ -18,6 +18,7 @@ import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBAttribute;
 import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBDocument;
 import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBHashKey;
 import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBRangeKey;
+import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTypeConverted;
 
 /**
  * DynamoDBItem for items that can be serialized as DynamoDB string
@@ -49,6 +50,7 @@ public class DynamoDBStringItem extends AbstractDynamoDBItem<String> {
 
     @Override
     @DynamoDBRangeKey(attributeName = ATTRIBUTE_NAME_TIMEUTC)
+    @DynamoDBTypeConverted(converter = ZonedDateTimeConverter.class)
     public ZonedDateTime getTime() {
         return time;
     }

bundles/org.openhab.persistence.dynamodb/src/test/java/org/openhab/persistence/dynamodb/internal/DateTimeItemIntegrationTest.java

@@ -12,6 +12,8 @@
  */
 package org.openhab.persistence.dynamodb.internal;
 
+import java.time.ZoneId;
+import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 
 import org.eclipse.jdt.annotation.NonNullByDefault;
@@ -34,8 +36,10 @@ public class DateTimeItemIntegrationTest extends AbstractTwoItemIntegrationTest
     private static final ZonedDateTime ZDT2 = ZonedDateTime.parse("2016-06-15T16:00:00.123Z");
     private static final ZonedDateTime ZDT_BETWEEN = ZonedDateTime.parse("2016-06-15T14:00:00Z");
 
+    // State1 stored as DateTimeType wrapping ZonedDateTime specified in UTC
     private static final DateTimeType STATE1 = new DateTimeType(ZDT1);
-    private static final DateTimeType STATE2 = new DateTimeType(ZDT2);
+    // State2 stored as DateTimeType wrapping ZonedDateTime specified in UTC+5
+    private static final DateTimeType STATE2 = new DateTimeType(ZDT2.withZoneSameInstant(ZoneOffset.ofHours(5)));
     private static final DateTimeType STATE_BETWEEN = new DateTimeType(ZDT_BETWEEN);
 
     @BeforeAll
@@ -65,12 +69,24 @@ public class DateTimeItemIntegrationTest extends AbstractTwoItemIntegrationTest
 
     @Override
     protected State getFirstItemState() {
-        return STATE1;
+        // The persistence converts to system default timezone
+        // Thus we need to convert here as well for comparison
+        // In the logs:
+        // [main] TRACE org.openhab.persistence.dynamodb.internal.DynamoDBPersistenceService - Dynamo item datetime
+        // (Type=DateTimeItem, State=2016-06-15T16:00:00.123+0000, Label=null, Category=null) converted to historic
+        // item: datetime: 2020-11-28T11:29:54.326Z: 2016-06-15T19:00:00.123+0300
+        return STATE1.toZone(ZoneId.systemDefault());
     }
 
     @Override
     protected State getSecondItemState() {
-        return STATE2;
+        // The persistence converts to system default timezone
+        // Thus we need to convert here as well for comparison
+        // In the logs:
+        // [main] TRACE org.openhab.persistence.dynamodb.internal.DynamoDBPersistenceService - Dynamo item datetime
+        // (Type=DateTimeItem, State=2016-06-15T16:00:00.123+0000, Label=null, Category=null) converted to historic
+        // item: datetime: 2020-11-28T11:29:54.326Z: 2016-06-15T19:00:00.123+0300
+        return STATE2.toZone(ZoneId.systemDefault());
     }
 
     @Override