Allow pre-escaped URLs for http binding (#12350)

Signed-off-by: James Melville <jamesmelville@gmail.com>

bundles/org.openhab.binding.http/README.md

@@ -23,7 +23,7 @@ It can be extended with different channels.
 | `commandMethod`   | no       |   GET   | Method used for sending commands: `GET`, `PUT`, `POST`. |
 | `contentType`     | yes      |    -    | MIME content-type of the command requests. Only used for  `PUT` and `POST`. |
 | `encoding`        | yes      |    -    | Encoding to be used if no encoding is found in responses (advanced parameter). |
-| `headers`         | yes      |    -    | Additional headers that are sent along with the request. Format is "header=value". Multiple values can be stored as `headers="key1=value1", "key2=value2", "key3=value3",`|
+| `headers`         | yes      |    -    | Additional headers that are sent along with the request. Format is "header=value". Multiple values can be stored as `headers="key1=value1", "key2=value2", "key3=value3",`. When using text based configuration include at minimum 2 headers to avoid parsing errors.|
 | `ignoreSSLErrors` | no       |  false  | If set to true ignores invalid SSL certificate errors. This is potentially dangerous.|
 
 *Note:* Optional "no" means that you have to configure a value unless a default is provided and you are ok with that setting.
@@ -35,16 +35,18 @@ Authentication might fail if redirections are involved as headers are stripper p
 
 *Note:* If you rate-limit requests by using the `delay` parameter you have to make sure that the time between two refreshes is larger than the time needed for one refresh cycle.
 
-**Attention:** `baseUrl` (and `stateExtension`/`commandExtension`) should not use escaping (e.g. `%22` instead of `"` or `%2c` instead of `,`).
+**Attention:** `baseUrl` (and `stateExtension`/`commandExtension`) should not normally use escaping (e.g. `%22` instead of `"` or `%2c` instead of `,`).
 URLs are properly escaped by the binding itself before the request is sent.
 Using escaped strings in URL parameters may lead to problems with the formatting (see below).
 
+In certain scenarios you may need to manually escape your URL, for example if you need to include an escaped `=` (`%3D`) in this scenario include `%%3D` in the URL to preserve the `%` during formatting, and set the parameter `escapedUrl` to true on the channel.
+
 ## Channels
 
 Each item type has its own channel-type.
 Depending on the channel-type, channels have different configuration options.
 All channel-types (except `image`) have `stateExtension`, `commandExtension`, `stateTransformation`, `commandTransformation` and `mode` parameters.
-The `image` channel-type supports `stateExtension` only.
+The `image` channel-type supports `stateExtension`, `stateContent` and `escapedUrl` only.
 
 | parameter               | optional | default     | description |
 |-------------------------|----------|-------------|-------------|
@@ -52,6 +54,7 @@ The `image` channel-type supports `stateExtension` only.
 | `commandExtension`      | yes      |      -      | Appended to the `baseURL` for sending commands. If empty, same as `stateExtension`. |
 | `stateTransformation  ` | yes      |      -      | One or more transformation applied to received values before updating channel. |
 | `commandTransformation` | yes      |      -      | One or more transformation applied to channel value before sending to a remote. |
+| `escapedUrl`            | yes      |      -      | This specifies whether the URL is already escaped. |
 | `stateContent`          | yes      |      -      | Content for state requests (if method is `PUT` or `POST`) |
 | `mode`                  | no       | `READWRITE` | Mode this channel is allowed to operate. `READONLY` means receive state, `WRITEONLY` means send commands. |
 

bundles/org.openhab.binding.http/src/main/java/org/openhab/binding/http/internal/HttpThingHandler.java

@@ -292,8 +292,11 @@ public class HttpThingHandler extends BaseThingHandler {
             // we need a key consisting of stateContent and URL, only if both are equal, we can use the same cache
             String key = channelConfig.stateContent + "$" + stateUrl;
             channelUrls.put(channelUID, key);
-            urlHandlers.computeIfAbsent(key, k -> new RefreshingUrlCache(scheduler, rateLimitedHttpClient, stateUrl,
+            urlHandlers
-                    config, channelConfig.stateContent)).addConsumer(itemValueConverter::process);
+                    .computeIfAbsent(key,
+                            k -> new RefreshingUrlCache(scheduler, rateLimitedHttpClient, stateUrl,
+                                    channelConfig.escapedUrl, config, channelConfig.stateContent))
+                    .addConsumer(itemValueConverter::process);
         }
 
         StateDescription stateDescription = StateDescriptionFragmentBuilder.create()
@@ -304,14 +307,15 @@ public class HttpThingHandler extends BaseThingHandler {
         }
     }
 
-    private void sendHttpValue(String commandUrl, String command) {
+    private void sendHttpValue(String commandUrl, boolean escapedUrl, String command) {
-        sendHttpValue(commandUrl, command, false);
+        sendHttpValue(commandUrl, escapedUrl, command, false);
     }
 
-    private void sendHttpValue(String commandUrl, String command, boolean isRetry) {
+    private void sendHttpValue(String commandUrl, boolean escapedUrl, String command, boolean isRetry) {
         try {
             // format URL
-            URI uri = Util.uriFromString(String.format(commandUrl, new Date(), command));
+            String url = String.format(commandUrl, new Date(), command);
+            URI uri = escapedUrl ? new URI(url) : Util.uriFromString(url);
 
             // build request
             Request request = httpClient.newRequest(uri).timeout(config.timeout, TimeUnit.MILLISECONDS)
@@ -349,7 +353,7 @@ public class HttpThingHandler extends BaseThingHandler {
                         if (authResult != null) {
                             authStore.removeAuthenticationResult(authResult);
                             logger.debug("Cleared authentication result for '{}', retrying immediately", uri);
-                            sendHttpValue(commandUrl, command, true);
+                            sendHttpValue(commandUrl, escapedUrl, command, true);
                         } else {
                             logger.warn("Could not find authentication result for '{}', failing here", uri);
                         }
@@ -379,7 +383,7 @@ public class HttpThingHandler extends BaseThingHandler {
     private ItemValueConverter createItemConverter(AbstractTransformingItemConverter.Factory factory, String commandUrl,
             ChannelUID channelUID, HttpChannelConfig channelConfig) {
         return factory.create(state -> updateState(channelUID, state), command -> postCommand(channelUID, command),
-                command -> sendHttpValue(commandUrl, command),
+                command -> sendHttpValue(commandUrl, channelConfig.escapedUrl, command),
                 valueTransformationProvider.getValueTransformation(channelConfig.stateTransformation),
                 valueTransformationProvider.getValueTransformation(channelConfig.commandTransformation), channelConfig);
     }

bundles/org.openhab.binding.http/src/main/java/org/openhab/binding/http/internal/config/HttpChannelConfig.java

@@ -46,6 +46,7 @@ public class HttpChannelConfig {
     public @Nullable String stateTransformation;
     public @Nullable String commandTransformation;
     public String stateContent = "";
+    public boolean escapedUrl = false;
 
     public HttpChannelMode mode = HttpChannelMode.READWRITE;
 

bundles/org.openhab.binding.http/src/main/java/org/openhab/binding/http/internal/http/RefreshingUrlCache.java

@@ -48,6 +48,7 @@ public class RefreshingUrlCache {
     private final Logger logger = LoggerFactory.getLogger(RefreshingUrlCache.class);
 
     private final String url;
+    private final boolean escapedUrl;
     private final RateLimitedHttpClient httpClient;
     private final int timeout;
     private final int bufferSize;
@@ -61,9 +62,10 @@ public class RefreshingUrlCache {
     private @Nullable Content lastContent;
 
     public RefreshingUrlCache(ScheduledExecutorService executor, RateLimitedHttpClient httpClient, String url,
-            HttpThingConfig thingConfig, String httpContent) {
+            boolean escapedUrl, HttpThingConfig thingConfig, String httpContent) {
         this.httpClient = httpClient;
         this.url = url;
+        this.escapedUrl = escapedUrl;
         this.timeout = thingConfig.timeout;
         this.bufferSize = thingConfig.bufferSize;
         this.headers = thingConfig.headers;
@@ -87,7 +89,8 @@ public class RefreshingUrlCache {
 
         // format URL
         try {
-            URI uri = Util.uriFromString(String.format(this.url, new Date()));
+            String url = String.format(this.url, new Date());
+            URI uri = escapedUrl ? new URI(url) : Util.uriFromString(url);
             logger.trace("Requesting refresh (retry={}) from '{}' with timeout {}ms", isRetry, uri, timeout);
 
             httpClient.newRequest(uri, httpMethod, httpContent).thenAccept(request -> {

bundles/org.openhab.binding.http/src/main/resources/OH-INF/config/config.xml

@@ -25,6 +25,13 @@
 			<description>This value is added to the base URL configured in the thing for sending values.</description>
 			<advanced>true</advanced>
 		</parameter>
+		<parameter name="escapedUrl" type="boolean">
+			<label>Escaped URL</label>
+			<description>This specifies whether the URL is already escaped. Applies to the base URL, commandExtension and
+				stateExtension.</description>
+			<advanced>true</advanced>
+			<default>false</default>
+		</parameter>
 		<parameter name="stateContent" type="text">
 			<label>State Content</label>
 			<description>Content for state request (only used if method is POST/PUT)</description>
@@ -64,6 +71,13 @@
 			<description>This value is added to the base URL configured in the thing for sending values.</description>
 			<advanced>true</advanced>
 		</parameter>
+		<parameter name="escapedUrl" type="boolean">
+			<label>Escaped URL</label>
+			<description>This specifies whether the URL is already escaped. Applies to the base URL, commandExtension and
+				stateExtension.</description>
+			<advanced>true</advanced>
+			<default>false</default>
+		</parameter>
 		<parameter name="stateContent" type="text">
 			<label>State Content</label>
 			<description>Content for state request (only used if method is POST/PUT)</description>
@@ -135,6 +149,13 @@
 			<description>This value is added to the base URL configured in the thing for sending values.</description>
 			<advanced>true</advanced>
 		</parameter>
+		<parameter name="escapedUrl" type="boolean">
+			<label>Escaped URL</label>
+			<description>This specifies whether the URL is already escaped. Applies to the base URL, commandExtension and
+				stateExtension.</description>
+			<advanced>true</advanced>
+			<default>false</default>
+		</parameter>
 		<parameter name="stateContent" type="text">
 			<label>State Content</label>
 			<description>Content for state request (only used if method is POST/PUT)</description>
@@ -182,6 +203,13 @@
 			<description>This value is added to the base URL configured in the thing for sending values.</description>
 			<advanced>true</advanced>
 		</parameter>
+		<parameter name="escapedUrl" type="boolean">
+			<label>Escaped URL</label>
+			<description>This specifies whether the URL is already escaped. Applies to the base URL, commandExtension and
+				stateExtension.</description>
+			<advanced>true</advanced>
+			<default>false</default>
+		</parameter>
 		<parameter name="stateContent" type="text">
 			<label>State Content</label>
 			<description>Content for state request (only used if method is POST/PUT)</description>
@@ -228,6 +256,12 @@
 			<description>This value is added to the base URL configured in the thing for retrieving values.</description>
 			<advanced>true</advanced>
 		</parameter>
+		<parameter name="escapedUrl" type="boolean">
+			<label>Escaped URL</label>
+			<description>This specifies whether the URL is already escaped. Applies to the base URL and stateExtension.</description>
+			<advanced>true</advanced>
+			<default>false</default>
+		</parameter>
 		<parameter name="stateContent" type="text">
 			<label>State Content</label>
 			<description>Content for state request (only used if method is POST/PUT)</description>
@@ -256,6 +290,13 @@
 			<description>This value is added to the base URL configured in the thing for sending values.</description>
 			<advanced>true</advanced>
 		</parameter>
+		<parameter name="escapedUrl" type="boolean">
+			<label>Escaped URL</label>
+			<description>This specifies whether the URL is already escaped. Applies to the base URL, commandExtension and
+				stateExtension.</description>
+			<advanced>true</advanced>
+			<default>false</default>
+		</parameter>
 		<parameter name="stateContent" type="text">
 			<label>State Content</label>
 			<description>Content for state request (only used if method is POST/PUT)</description>
@@ -300,6 +341,13 @@
 			<description>This value is added to the base URL configured in the thing for sending values.</description>
 			<advanced>true</advanced>
 		</parameter>
+		<parameter name="escapedUrl" type="boolean">
+			<label>Escaped URL</label>
+			<description>This specifies whether the URL is already escaped. Applies to the base URL, commandExtension and
+				stateExtension.</description>
+			<advanced>true</advanced>
+			<default>false</default>
+		</parameter>
 		<parameter name="stateContent" type="text">
 			<label>State Content</label>
 			<description>Content for state request (only used if method is POST/PUT)</description>
@@ -363,6 +411,13 @@
 			<description>This value is added to the base URL configured in the thing for sending values.</description>
 			<advanced>true</advanced>
 		</parameter>
+		<parameter name="escapedUrl" type="boolean">
+			<label>Escaped URL</label>
+			<description>This specifies whether the URL is already escaped. Applies to the base URL, commandExtension and
+				stateExtension.</description>
+			<advanced>true</advanced>
+			<default>false</default>
+		</parameter>
 		<parameter name="stateContent" type="text">
 			<label>State Content</label>
 			<description>Content for state request (only used if method is POST/PUT)</description>
@@ -418,6 +473,13 @@
 			<description>This value is added to the base URL configured in the thing for sending values.</description>
 			<advanced>true</advanced>
 		</parameter>
+		<parameter name="escapedUrl" type="boolean">
+			<label>Escaped URL</label>
+			<description>This specifies whether the URL is already escaped. Applies to the base URL, commandExtension and
+				stateExtension.</description>
+			<advanced>true</advanced>
+			<default>false</default>
+		</parameter>
 		<parameter name="stateContent" type="text">
 			<label>State Content</label>
 			<description>Content for state request (only used if method is POST/PUT)</description>

bundles/org.openhab.binding.http/src/main/resources/OH-INF/i18n/http.properties

@@ -80,6 +80,8 @@ channel-type.config.http.channel-config-color.decreaseValue.label = Decrease Val
 channel-type.config.http.channel-config-color.decreaseValue.description = The value that represents DECREASE
 channel-type.config.http.channel-config-color.increaseValue.label = Increase Value
 channel-type.config.http.channel-config-color.increaseValue.description = The value that represents INCREASE
+channel-type.config.http.channel-config-color.escapedUrl.label = Escaped URL
+channel-type.config.http.channel-config-color.escapedUrl.description = This specifies whether the URL is already escaped. Applies to the base URL, commandExtension and stateExtension.
 channel-type.config.http.channel-config-color.mode.label = Read/Write Mode
 channel-type.config.http.channel-config-color.mode.option.READWRITE = Read/Write
 channel-type.config.http.channel-config-color.mode.option.READONLY = Read Only
@@ -102,6 +104,8 @@ channel-type.config.http.channel-config-contact.commandExtension.label = Command
 channel-type.config.http.channel-config-contact.commandExtension.description = This value is added to the base URL configured in the thing for sending values.
 channel-type.config.http.channel-config-contact.commandTransformation.label = Command Transformation
 channel-type.config.http.channel-config-contact.commandTransformation.description = Transformation pattern used when sending values. Chain multiple transformations with the mathematical intersection character "∩".
+channel-type.config.http.channel-config-contact.escapedUrl.label = Escaped URL
+channel-type.config.http.channel-config-contact.escapedUrl.description = This specifies whether the URL is already escaped. Applies to the base URL, commandExtension and stateExtension.
 channel-type.config.http.channel-config-contact.mode.label = Read/Write Mode
 channel-type.config.http.channel-config-contact.mode.option.READWRITE = Read/Write
 channel-type.config.http.channel-config-contact.mode.option.READONLY = Read Only
@@ -122,6 +126,8 @@ channel-type.config.http.channel-config-dimmer.decreaseValue.label = Decrease Va
 channel-type.config.http.channel-config-dimmer.decreaseValue.description = The value that represents DECREASE
 channel-type.config.http.channel-config-dimmer.increaseValue.label = Increase Value
 channel-type.config.http.channel-config-dimmer.increaseValue.description = The value that represents INCREASE
+channel-type.config.http.channel-config-dimmer.escapedUrl.label = Escaped URL
+channel-type.config.http.channel-config-dimmer.escapedUrl.description = This specifies whether the URL is already escaped. Applies to the base URL, commandExtension and stateExtension.
 channel-type.config.http.channel-config-dimmer.mode.label = Read/Write Mode
 channel-type.config.http.channel-config-dimmer.mode.option.READWRITE = Read/Write
 channel-type.config.http.channel-config-dimmer.mode.option.READONLY = Read Only
@@ -138,6 +144,8 @@ channel-type.config.http.channel-config-dimmer.stateTransformation.label = State
 channel-type.config.http.channel-config-dimmer.stateTransformation.description = Transformation pattern used when receiving values. Chain multiple transformations with the mathematical intersection character "∩".
 channel-type.config.http.channel-config-dimmer.step.label = Increase/Decrease Step
 channel-type.config.http.channel-config-dimmer.step.description = The value by which the current brightness is increased/decreased if the corresponding command is received
+channel-type.config.http.channel-config-image.escapedUrl.label = Escaped URL
+channel-type.config.http.channel-config-image.escapedUrl.description = This specifies whether the URL is already escaped. Applies to the base URL and stateExtension.
 channel-type.config.http.channel-config-image.stateContent.label = State Content
 channel-type.config.http.channel-config-image.stateContent.description = Content for state request (only used if method is POST/PUT)
 channel-type.config.http.channel-config-image.stateExtension.label = State URL Extension
@@ -146,6 +154,8 @@ channel-type.config.http.channel-config-number.commandExtension.label = Command
 channel-type.config.http.channel-config-number.commandExtension.description = This value is added to the base URL configured in the thing for sending values.
 channel-type.config.http.channel-config-number.commandTransformation.label = Command Transformation
 channel-type.config.http.channel-config-number.commandTransformation.description = Transformation pattern used when sending values. Chain multiple transformations with the mathematical intersection character "∩".
+channel-type.config.http.channel-config-number.escapedUrl.label = Escaped URL
+channel-type.config.http.channel-config-number.escapedUrl.description = This specifies whether the URL is already escaped. Applies to the base URL, commandExtension and stateExtension.
 channel-type.config.http.channel-config-number.mode.label = Read/Write Mode
 channel-type.config.http.channel-config-number.mode.option.READWRITE = Read/Write
 channel-type.config.http.channel-config-number.mode.option.READONLY = Read Only
@@ -164,6 +174,8 @@ channel-type.config.http.channel-config-player.commandTransformation.label = Com
 channel-type.config.http.channel-config-player.commandTransformation.description = Transformation pattern used when sending values. Chain multiple transformations with the mathematical intersection character "∩".
 channel-type.config.http.channel-config-player.fastforwardValue.label = Fast Forward Value
 channel-type.config.http.channel-config-player.fastforwardValue.description = The value that represents FASTFORWARD
+channel-type.config.http.channel-config-player.escapedUrl.label = Escaped URL
+channel-type.config.http.channel-config-player.escapedUrl.description = This specifies whether the URL is already escaped. Applies to the base URL, commandExtension and stateExtension.
 channel-type.config.http.channel-config-player.mode.label = Read/Write Mode
 channel-type.config.http.channel-config-player.mode.option.READWRITE = Read/Write
 channel-type.config.http.channel-config-player.mode.option.READONLY = Read Only
@@ -190,6 +202,8 @@ channel-type.config.http.channel-config-rollershutter.commandTransformation.labe
 channel-type.config.http.channel-config-rollershutter.commandTransformation.description = Transformation pattern used when sending values Chain multiple transformations with the mathematical intersection character "∩"..
 channel-type.config.http.channel-config-rollershutter.downValue.label = Down Value
 channel-type.config.http.channel-config-rollershutter.downValue.description = The value that represents DOWN
+channel-type.config.http.channel-config-rollershutter.escapedUrl.label = Escaped URL
+channel-type.config.http.channel-config-rollershutter.escapedUrl.description = This specifies whether the URL is already escaped. Applies to the base URL, commandExtension and stateExtension.
 channel-type.config.http.channel-config-rollershutter.mode.label = Read/Write Mode
 channel-type.config.http.channel-config-rollershutter.mode.option.READWRITE = Read/Write
 channel-type.config.http.channel-config-rollershutter.mode.option.READONLY = Read Only
@@ -210,6 +224,8 @@ channel-type.config.http.channel-config-switch.commandExtension.label = Command
 channel-type.config.http.channel-config-switch.commandExtension.description = This value is added to the base URL configured in the thing for sending values.
 channel-type.config.http.channel-config-switch.commandTransformation.label = Command Transformation
 channel-type.config.http.channel-config-switch.commandTransformation.description = Transformation pattern used when sending values. Chain multiple transformations with the mathematical intersection character "∩".
+channel-type.config.http.channel-config-switch.escapedUrl.label = Escaped URL
+channel-type.config.http.channel-config-switch.escapedUrl.description = This specifies whether the URL is already escaped. Applies to the base URL, commandExtension and stateExtension.
 channel-type.config.http.channel-config-switch.mode.label = Read/Write Mode
 channel-type.config.http.channel-config-switch.mode.option.READWRITE = Read/Write
 channel-type.config.http.channel-config-switch.mode.option.READONLY = Read Only
@@ -228,6 +244,8 @@ channel-type.config.http.channel-config.commandExtension.label = Command URL Ext
 channel-type.config.http.channel-config.commandExtension.description = This value is added to the base URL configured in the thing for sending values.
 channel-type.config.http.channel-config.commandTransformation.label = Command Transformation
 channel-type.config.http.channel-config.commandTransformation.description = Transformation pattern used when sending values. Chain multiple transformations with the mathematical intersection character "∩".
+channel-type.config.http.channel-config.escapedUrl.label = Escaped URL
+channel-type.config.http.channel-config.escapedUrl.description = This specifies whether the URL is already escaped. Applies to the base URL, commandExtension and stateExtension.
 channel-type.config.http.channel-config.mode.label = Read/Write Mode
 channel-type.config.http.channel-config.mode.option.READWRITE = Read/Write
 channel-type.config.http.channel-config.mode.option.READONLY = Read Only