Commit 2d958750 authored by Johan Ström's avatar Johan Ström Committed by Marcel Stör
Browse files

Handle large/chunked/fragmented MQTT messages properly (#2571)

* MQTT: handle large/chunked/fragmented messages properly

If a message spans multiple TCP packets it must be buffered before
delivered to LUA. Prior code did not do this at all, so this "patch"
really adds proper handling of fragmented MQTT packets.
This could also occur if multiple small messages was sent in a
single TCP packet, and the last message did not completely fit in that
packet.

Introduces a new option to the mqtt.Client constructor:
max_publish_length which defaults to 1024

Introduces a new 'overflow' callback.

Fixes issue #2308 and proper fix for PR #2544.

* mqtt.md: clarified heap allocation

* mqtt: ensure ack is sent for overflowed publish

If QoS is used we should still acknowledge that we received it, or server might retransmit it later.
parent b77033f9
This diff is collapsed.
...@@ -126,12 +126,16 @@ void mqtt_msg_init(mqtt_connection_t* connection, uint8_t* buffer, uint16_t buff ...@@ -126,12 +126,16 @@ void mqtt_msg_init(mqtt_connection_t* connection, uint8_t* buffer, uint16_t buff
connection->buffer_length = buffer_length; connection->buffer_length = buffer_length;
} }
int mqtt_get_total_length(uint8_t* buffer, uint16_t length) // Returns total length of message, or -1 if not enough bytes are available
int32_t mqtt_get_total_length(uint8_t* buffer, uint16_t buffer_length)
{ {
int i; int i;
int totlen = 0; int totlen = 0;
for(i = 1; i < length; ++i) if(buffer_length == 1)
return -1;
for(i = 1; i < buffer_length; ++i)
{ {
totlen += (buffer[i] & 0x7f) << (7 * (i - 1)); totlen += (buffer[i] & 0x7f) << (7 * (i - 1));
if((buffer[i] & 0x80) == 0) if((buffer[i] & 0x80) == 0)
...@@ -139,19 +143,23 @@ int mqtt_get_total_length(uint8_t* buffer, uint16_t length) ...@@ -139,19 +143,23 @@ int mqtt_get_total_length(uint8_t* buffer, uint16_t length)
++i; ++i;
break; break;
} }
if(i == buffer_length)
return -1;
} }
totlen += i; totlen += i;
return totlen; return totlen;
} }
const char* mqtt_get_publish_topic(uint8_t* buffer, uint16_t* length) const char* mqtt_get_publish_topic(uint8_t* buffer, uint16_t* buffer_length)
{ {
int i; int i;
int totlen = 0; int totlen = 0;
int topiclen; int topiclen;
for(i = 1; i < *length; ++i) for(i = 1; i < *buffer_length; ++i)
{ {
totlen += (buffer[i] & 0x7f) << (7 * (i -1)); totlen += (buffer[i] & 0x7f) << (7 * (i -1));
if((buffer[i] & 0x80) == 0) if((buffer[i] & 0x80) == 0)
...@@ -162,25 +170,25 @@ const char* mqtt_get_publish_topic(uint8_t* buffer, uint16_t* length) ...@@ -162,25 +170,25 @@ const char* mqtt_get_publish_topic(uint8_t* buffer, uint16_t* length)
} }
totlen += i; totlen += i;
if(i + 2 > *length) if(i + 2 > *buffer_length)
return NULL; return NULL;
topiclen = buffer[i++] << 8; topiclen = buffer[i++] << 8;
topiclen |= buffer[i++]; topiclen |= buffer[i++];
if(i + topiclen > *length) if(i + topiclen > *buffer_length)
return NULL; return NULL;
*length = topiclen; *buffer_length = topiclen;
return (const char*)(buffer + i); return (const char*)(buffer + i);
} }
const char* mqtt_get_publish_data(uint8_t* buffer, uint16_t* length) const char* mqtt_get_publish_data(uint8_t* buffer, uint16_t* buffer_length)
{ {
int i; int i;
int totlen = 0; int totlen = 0;
int topiclen; int topiclen;
for(i = 1; i < *length; ++i) for(i = 1; i < *buffer_length; ++i)
{ {
totlen += (buffer[i] & 0x7f) << (7 * (i - 1)); totlen += (buffer[i] & 0x7f) << (7 * (i - 1));
if((buffer[i] & 0x80) == 0) if((buffer[i] & 0x80) == 0)
...@@ -191,20 +199,20 @@ const char* mqtt_get_publish_data(uint8_t* buffer, uint16_t* length) ...@@ -191,20 +199,20 @@ const char* mqtt_get_publish_data(uint8_t* buffer, uint16_t* length)
} }
totlen += i; totlen += i;
if(i + 2 > *length) if(i + 2 > *buffer_length)
return NULL; return NULL;
topiclen = buffer[i++] << 8; topiclen = buffer[i++] << 8;
topiclen |= buffer[i++]; topiclen |= buffer[i++];
if(i + topiclen > *length){ if(i + topiclen > *buffer_length){
*length = 0; *buffer_length = 0;
return NULL; return NULL;
} }
i += topiclen; i += topiclen;
if(mqtt_get_qos(buffer) > 0) if(mqtt_get_qos(buffer) > 0)
{ {
if(i + 2 > *length) if(i + 2 > *buffer_length)
return NULL; return NULL;
i += 2; i += 2;
} }
...@@ -212,16 +220,16 @@ const char* mqtt_get_publish_data(uint8_t* buffer, uint16_t* length) ...@@ -212,16 +220,16 @@ const char* mqtt_get_publish_data(uint8_t* buffer, uint16_t* length)
if(totlen < i) if(totlen < i)
return NULL; return NULL;
if(totlen <= *length) if(totlen <= *buffer_length)
*length = totlen - i; *buffer_length = totlen - i;
else else
*length = *length - i; *buffer_length = *buffer_length - i;
return (const char*)(buffer + i); return (const char*)(buffer + i);
} }
uint16_t mqtt_get_id(uint8_t* buffer, uint16_t length) uint16_t mqtt_get_id(uint8_t* buffer, uint16_t buffer_length)
{ {
if(length < 1) if(buffer_length < 1)
return 0; return 0;
switch(mqtt_get_type(buffer)) switch(mqtt_get_type(buffer))
...@@ -234,7 +242,7 @@ uint16_t mqtt_get_id(uint8_t* buffer, uint16_t length) ...@@ -234,7 +242,7 @@ uint16_t mqtt_get_id(uint8_t* buffer, uint16_t length)
if(mqtt_get_qos(buffer) <= 0) if(mqtt_get_qos(buffer) <= 0)
return 0; return 0;
for(i = 1; i < length; ++i) for(i = 1; i < buffer_length; ++i)
{ {
if((buffer[i] & 0x80) == 0) if((buffer[i] & 0x80) == 0)
{ {
...@@ -243,16 +251,16 @@ uint16_t mqtt_get_id(uint8_t* buffer, uint16_t length) ...@@ -243,16 +251,16 @@ uint16_t mqtt_get_id(uint8_t* buffer, uint16_t length)
} }
} }
if(i + 2 > length) if(i + 2 > buffer_length)
return 0; return 0;
topiclen = buffer[i++] << 8; topiclen = buffer[i++] << 8;
topiclen |= buffer[i++]; topiclen |= buffer[i++];
if(i + topiclen > length) if(i + topiclen > buffer_length)
return 0; return 0;
i += topiclen; i += topiclen;
if(i + 2 > length) if(i + 2 > buffer_length)
return 0; return 0;
return (buffer[i] << 8) | buffer[i + 1]; return (buffer[i] << 8) | buffer[i + 1];
...@@ -267,7 +275,7 @@ uint16_t mqtt_get_id(uint8_t* buffer, uint16_t length) ...@@ -267,7 +275,7 @@ uint16_t mqtt_get_id(uint8_t* buffer, uint16_t length)
{ {
// This requires the remaining length to be encoded in 1 byte, // This requires the remaining length to be encoded in 1 byte,
// which it should be. // which it should be.
if(length >= 4 && (buffer[1] & 0x80) == 0) if(buffer_length >= 4 && (buffer[1] & 0x80) == 0)
return (buffer[2] << 8) | buffer[3]; return (buffer[2] << 8) | buffer[3];
else else
return 0; return 0;
......
...@@ -108,21 +108,22 @@ typedef struct mqtt_connect_info ...@@ -108,21 +108,22 @@ typedef struct mqtt_connect_info
int will_qos; int will_qos;
int will_retain; int will_retain;
int clean_session; int clean_session;
uint16_t max_message_length;
} mqtt_connect_info_t; } mqtt_connect_info_t;
static inline int mqtt_get_type(uint8_t* buffer) { return (buffer[0] & 0xf0) >> 4; } static inline uint8_t mqtt_get_type(uint8_t* buffer) { return (buffer[0] & 0xf0) >> 4; }
static inline int mqtt_get_dup(uint8_t* buffer) { return (buffer[0] & 0x08) >> 3; } static inline uint8_t mqtt_get_dup(uint8_t* buffer) { return (buffer[0] & 0x08) >> 3; }
static inline int mqtt_get_qos(uint8_t* buffer) { return (buffer[0] & 0x06) >> 1; } static inline uint8_t mqtt_get_qos(uint8_t* buffer) { return (buffer[0] & 0x06) >> 1; }
static inline int mqtt_get_retain(uint8_t* buffer) { return (buffer[0] & 0x01); } static inline uint8_t mqtt_get_retain(uint8_t* buffer) { return (buffer[0] & 0x01); }
static inline int mqtt_get_connect_ret_code(uint8_t* buffer) { return (buffer[3]); } static inline uint8_t mqtt_get_connect_ret_code(uint8_t* buffer) { return (buffer[3]); }
void mqtt_msg_init(mqtt_connection_t* connection, uint8_t* buffer, uint16_t buffer_length); void mqtt_msg_init(mqtt_connection_t* connection, uint8_t* buffer, uint16_t buffer_length);
int mqtt_get_total_length(uint8_t* buffer, uint16_t length); int32_t mqtt_get_total_length(uint8_t* buffer, uint16_t buffer_length);
const char* mqtt_get_publish_topic(uint8_t* buffer, uint16_t* length); const char* mqtt_get_publish_topic(uint8_t* buffer, uint16_t* buffer_length);
const char* mqtt_get_publish_data(uint8_t* buffer, uint16_t* length); const char* mqtt_get_publish_data(uint8_t* buffer, uint16_t* buffer_length);
uint16_t mqtt_get_id(uint8_t* buffer, uint16_t length); uint16_t mqtt_get_id(uint8_t* buffer, uint16_t buffer_length);
mqtt_message_t* mqtt_msg_connect(mqtt_connection_t* connection, mqtt_connect_info_t* info); mqtt_message_t* mqtt_msg_connect(mqtt_connection_t* connection, mqtt_connect_info_t* info);
mqtt_message_t* mqtt_msg_publish(mqtt_connection_t* connection, const char* topic, const char* data, int data_length, int qos, int retain, uint16_t* message_id); mqtt_message_t* mqtt_msg_publish(mqtt_connection_t* connection, const char* topic, const char* data, int data_length, int qos, int retain, uint16_t* message_id);
......
...@@ -11,7 +11,7 @@ The client adheres to version 3.1.1 of the [MQTT](https://en.wikipedia.org/wiki/ ...@@ -11,7 +11,7 @@ The client adheres to version 3.1.1 of the [MQTT](https://en.wikipedia.org/wiki/
Creates a MQTT client. Creates a MQTT client.
#### Syntax #### Syntax
`mqtt.Client(clientid, keepalive[, username, password, cleansession])` `mqtt.Client(clientid, keepalive[, username, password, cleansession, max_message_length])`
#### Parameters #### Parameters
- `clientid` client ID - `clientid` client ID
...@@ -19,10 +19,38 @@ Creates a MQTT client. ...@@ -19,10 +19,38 @@ Creates a MQTT client.
- `username` user name - `username` user name
- `password` user password - `password` user password
- `cleansession` 0/1 for `false`/`true`. Default is 1 (`true`). - `cleansession` 0/1 for `false`/`true`. Default is 1 (`true`).
- `max_message_length`, how large messages to accept. Default is 1024.
#### Returns #### Returns
MQTT client MQTT client
#### Notes
According to MQTT specification the max PUBLISH length is 256Mb. This is too large for NodeMCU to realistically handle. To avoid
an out-of-memory situation, there is a limit on how big messages to accept. This is controlled by the `max_message_length` parameter.
In practice, this only affects incoming PUBLISH messages since all regular control packets are small.
The default 1024 was chosen as this was the implicit limit in NodeMCU 2.2.1 and older (where this was not handled at all).
Note that "message length" refers to the full MQTT message size, including fixed & variable headers, topic name, packet ID (if applicable),
and payload. For exact details, please see [the MQTT specification](http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html#_Toc398718037).
Any message *larger* than `max_message_length` will be (partially) delivered to the `overflow` callback, if defined. The rest
of the message will be discarded. Any subsequent messages should be handled as expected.
Discarded messages will still be ACK'ed if QoS level 1 or 2 was requested, even if the application stack cannot handle them.
Heap memory will be used to buffer any message which spans more than a single TCP packet. A single allocation for the full
message will be performed when the message header is first seen, to avoid heap fragmentation.
If allocation fails, the MQTT session will be disconnected.
Naturally, messages larger than `max_message_length` will not be stored.
Note that heap allocation may occur even if the individual messages are not larger than the configured max! For example,
the broker may send multiple smaller messages in quick succession, which could go into the same TCP packet. If the last message
in the TCP packet did not fit fully, a heap buffer will be allocated to hold the incomplete message while waiting for the next TCP packet.
The typical maximum size for a message to fit into a single TCP packet is 1460 bytes, but this depends on the network's MTU
configuration, any packet fragmentation, and as described above, multiple messages in the same TCP packet.
#### Example #### Example
```lua ```lua
-- init mqtt client without logins, keepalive timer 120s -- init mqtt client without logins, keepalive timer 120s
...@@ -47,6 +75,11 @@ m:on("message", function(client, topic, data) ...@@ -47,6 +75,11 @@ m:on("message", function(client, topic, data)
end end
end) end)
-- on publish overflow receive event
m:on("overflow", function(client, topic, data)
print(topic .. " partial overflowed message: " .. data )
end)
-- for TLS: m:connect("192.168.11.118", secure-port, 1) -- for TLS: m:connect("192.168.11.118", secure-port, 1)
m:connect("192.168.11.118", 1883, 0, function(client) m:connect("192.168.11.118", 1883, 0, function(client)
print("connected") print("connected")
...@@ -180,7 +213,7 @@ Registers a callback function for an event. ...@@ -180,7 +213,7 @@ Registers a callback function for an event.
`mqtt:on(event, function(client[, topic[, message]]))` `mqtt:on(event, function(client[, topic[, message]]))`
#### Parameters #### Parameters
- `event` can be "connect", "message" or "offline" - `event` can be "connect", "message", "offline" or "overflow"
- `function(client[, topic[, message]])` callback function. The first parameter is the client. If event is "message", the 2nd and 3rd param are received topic and message (strings). - `function(client[, topic[, message]])` callback function. The first parameter is the client. If event is "message", the 2nd and 3rd param are received topic and message (strings).
#### Returns #### Returns
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment