ESP32 Over-the-air (OTA) updates

In this tutorial, we will look at how to do over-the-air (OTA) update for ESP32.

Prerequisites

  1. You know how to connect your ESP to the Kaa platform.

Playbook

Compile and host new firmware binary

Compile and upload the new firmware binary file to Kaa File Management.

If you use Arduino IDE choose Sketch -> Export compiled Binary.

Export compiled Binary

After the compilation is done, choose Sketch -> Show Sketch Folder.

Compiled binary

This is a new firmware that we are going to update our ESP32 to.

Next, we should host it on the HTTP server. ESP32 is also able to download its firmware from the HTTPS server but to keep things simple we chose HTTP.

Let’s host compiled binary.

Go to your Kaa Cloud account -> Files and choose bucket suffixed with the public. All files stored in that bucket are publicly available. This is exactly what we need as in this tutorial we want our ESP32 to download firmware binary without extra authentication.

Upload the binary and get its sharable link. Remember the link, we will need it in a moment.

Host firmware

Initial firmware definition provisioning

Firstly, we should provision the initial firmware that our device has by default, from the factory, for example.

Go to Kaa UI -> Device Management -> Software OTA and add new software definition pressing Add software version.

Set 1.0.0 for Semantic software version and The initial firmware for the Upgradable from. Press Create.

Create initial firmware

New firmware definition provisioning

Now we want to provision a new firmware and define the relationship between the initial firmware and this one.

Again, go to Kaa UI -> Device Management -> Software OTA and add a new software definition pressing Add software version.

This time set 1.0.1 for Semantic software version and Second firmware for the Upgradable from, choose 1.0.0 for Upgradable from. Paste the earlier copied link to firmware binary in the Download link field. Update the link schema from HTTPS to HTTP, otherwise, ESP won’t be able to download the firmware over the HTTPS protocol. Also, remove extra params from the URL so that it looks like the below:

http://minio.cloud.kaaiot.com/59433264-b474-4de1-bfe6-9c5ca3e86ddc-public/sketch_apr30a.ino.esp32.bin

Choose All Rollout configuration checkbox. This rollout configuration strategy means that all endpoints in that application will be able to update to this firmware. Press Create.

Create the second firmware

Update ESP32 firmware

Now we are ready to do over-the-air (OTA) update on ESP32. The below firmware connects to Kaa, reports its current firmware version, which is 1.0.0, and requests the new one and updates to it if it exists.

Specify your WiFi SSID, password, and token and application version of your endpoint.

#include <WiFi.h>
#include <PubSubClient.h>
#include <ArduinoJson.h>
#include <HTTPClient.h>
#include <HTTPUpdate.h>

const char* ssid = "";        // WiFi name
const char* password = "";    // WiFi password

const char* mqttServer = "mqtt.cloud.kaaiot.com";

const String TOKEN = "";        // Endpoint token - you get (or specify) it during device provisioning
const String APP_VERSION = "";  // Application version - you specify it during device provisioning

WiFiClient espClient;
PubSubClient client(espClient);

void setup() {
  Serial.begin(115200);
  client.setServer(mqttServer, 1883);
  client.setCallback(handleOtaUpdate);
  initServerConnection();

  if (client.setBufferSize(1023)) {
    Serial.println("Successfully reallocated internal buffer size");
  } else {
    Serial.println("Failed to reallocated internal buffer size");
  }

  delay(1000);
  reportCurrentFirmwareVersion();
  requestNewFirmware();
}

void loop() {
  // Do work here
  initServerConnection();
  delay(1000);
}

void reportCurrentFirmwareVersion() {
  String reportTopic = "kp1/" + APP_VERSION + "/cmx_ota/" + TOKEN + "/applied/json";
  String reportPayload = "{\"configId\":\"1.0.0\"}";
  Serial.println("Reporting current firmware version on topic: " + reportTopic + " and payload: " + reportPayload);
  client.publish(reportTopic.c_str(), reportPayload.c_str());
}

void requestNewFirmware() {
  int requestID = random(0, 99);
  String firmwareRequestTopic = "kp1/" + APP_VERSION + "/cmx_ota/" + TOKEN + "/config/json/" + requestID;
  Serial.println("Requesting firmware using topic: " + firmwareRequestTopic);
  client.publish(firmwareRequestTopic.c_str(), "{\"observe\":true}"); // observe is used to specify whether the client wants to accept server pushes
}

void initServerConnection() {
  setupWifi();
  if (!client.connected()) {
    reconnect();
  }
  client.loop();
}

void handleOtaUpdate(char* topic, byte* payload, unsigned int length) {
  Serial.printf("\nHandling firmware update message on topic: %s and payload: ", topic);

  DynamicJsonDocument doc(1023);
  deserializeJson(doc, payload, length);
  JsonVariant json_var = doc.as<JsonVariant>();
  Serial.println(json_var.as<String>());
  if (json_var.isNull()) {
    Serial.println("No new firmware version is available");
    return;
  }

  unsigned int statusCode = json_var["statusCode"].as<unsigned int>();
  if (statusCode != 200) {
    Serial.printf("Firmware message's status code is not 200, but: %d\n", statusCode);
    return;
  }

  String firmwareLink = json_var["config"]["link"].as<String>();

  t_httpUpdate_return ret = httpUpdate.update(espClient, firmwareLink.c_str());

  switch (ret) {
    case HTTP_UPDATE_FAILED:
      Serial.printf("HTTP_UPDATE_FAILED Error (%d): %s\n", httpUpdate.getLastError(), httpUpdate.getLastErrorString().c_str());
      break;

    case HTTP_UPDATE_NO_UPDATES:
      Serial.println("HTTP_UPDATE_NO_UPDATES");
      break;

    case HTTP_UPDATE_OK:
      Serial.println("HTTP_UPDATE_OK");
      break;
  }
}

void setupWifi() {
  if (WiFi.status() != WL_CONNECTED) {
    delay(200);
    Serial.println();
    Serial.printf("Connecting to [%s]", ssid);
    WiFi.begin(ssid, password);
    connectWiFi();
  }
}

void connectWiFi() {
  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }
  Serial.println();
  Serial.println("WiFi connected");
  Serial.println("IP address: ");
  Serial.println(WiFi.localIP());
}

void reconnect() {
  while (!client.connected()) {
    Serial.println("Attempting MQTT connection...");
    char *client_id = "bqf1uai03p4cop6jr3u0";
    if (client.connect(client_id)) {
      Serial.println("Connected to WiFi");
      subscribeToFirmwareUpdates();
    } else {
      Serial.print("failed, rc=");
      Serial.print(client.state());
      Serial.println(" try again in 5 seconds");
      // Wait 5 seconds before retrying
      delay(5000);
    }
  }
}

void subscribeToFirmwareUpdates() {
  String serverPushOnConnect = "kp1/" + APP_VERSION + "/cmx_ota/" + TOKEN + "/config/json/#";
  client.subscribe(serverPushOnConnect.c_str());
  Serial.println("Subscribed to server firmware push on topic: " + serverPushOnConnect);

  String serverFirmwareResponse = "kp1/" + APP_VERSION + "/cmx_ota/" + TOKEN + "/config/json/status/#";
  client.subscribe(serverFirmwareResponse.c_str());
  Serial.println("Subscribed to server firmware response on topic: " + serverFirmwareResponse);

  String serverFirmwareErrorResponse = "kp1/" + APP_VERSION + "/cmx_ota/" + TOKEN + "/config/json/status/error";
  client.subscribe(serverFirmwareErrorResponse.c_str());
  Serial.println("Subscribed to server firmware response on topic: " + serverFirmwareErrorResponse);
}

Upload the sketch and check the Serial Monitor in Arduino IDE.

ESP32 OTA logs

Troubleshooting

We provide the below script written in Python that you can use to troubleshoot software exchange between a client and the Kaa platform.

import itertools
import json
import queue
import random
import string
import sys
import time

import paho.mqtt.client as mqtt

KPC_HOST = "mqtt.cloud.kaaiot.com"  # Kaa Cloud plain MQTT host
KPC_PORT = 1883                     # Kaa Cloud plain MQTT port

CURRENT_SOFTWARE_VERSION = ""   # Specify software that device currently uses (e.g., 0.0.1)

APPLICATION_VERSION = ""     # Paste your application version
ENDPOINT_TOKEN = ""          # Paste your endpoint token


class SoftwareClient:

    def __init__(self, client):
        self.client = client
        self.software_by_request_id = {}
        self.global_request_id = itertools.count()
        get_software_topic = f'kp1/{APPLICATION_VERSION}/cmx_ota/{ENDPOINT_TOKEN}/config/json/#'
        self.client.message_callback_add(get_software_topic, self.handle_software)

    def handle_software(self, client, userdata, message):
        if message.topic.split('/')[-1] == 'status':
            topic_part = message.topic.split('/')[-2]
            if topic_part.isnumeric():
                request_id = int(topic_part)
                print(f'<--- Received software response on topic {message.topic}')
                software_queue = self.software_by_request_id[request_id]
                software_queue.put_nowait(message.payload)
            else:
                print(f'<--- Received software push on topic {message.topic}:\n{str(message.payload.decode("utf-8"))}')
        else:
            print(f'<--- Received bad software response on topic {message.topic}:\n{str(message.payload.decode("utf-8"))}')

    def get_software(self):
        request_id = next(self.global_request_id)
        get_software_topic = f'kp1/{APPLICATION_VERSION}/cmx_ota/{ENDPOINT_TOKEN}/config/json/{request_id}'

        software_queue = queue.Queue()
        self.software_by_request_id[request_id] = software_queue

        print(f'---> Requesting software by topic {get_software_topic}')
        payload = {
            "configId": CURRENT_SOFTWARE_VERSION
        }
        self.client.publish(topic=get_software_topic, payload=json.dumps(payload))

        try:
            software = software_queue.get(True, 5)
            del self.software_by_request_id[request_id]
            return str(software.decode("utf-8"))
        except queue.Empty:
            print('Timed out waiting for software response from server')
            sys.exit()

def main():
    # Initiate server connection
    print(f'Connecting to Kaa server at {KPC_HOST}:{KPC_PORT} using application version {APPLICATION_VERSION} and endpoint token {ENDPOINT_TOKEN}')

    client_id = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6))
    client = mqtt.Client(client_id=client_id)
    client.connect(KPC_HOST, KPC_PORT, 60)
    client.loop_start()

    software_client = SoftwareClient(client)

    # Fetch available software
    retrieved_software = software_client.get_software()
    print(f'Retrieved software from server: {retrieved_software}')

    time.sleep(5)
    client.disconnect()


if __name__ == '__main__':
    main()