Unable to make HTTPS POST with client certificate

[cezarg1410]

Hello,
I have really strange issue when trying to make HTTP POST with SSL from ESP32 with SIM7600G.
The thing is it works from time to time. Probably 1-2 out of 10 calls are successfull. For the rest the library returns -2 or -3 status code.

I can also add, that using that code without SSL auth works perfectly. Also using WiFiClientSecure works really fine.

Here is my full code:

#include <SSLClient.h>
#include "certs.h"
#include <HardwareSerial.h>

#define GSM_RX 27
#define GSM_TX 14

#define TEENSY_RX 16
#define TEENSY_TX 17

#define MODEM_PWRKEY 25

// Select your modem:
#define TINY_GSM_MODEM_SIM7600
#define TINY_GSM_RX_BUFFER 1024

#include <TinyGsmClient.h>
#include <ArduinoHttpClient.h>

HardwareSerial SerialAT(1);      //gsm
HardwareSerial SerialTeensy(2);  //teensy

//============================================
#define TINY_GSM_DEBUG SerialMon

#ifdef TINY_GSM_DEBUG
#define SerialMon Serial
#endif

#define TINY_GSM_USE_GPRS true
//============================================
//#define DUMP_AT_COMMANDS
#ifdef DUMP_AT_COMMANDS
#include <StreamDebugger.h>
StreamDebugger debugger(SerialAT, SerialMon);
TinyGsm modem(debugger);
#else
TinyGsm modem(SerialAT);
#endif
//============================================

TinyGsmClient gsm_transpor_layer(modem, 0);
SSLClient secure_presentation_layer(&gsm_transpor_layer);
HttpClient client = HttpClient(secure_presentation_layer, server, 443);

unsigned long prevFinished = 0;

void setup() {
#ifdef TINY_GSM_DEBUG
  SerialMon.begin(115200);
  delay(10);
#endif

  logTextLine("Wait...");

  SerialAT.begin(115200, SERIAL_8N1, GSM_RX, GSM_TX);
  while (!SerialAT)
    ;

  SerialTeensy.begin(115200, SERIAL_8N1, TEENSY_RX, TEENSY_TX);
  while (!SerialTeensy)
    ;

  setupModem();
  secure_presentation_layer.setCACert(ca);
  secure_presentation_layer.setCertificate(client_crt);
  secure_presentation_layer.setPrivateKey(client_key);
}

bool sending = false;
String dataToSend = "";

void loop() {
#ifdef TINY_GSM_DEBUG
  String text = ">dataToSend";
#else
  String text = SerialTeensy.readStringUntil('<');
#endif
  if (!sending && text.startsWith(">") && text.length() > 2 && (millis() - prevFinished > 15000)) {
    logText("Received >>");
    logText(text);
    logTextLine("<<");
    if (!modem.init()) {
      logTextLine(" fail... restarting modem...");
      setupModem();
      // Restart takes quite some time
      // Use modem.init() if you don't need the complete restart
      if (!modem.restart()) {
        logTextLine(" fail... even after restart");
        return;
      }
    }
    logTextLine(" Modem initialized");

    sending = true;
    dataToSend = text.substring(1);
    logText("Sending: ");
    logTextLine(dataToSend);

    if (send()) {
      logTextLine("Successfully send data");
    } else {
      logTextLine("Sending data failed");
    }
    sending = false;
    dataToSend = "";
    prevFinished = millis();
  }
}
bool send() {
  if (!waitForNetwork() || !connectToNetwork() || !setupGprs()) {
    return false;
  }

  logTextLine("Performing HTTPS POST request... ");

  String jsonData = "{\"data\":\"" + dataToSend + "\"}";
   
  client.beginRequest();
  client.post(resource);
  client.sendHeader("Content-Type", "application/json");
  client.sendHeader("Authorization", "Basic xxx");
  client.sendHeader("Content-Length", String(jsonData.length()));
  client.beginBody();
  client.print(jsonData);
  client.endRequest();

  int status_code = client.responseStatusCode();

  logText("Status code: ");
  logTextLine(status_code);

  if (status_code <= 0) {
    setupModem();
    return false;
  }

  client.stop();
  modem.gprsDisconnect();
  modem.poweroff();
  logTextLine("Server disconnected");

  return status_code == 200;
}

void turnModemOn() {
  pinMode(MODEM_PWRKEY, OUTPUT);
  digitalWrite(MODEM_PWRKEY, LOW);
  delay(1000);  //Datasheet Ton mintues = 1S
  digitalWrite(MODEM_PWRKEY, HIGH);
}

void turnModemOff() {
  digitalWrite(MODEM_PWRKEY, LOW);
  delay(1500);  //Datasheet Ton mintues = 1.2S
  digitalWrite(MODEM_PWRKEY, HIGH);
}

void setupModem() {
  logTextLine("Restarting modem");
  turnModemOff();
  delay(1000);
  turnModemOn();
  delay(5000);

  logTextLine("Initializing modem...");
  modem.restart();
  modem.setNetworkMode(51);   //2 auto , 13 - gsm, 38 - lte, 51 - gsm and lte

  String modemInfo = modem.getModemInfo();
  logText("Modem Info: ");
  logTextLine(modemInfo);

  logText("Signal strength: ");
  logTextLine(modem.getSignalQuality());
}


bool waitForNetwork() {
  logText("Waiting for network...");
  if (!modem.waitForNetwork()) {
    logTextLine(" fail");
    return false;
  }
  logTextLine(" success");
  return true;
}

bool connectToNetwork() {
  if (modem.isNetworkConnected()) {
    logTextLine("Network connected");
    return true;
  } else {
    return false;
  }
}

bool setupGprs() {
  // GPRS connection parameters are usually set after network registration
  logText("Connecting to ");
  logTextLine(apn);

  modem.sendAT(GF("+CGDCONT=1"));
  modem.waitResponse();
  modem.sendAT(GF("+CGDCONT=2"));
  modem.waitResponse();
  modem.sendAT(GF("+CGDCONT=3"));
  modem.waitResponse();

  logTextLine("PDP reset done");

  if (!modem.gprsConnect(apn, gprsUser, gprsPass)) {
    logTextLine(" fail");
    return false;
  }
  logTextLine(" success");

  if (modem.isGprsConnected()) {
    logTextLine("GPRS connected");
    return true;
  }
  return false;
}


void logTextLine(String line) {
#ifdef TINY_GSM_DEBUG
  SerialMon.println(line);
#endif
}

void logTextLine(int line) {
#ifdef TINY_GSM_DEBUG
  SerialMon.println(line);
#endif
}

void logText(String line) {
#ifdef TINY_GSM_DEBUG
  SerialMon.print(line);
#endif
}

void logText(int line) {
#ifdef TINY_GSM_DEBUG
  SerialMon.print(line);
#endif
}

Ane the result:

Waiting for network... success
Network connected
Connecting to darmowy
PDP reset done
 success
GPRS connected
Performing HTTPS POST request... 
[536307][V][SSLClient.cpp:161] connect(): connect with CA
[536307][V][SSLClient.cpp:219] connect(): Connecting to *************
[536312][V][SSLClient.cpp:220] connect(): Timeout value: 0
[536594][V][ssl_client.cpp:405] seed_random_number_generator(): Entropy context initialized
[536600][V][ssl_client.cpp:308] start_ssl_client(): Random number generator seeded, ret: 0
[536608][V][ssl_client.cpp:425] set_up_tls_defaults(): Setting up the SSL/TLS defaults...
[536616][V][ssl_client.cpp:313] start_ssl_client(): SSL config defaults set, ret: 0
[536623][V][ssl_client.cpp:461] auth_root_ca_buff(): Loading CA cert
[536633][V][ssl_client.cpp:318] start_ssl_client(): SSL auth mode set, ret: 0
[536636][V][ssl_client.cpp:546] auth_client_cert_key(): Loading CRT cert
[536646][V][ssl_client.cpp:555] auth_client_cert_key(): Loading private key
[536654][V][ssl_client.cpp:323] start_ssl_client(): SSL client cert and key set, ret: 0
[536657][V][ssl_client.cpp:601] set_hostname_for_tls(): Setting hostname for TLS session...
[536665][V][ssl_client.cpp:328] start_ssl_client(): SSL hostname set, ret: 0
[536672][V][ssl_client.cpp:653] set_io_callbacks_and_timeout(): Setting up IO callbacks...
[536680][V][ssl_client.cpp:656] set_io_callbacks_and_timeout(): Setting timeout to 0
[536687][V][ssl_client.cpp:333] start_ssl_client(): SSL IO callbacks and timeout set, ret: 0
[536695][V][ssl_client.cpp:699] perform_ssl_handshake(): Performing the SSL/TLS handshake, timeout 120000 ms
[536705][D][ssl_client.cpp:701] perform_ssl_handshake(): calling mbedtls_ssl_handshake with ssl_ctx address 0x3ffba7d8
[536790][V][ssl_client.cpp:194] client_net_send(): SSL client TX res=310 len=310
[536791][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[536804][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[536814][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[536824][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[536834][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[536844][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[536975][V][ssl_client.cpp:135] client_net_recv_timeout(): SSL client RX (received=5 expected=5 in 131ms)
[536975][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[536979][V][ssl_client.cpp:135] client_net_recv_timeout(): SSL client RX (received=67 expected=67 in 0ms)
[536989][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[536995][V][ssl_client.cpp:135] client_net_recv_timeout(): SSL client RX (received=5 expected=5 in 0ms)
[537005][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[537202][V][ssl_client.cpp:135] client_net_recv_timeout(): SSL client RX (received=2321 expected=2321 in 191ms)
[537642][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[537642][V][ssl_client.cpp:135] client_net_recv_timeout(): SSL client RX (received=5 expected=5 in 0ms)
[537647][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[537654][V][ssl_client.cpp:135] client_net_recv_timeout(): SSL client RX (received=115 expected=115 in 0ms)
[538101][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[538101][V][ssl_client.cpp:135] client_net_recv_timeout(): SSL client RX (received=5 expected=5 in 0ms)
[538106][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[538112][V][ssl_client.cpp:135] client_net_recv_timeout(): SSL client RX (received=374 expected=374 in 0ms)
[538122][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[538128][V][ssl_client.cpp:135] client_net_recv_timeout(): SSL client RX (received=5 expected=5 in 0ms)
[538138][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[538144][V][ssl_client.cpp:135] client_net_recv_timeout(): SSL client RX (received=4 expected=4 in 0ms)
[538306][V][ssl_client.cpp:194] client_net_send(): SSL client TX res=1064 len=1064
[538797][V][ssl_client.cpp:194] client_net_send(): SSL client TX res=42 len=42
[539343][V][ssl_client.cpp:194] client_net_send(): SSL client TX res=269 len=269
[539363][V][ssl_client.cpp:194] client_net_send(): SSL client TX res=6 len=6
[539387][V][ssl_client.cpp:194] client_net_send(): SSL client TX res=45 len=45
[539387][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[539400][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[539410][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[539420][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[539430][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[539560][V][ssl_client.cpp:135] client_net_recv_timeout(): SSL client RX (received=5 expected=5 in 130ms)
[539560][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[539603][V][ssl_client.cpp:135] client_net_recv_timeout(): SSL client RX (received=1258 expected=1258 in 38ms)
[539604][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[539609][V][ssl_client.cpp:135] client_net_recv_timeout(): SSL client RX (received=5 expected=5 in 0ms)
[539618][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[539625][V][ssl_client.cpp:135] client_net_recv_timeout(): SSL client RX (received=1 expected=1 in 0ms)
[539634][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[539640][V][ssl_client.cpp:135] client_net_recv_timeout(): SSL client RX (received=5 expected=5 in 0ms)
[539649][V][ssl_client.cpp:118] client_net_recv_timeout(): Timeout set to 0
[539656][V][ssl_client.cpp:135] client_net_recv_timeout(): SSL client RX (received=40 expected=40 in 0ms)
[539666][V][ssl_client.cpp:722] perform_ssl_handshake(): Protocol is TLSv1.2 Ciphersuite is TLS-ECDHE-ECDSA-WITH-AES-128-GCM-SHA256
[539677][E][ssl_client.cpp:728] perform_ssl_handshake(): mbedtls_ssl_get_record_expansion returned -0xffffffe3
[539687][D][ssl_client.cpp:791] stop_ssl_socket(): Cleaning SSL connection.
[539693][D][ssl_client.cpp:795] stop_ssl_socket(): Stopping SSL client. Current client pointer address: 0x3ffc2468
[539713][D][ssl_client.cpp:800] stop_ssl_socket(): Freeing CA cert. Current ca_cert address: 0x3ffbada8
[539713][D][ssl_client.cpp:808] stop_ssl_socket(): Freeing client cert and client key. Current client_cert address: 0x3ffbaf00, client_key address: 0x3ffbb058
[539726][D][ssl_client.cpp:816] stop_ssl_socket(): Freeing SSL context. Current ssl_ctx address: 0x3ffba7d8
[539736][D][ssl_client.cpp:819] stop_ssl_socket(): Freeing SSL config. Current ssl_conf address: 0x3ffba9f8
[539745][D][ssl_client.cpp:822] stop_ssl_socket(): Freeing DRBG context. Current drbg_ctx address: 0x3ffbaae0
[539755][D][ssl_client.cpp:825] stop_ssl_socket(): Freeing entropy context. Current entropy_ctx address: 0x3ffbab30
[539765][D][ssl_client.cpp:828] stop_ssl_socket(): Finished cleaning SSL connection.
[539773][D][ssl_client.cpp:249] cleanup(): Free internal heap after TLS 326884
[539780][E][ssl_client.cpp:45] _handle_error(): [start_ssl_client():353]: (29) UNKNOWN ERROR CODE (001D)
[539789][V][SSLClient.cpp:232] connect(): Return value from start_ssl_client: 0
[539796][E][SSLClient.cpp:235] connect(): start_ssl_client failed: 0
[539802][D][SSLClient.cpp:90] stop(): Stopping ssl client
[539807][D][ssl_client.cpp:791] stop_ssl_socket(): Cleaning SSL connection.
[539814][D][ssl_client.cpp:795] stop_ssl_socket(): Stopping SSL client. Current client pointer address: 0x3ffc2468
[539885][D][ssl_client.cpp:816] stop_ssl_socket(): Freeing SSL context. Current ssl_ctx address: 0x3ffba7d8
[539885][D][ssl_client.cpp:819] stop_ssl_socket(): Freeing SSL config. Current ssl_conf address: 0x3ffba9f8
[539893][D][ssl_client.cpp:822] stop_ssl_socket(): Freeing DRBG context. Current drbg_ctx address: 0x3ffbaae0
[539903][D][ssl_client.cpp:825] stop_ssl_socket(): Freeing entropy context. Current entropy_ctx address: 0x3ffbab30
[539913][D][ssl_client.cpp:828] stop_ssl_socket(): Finished cleaning SSL connection.
[539920][W][SSLClient.cpp:369] write(): SSLClient is not connected.
[539926][W][SSLClient.cpp:369] write(): SSLClient is not connected.
[539932][W][SSLClient.cpp:369] write(): SSLClient is not connected.
[539938][W][SSLClient.cpp:369] write(): SSLClient is not connected.
[539944][W][SSLClient.cpp:369] write(): SSLClient is not connected.
[539950][W][SSLClient.cpp:369] write(): SSLClient is not connected.
[539956][W][SSLClient.cpp:369] write(): SSLClient is not connected.
[539962][W][SSLClient.cpp:369] write(): SSLClient is not connected.
[539968][W][SSLClient.cpp:369] write(): SSLClient is not connected.
[539974][W][SSLClient.cpp:369] write(): SSLClient is not connected.
[539980][W][SSLClient.cpp:369] write(): SSLClient is not connected.
[539986][W][SSLClient.cpp:369] write(): SSLClient is not connected.
[539992][W][SSLClient.cpp:369] write(): SSLClient is not connected.
[539998][W][SSLClient.cpp:369] write(): SSLClient is not connected.
Status code: -3

[RobertByrnes]

Hi, the -3 looks like a timeout coming from the ArduinoHttpClient. Also, it looks due to the ssl handshake failing.

I have had this before and fixed when using ArduinoHttpClient the call to client.connectionKeepAlive() before startRequest is called. Although technically this shouldn't be necessary.

Also, check your ArduinoHttpClient and SSLClient timeout values and play at adjusting them to longer. Maybe go higher and work back.

Your post request structure for using the ArduinoHttpClient looks good.

With TinyGsm is tend to log out a connection strength operator periodically as well to help diagnose connection issues.

[ilgarbenli]

Hi, I'm also working on with sim800L model modul with sending json request to googleapis.com. But I couldn't build proper structure of POST request.

This is my working structure with another library. How can I implement this structure with sslclient.h library

    SerialMon.println("Performing HTTP POST request...");
    String body = "{\"considerIP\":false,\"wifiAccessPoints\":" + getWifisJSON() + "}";
    String request = "POST " + String(googleApiUrl_) + "?key=" + googleApiKey_ + " HTTP/1.1\r\n";
    request += "Host: " + String(googleApisHost_) + "\r\n";
    request += "User-Agent: ESP32\r\n";
    request += "Content-Type:application/json\r\n";
    request += "Content-Length:" + String(body.length()) + "\r\n";
    request += "Connection: keep-alive\r\n\r\n";
    request += body;
    client.println(request);

I couldnt use this code with client.post() function. Can anybody help?

[RobertByrnes]

Hi, @ilgarbenli I have used this lib with SIM800L and ArduinoHttpClient successfully A LOT.

Just using client.post() method (look at ArduinoHttpClient examples.

Using client.write() only

And also (my favourite method for completeness, structure and clarity) using a complex and explicit use of ArduinoHttpClient - one of their example shows this. Pretty sure you are better off in this order.

1. client.connectionKeepAlive - or send the header yourself, both do the same.
2. client.connect
3. client.setHttpResponseTimeout
4. client.beginRequest(endpoint, method)
5. client.sendHeader() content type
6. client.sendHeader() content len
7. client.write(body, bodyLen)
8. client.endRequest()

Then do your check for response codes and client availability, etc... and read your responseBody() as a String() (personally yuk!) or as bytes (client.read) into a buffer.

The steps above - a good habit is to check return value of any non void function. E.g. client.write returning 0 would mean no bytes were written.

Also, good to get familiar with response codes like -2 or -3. Have a look in the ArduinoHttpClient itself, they are defined in there and can refer more often to TLS layer issues caused by poor connection. One of these codes has not come from the server!

Hope that helps

[ilgarbenli]

@RobertByrnes thank you for your respond. I tried several things and couldn't make it unfortunately. I guess I'm doing something wrong with my code. I'll open new issue of mine with my full of code. Would you review it when you have a chance

[ilgarbenli]

@cezarg1410 what is resource in client.post(resource); ?

[RobertByrnes]

Resource = "/something"

As in https://api.example.com/something

You have already given the host when you connected in the connect method as "api.example.com" so now the endpoint or resource is "/something"

[ilgarbenli]

Resource = "/something"

As in https://api.example.com/something

You have already given the host when you connected in the connect method as "api.example.com" so now the endpoint or resource is "/something"

Thank you for your respond and thank you for your help. My code is working now.

[cezarg1410]

Unfortunately - no luck for me.
I tried option with connectionKeepAlive() but it does not change anything.
SSL Handshake should be fine - i am able to POST identical request from postman without issues.

Looks like, the library or the sim7600 is not able to handle the client certificate - when i remove those two lines:

  secure_presentation_layer.setCertificate(client_crt);
  secure_presentation_layer.setPrivateKey(client_key);

then it works fine every time (of course i am getting then HTTP 403 response because my server requires proper client certificate).

I updated all libraries (TinyGSM, SSLClient and ArduinoHttpClient) to the newest versions and something has changed. Now i am getting -3 error every single time : D

I am starting to loose hope :) Looks like i should resign from authenticating client with its certificate, but obviously i would rather not to do it - it is in my opinion the best and the most secure method for exposing API to the internet.

[ilgarbenli]

@cezarg1410 may it works for your case? https://github.com/Samorange1/Connect-ESP32-to-Firebase-via-GSM
it bypass ssl connecction with a server

[cezarg1410]

Thanks for that! Looks interesting, will get deep dive into it

[ilgarbenli]

Thanks for that! Looks interesting, will get deep dive into it

please keep contact together, It worked for me. But also I'm struggling about 403 forbidden issue for another client. Maybe we can help together

[udayalawa]

any updates?

[cezarg1410]

No luck for me.
I decided to remove client certificate authentication from my solution and instead i am using API token authentication on my rest API with HTTPS.
It works just fine and i stopped wasting time trying to make it working.

[RobertByrnes]

solved by v1.2.0