Merge branch 'develop' into zps-module

This reverts commit 391928ed08.

Since esp32_https_server commit e2c9f98, this is no longer necessary.

Also, the esp32-common.ini '-include mbedtls/error.h' causes a build error
under IDF v5 (https://github.com/meshtastic/firmware/pull/9122#issuecomment-4185767085):

  <command-line>: fatal error: mbedtls/error.h: No such file or directory

so this change fixes that error.

platformio.ini

@@ -50,6 +50,7 @@ build_flags = -Wno-missing-field-initializers
 	-DRADIOLIB_EXCLUDE_APRS=1
 	-DRADIOLIB_EXCLUDE_LORAWAN=1
 	-DMESHTASTIC_EXCLUDE_DROPZONE=1
+	-DMESHTASTIC_EXCLUDE_ZPS=1
 	-DMESHTASTIC_EXCLUDE_REPLYBOT=1
 	-DMESHTASTIC_EXCLUDE_REMOTEHARDWARE=1
 	-DMESHTASTIC_EXCLUDE_HEALTH_TELEMETRY=1

src/RedirectablePrint.cpp

@@ -137,7 +137,7 @@ void RedirectablePrint::log_to_serial(const char *logLevel, const char *format,
         if (color) {
             ::printf("\u001b[0m");
         }
-        ::printf("| %02d:%02d:%02d %u ", hour, min, sec, millis() / 1000);
+        ::printf("| %02d:%02d:%02d %u.%03u ", hour, min, sec, millis() / 1000, millis() % 1000);
 #else
         printf("%s ", logLevel);
         if (color) {
@@ -151,7 +151,7 @@ void RedirectablePrint::log_to_serial(const char *logLevel, const char *format,
         if (color) {
             ::printf("\u001b[0m");
         }
-        ::printf("| ??:??:?? %u ", millis() / 1000);
+        ::printf("| ??:??:?? %u.%03u ", millis() / 1000, millis() % 1000);
 #else
         printf("%s ", logLevel);
         if (color) {

src/graphics/draw/MenuHandler.cpp

@@ -163,9 +163,11 @@ void menuHandler::LoraRegionPicker(uint32_t duration)
 
             // Guard: without a reboot, reconfigure() applies the region directly.
             // Reject LORA_24 on sub-GHz-only hardware — getRadio() used to catch this post-reboot.
+            // TODO: change this to either use the validateLoraConfig() logic or at least check the region for wideLora
+            // rather than a hardcoded check for LORA_24.
             if (selectedRegion == meshtastic_Config_LoRaConfig_RegionCode_LORA_24 &&
                 !(RadioLibInterface::instance && RadioLibInterface::instance->wideLora())) {
-                LOG_WARN("Radio hardware does not support 2.4 GHz; ignoring LORA_24 selection");
+                LOG_WARN("Radio hardware does not support 2.4 GHz; ignoring region selection");
                 return;
             }
 

src/mesh/Default.h

@@ -32,8 +32,8 @@
 #define default_map_publish_interval_secs 60 * 60
 
 // Traffic management defaults
-#define default_traffic_mgmt_position_precision_bits 24         // ~10m grid cells
-#define default_traffic_mgmt_position_min_interval_secs ONE_DAY // 1 day between identical positions
+#define default_traffic_mgmt_position_precision_bits 24               // ~10m grid cells
+#define default_traffic_mgmt_position_min_interval_secs (ONE_DAY / 2) // 12 hours between identical positions
 
 #ifdef USERPREFS_RINGTONE_NAG_SECS
 #define default_ringtone_nag_secs USERPREFS_RINGTONE_NAG_SECS

src/mesh/LR11x0Interface.cpp

@@ -71,12 +71,10 @@ template <typename T> bool LR11x0Interface<T>::init()
 
     RadioLibInterface::init();
 
-    limitPower(LR1110_MAX_POWER);
-
-    if ((power > LR1120_MAX_POWER) &&
-        (config.lora.region == meshtastic_Config_LoRaConfig_RegionCode_LORA_24)) { // clamp again if wide freq range
-        power = LR1120_MAX_POWER;
-        preambleLength = 12; // 12 is the default for operation above 2GHz
+    if (config.lora.region == meshtastic_Config_LoRaConfig_RegionCode_LORA_24) { // clamp if wide freq range
+        limitPower(LR1120_MAX_POWER);
+    } else {
+        limitPower(LR1110_MAX_POWER); // default clamp for non-wide freq range
     }
 
 #ifdef LR11X0_RF_SWITCH_SUBGHZ
@@ -177,6 +175,12 @@ template <typename T> bool LR11x0Interface<T>::reconfigure()
     err = lora.setSyncWord(syncWord);
     assert(err == RADIOLIB_ERR_NONE);
 
+    if (config.lora.region == meshtastic_Config_LoRaConfig_RegionCode_LORA_24) { // clamp if wide freq range
+        limitPower(LR1120_MAX_POWER);
+    } else {
+        limitPower(LR1110_MAX_POWER); // default clamp for non-wide freq range
+    }
+
     err = lora.setPreambleLength(preambleLength);
     assert(err == RADIOLIB_ERR_NONE);
 
@@ -184,11 +188,6 @@ template <typename T> bool LR11x0Interface<T>::reconfigure()
     if (err != RADIOLIB_ERR_NONE)
         RECORD_CRITICALERROR(meshtastic_CriticalErrorCode_INVALID_RADIO_SETTING);
 
-    if (power > LR1110_MAX_POWER) // This chip has lower power limits than some
-        power = LR1110_MAX_POWER;
-    if ((power > LR1120_MAX_POWER) && (config.lora.region == meshtastic_Config_LoRaConfig_RegionCode_LORA_24)) // 2.4G power limit
-        power = LR1120_MAX_POWER;
-
     err = lora.setOutputPower(power);
     assert(err == RADIOLIB_ERR_NONE);
 

src/mesh/PointerQueue.h

@@ -17,14 +17,4 @@ template <class T> class PointerQueue : public TypedQueue<T *>
 
         return this->dequeue(&p, maxWait) ? p : nullptr;
     }
-
-#ifdef HAS_FREE_RTOS
-    // returns a ptr or null if the queue was empty
-    T *dequeuePtrFromISR(BaseType_t *higherPriWoken)
-    {
-        T *p;
-
-        return this->dequeueFromISR(&p, higherPriWoken) ? p : nullptr;
-    }
-#endif
 };

src/mesh/RF95Interface.cpp

@@ -240,8 +240,7 @@ bool RF95Interface::reconfigure()
     if (err != RADIOLIB_ERR_NONE)
         RECORD_CRITICALERROR(meshtastic_CriticalErrorCode_INVALID_RADIO_SETTING);
 
-    if (power > RF95_MAX_POWER) // This chip has lower power limits than some
-        power = RF95_MAX_POWER;
+    limitPower(RF95_MAX_POWER);
 
 #ifdef USE_RF95_RFO
     err = lora->setOutputPower(power, true);

src/mesh/RadioInterface.cpp

@@ -1038,6 +1038,13 @@ void RadioInterface::applyModemConfig()
     saveFreq(freq + loraConfig.frequency_offset);
     const char *channelName = channels.getName(channels.getPrimaryIndex());
 
+    if (newRegion->wideLora) {                          // clamp if wide freq range
+        preambleLength = wideLoraPreambleLengthDefault; // 12 is the default for operation above 2GHz
+    } else {
+        preambleLength =
+            preambleLengthDefault; // 8 is default, but we use longer to increase the amount of sleep time when receiving
+    }
+
     slotTimeMsec = computeSlotTimeMsec();
     preambleTimeMsec = preambleLength * (pow_of_2(sf) / bw);
 

src/mesh/RadioInterface.h

@@ -92,15 +92,20 @@ class RadioInterface
     uint8_t sf = 9;
     uint8_t cr = 5;
 
-    const uint8_t NUM_SYM_CAD = 2;       // Number of symbols used for CAD, 2 is the default since RadioLib 6.3.0 as per AN1200.48
-    const uint8_t NUM_SYM_CAD_24GHZ = 4; // Number of symbols used for CAD in 2.4 GHz, 4 is recommended in AN1200.22 of SX1280
+    static constexpr uint8_t NUM_SYM_CAD =
+        2; // Number of symbols used for CAD, 2 is the default since RadioLib 6.3.0 as per AN1200.48
+    static constexpr uint8_t NUM_SYM_CAD_24GHZ =
+        4; // Number of symbols used for CAD in 2.4 GHz, 4 is recommended in AN1200.22 of SX1280
     uint32_t slotTimeMsec = computeSlotTimeMsec();
-    uint16_t preambleLength = 16;    // 8 is default, but we use longer to increase the amount of sleep time when receiving
-    uint32_t preambleTimeMsec = 165; // calculated on startup, this is the default for LongFast
-    const uint32_t PROCESSING_TIME_MSEC =
-        4500;                // time to construct, process and construct a packet again (empirically determined)
-    const uint8_t CWmin = 3; // minimum CWsize
-    const uint8_t CWmax = 8; // maximum CWsize
+    uint16_t preambleLength = 16; // 8 is default, but we use longer to increase the amount of sleep time when receiving
+    static constexpr uint16_t preambleLengthDefault =
+        16; // 8 is default, but we use longer to increase the amount of sleep time when receiving
+    static constexpr uint16_t wideLoraPreambleLengthDefault = 12; // 12 is default for wide Lora
+    uint32_t preambleTimeMsec = 165;                              // calculated on startup, this is the default for LongFast
+    static constexpr uint32_t PROCESSING_TIME_MSEC =
+        4500;                           // time to construct, process and construct a packet again (empirically determined)
+    static constexpr uint8_t CWmin = 3; // minimum CWsize
+    static constexpr uint8_t CWmax = 8; // maximum CWsize
 
     meshtastic_MeshPacket *sendingPacket = NULL; // The packet we are currently sending
     uint32_t lastTxStart = 0L;

src/mesh/SX126xInterface.cpp

@@ -245,8 +245,10 @@ template <typename T> bool SX126xInterface<T>::reconfigure()
     if (err != RADIOLIB_ERR_NONE)
         RECORD_CRITICALERROR(meshtastic_CriticalErrorCode_INVALID_RADIO_SETTING);
 
-    if (power > SX126X_MAX_POWER) // This chip has lower power limits than some
-        power = SX126X_MAX_POWER;
+    limitPower(SX126X_MAX_POWER);
+    // Make sure we reach the minimum power supported to turn the chip on (-9dBm)
+    if (power < -9)
+        power = -9;
 
     err = lora.setOutputPower(power);
     if (err != RADIOLIB_ERR_NONE)

src/mesh/SX128xInterface.cpp

@@ -144,8 +144,7 @@ template <typename T> bool SX128xInterface<T>::reconfigure()
     if (err != RADIOLIB_ERR_NONE)
         RECORD_CRITICALERROR(meshtastic_CriticalErrorCode_INVALID_RADIO_SETTING);
 
-    if (power > SX128X_MAX_POWER) // This chip has lower power limits than some
-        power = SX128X_MAX_POWER;
+    limitPower(SX128X_MAX_POWER);
 
     err = lora.setOutputPower(power);
     if (err != RADIOLIB_ERR_NONE)

src/mesh/compression/unishox2.h

@@ -0,0 +1,347 @@
+/*
+ * Copyright (C) 2020 Siara Logics (cc)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @author Arundale Ramanathan
+ *
+ * Port for Particle (particle.io) / Aruino - Jonathan Greenblatt
+ *
+ * This file describes each function of the Unishox2 API \n
+ * For finding out how this API can be used in your program, \n
+ * please see test_unishox2.c.
+ */
+
+#ifndef unishox2
+#define unishox2
+
+#define UNISHOX_VERSION "2.0" ///< Unicode spec version
+
+/**
+ * Macro switch to enable/disable output buffer length parameter in low level api \n
+ * Disabled by default \n
+ * When this macro is defined, the all the API functions \n
+ * except the simple API functions accept an additional parameter olen \n
+ * that enables the developer to pass the size of the output buffer provided \n
+ * so that the api function may not write beyond that length. \n
+ * This can be disabled if the developer knows that the buffer provided is sufficient enough \n
+ * so no additional parameter is passed and the program is faster since additional check \n
+ * for output length is not performed at each step \n
+ * The simple api, i.e. unishox2_(de)compress_simple will always omit the buffer length
+ */
+#ifndef UNISHOX_API_WITH_OUTPUT_LEN
+#define UNISHOX_API_WITH_OUTPUT_LEN 1
+#endif
+
+/// Upto 8 bits of initial magic bit sequence can be included. Bit count can be specified with UNISHOX_MAGIC_BIT_LEN
+#ifndef UNISHOX_MAGIC_BITS
+#define UNISHOX_MAGIC_BITS 0xFF
+#endif
+
+/// Desired length of Magic bits defined by UNISHOX_MAGIC_BITS
+#ifdef UNISHOX_MAGIC_BIT_LEN
+#if UNISHOX_MAGIC_BIT_LEN < 0 || 9 <= UNISHOX_MAGIC_BIT_LEN
+#error "UNISHOX_MAGIC_BIT_LEN need between [0, 8)"
+#endif
+#else
+#define UNISHOX_MAGIC_BIT_LEN 1
+#endif
+
+// enum {USX_ALPHA = 0, USX_SYM, USX_NUM, USX_DICT, USX_DELTA};
+
+/// Default Horizontal codes. When composition of text is know beforehand, the other hcodes in this section can be used to achieve
+/// more compression.
+#define USX_HCODES_DFLT                                                                                                          \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        0x00, 0x40, 0x80, 0xC0, 0xE0                                                                                             \
+    }
+/// Length of each default hcode
+#define USX_HCODE_LENS_DFLT                                                                                                      \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        2, 2, 2, 3, 3                                                                                                            \
+    }
+
+/// Horizontal codes preset for English Alphabet content only
+#define USX_HCODES_ALPHA_ONLY                                                                                                    \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        0x00, 0x00, 0x00, 0x00, 0x00                                                                                             \
+    }
+/// Length of each Alpha only hcode
+#define USX_HCODE_LENS_ALPHA_ONLY                                                                                                \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        0, 0, 0, 0, 0                                                                                                            \
+    }
+
+/// Horizontal codes preset for Alpha Numeric content only
+#define USX_HCODES_ALPHA_NUM_ONLY                                                                                                \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        0x00, 0x00, 0x80, 0x00, 0x00                                                                                             \
+    }
+/// Length of each Alpha numeric hcode
+#define USX_HCODE_LENS_ALPHA_NUM_ONLY                                                                                            \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        1, 0, 1, 0, 0                                                                                                            \
+    }
+
+/// Horizontal codes preset for Alpha Numeric and Symbol content only
+#define USX_HCODES_ALPHA_NUM_SYM_ONLY                                                                                            \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        0x00, 0x80, 0xC0, 0x00, 0x00                                                                                             \
+    }
+/// Length of each Alpha numeric and symbol hcodes
+#define USX_HCODE_LENS_ALPHA_NUM_SYM_ONLY                                                                                        \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        1, 2, 2, 0, 0                                                                                                            \
+    }
+
+/// Horizontal codes preset favouring Alphabet content
+#define USX_HCODES_FAVOR_ALPHA                                                                                                   \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        0x00, 0x80, 0xA0, 0xC0, 0xE0                                                                                             \
+    }
+/// Length of each hcode favouring Alpha content
+#define USX_HCODE_LENS_FAVOR_ALPHA                                                                                               \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        1, 3, 3, 3, 3                                                                                                            \
+    }
+
+/// Horizontal codes preset favouring repeating sequences
+#define USX_HCODES_FAVOR_DICT                                                                                                    \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        0x00, 0x40, 0xC0, 0x80, 0xE0                                                                                             \
+    }
+/// Length of each hcode favouring repeating sequences
+#define USX_HCODE_LENS_FAVOR_DICT                                                                                                \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        2, 2, 3, 2, 3                                                                                                            \
+    }
+
+/// Horizontal codes preset favouring symbols
+#define USX_HCODES_FAVOR_SYM                                                                                                     \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        0x80, 0x00, 0xA0, 0xC0, 0xE0                                                                                             \
+    }
+/// Length of each hcode favouring symbols
+#define USX_HCODE_LENS_FAVOR_SYM                                                                                                 \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        3, 1, 3, 3, 3                                                                                                            \
+    }
+
+// #define USX_HCODES_FAVOR_UMLAUT {0x00, 0x40, 0xE0, 0xC0, 0x80}
+// #define USX_HCODE_LENS_FAVOR_UMLAUT {2, 2, 3, 3, 2}
+
+/// Horizontal codes preset favouring umlaut letters
+#define USX_HCODES_FAVOR_UMLAUT                                                                                                  \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        0x80, 0xA0, 0xC0, 0xE0, 0x00                                                                                             \
+    }
+/// Length of each hcode favouring umlaut letters
+#define USX_HCODE_LENS_FAVOR_UMLAUT                                                                                              \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        3, 3, 3, 3, 1                                                                                                            \
+    }
+
+/// Horizontal codes preset for no repeating sequences
+#define USX_HCODES_NO_DICT                                                                                                       \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        0x00, 0x40, 0x80, 0x00, 0xC0                                                                                             \
+    }
+/// Length of each hcode for no repeating sequences
+#define USX_HCODE_LENS_NO_DICT                                                                                                   \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        2, 2, 2, 0, 2                                                                                                            \
+    }
+
+/// Horizontal codes preset for no Unicode characters
+#define USX_HCODES_NO_UNI                                                                                                        \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        0x00, 0x40, 0x80, 0xC0, 0x00                                                                                             \
+    }
+/// Length of each hcode for no Unicode characters
+#define USX_HCODE_LENS_NO_UNI                                                                                                    \
+    (const unsigned char[])                                                                                                      \
+    {                                                                                                                            \
+        2, 2, 2, 2, 0                                                                                                            \
+    }
+
+extern const char *USX_FREQ_SEQ_DFLT[];
+extern const char *USX_FREQ_SEQ_TXT[];
+extern const char *USX_FREQ_SEQ_URL[];
+extern const char *USX_FREQ_SEQ_JSON[];
+extern const char *USX_FREQ_SEQ_HTML[];
+extern const char *USX_FREQ_SEQ_XML[];
+extern const char *USX_TEMPLATES[];
+
+/// Default preset parameter set. When composition of text is know beforehand, the other parameter sets in this section can be
+/// used to achieve more compression.
+#define USX_PSET_DFLT USX_HCODES_DFLT, USX_HCODE_LENS_DFLT, USX_FREQ_SEQ_DFLT, USX_TEMPLATES
+/// Preset parameter set for English Alphabet only content
+#define USX_PSET_ALPHA_ONLY USX_HCODES_ALPHA_ONLY, USX_HCODE_LENS_ALPHA_ONLY, USX_FREQ_SEQ_TXT, USX_TEMPLATES
+/// Preset parameter set for Alpha numeric content
+#define USX_PSET_ALPHA_NUM_ONLY USX_HCODES_ALPHA_NUM_ONLY, USX_HCODE_LENS_ALPHA_NUM_ONLY, USX_FREQ_SEQ_TXT, USX_TEMPLATES
+/// Preset parameter set for Alpha numeric and symbol content
+#define USX_PSET_ALPHA_NUM_SYM_ONLY                                                                                              \
+    USX_HCODES_ALPHA_NUM_SYM_ONLY, USX_HCODE_LENS_ALPHA_NUM_SYM_ONLY, USX_FREQ_SEQ_DFLT, USX_TEMPLATES
+/// Preset parameter set for Alpha numeric symbol content having predominantly text
+#define USX_PSET_ALPHA_NUM_SYM_ONLY_TXT                                                                                          \
+    USX_HCODES_ALPHA_NUM_SYM_ONLY, USX_HCODE_LENS_ALPHA_NUM_SYM_ONLY, USX_FREQ_SEQ_DFLT, USX_TEMPLATES
+/// Preset parameter set favouring Alphabet content
+#define USX_PSET_FAVOR_ALPHA USX_HCODES_FAVOR_ALPHA, USX_HCODE_LENS_FAVOR_ALPHA, USX_FREQ_SEQ_TXT, USX_TEMPLATES
+/// Preset parameter set favouring repeating sequences
+#define USX_PSET_FAVOR_DICT USX_HCODES_FAVOR_DICT, USX_HCODE_LENS_FAVOR_DICT, USX_FREQ_SEQ_DFLT, USX_TEMPLATES
+/// Preset parameter set favouring symbols
+#define USX_PSET_FAVOR_SYM USX_HCODES_FAVOR_SYM, USX_HCODE_LENS_FAVOR_SYM, USX_FREQ_SEQ_DFLT, USX_TEMPLATES
+/// Preset parameter set favouring unlaut letters
+#define USX_PSET_FAVOR_UMLAUT USX_HCODES_FAVOR_UMLAUT, USX_HCODE_LENS_FAVOR_UMLAUT, USX_FREQ_SEQ_DFLT, USX_TEMPLATES
+/// Preset parameter set for when there are no repeating sequences
+#define USX_PSET_NO_DICT USX_HCODES_NO_DICT, USX_HCODE_LENS_NO_DICT, USX_FREQ_SEQ_DFLT, USX_TEMPLATES
+/// Preset parameter set for when there are no unicode symbols
+#define USX_PSET_NO_UNI USX_HCODES_NO_UNI, USX_HCODE_LENS_NO_UNI, USX_FREQ_SEQ_DFLT, USX_TEMPLATES
+/// Preset parameter set for when there are no unicode symbols favouring text
+#define USX_PSET_NO_UNI_FAVOR_TEXT USX_HCODES_NO_UNI, USX_HCODE_LENS_NO_UNI, USX_FREQ_SEQ_TXT, USX_TEMPLATES
+/// Preset parameter set favouring URL content
+#define USX_PSET_URL USX_HCODES_DFLT, USX_HCODE_LENS_DFLT, USX_FREQ_SEQ_URL, USX_TEMPLATES
+/// Preset parameter set favouring JSON content
+#define USX_PSET_JSON USX_HCODES_DFLT, USX_HCODE_LENS_DFLT, USX_FREQ_SEQ_JSON, USX_TEMPLATES
+/// Preset parameter set favouring JSON content having no Unicode symbols
+#define USX_PSET_JSON_NO_UNI USX_HCODES_NO_UNI, USX_HCODE_LENS_NO_UNI, USX_FREQ_SEQ_JSON, USX_TEMPLATES
+/// Preset parameter set favouring XML content
+#define USX_PSET_XML USX_HCODES_DFLT, USX_HCODE_LENS_DFLT, USX_FREQ_SEQ_XML, USX_TEMPLATES
+/// Preset parameter set favouring HTML content
+#define USX_PSET_HTML USX_HCODES_DFLT, USX_HCODE_LENS_DFLT, USX_FREQ_SEQ_HTML, USX_TEMPLATES
+
+/**
+ * This structure is used when a string array needs to be compressed.
+ * This is passed as a parameter to the unishox2_decompress_lines() function
+ */
+struct us_lnk_lst {
+    char *data;
+    struct us_lnk_lst *previous;
+};
+
+/**
+ * This macro is for internal use, but builds upon the macro UNISHOX_API_WITH_OUTPUT_LEN
+ * When the macro UNISHOX_API_WITH_OUTPUT_LEN is defined, the all the API functions
+ * except the simple API functions accept an additional parameter olen
+ * that enables the developer to pass the size of the output buffer provided
+ * so that the api function may not write beyond that length.
+ * This can be disabled if the developer knows that the buffer provided is sufficient enough
+ * so no additional parameter is passed and the program is faster since additional check
+ * for output length is not performed at each step
+ */
+#if defined(UNISHOX_API_WITH_OUTPUT_LEN) && UNISHOX_API_WITH_OUTPUT_LEN != 0
+#define UNISHOX_API_OUT_AND_LEN(out, olen) out, olen
+#else
+#define UNISHOX_API_OUT_AND_LEN(out, olen) out
+#endif
+
+/**
+ * Simple API for compressing a string
+ * @param[in] in    Input ASCII / UTF-8 string
+ * @param[in] len   length in bytes
+ * @param[out] out  output buffer - should be large enough to hold compressed output
+ */
+extern int unishox2_compress_simple(const char *in, int len, char *out);
+/**
+ * Simple API for decompressing a string
+ * @param[in] in    Input compressed bytes (output of unishox2_compress functions)
+ * @param[in] len   length of 'in' in bytes
+ * @param[out] out  output buffer for ASCII / UTF-8 string - should be large enough
+ */
+extern int unishox2_decompress_simple(const char *in, int len, char *out);
+/**
+ * Comprehensive API for compressing a string
+ *
+ * Presets are available for the last four parameters so they can be passed as single parameter. \n
+ * See USX_PSET_* macros. Example call: \n
+ *    unishox2_compress(in, len, out, olen, USX_PSET_ALPHA_ONLY);
+ *
+ * @param[in] in             Input ASCII / UTF-8 string
+ * @param[in] len            length in bytes
+ * @param[out] out           output buffer - should be large enough to hold compressed output
+ * @param[in] olen           length of 'out' buffer in bytes. Can be omitted if sufficient buffer is provided
+ * @param[in] usx_hcodes     Horizontal codes (array of bytes). See macro section for samples.
+ * @param[in] usx_hcode_lens Length of each element in usx_hcodes array
+ * @param[in] usx_freq_seq   Frequently occurring sequences. See USX_FREQ_SEQ_* macros for samples
+ * @param[in] usx_templates  Templates of frequently occurring patterns. See USX_TEMPLATES macro.
+ */
+extern int unishox2_compress(const char *in, int len, UNISHOX_API_OUT_AND_LEN(char *out, int olen),
+                             const unsigned char usx_hcodes[], const unsigned char usx_hcode_lens[], const char *usx_freq_seq[],
+                             const char *usx_templates[]);
+/**
+ * Comprehensive API for de-compressing a string
+ *
+ * Presets are available for the last four parameters so they can be passed as single parameter. \n
+ * See USX_PSET_* macros. Example call: \n
+ *    unishox2_decompress(in, len, out, olen, USX_PSET_ALPHA_ONLY);
+ *
+ * @param[in] in             Input compressed bytes (output of unishox2_compress functions)
+ * @param[in] len            length of 'in' in bytes
+ * @param[out] out           output buffer - should be large enough to hold de-compressed output
+ * @param[in] olen           length of 'out' buffer in bytes. Can be omitted if sufficient buffer is provided
+ * @param[in] usx_hcodes     Horizontal codes (array of bytes). See macro section for samples.
+ * @param[in] usx_hcode_lens Length of each element in usx_hcodes array
+ * @param[in] usx_freq_seq   Frequently occurring sequences. See USX_FREQ_SEQ_* macros for samples
+ * @param[in] usx_templates  Templates of frequently occurring patterns. See USX_TEMPLATES macro.
+ */
+extern int unishox2_decompress(const char *in, int len, UNISHOX_API_OUT_AND_LEN(char *out, int olen),
+                               const unsigned char usx_hcodes[], const unsigned char usx_hcode_lens[], const char *usx_freq_seq[],
+                               const char *usx_templates[]);
+/**
+ * More Comprehensive API for compressing array of strings
+ *
+ * See unishox2_compress() function for parameter definitions. \n
+ * This function takes an additional parameter, i.e. 'prev_lines' - the usx_lnk_lst structure \n
+ * See -g parameter in test_unishox2.c to find out how this can be used. \n
+ * This function is used when an array of strings need to be compressed \n
+ * and stored in a compressed array of bytes for use as a constant in other programs \n
+ * where each element of the array can be decompressed and used at runtime.
+ */
+extern int unishox2_compress_lines(const char *in, int len, UNISHOX_API_OUT_AND_LEN(char *out, int olen),
+                                   const unsigned char usx_hcodes[], const unsigned char usx_hcode_lens[],
+                                   const char *usx_freq_seq[], const char *usx_templates[], struct us_lnk_lst *prev_lines);
+/**
+ * More Comprehensive API for de-compressing array of strings \n
+ * This function is not be used in conjuction with unishox2_compress_lines()
+ *
+ * See unishox2_decompress() function for parameter definitions. \n
+ * Typically an array is compressed using unishox2_compress_lines() and \n
+ * a header (.h) file is generated using the resultant compressed array. \n
+ * This header file can be used in another program with another decompress \n
+ * routine which takes this compressed array as parameter and index to be \n
+ * decompressed.
+ */
+extern int unishox2_decompress_lines(const char *in, int len, UNISHOX_API_OUT_AND_LEN(char *out, int olen),
+                                     const unsigned char usx_hcodes[], const unsigned char usx_hcode_lens[],
+                                     const char *usx_freq_seq[], const char *usx_templates[], struct us_lnk_lst *prev_lines);
+
+#endif

src/modules/AtakPluginModule.cpp

@@ -1,14 +1,207 @@
 #include "AtakPluginModule.h"
+#include "Default.h"
+#include "MeshService.h"
+#include "NodeDB.h"
+#include "PowerFSM.h"
+#include "configuration.h"
+#include "main.h"
+#include "mesh/compression/unishox2.h"
+#include "meshUtils.h"
+#include "meshtastic/atak.pb.h"
 
 AtakPluginModule *atakPluginModule;
 
 AtakPluginModule::AtakPluginModule()
-    : SinglePortModule("atak", meshtastic_PortNum_ATAK_PLUGIN_V2)
+    : ProtobufModule("atak", meshtastic_PortNum_ATAK_PLUGIN, &meshtastic_TAKPacket_msg), concurrency::OSThread("AtakPlugin")
 {
+    ourPortNum = meshtastic_PortNum_ATAK_PLUGIN;
 }
 
-ProcessMessage AtakPluginModule::handleReceived(const meshtastic_MeshPacket &mp)
+/*
+Encompasses the full construction and sending packet to mesh
+Will be used for broadcast.
+*/
+int32_t AtakPluginModule::runOnce()
 {
-    (void)mp; // Passthrough — no processing needed, apps handle compression/decompression
-    return ProcessMessage::CONTINUE;
+    return default_broadcast_interval_secs;
 }
+
+bool AtakPluginModule::handleReceivedProtobuf(const meshtastic_MeshPacket &mp, meshtastic_TAKPacket *r)
+{
+    return false;
+}
+
+meshtastic_TAKPacket AtakPluginModule::cloneTAKPacketData(meshtastic_TAKPacket *t)
+{
+    meshtastic_TAKPacket clone = meshtastic_TAKPacket_init_zero;
+    if (t->has_group) {
+        clone.has_group = true;
+        clone.group = t->group;
+    }
+    if (t->has_status) {
+        clone.has_status = true;
+        clone.status = t->status;
+    }
+    if (t->has_contact) {
+        clone.has_contact = true;
+        clone.contact = {0};
+    }
+
+    if (t->which_payload_variant == meshtastic_TAKPacket_pli_tag) {
+        clone.which_payload_variant = meshtastic_TAKPacket_pli_tag;
+        clone.payload_variant.pli = t->payload_variant.pli;
+    } else if (t->which_payload_variant == meshtastic_TAKPacket_chat_tag) {
+        clone.which_payload_variant = meshtastic_TAKPacket_chat_tag;
+        clone.payload_variant.chat = {0};
+    } else if (t->which_payload_variant == meshtastic_TAKPacket_detail_tag) {
+        clone.which_payload_variant = meshtastic_TAKPacket_detail_tag;
+        clone.payload_variant.detail.size = t->payload_variant.detail.size;
+        memcpy(clone.payload_variant.detail.bytes, t->payload_variant.detail.bytes, t->payload_variant.detail.size);
+    }
+
+    return clone;
+}
+
+void AtakPluginModule::alterReceivedProtobuf(meshtastic_MeshPacket &mp, meshtastic_TAKPacket *t)
+{
+    // From Phone (EUD)
+    if (mp.from == 0) {
+        LOG_DEBUG("Received uncompressed TAK payload from phone: %d bytes", mp.decoded.payload.size);
+        // Compress for LoRA transport
+        auto compressed = cloneTAKPacketData(t);
+        compressed.is_compressed = true;
+        if (t->has_contact) {
+            auto length = unishox2_compress_lines(
+                t->contact.callsign, pb_string_length(t->contact.callsign, sizeof(t->contact.callsign)),
+                compressed.contact.callsign, sizeof(compressed.contact.callsign) - 1, USX_PSET_DFLT, NULL);
+            if (length < 0) {
+                LOG_WARN("Compress overflow contact.callsign. Revert to uncompressed packet");
+                return;
+            }
+            LOG_DEBUG("Compressed callsign: %d bytes", length);
+            length = unishox2_compress_lines(
+                t->contact.device_callsign, pb_string_length(t->contact.device_callsign, sizeof(t->contact.device_callsign)),
+                compressed.contact.device_callsign, sizeof(compressed.contact.device_callsign) - 1, USX_PSET_DFLT, NULL);
+            if (length < 0) {
+                LOG_WARN("Compress overflow contact.device_callsign. Revert to uncompressed packet");
+                return;
+            }
+            LOG_DEBUG("Compressed device_callsign: %d bytes", length);
+        }
+        if (t->which_payload_variant == meshtastic_TAKPacket_chat_tag) {
+            auto length = unishox2_compress_lines(
+                t->payload_variant.chat.message,
+                pb_string_length(t->payload_variant.chat.message, sizeof(t->payload_variant.chat.message)),
+                compressed.payload_variant.chat.message, sizeof(compressed.payload_variant.chat.message) - 1, USX_PSET_DFLT,
+                NULL);
+            if (length < 0) {
+                LOG_WARN("Compress overflow chat.message. Revert to uncompressed packet");
+                return;
+            }
+            LOG_DEBUG("Compressed chat message: %d bytes", length);
+
+            if (t->payload_variant.chat.has_to) {
+                compressed.payload_variant.chat.has_to = true;
+                length = unishox2_compress_lines(
+                    t->payload_variant.chat.to, pb_string_length(t->payload_variant.chat.to, sizeof(t->payload_variant.chat.to)),
+                    compressed.payload_variant.chat.to, sizeof(compressed.payload_variant.chat.to) - 1, USX_PSET_DFLT, NULL);
+                if (length < 0) {
+                    LOG_WARN("Compress overflow chat.to. Revert to uncompressed packet");
+                    return;
+                }
+                LOG_DEBUG("Compressed chat to: %d bytes", length);
+            }
+
+            if (t->payload_variant.chat.has_to_callsign) {
+                compressed.payload_variant.chat.has_to_callsign = true;
+                length = unishox2_compress_lines(
+                    t->payload_variant.chat.to_callsign,
+                    pb_string_length(t->payload_variant.chat.to_callsign, sizeof(t->payload_variant.chat.to_callsign)),
+                    compressed.payload_variant.chat.to_callsign, sizeof(compressed.payload_variant.chat.to_callsign) - 1,
+                    USX_PSET_DFLT, NULL);
+                if (length < 0) {
+                    LOG_WARN("Compress overflow chat.to_callsign. Revert to uncompressed packet");
+                    return;
+                }
+                LOG_DEBUG("Compressed chat to_callsign: %d bytes", length);
+            }
+        }
+        mp.decoded.payload.size = pb_encode_to_bytes(mp.decoded.payload.bytes, sizeof(mp.decoded.payload.bytes),
+                                                     meshtastic_TAKPacket_fields, &compressed);
+        LOG_DEBUG("Final payload: %d bytes", mp.decoded.payload.size);
+    } else {
+        if (!t->is_compressed) {
+            // Not compressed. Something is wrong
+            LOG_WARN("Received uncompressed TAKPacket over radio! Skip");
+            return;
+        }
+
+        // Decompress for Phone (EUD)
+        auto uncompressed = cloneTAKPacketData(t);
+        uncompressed.is_compressed = false;
+        if (t->has_contact) {
+            auto length = unishox2_decompress_lines(
+                t->contact.callsign, pb_string_length(t->contact.callsign, sizeof(t->contact.callsign)),
+                uncompressed.contact.callsign, sizeof(uncompressed.contact.callsign) - 1, USX_PSET_DFLT, NULL);
+            if (length < 0) {
+                LOG_WARN("Decompress overflow contact.callsign. Bailing out");
+                return;
+            }
+            LOG_DEBUG("Decompressed callsign: %d bytes", length);
+
+            length = unishox2_decompress_lines(
+                t->contact.device_callsign, pb_string_length(t->contact.device_callsign, sizeof(t->contact.device_callsign)),
+                uncompressed.contact.device_callsign, sizeof(uncompressed.contact.device_callsign) - 1, USX_PSET_DFLT, NULL);
+            if (length < 0) {
+                LOG_WARN("Decompress overflow contact.device_callsign. Bailing out");
+                return;
+            }
+            LOG_DEBUG("Decompressed device_callsign: %d bytes", length);
+        }
+        if (uncompressed.which_payload_variant == meshtastic_TAKPacket_chat_tag) {
+            auto length = unishox2_decompress_lines(
+                t->payload_variant.chat.message,
+                pb_string_length(t->payload_variant.chat.message, sizeof(t->payload_variant.chat.message)),
+                uncompressed.payload_variant.chat.message, sizeof(uncompressed.payload_variant.chat.message) - 1, USX_PSET_DFLT,
+                NULL);
+            if (length < 0) {
+                LOG_WARN("Decompress overflow chat.message. Bailing out");
+                return;
+            }
+            LOG_DEBUG("Decompressed chat message: %d bytes", length);
+
+            if (t->payload_variant.chat.has_to) {
+                uncompressed.payload_variant.chat.has_to = true;
+                length = unishox2_decompress_lines(
+                    t->payload_variant.chat.to, pb_string_length(t->payload_variant.chat.to, sizeof(t->payload_variant.chat.to)),
+                    uncompressed.payload_variant.chat.to, sizeof(uncompressed.payload_variant.chat.to) - 1, USX_PSET_DFLT, NULL);
+                if (length < 0) {
+                    LOG_WARN("Decompress overflow chat.to. Bailing out");
+                    return;
+                }
+                LOG_DEBUG("Decompressed chat to: %d bytes", length);
+            }
+
+            if (t->payload_variant.chat.has_to_callsign) {
+                uncompressed.payload_variant.chat.has_to_callsign = true;
+                length = unishox2_decompress_lines(
+                    t->payload_variant.chat.to_callsign,
+                    pb_string_length(t->payload_variant.chat.to_callsign, sizeof(t->payload_variant.chat.to_callsign)),
+                    uncompressed.payload_variant.chat.to_callsign, sizeof(uncompressed.payload_variant.chat.to_callsign) - 1,
+                    USX_PSET_DFLT, NULL);
+                if (length < 0) {
+                    LOG_WARN("Decompress overflow chat.to_callsign. Bailing out");
+                    return;
+                }
+                LOG_DEBUG("Decompressed chat to_callsign: %d bytes", length);
+            }
+        }
+        auto decompressedCopy = packetPool.allocCopy(mp);
+        decompressedCopy->decoded.payload.size =
+            pb_encode_to_bytes(decompressedCopy->decoded.payload.bytes, sizeof(decompressedCopy->decoded.payload),
+                               meshtastic_TAKPacket_fields, &uncompressed);
+
+        service->sendToPhone(decompressedCopy);
+    }
+    return;
+}

src/modules/AtakPluginModule.h

@@ -1,15 +1,11 @@
 #pragma once
-#include "SinglePortModule.h"
+#include "ProtobufModule.h"
+#include "meshtastic/atak.pb.h"
 
 /**
- * ATAK Plugin V2 module - passthrough for ATAK_PLUGIN_V2 payloads.
- * The wire format includes a leading flags byte followed by opaque payload bytes.
- * Depending on the flags, the payload may be zstd dictionary-compressed or raw/uncompressed protobuf.
- * Compression/decompression and payload interpretation are handled by the apps
- * (Android, iOS, ATAK plugin); firmware forwards the bytes unchanged on the
- * ATAK_PLUGIN_V2 port.
+ * Waypoint message handling for meshtastic
  */
-class AtakPluginModule : public SinglePortModule
+class AtakPluginModule : public ProtobufModule<meshtastic_TAKPacket>, private concurrency::OSThread
 {
   public:
     /** Constructor
@@ -18,7 +14,13 @@ class AtakPluginModule : public SinglePortModule
     AtakPluginModule();
 
   protected:
-    virtual ProcessMessage handleReceived(const meshtastic_MeshPacket &mp) override;
+    virtual bool handleReceivedProtobuf(const meshtastic_MeshPacket &mp, meshtastic_TAKPacket *t) override;
+    virtual void alterReceivedProtobuf(meshtastic_MeshPacket &mp, meshtastic_TAKPacket *t) override;
+    /* Does our periodic broadcast */
+    int32_t runOnce() override;
+
+  private:
+    meshtastic_TAKPacket cloneTAKPacketData(meshtastic_TAKPacket *t);
 };
 
-extern AtakPluginModule *atakPluginModule;
+extern AtakPluginModule *atakPluginModule;

src/modules/Modules.cpp

@@ -104,6 +104,10 @@
 #include "modules/StatusMessageModule.h"
 #endif
 
+#if !MESHTASTIC_EXCLUDE_ZPS
+#include "modules/esp32/ZPSModule.h"
+#endif
+
 #if defined(HAS_HARDWARE_WATCHDOG)
 #include "watchdog/watchdogThread.h"
 #endif
@@ -174,6 +178,9 @@ void setupModules()
 #if !MESHTASTIC_EXCLUDE_STATUS
     statusMessageModule = new StatusMessageModule();
 #endif
+#if !MESHTASTIC_EXCLUDE_ZPS
+        zpsModule = new ZPSModule();
+#endif
 #if !MESHTASTIC_EXCLUDE_GENERIC_THREAD_MODULE
     new GenericThreadModule();
 #endif

src/modules/PositionModule.cpp

@@ -12,6 +12,7 @@
 #include "configuration.h"
 #include "gps/GeoCoord.h"
 #include "main.h"
+#include "mesh/compression/unishox2.h"
 #include "meshUtils.h"
 #include "meshtastic/atak.pb.h"
 #include "sleep.h"
@@ -306,77 +307,37 @@ meshtastic_MeshPacket *PositionModule::allocReply()
 
 meshtastic_MeshPacket *PositionModule::allocAtakPli()
 {
-    LOG_INFO("Send TAK V2 PLI packet");
+    LOG_INFO("Send TAK PLI packet");
     meshtastic_MeshPacket *mp = allocDataPacket();
-    mp->decoded.portnum = meshtastic_PortNum_ATAK_PLUGIN_V2;
+    mp->decoded.portnum = meshtastic_PortNum_ATAK_PLUGIN;
 
-    meshtastic_TAKPacketV2 takPacket = meshtastic_TAKPacketV2_init_zero;
-    takPacket.cot_type_id = meshtastic_CotType_CotType_a_f_G_U_C;
+    meshtastic_TAKPacket takPacket = {.is_compressed = true,
+                                      .has_contact = true,
+                                      .contact = meshtastic_Contact_init_default,
+                                      .has_group = true,
+                                      .group = {meshtastic_MemberRole_TeamMember, meshtastic_Team_Cyan},
+                                      .has_status = true,
+                                      .status =
+                                          {
+                                              .battery = powerStatus->getBatteryChargePercent(),
+                                          },
+                                      .which_payload_variant = meshtastic_TAKPacket_pli_tag,
+                                      .payload_variant = {.pli = {
+                                                              .latitude_i = localPosition.latitude_i,
+                                                              .longitude_i = localPosition.longitude_i,
+                                                              .altitude = localPosition.altitude_hae,
+                                                              .speed = localPosition.ground_speed,
+                                                              .course = static_cast<uint16_t>(localPosition.ground_track),
+                                                          }}};
 
-    // Use TAK config for team/role if configured, otherwise use defaults (Cyan/TeamMember)
-    if (moduleConfig.has_tak && moduleConfig.tak.team != meshtastic_Team_Unspecifed_Color) {
-        takPacket.team = moduleConfig.tak.team;
-    } else {
-        takPacket.team = meshtastic_Team_Cyan;
-    }
-
-    if (moduleConfig.has_tak && moduleConfig.tak.role != meshtastic_MemberRole_Unspecifed) {
-        takPacket.role = moduleConfig.tak.role;
-    } else {
-        takPacket.role = meshtastic_MemberRole_TeamMember;
-    }
-    takPacket.latitude_i = localPosition.latitude_i;
-    takPacket.longitude_i = localPosition.longitude_i;
-    takPacket.altitude = localPosition.altitude_hae;
-    takPacket.speed = localPosition.ground_speed;
-    // ground_track is stored as degrees * 1e5, course field expects degrees * 100
-    int32_t course = localPosition.ground_track / 1000;
-    if (course < 0)
-        course = 0;
-    else if (course > 36000)
-        course = 36000;
-    takPacket.course = static_cast<uint16_t>(course);
-    takPacket.battery = powerStatus->getBatteryChargePercent();
-
-    // Map position source to CoT how/geo_src/alt_src
-    if (config.position.fixed_position || localPosition.location_source == meshtastic_Position_LocSource_LOC_MANUAL) {
-        takPacket.how = meshtastic_CotHow_CotHow_h_e;
-        takPacket.geo_src = meshtastic_GeoPointSource_GeoPointSource_USER;
-        takPacket.alt_src = meshtastic_GeoPointSource_GeoPointSource_USER;
-    } else {
-        takPacket.how = meshtastic_CotHow_CotHow_m_g;
-        takPacket.geo_src = meshtastic_GeoPointSource_GeoPointSource_GPS;
-        takPacket.alt_src = meshtastic_GeoPointSource_GeoPointSource_GPS;
-    }
-    takPacket.which_payload_variant = meshtastic_TAKPacketV2_pli_tag;
-    takPacket.payload_variant.pli = true;
-
-    // Callsign - stored as plain string (no compression, apps handle that)
-    strncpy(takPacket.callsign, owner.long_name, sizeof(takPacket.callsign) - 1);
-    takPacket.callsign[sizeof(takPacket.callsign) - 1] = '\0';
-    strncpy(takPacket.device_callsign, owner.long_name, sizeof(takPacket.device_callsign) - 1);
-    takPacket.device_callsign[sizeof(takPacket.device_callsign) - 1] = '\0';
-
-    // Encode TAKPacketV2 protobuf, leaving room for flags byte prefix
-    uint8_t protobuf_bytes[sizeof(mp->decoded.payload.bytes) - 1];
-    size_t proto_size = pb_encode_to_bytes(protobuf_bytes, sizeof(protobuf_bytes), &meshtastic_TAKPacketV2_msg, &takPacket);
-
-    if (proto_size == 0) {
-        LOG_ERROR("Failed to encode TAK V2 PLI packet");
-        packetPool.release(mp);
-        return nullptr;
-    }
-
-    // Wire format: [flags byte][protobuf bytes]
-    // Flags byte 0xFF is a reserved sentinel meaning the remainder is an
-    // uncompressed raw protobuf payload (that is, no zstd dictionary ID is
-    // encoded in the flags byte). Decoders must check for this sentinel before
-    // applying any dictionary-ID masking/interpretation.
-    mp->decoded.payload.bytes[0] = 0xFF;
-    memcpy(mp->decoded.payload.bytes + 1, protobuf_bytes, proto_size);
-    mp->decoded.payload.size = proto_size + 1;
-
-    LOG_DEBUG("TAK V2 PLI payload: %zu bytes (1 flags + %zu protobuf)", mp->decoded.payload.size, proto_size);
+    auto length = unishox2_compress_lines(owner.long_name, strlen(owner.long_name), takPacket.contact.device_callsign,
+                                          sizeof(takPacket.contact.device_callsign) - 1, USX_PSET_DFLT, NULL);
+    LOG_DEBUG("Uncompressed device_callsign '%s' - %d bytes", owner.long_name, strlen(owner.long_name));
+    LOG_DEBUG("Compressed device_callsign '%s' - %d bytes", takPacket.contact.device_callsign, length);
+    length = unishox2_compress_lines(owner.long_name, strlen(owner.long_name), takPacket.contact.callsign,
+                                     sizeof(takPacket.contact.callsign) - 1, USX_PSET_DFLT, NULL);
+    mp->decoded.payload.size =
+        pb_encode_to_bytes(mp->decoded.payload.bytes, sizeof(mp->decoded.payload.bytes), &meshtastic_TAKPacket_msg, &takPacket);
     return mp;
 }
 

src/modules/TrafficManagementModule.cpp

@@ -7,6 +7,7 @@
 #include "NodeDB.h"
 #include "Router.h"
 #include "TypeConversions.h"
+#include "airtime.h"
 #include "concurrency/LockGuard.h"
 #include "configuration.h"
 #include "mesh-pb-constants.h"
@@ -1001,7 +1002,11 @@ void TrafficManagementModule::alterReceived(meshtastic_MeshPacket &mp)
     const auto &cfg = moduleConfig.traffic_management;
     const bool isTelemetry = mp.decoded.portnum == meshtastic_PortNum_TELEMETRY_APP;
     const bool isPosition = mp.decoded.portnum == meshtastic_PortNum_POSITION_APP;
-    const bool shouldExhaust = (isTelemetry && cfg.exhaust_hop_telemetry) || (isPosition && cfg.exhaust_hop_position);
+    // Only exhaust telemetry hops when channel is actually congested, mirroring the same
+    // airtime checks that gate self-generated telemetry in the telemetry modules.
+    const bool channelBusy = airTime && (!airTime->isTxAllowedChannelUtil(true) || !airTime->isTxAllowedAirUtil());
+    const bool shouldExhaust =
+        ((channelBusy && isTelemetry && cfg.exhaust_hop_telemetry) || (isPosition && cfg.exhaust_hop_position));
 
     if (!shouldExhaust || !isBroadcast(mp.to))
         return;

src/modules/esp32/ZPSModule.cpp

@@ -0,0 +1,419 @@
+/*
+ * ZPS - Zero-GPS Positioning System for standalone Meshtastic devices
+ * - experimental tools for estimating own position without a GPS -
+ *
+ * Copyright 2021 all rights reserved by https://github.com/a-f-G-U-C
+ * Released under GPL v3 (see LICENSE file for details)
+ */
+
+#include "ZPSModule.h"
+#include "Default.h"
+#include "MeshService.h"
+#include "NodeDB.h"
+#include "NodeStatus.h"
+#include "Router.h"
+#include "configuration.h"
+#include "gps/RTC.h"
+#include <WiFi.h>
+
+#if !defined(MESHTASTIC_EXCLUDE_BLUETOOTH)
+
+#include "NimBLEDevice.h"
+
+#define BLE_MAX_REC 15
+#define BLE_NO_RESULTS -1 // Indicates a BLE scan is in progress
+
+uint8_t bleCounter = 0; // used internally by the ble scanner
+uint64_t bleResult[BLE_MAX_REC + 1];
+int bleResSize = BLE_NO_RESULTS;
+
+uint64_t scanStart = 0;
+
+ZPSModule *zpsModule;
+
+// Mini BLE scanner, NIMBLE based and modelled loosely after the Wifi scanner
+static int ble_scan(uint32_t duration, bool passive = true, bool dedup = true);
+
+// ZPSModule::ZPSModule()
+//     : ProtobufModule("ZPS", ZPS_PORTNUM, Position_fields), concurrency::OSThread("ZPSModule")
+ZPSModule::ZPSModule() : SinglePortModule("ZPS", ZPS_PORTNUM), concurrency::OSThread("ZPSModule")
+{
+    setIntervalFromNow(ZPS_STARTUP_DELAY); // Delay startup by 10 seconds, no need to race :)
+
+    wantBSS = true;
+    wantBLE = true;
+
+    WiFi.mode(WIFI_STA);
+    WiFi.disconnect();
+    WiFi.scanNetworks(true, true); // nonblock, showhidden
+    scanState = SCAN_BSS_RUN;
+}
+
+ProcessMessage ZPSModule::handleReceived(const meshtastic_MeshPacket &mp)
+{
+    meshtastic_Position pos = meshtastic_Position_init_default;
+
+    auto &pd = mp.decoded;
+    uint8_t nRecs = pd.payload.size >> 3;
+
+    LOG_DEBUG("handleReceived %s 0x%0x->0x%0x, id=0x%x, port=%d, len=%d, rec=%d\n", name, mp.from, mp.to, mp.id, pd.portnum,
+              pd.payload.size, nRecs);
+    if (nRecs > ZPS_DATAPKT_MAXITEMS)
+        nRecs = ZPS_DATAPKT_MAXITEMS;
+    memcpy(&netData, pd.payload.bytes, nRecs << 3);
+
+    // Currently we are unable to act as a position server, so we're
+    // not interested in broadcasts (this will change later)
+    if (mp.to != nodeDB->getNodeNum()) {
+        // Message is not for us, won't process
+        return ProcessMessage::CONTINUE;
+    }
+
+#ifdef ZPS_EXTRAVERBOSE
+    for (int i = 0; i < nRecs; i++) {
+        LOG_DEBUG("ZPS[%d]: %08x"
+                  "%08x\n",
+                  i, (uint32_t)(netData[i] >> 32), (uint32_t)netData[i]);
+    }
+#endif
+
+    if ((netData[0] & 0x800000000000) && (nRecs >= 2)) {
+        // message contains a position
+        pos.PDOP = (netData[0] >> 40) & 0x7f;
+        pos.timestamp = netData[0] & 0xffffffff;
+        // second int64 encodes lat and lon
+        pos.longitude_i = (int32_t)(netData[1] & 0xffffffff);
+        pos.latitude_i = (int32_t)((netData[1] >> 32) & 0xffffffff);
+
+        // FIXME should be conditional, to ensure we don't overwrite a good GPS fix!
+        LOG_DEBUG("ZPS lat/lon/dop/pts %d/%d/%d/%d\n", pos.latitude_i, pos.longitude_i, pos.PDOP, pos.timestamp);
+
+        // Some required fields
+        pos.time = getTime();
+        pos.location_source = meshtastic_Position_LocSource_LOC_EXTERNAL;
+
+        // don't update position if my gps fix is valid
+        if (nodeDB->hasValidPosition(nodeDB->getMeshNode(nodeDB->getNodeNum()))) {
+            LOG_DEBUG("ZPSModule::handleReceived: ignoring position update, GPS is valid\n");
+            return ProcessMessage::CONTINUE;
+        }
+        nodeDB->updatePosition(nodeDB->getNodeNum(), pos);
+    } else {
+        // nothing we can do - for now
+        return ProcessMessage::CONTINUE;
+    }
+
+    return ProcessMessage::CONTINUE; // Let others look at this message also if they want
+}
+
+meshtastic_MeshPacket *ZPSModule::allocReply()
+{
+    meshtastic_MeshPacket *p = allocDataPacket();
+    p->decoded.payload.size = (netRecs + 2) << 3; // actually can be only +1 if no GPS data
+
+    LOG_DEBUG("Allocating dataPacket for %d items, %d bytes\n", netRecs, p->decoded.payload.size);
+    memcpy(p->decoded.payload.bytes, &netData, p->decoded.payload.size);
+
+    return (p);
+}
+
+void ZPSModule::sendDataPacket(NodeNum dest, bool wantReplies)
+{
+    // cancel any not yet sent (now stale) position packets
+    if (prevPacketId)
+        service->cancelSending(prevPacketId);
+
+    meshtastic_MeshPacket *p = allocReply();
+    p->to = dest;
+    p->decoded.portnum = meshtastic_PortNum_ZPS_APP;
+    p->decoded.want_response = wantReplies;
+    p->priority = meshtastic_MeshPacket_Priority_BACKGROUND;
+    prevPacketId = p->id;
+
+    service->sendToMesh(p, RX_SRC_LOCAL);
+}
+
+int32_t ZPSModule::runOnce()
+{
+    meshtastic_NodeInfoLite *node = nodeDB->getMeshNode(nodeDB->getNodeNum());
+    assert(node);
+
+    // LOG_DEBUG("ZPSModule::runOnce() START, scanState: %d\n", (int) scanState);
+
+    int numWifi = 0;
+
+    if (scanState == SCAN_BSS_RUN) {
+        // check completion status of any running Wifi scan
+        numWifi = WiFi.scanComplete();
+
+        if (numWifi >= 0) {
+            // scan is complete
+            LOG_DEBUG("%d BSS found\n", numWifi);
+            LOG_DEBUG("BSS scan done in %d millis\n", millis() - scanStart);
+
+            if (wantBSS && haveBSS) {
+                // old data exists, overwrite it
+                netRecs = 0;
+                haveBSS = haveBLE = false;
+            }
+
+            for (int i = 0; i < numWifi; i++) {
+                // pack each Wifi network record into a 64-bit int
+                uint64_t netBytes = encodeBSS(WiFi.BSSID(i), WiFi.channel(i), abs(WiFi.RSSI(i)));
+
+                if (wantBSS) {
+                    // load into outbound array if needed
+                    outBufAdd(netBytes);
+                    haveBSS = true;
+                }
+#ifdef ZPS_EXTRAVERBOSE
+                LOG_DEBUG("BSS[%02d]: %08x"
+                          "%08x\n",
+                          i, (uint32_t)(netBytes >> 32), (uint32_t)netBytes);
+#endif
+            }
+
+            WiFi.scanDelete();
+            scanState = SCAN_BSS_DONE;
+
+#ifdef ZPS_EXTRAVERBOSE
+        } else if (numWifi == -1) {
+            // LOG_DEBUG("BSS scan in-progress\n");
+        } else {
+            LOG_DEBUG("BSS scan state=%d\n", numWifi);
+#endif
+        }
+    }
+
+    if ((scanState == SCAN_BLE_RUN) && (bleResSize >= 0)) {
+        // completion status checked above (bleResSize >= 0)
+        LOG_DEBUG("BLE scan done in %d millis\n", millis() - scanStart);
+        scanState = SCAN_BLE_DONE;
+
+        if (wantBLE && haveBLE) {
+            // old data exists, overwrite it
+            netRecs = 0;
+            haveBSS = haveBLE = false;
+        }
+
+        for (int i = 0; i < bleResSize; i++) {
+            // load data into output array if needed
+            if (wantBLE) {
+                outBufAdd(bleResult[i]);
+                haveBLE = true;
+            }
+#ifdef ZPS_EXTRAVERBOSE
+            LOG_DEBUG("BLE[%d]: %08x"
+                      "%08x\n",
+                      i, (uint32_t)(bleResult[i] >> 32), (uint32_t)bleResult[i]);
+#endif
+        }
+
+        // Reset the counter once we're done with the dataset
+        bleResSize = BLE_NO_RESULTS;
+    }
+
+    // Are we finished assembling that packet? Then send it out
+    if ((wantBSS == haveBSS) && (wantBLE == haveBLE) &&
+        airTime->isTxAllowedChannelUtil(config.device.role != meshtastic_Config_DeviceConfig_Role_SENSOR) &&
+        airTime->isTxAllowedAirUtil() &&
+        (lastSend == 0 || millis() - lastSend >= Default::getConfiguredOrDefaultMsScaled(config.position.position_broadcast_secs,
+                                                                                         default_broadcast_interval_secs,
+                                                                                         nodeStatus->getNumOnline()))) {
+
+        haveBSS = haveBLE = false;
+        sendDataPacket(NODENUM_BROADCAST, false); // no replies
+        lastSend = millis();
+        netRecs = 0; // reset packet
+    }
+
+    /*
+     * State machine transitions
+     *
+     * FIXME could be managed better, for example: check if we require
+     *   each type of scan (wantBSS/wantBLE), and if not, don't start it!
+     */
+    if (scanState == SCAN_BLE_DONE) {
+        // BLE done, transition to BSS scanning
+        scanStart = millis();
+        LOG_DEBUG("BSS scan start t=%d\n", scanStart);
+        if (WiFi.scanNetworks(true, true) == WIFI_SCAN_RUNNING) // nonblock, showhidden
+            scanState = SCAN_BSS_RUN;
+
+    } else if (scanState == SCAN_BSS_DONE) {
+        // BSS done, transition to BLE scanning
+        scanStart = millis();
+        LOG_DEBUG("BLE scan start t=%d\n", scanStart);
+        if (ble_scan(ZPS_BLE_SCANTIME) == 0)
+            scanState = SCAN_BLE_RUN;
+    }
+
+    // LOG_DEBUG("ZPSModule::runOnce() DONE, scanState=%d\n", scanState);
+    if ((scanState == SCAN_BSS_RUN) || (scanState == SCAN_BLE_RUN)) {
+        return 1000; // scan in progress, re-check soon
+    }
+
+    return 5000;
+}
+
+uint64_t encodeBSS(uint8_t *bssid, uint8_t chan, uint8_t absRSSI)
+{
+    uint64_t netBytes = absRSSI & 0xff;
+    netBytes <<= 8;
+    netBytes |= (chan & 0xff);
+
+    for (uint8_t b = 0; b < 6; b++) {
+        netBytes <<= 8;
+        netBytes |= bssid[b];
+    }
+
+    return netBytes;
+}
+
+uint64_t encodeBLE(uint8_t *addr, uint8_t absRSSI)
+{
+    uint64_t netBytes = absRSSI & 0xff;
+    netBytes <<= 8;
+    netBytes |= 0xff; // "channel" byte reserved in BLE records
+
+    for (uint8_t b = 0; b < 6; b++) {
+        netBytes <<= 8;
+        netBytes |= addr[5 - b] & 0xff;
+    }
+
+    return netBytes;
+}
+
+/**
+ * Event handler
+ */
+static int ble_gap_event(struct ble_gap_event *event, void *arg)
+{
+    // Adverts matching certain patterns are useless for positioning purposes
+    //  (ephemeral MAC etc), so try excluding them if possible
+    //
+    // TODO: Expand the list of reject patterns for BLE adverts.
+    //   There are likely more than 10 patterns to test and reject, including most Apple devices and others.
+    //
+    // TODO: Implement full packet search for reject patterns (use memmem() or similar),
+    //   not just at the beginning (currently uses memcmp()).
+
+    const uint8_t rejPat[] = {0x1e, 0xff, 0x06, 0x00, 0x01}; // one of many
+
+    struct ble_hs_adv_fields fields;
+    int rc;
+    int i = 0;
+
+    uint64_t netBytes = 0;
+
+    switch (event->type) {
+    case BLE_GAP_EVENT_DISC:
+        // called once for every BLE advert received
+        rc = ble_hs_adv_parse_fields(&fields, event->disc.data, event->disc.length_data);
+        if (rc != 0)
+            return 0;
+
+        if (bleResSize != BLE_NO_RESULTS)
+            // as far as we know, we're not in the middle of a BLE scan!
+            LOG_DEBUG("Unexpected BLE_GAP_EVENT_DISC!\n");
+
+#ifdef ZPS_EXTRAVERBOSE
+        // Dump the advertisement packet
+        DEBUG_PORT.hexDump("DEBUG", (unsigned char *)event->disc.data, event->disc.length_data);
+#endif
+        // Reject beacons known to be unreliable (ephemeral etc)
+        if (memcmp(event->disc.data, rejPat, sizeof(rejPat)) == 0) {
+            LOG_DEBUG("(BLE item filtered by pattern)\n");
+            return 0; // Processing-wise, it's still a success
+        }
+
+        //
+        // STORE THE RESULTS IN A SORTED LIST
+        //
+
+        // first, pack each BLE item reading into a 64-bit int
+        netBytes = encodeBLE(event->disc.addr.val, abs(event->disc.rssi));
+
+        // SOME DUPLICATES SURVIVE through filter_duplicates = 1, catch them here
+        // Duplicate filtering is now handled in the sorting loop below,
+        // but right now we write for clarity not optimization
+        for (i = 0; i < bleCounter; i++) {
+            if ((bleResult[i] & 0xffffffffffff) == (netBytes & 0xffffffffffff)) {
+                LOG_DEBUG("(BLE duplicate filtered)\n");
+                return 0;
+            }
+        }
+
+#ifdef ZPS_EXTRAVERBOSE
+        // redundant extraverbosity, but I need it for duplicate hunting
+        LOG_DEBUG("BL_[%02d]: %08x"
+                  "%08x\n",
+                  bleCounter, (uint32_t)(netBytes >> 32), (uint32_t)netBytes);
+#endif
+        // then insert item into a list (up to BLE_MAX_REC records), sorted by RSSI
+        for (i = 0; i < bleCounter; i++) {
+            // find first element greater than ours, that will be our insertion point
+            if (bleResult[i] > netBytes)
+                break;
+        }
+        // any other records move down one position to vacate res[i]
+        for (int j = bleCounter; j > i; j--)
+            bleResult[j] = bleResult[j - 1];
+        // write new element at insertion point
+        bleResult[i] = netBytes;
+
+        // advance tail of list, but not beyond limit
+        if (bleCounter < BLE_MAX_REC)
+            bleCounter++;
+
+        return 0; // SUCCESS
+
+    case BLE_GAP_EVENT_DISC_COMPLETE:
+        LOG_DEBUG("EVENT_DISC_COMPLETE in %d millis\n", (millis() - scanStart));
+        LOG_DEBUG("%d BLE found\n", bleCounter);
+        bleResSize = bleCounter;
+
+        bleCounter = 0; // reset counter
+        return 0;       // SUCCESS
+
+    default:
+        return 0; // SUCCESS
+    }
+}
+
+/**
+ * Initiates the GAP general discovery procedure (non-blocking)
+ */
+static int ble_scan(uint32_t duration, bool passive, bool dedup)
+{
+    uint8_t own_addr_type;
+    struct ble_gap_disc_params disc_params;
+    int rc;
+
+    // Figure out address type to use
+    rc = ble_hs_id_infer_auto(0, &own_addr_type);
+    if (rc != 0) {
+        LOG_DEBUG("error determining address type; rc=%d\n", rc);
+        return rc;
+    }
+
+    // Scanning parameters, these are mostly default
+    disc_params.itvl = 0;
+    disc_params.window = 0;
+    disc_params.filter_policy = 0;
+    disc_params.limited = 0;
+
+    // These two params are the more interesting ones
+    disc_params.filter_duplicates = dedup; // self-explanatory
+    disc_params.passive = passive;         // passive uses less power
+
+    // Start scanning process (non-blocking) and return
+    rc = ble_gap_disc(own_addr_type, duration, &disc_params, ble_gap_event, NULL);
+    if (rc != 0) {
+        LOG_DEBUG("error initiating GAP discovery; rc=%d\n", rc);
+    }
+
+    return rc;
+}
+
+#endif // MESHTASTIC_EXCLUDE_BLUETOOTH

src/modules/esp32/ZPSModule.h

@@ -0,0 +1,86 @@
+#pragma once
+#include "SinglePortModule.h"
+#include "concurrency/OSThread.h"
+#include "gps/RTC.h"
+
+#define ZPS_PORTNUM meshtastic_PortNum_ZPS_APP
+
+#define ZPS_DATAPKT_MAXITEMS 20 // max number of records to pack in an outbound packet (~10)
+#define ZPS_STARTUP_DELAY 10000 // Module startup delay in millis
+
+// Duration of a BLE scan in millis.
+// We want this number to be SLIGHTLY UNDER an integer number of seconds,
+//  to be able to catch the result as fresh as possible on a 1-second polling loop
+#define ZPS_BLE_SCANTIME 2900 // millis
+
+enum SCANSTATE { SCAN_NONE, SCAN_BSS_RUN, SCAN_BSS_DONE, SCAN_BLE_RUN, SCAN_BLE_DONE };
+
+/*
+ * Data packing "compression" functions
+ * Ingest a WiFi BSSID, channel and RSSI (or BLE address and RSSI)
+ *   and encode them into a packed uint64
+ */
+uint64_t encodeBSS(uint8_t *bssid, uint8_t chan, uint8_t absRSSI);
+uint64_t encodeBLE(uint8_t *addr, uint8_t absRSSI);
+
+class ZPSModule : public SinglePortModule, private concurrency::OSThread
+{
+    /// The id of the last packet we sent, to allow us to cancel it if we make something fresher
+    PacketId prevPacketId = 0;
+
+    /// We limit our broadcasts to a max rate
+    uint32_t lastSend = 0;
+
+    bool wantBSS = true;
+    bool haveBSS = false;
+
+    bool wantBLE = true;
+    bool haveBLE = false;
+
+  public:
+    /** Constructor
+     * name is for debugging output
+     */
+    ZPSModule();
+
+    /**
+     * Send our radio environment data into the mesh
+     */
+    void sendDataPacket(NodeNum dest = NODENUM_BROADCAST, bool wantReplies = false);
+
+  protected:
+    /** Called to handle a particular incoming message
+    @return true if you've guaranteed you've handled this message and no other handlers should be considered for it
+    */
+    virtual ProcessMessage handleReceived(const meshtastic_MeshPacket &mp);
+
+    /** Messages can be received that have the want_response bit set.  If set, this callback will be invoked
+     * so that subclasses can (optionally) send a response back to the original sender.  */
+    virtual meshtastic_MeshPacket *allocReply();
+
+    /** Does our periodic broadcast */
+    virtual int32_t runOnce();
+
+  private:
+    // outbound data packet staging buffer and record counter
+    uint64_t netData[ZPS_DATAPKT_MAXITEMS + 2] = {0};
+    uint8_t netRecs = 0;
+
+    // mini state machine to alternate between BSS(Wifi) and BLE scanning
+    SCANSTATE scanState = SCAN_NONE;
+
+    inline void outBufAdd(uint64_t netBytes)
+    {
+        // If this is the first record, initialize the header with the current time and reset the record count.
+        if (!netRecs) {
+            netData[0] = getTime();
+            netData[1] = 0;
+        }
+
+        // push to buffer and update counter
+        if (netRecs < ZPS_DATAPKT_MAXITEMS)
+            netData[2 + (netRecs++)] = netBytes;
+    }
+};
+
+extern ZPSModule *zpsModule;

variants/esp32/esp32-common.ini

@@ -39,7 +39,6 @@ build_flags =
   -Wall
   -Wextra
   -Isrc/platform/esp32
-  -include mbedtls/error.h
   -std=gnu++17
   -DLOG_LOCAL_LEVEL=ESP_LOG_DEBUG
   -DCORE_DEBUG_LEVEL=ARDUHAL_LOG_LEVEL_DEBUG
@@ -68,7 +67,7 @@ lib_deps =
   ${environmental_extra.lib_deps}
   ${radiolib_base.lib_deps}
   # renovate: datasource=git-refs depName=meshtastic-esp32_https_server packageName=https://github.com/meshtastic/esp32_https_server gitBranch=master
-  https://github.com/meshtastic/esp32_https_server/archive/b78f12c86ea65c3ca08968840ff554ff7ed69b60.zip
+  https://github.com/meshtastic/esp32_https_server/archive/0c71f380390ad483ff134ad938e07f6cf1226c5b.zip
   # renovate: datasource=custom.pio depName=NimBLE-Arduino packageName=h2zero/library/NimBLE-Arduino
   h2zero/NimBLE-Arduino@1.4.3
   # renovate: datasource=git-refs depName=libpax packageName=https://github.com/dbinfrago/libpax gitBranch=master