Add device-side AEC to EchoEar

CMakeLists.txt

@@ -4,7 +4,7 @@
 # CMakeLists in this exact order for cmake to work correctly
 cmake_minimum_required(VERSION 3.16)
 
-set(PROJECT_VER "1.9.0")
+set(PROJECT_VER "1.9.2")
 
 # Add this line to disable the specific warning
 add_compile_options(-Wno-missing-field-initializers)

main/Kconfig.projbuild

@@ -428,24 +428,24 @@ endchoice
 choice DISPLAY_ESP32S3_KORVO2_V3
     depends on BOARD_TYPE_ESP32S3_KORVO2_V3
     prompt "ESP32S3_KORVO2_V3 LCD Type"
-    default LCD_ST7789
+    default ESP32S3_KORVO2_V3_LCD_ST7789
     help
         屏幕类型选择
-    config LCD_ST7789
+    config ESP32S3_KORVO2_V3_LCD_ST7789
         bool "ST7789, 分辨率240*280"
-    config LCD_ILI9341
+    config ESP32S3_KORVO2_V3_LCD_ILI9341
         bool "ILI9341, 分辨率240*320"
 endchoice
 
 choice DISPLAY_ESP32S3_AUDIO_BOARD
     depends on BOARD_TYPE_ESP32S3_AUDIO_BOARD
     prompt "ESP32S3_AUDIO_BOARD LCD Type"
-    default LCD_JD9853
+    default AUDIO_BOARD_LCD_JD9853
     help
         屏幕类型选择
-    config LCD_JD9853
+    config AUDIO_BOARD_LCD_JD9853
         bool "JD9853, 分辨率320*172"
-    config LCD_ST7789
+    config AUDIO_BOARD_LCD_ST7789
         bool "ST7789, 分辨率240*320"
 endchoice
 
@@ -508,7 +508,10 @@ config USE_AUDIO_PROCESSOR
 config USE_DEVICE_AEC
     bool "Enable Device-Side AEC"
     default n
-        depends on USE_AUDIO_PROCESSOR && (BOARD_TYPE_ESP_BOX_3 || BOARD_TYPE_ESP_BOX || BOARD_TYPE_ESP_BOX_LITE || BOARD_TYPE_LICHUANG_DEV || BOARD_TYPE_ESP32S3_KORVO2_V3 || BOARD_TYPE_ESP32S3_Touch_AMOLED_1_75 || BOARD_TYPE_ESP32S3_Touch_AMOLED_2_06 || BOARD_TYPE_ESP32P4_WIFI6_Touch_LCD_4B || BOARD_TYPE_ESP32P4_WIFI6_Touch_LCD_XC || BOARD_TYPE_ESP_S3_LCD_EV_Board_2)
+        depends on USE_AUDIO_PROCESSOR && (BOARD_TYPE_ESP_BOX_3 || BOARD_TYPE_ESP_BOX || BOARD_TYPE_ESP_BOX_LITE || BOARD_TYPE_LICHUANG_DEV \
+            || BOARD_TYPE_ESP32S3_KORVO2_V3 || BOARD_TYPE_ESP32S3_Touch_AMOLED_1_75 || BOARD_TYPE_ESP32S3_Touch_AMOLED_2_06 \
+            || BOARD_TYPE_ESP32P4_WIFI6_Touch_LCD_4B || BOARD_TYPE_ESP32P4_WIFI6_Touch_LCD_XC || BOARD_TYPE_ESP_S3_LCD_EV_Board_2 \
+            || BOARD_TYPE_ECHOEAR)
     help
         因为性能不够，不建议和微信聊天界面风格同时开启
 

main/application.cc

@@ -332,6 +332,9 @@ void Application::Start() {
     /* Setup the display */
     auto display = board.GetDisplay();
 
+    // Print board name/version info
+    display->SetChatMessage("system", SystemInfo::GetUserAgent().c_str());
+
     /* Setup the audio service */
     auto codec = board.GetAudioCodec();
     audio_service_.Initialize(codec);
@@ -349,6 +352,12 @@ void Application::Start() {
     };
     audio_service_.SetCallbacks(callbacks);
 
+    // Start the main event loop task with priority 3
+    xTaskCreate([](void* arg) {
+        ((Application*)arg)->MainEventLoop();
+        vTaskDelete(NULL);
+    }, "main_event_loop", 2048 * 4, this, 3, &main_event_loop_task_handle_);
+
     /* Start the clock timer to update the status bar */
     esp_timer_start_periodic(clock_timer_handle_, 1000000);
 
@@ -523,9 +532,6 @@ void Application::Schedule(std::function<void()> callback) {
 // If other tasks need to access the websocket or chat state,
 // they should use Schedule to call this function
 void Application::MainEventLoop() {
-    // Raise the priority of the main event loop to avoid being interrupted by background tasks (which has priority 2)
-    vTaskPrioritySet(NULL, 3);
-
     while (true) {
         auto bits = xEventGroupWaitBits(event_group_, MAIN_EVENT_SCHEDULE |
             MAIN_EVENT_SEND_AUDIO |
@@ -737,11 +743,20 @@ bool Application::CanEnterSleepMode() {
 }
 
 void Application::SendMcpMessage(const std::string& payload) {
-    Schedule([this, payload]() {
-        if (protocol_) {
+    if (protocol_ == nullptr) {
+        return;
+    }
+
+    // Make sure you are using main thread to send MCP message
+    if (xTaskGetCurrentTaskHandle() == main_event_loop_task_handle_) {
+        ESP_LOGI(TAG, "Send MCP message in main thread");
+        protocol_->SendMcpMessage(payload);
+    } else {
+        ESP_LOGI(TAG, "Send MCP message in sub thread");
+        Schedule([this, payload = std::move(payload)]() {
             protocol_->SendMcpMessage(payload);
-        }
-    });
+        });
+    }
 }
 
 void Application::SetAecMode(AecMode mode) {

main/application.h

@@ -82,6 +82,7 @@ private:
     bool aborted_ = false;
     int clock_ticks_ = 0;
     TaskHandle_t check_new_version_task_handle_ = nullptr;
+    TaskHandle_t main_event_loop_task_handle_ = nullptr;
 
     void OnWakeWordDetected();
     void CheckNewVersion(Ota& ota);
@@ -89,4 +90,19 @@ private:
     void SetListeningMode(ListeningMode mode);
 };
 
+
+class TaskPriorityReset {
+public:
+    TaskPriorityReset(BaseType_t priority) {
+        original_priority_ = uxTaskPriorityGet(NULL);
+        vTaskPrioritySet(NULL, priority);
+    }
+    ~TaskPriorityReset() {
+        vTaskPrioritySet(NULL, original_priority_);
+    }
+
+private:
+    BaseType_t original_priority_;
+};
+
 #endif // _APPLICATION_H_

main/audio/audio_service.cc

@@ -100,11 +100,11 @@ void AudioService::Start() {
 
 #if CONFIG_USE_AUDIO_PROCESSOR
     /* Start the audio input task */
-    xTaskCreatePinnedToCore([](void* arg) {
+    xTaskCreate([](void* arg) {
         AudioService* audio_service = (AudioService*)arg;
         audio_service->AudioInputTask();
         vTaskDelete(NULL);
-    }, "audio_input", 2048 * 3, this, 8, &audio_input_task_handle_, 1);
+    }, "audio_input", 2048 * 3, this, 8, &audio_input_task_handle_);
 
     /* Start the audio output task */
     xTaskCreate([](void* arg) {

main/boards/echoear/config.json

@@ -4,7 +4,8 @@
         {
             "name": "echoear",
             "sdkconfig_append": [
-                "CONFIG_PARTITION_TABLE_CUSTOM_FILENAME=\"partitions/v1/16m_echoear.csv\""
+                "CONFIG_PARTITION_TABLE_CUSTOM_FILENAME=\"partitions/v1/16m_echoear.csv\"",
+                "CONFIG_USE_DEVICE_AEC=y"
             ]
         }
     ]

main/boards/esp32s3-korvo2-v3/config.h

@@ -26,7 +26,7 @@
 #define VOLUME_UP_BUTTON_GPIO   GPIO_NUM_NC
 #define VOLUME_DOWN_BUTTON_GPIO GPIO_NUM_NC
 
-#ifdef CONFIG_LCD_ST7789
+#ifdef CONFIG_ESP32S3_KORVO2_V3_LCD_ST7789
 #define DISPLAY_SDA_PIN GPIO_NUM_NC
 #define DISPLAY_SCL_PIN GPIO_NUM_NC
 #define DISPLAY_WIDTH   280
@@ -40,7 +40,7 @@
 #define DISPLAY_OFFSET_Y  0
 #endif
 
-#ifdef CONFIG_LCD_ILI9341
+#ifdef CONFIG_ESP32S3_KORVO2_V3_LCD_ILI9341
 #define LCD_TYPE_ILI9341_SERIAL
 #define DISPLAY_SDA_PIN GPIO_NUM_NC
 #define DISPLAY_SCL_PIN GPIO_NUM_NC
@@ -78,4 +78,4 @@
 #define CAMERA_PIN_PCLK 11
 
 #define XCLK_FREQ_HZ 20000000
-#endif // _BOARD_CONFIG_H_
+#endif // _BOARD_CONFIG_H_

main/boards/surfer-c3-1.14tft/surfer-c3-1.14tft.cc

@@ -21,8 +21,8 @@
 
 #define TAG "SURFERC3114TFT"
 
-LV_FONT_DECLARE(font_puhui_16_4);
-LV_FONT_DECLARE(font_awesome_16_4);
+LV_FONT_DECLARE(font_puhui_20_4);
+LV_FONT_DECLARE(font_awesome_20_4);
 
 class SurferC3114TFT : public WifiBoard {
 private:
@@ -148,8 +148,8 @@ private:
         display_ = new SpiLcdDisplay(panel_io_, panel_,
                                     DISPLAY_WIDTH, DISPLAY_HEIGHT, DISPLAY_OFFSET_X, DISPLAY_OFFSET_Y, DISPLAY_MIRROR_X, DISPLAY_MIRROR_Y, DISPLAY_SWAP_XY,
                                     {
-                                        .text_font = &font_puhui_16_4,
-                                        .icon_font = &font_awesome_16_4,
+                                        .text_font = &font_puhui_20_4,
+                                        .icon_font = &font_awesome_20_4,
                                         .emoji_font = font_emoji_32_init(),
                                     });
     }

main/boards/waveshare-s3-audio-board/config.h

@@ -62,7 +62,7 @@
 
 
 
-#ifdef CONFIG_LCD_JD9853
+#ifdef CONFIG_AUDIO_BOARD_LCD_JD9853
 #define LCD_TYPE_JD9853_SERIAL
 #define DISPLAY_WIDTH   320
 #define DISPLAY_HEIGHT  172
@@ -76,7 +76,7 @@
 #define DISPLAY_OFFSET_Y  0
 #endif
 
-#ifdef CONFIG_LCD_ST7789
+#ifdef CONFIG_AUDIO_BOARD_LCD_ST7789
 #define LCD_TYPE_ST7789_SERIAL
 #define DISPLAY_WIDTH   240
 #define DISPLAY_HEIGHT  320
@@ -92,4 +92,4 @@
 
 
 
-#endif // _BOARD_CONFIG_H_
+#endif // _BOARD_CONFIG_H_

main/boards/waveshare-s3-touch-lcd-3.5b/custom_lcd_display.cc

@@ -3,7 +3,6 @@
 #include "lcd_display.h"
 
 #include <vector>
-#include <font_awesome_symbols.h>
 #include <esp_log.h>
 #include <esp_err.h>
 #include <esp_lvgl_port.h>
@@ -350,4 +349,4 @@ CustomLcdDisplay::CustomLcdDisplay(esp_lcd_panel_io_handle_t panel_io, esp_lcd_p
     }
 
     SetupUI();
-}
+}

main/display/lcd_display.cc

@@ -117,6 +117,9 @@ SpiLcdDisplay::SpiLcdDisplay(esp_lcd_panel_io_handle_t panel_io, esp_lcd_panel_h
     ESP_LOGI(TAG, "Initialize LVGL port");
     lvgl_port_cfg_t port_cfg = ESP_LVGL_PORT_INIT_CONFIG();
     port_cfg.task_priority = 1;
+#if CONFIG_SOC_CPU_CORES_NUM > 1
+    port_cfg.task_affinity = 1;
+#endif
     lvgl_port_init(&port_cfg);
 
     ESP_LOGI(TAG, "Adding LCD display");
@@ -178,7 +181,9 @@ RgbLcdDisplay::RgbLcdDisplay(esp_lcd_panel_io_handle_t panel_io, esp_lcd_panel_h
     ESP_LOGI(TAG, "Initialize LVGL port");
     lvgl_port_cfg_t port_cfg = ESP_LVGL_PORT_INIT_CONFIG();
     port_cfg.task_priority = 1;
-    port_cfg.timer_period_ms = 50;
+#if CONFIG_SOC_CPU_CORES_NUM > 1
+    port_cfg.task_affinity = 1;
+#endif
     lvgl_port_init(&port_cfg);
 
     ESP_LOGI(TAG, "Adding LCD display");
@@ -237,6 +242,10 @@ MipiLcdDisplay::MipiLcdDisplay(esp_lcd_panel_io_handle_t panel_io, esp_lcd_panel
 
     ESP_LOGI(TAG, "Initialize LVGL port");
     lvgl_port_cfg_t port_cfg = ESP_LVGL_PORT_INIT_CONFIG();
+    port_cfg.task_priority = 1;
+#if CONFIG_SOC_CPU_CORES_NUM > 1
+    port_cfg.task_affinity = 1;
+#endif
     lvgl_port_init(&port_cfg);
 
     ESP_LOGI(TAG, "Adding LCD display");
@@ -636,9 +645,6 @@ void LcdDisplay::SetPreviewImage(const lv_img_dsc_t* img_dsc) {
         // Create the image object inside the bubble
         lv_obj_t* preview_image = lv_image_create(img_bubble);
         
-        // Create the image object inside the bubble
-        lv_obj_t* preview_image = lv_image_create(img_bubble);
-        
         // Copy the image descriptor and data to avoid source data changes
         lv_img_dsc_t* copied_img_dsc = (lv_img_dsc_t*)heap_caps_malloc(sizeof(lv_img_dsc_t), MALLOC_CAP_8BIT);
         if (copied_img_dsc == nullptr) {

main/display/oled_display.cc

@@ -23,6 +23,9 @@ OledDisplay::OledDisplay(esp_lcd_panel_io_handle_t panel_io, esp_lcd_panel_handl
     lvgl_port_cfg_t port_cfg = ESP_LVGL_PORT_INIT_CONFIG();
     port_cfg.task_priority = 1;
     port_cfg.task_stack = 6144;
+#if CONFIG_SOC_CPU_CORES_NUM > 1
+    port_cfg.task_affinity = 1;
+#endif
     lvgl_port_init(&port_cfg);
 
     ESP_LOGI(TAG, "Adding OLED display");

main/idf_component.yml

@@ -15,7 +15,7 @@ dependencies:
   78/esp_lcd_nv3023: ~1.0.0
   78/esp-wifi-connect: ~2.5.2
   78/esp-opus-encoder: ~2.4.1
-  78/esp-ml307: ~3.3.1
+  78/esp-ml307: ~3.3.5
   78/xiaozhi-fonts: ~1.5.2
   espressif/led_strip: ~3.0.1
   espressif/esp_codec_dev: ~1.4.0
@@ -32,7 +32,7 @@ dependencies:
   esp_lvgl_port: ~2.6.0
   espressif/esp_io_expander_tca95xx_16bit: ^2.0.0
   espressif2022/image_player: ==1.1.0~1
-  espressif2022/esp_emote_gfx: ^1.0.0
+  espressif2022/esp_emote_gfx: ==1.0.0~2
   espressif/adc_mic: ^0.2.1
   espressif/esp_mmap_assets: '>=1.2'
   txp666/otto-emoji-gif-component: ~1.0.2

main/main.cc

@@ -27,5 +27,4 @@ extern "C" void app_main(void)
     // Launch the application
     auto& app = Application::GetInstance();
     app.Start();
-    app.MainEventLoop();
 }

main/mcp_server.cc

@@ -16,8 +16,6 @@
 
 #define TAG "MCP"
 
-#define DEFAULT_TOOLCALL_STACK_SIZE 6144
-
 McpServer::McpServer() {
 }
 
@@ -100,6 +98,9 @@ void McpServer::AddCommonTools() {
                 Property("question", kPropertyTypeString)
             }),
             [camera](const PropertyList& properties) -> ReturnValue {
+                // Lower the priority to do the camera capture
+                TaskPriorityReset priority_reset(1);
+
                 if (!camera->Capture()) {
                     return "{\"success\": false, \"message\": \"Failed to capture photo\"}";
                 }
@@ -235,13 +236,7 @@ void McpServer::ParseMessage(const cJSON* json) {
             ReplyError(id_int, "Invalid arguments");
             return;
         }
-        auto stack_size = cJSON_GetObjectItem(params, "stackSize");
-        if (stack_size != nullptr && !cJSON_IsNumber(stack_size)) {
-            ESP_LOGE(TAG, "tools/call: Invalid stackSize");
-            ReplyError(id_int, "Invalid stackSize");
-            return;
-        }
-        DoToolCall(id_int, std::string(tool_name->valuestring), tool_arguments, stack_size ? stack_size->valueint : DEFAULT_TOOLCALL_STACK_SIZE);
+        DoToolCall(id_int, std::string(tool_name->valuestring), tool_arguments);
     } else {
         ESP_LOGE(TAG, "Method not implemented: %s", method_str.c_str());
         ReplyError(id_int, "Method not implemented: " + method_str);
@@ -316,7 +311,7 @@ void McpServer::GetToolsList(int id, const std::string& cursor) {
     ReplyResult(id, json);
 }
 
-void McpServer::DoToolCall(int id, const std::string& tool_name, const cJSON* tool_arguments, int stack_size) {
+void McpServer::DoToolCall(int id, const std::string& tool_name, const cJSON* tool_arguments) {
     auto tool_iter = std::find_if(tools_.begin(), tools_.end(), 
                                  [&tool_name](const McpTool* tool) { 
                                      return tool->name() == tool_name; 
@@ -358,15 +353,9 @@ void McpServer::DoToolCall(int id, const std::string& tool_name, const cJSON* to
         return;
     }
 
-    // Start a task to receive data with stack size
-    esp_pthread_cfg_t cfg = esp_pthread_get_default_config();
-    cfg.thread_name = "tool_call";
-    cfg.stack_size = stack_size;
-    cfg.prio = 1;
-    esp_pthread_set_cfg(&cfg);
-
-    // Use a thread to call the tool to avoid blocking the main thread
-    tool_call_thread_ = std::thread([this, id, tool_iter, arguments = std::move(arguments)]() {
+    // Use main thread to call the tool
+    auto& app = Application::GetInstance();
+    app.Schedule([this, id, tool_iter, arguments = std::move(arguments)]() {
         try {
             ReplyResult(id, (*tool_iter)->Call(arguments));
         } catch (const std::exception& e) {
@@ -374,5 +363,4 @@ void McpServer::DoToolCall(int id, const std::string& tool_name, const cJSON* to
             ReplyError(id, e.what());
         }
     });
-    tool_call_thread_.detach();
 }

main/mcp_server.h

@@ -285,10 +285,9 @@ private:
     void ReplyError(int id, const std::string& message);
 
     void GetToolsList(int id, const std::string& cursor);
-    void DoToolCall(int id, const std::string& tool_name, const cJSON* tool_arguments, int stack_size);
+    void DoToolCall(int id, const std::string& tool_name, const cJSON* tool_arguments);
 
     std::vector<McpTool*> tools_;
-    std::thread tool_call_thread_;
 };
 
 #endif // MCP_SERVER_H

main/ota.cc

@@ -51,11 +51,9 @@ std::string Ota::GetCheckVersionUrl() {
 
 std::unique_ptr<Http> Ota::SetupHttp() {
     auto& board = Board::GetInstance();
-    auto app_desc = esp_app_get_description();
-
     auto network = board.GetNetwork();
     auto http = network->CreateHttp(0);
-    auto user_agent = std::string(BOARD_NAME "/") + app_desc->version;
+    auto user_agent = SystemInfo::GetUserAgent();
     http->SetHeader("Activation-Version", has_serial_number_ ? "2" : "1");
     http->SetHeader("Device-Id", SystemInfo::GetMacAddress().c_str());
     http->SetHeader("Client-Id", board.GetUuid());

main/system_info.cc

@@ -47,6 +47,12 @@ std::string SystemInfo::GetChipModelName() {
     return std::string(CONFIG_IDF_TARGET);
 }
 
+std::string SystemInfo::GetUserAgent() {
+    auto app_desc = esp_app_get_description();
+    auto user_agent = std::string(BOARD_NAME "/") + app_desc->version;
+    return user_agent;
+}
+
 esp_err_t SystemInfo::PrintTaskCpuUsage(TickType_t xTicksToWait) {
     #define ARRAY_SIZE_OFFSET 5
     TaskStatus_t *start_array = NULL, *end_array = NULL;

main/system_info.h

@@ -13,6 +13,7 @@ public:
     static size_t GetFreeHeapSize();
     static std::string GetMacAddress();
     static std::string GetChipModelName();
+    static std::string GetUserAgent();
     static esp_err_t PrintTaskCpuUsage(TickType_t xTicksToWait);
     static void PrintTaskList();
     static void PrintHeapStats();

sdkconfig.defaults

@@ -54,7 +54,6 @@ CONFIG_LV_USE_IMGFONT=y
 CONFIG_LV_USE_ASSERT_STYLE=y
 CONFIG_LV_USE_GIF=y
 CONFIG_LV_USE_LODEPNG=y
-CONFIG_LV_USE_TJPGD=y
 
 # Use compressed font
 CONFIG_LV_FONT_FMT_TXT_LARGE=y