Merge pull request #20 from 78/ui

v0.9.2

CMakeLists.txt

@@ -4,7 +4,7 @@
 # CMakeLists in this exact order for cmake to work correctly
 cmake_minimum_required(VERSION 3.16)
 
-set(PROJECT_VER "0.9.0")
+set(PROJECT_VER "0.9.2")
 
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 project(xiaozhi)

main/application.cc

@@ -61,6 +61,8 @@ void Application::CheckNewVersion() {
                 
                 display->SetIcon(FONT_AWESOME_DOWNLOAD);
                 display->SetStatus("新版本 " + ota_.GetFirmwareVersion());
+
+                // 预先关闭音频输出，避免升级过程有音频操作
                 board.GetAudioCodec()->EnableOutput(false);
 
                 ota_.StartUpgrade([display](int progress, size_t speed) {
@@ -121,20 +123,52 @@ void Application::ToggleChatState() {
     Schedule([this]() {
         if (chat_state_ == kChatStateIdle) {
             SetChatState(kChatStateConnecting);
-            if (protocol_->OpenAudioChannel()) {
-                opus_encoder_.ResetState();
-                SetChatState(kChatStateListening);
-            } else {
+            if (!protocol_->OpenAudioChannel()) {
+                ESP_LOGE(TAG, "Failed to open audio channel");
                 SetChatState(kChatStateIdle);
+                return;
             }
+
+            keep_listening_ = true;
+            protocol_->SendStartListening(kListeningModeAutoStop);
+            SetChatState(kChatStateListening);
         } else if (chat_state_ == kChatStateSpeaking) {
-            AbortSpeaking();
+            AbortSpeaking(kAbortReasonNone);
         } else if (chat_state_ == kChatStateListening) {
             protocol_->CloseAudioChannel();
         }
     });
 }
 
+void Application::StartListening() {
+    Schedule([this]() {
+        keep_listening_ = false;
+        if (chat_state_ == kChatStateIdle) {
+            if (!protocol_->IsAudioChannelOpened()) {
+                SetChatState(kChatStateConnecting);
+                if (!protocol_->OpenAudioChannel()) {
+                    SetChatState(kChatStateIdle);
+                    ESP_LOGE(TAG, "Failed to open audio channel");
+                    return;
+                }
+            }
+            protocol_->SendStartListening(kListeningModeManualStop);
+            SetChatState(kChatStateListening);
+        } else if (chat_state_ == kChatStateSpeaking) {
+            AbortSpeaking(kAbortReasonNone);
+            protocol_->SendStartListening(kListeningModeManualStop);
+            SetChatState(kChatStateListening);
+        }
+    });
+}
+
+void Application::StopListening() {
+    Schedule([this]() {
+        protocol_->SendStopListening();
+        SetChatState(kChatStateIdle);
+    });
+}
+
 void Application::Start() {
     auto& board = Board::GetInstance();
     board.Initialize();
@@ -246,26 +280,31 @@ void Application::Start() {
         });
     });
 
-    wake_word_detect_.OnWakeWordDetected([this]() {
-        Schedule([this]() {
+    wake_word_detect_.OnWakeWordDetected([this](const std::string& wake_word) {
+        Schedule([this, &wake_word]() {
             if (chat_state_ == kChatStateIdle) {
                 SetChatState(kChatStateConnecting);
                 wake_word_detect_.EncodeWakeWordData();
 
-                if (protocol_->OpenAudioChannel()) {
-                    std::string opus;
-                    // Encode and send the wake word data to the server
-                    while (wake_word_detect_.GetWakeWordOpus(opus)) {
-                        protocol_->SendAudio(opus);
-                    }
-                    opus_encoder_.ResetState();
-                    // Send a ready message to indicate the server that the wake word data is sent
-                    SetChatState(kChatStateWakeWordDetected);
-                } else {
+                if (!protocol_->OpenAudioChannel()) {
+                    ESP_LOGE(TAG, "Failed to open audio channel");
                     SetChatState(kChatStateIdle);
+                    wake_word_detect_.StartDetection();
+                    return;
                 }
+                
+                std::string opus;
+                // Encode and send the wake word data to the server
+                while (wake_word_detect_.GetWakeWordOpus(opus)) {
+                    protocol_->SendAudio(opus);
+                }
+                // Set the chat state to wake word detected
+                protocol_->SendWakeWordDetected(wake_word);
+                ESP_LOGI(TAG, "Wake word detected: %s", wake_word.c_str());
+                keep_listening_ = true;
+                SetChatState(kChatStateListening);
             } else if (chat_state_ == kChatStateSpeaking) {
-                AbortSpeaking();
+                AbortSpeaking(kAbortReasonWakeWordDetected);
             }
 
             // Resume detection
@@ -282,6 +321,9 @@ void Application::Start() {
 #else
     protocol_ = new MqttProtocol();
 #endif
+    protocol_->OnNetworkError([this](const std::string& message) {
+        Alert("Error", std::move(message));
+    });
     protocol_->OnIncomingAudio([this](const std::string& data) {
         std::lock_guard<std::mutex> lock(mutex_);
         audio_decode_queue_.emplace_back(std::move(data));
@@ -308,15 +350,23 @@ void Application::Start() {
             auto state = cJSON_GetObjectItem(root, "state");
             if (strcmp(state->valuestring, "start") == 0) {
                 Schedule([this]() {
-                    skip_to_end_ = false;
-                    SetChatState(kChatStateSpeaking);
+                    if (chat_state_ == kChatStateIdle || chat_state_ == kChatStateListening) {
+                        skip_to_end_ = false;
+                        opus_decoder_ctl(opus_decoder_, OPUS_RESET_STATE);
+                        SetChatState(kChatStateSpeaking);
+                    }
                 });
             } else if (strcmp(state->valuestring, "stop") == 0) {
                 Schedule([this]() {
                     auto codec = Board::GetInstance().GetAudioCodec();
                     codec->WaitForOutputDone();
                     if (chat_state_ == kChatStateSpeaking) {
-                        SetChatState(kChatStateListening);
+                        if (keep_listening_) {
+                            protocol_->SendStartListening(kListeningModeAutoStop);
+                            SetChatState(kChatStateListening);
+                        } else {
+                            SetChatState(kChatStateIdle);
+                        }
                     }
                 });
             } else if (strcmp(state->valuestring, "sentence_start") == 0) {
@@ -370,9 +420,9 @@ void Application::MainLoop() {
     }
 }
 
-void Application::AbortSpeaking() {
+void Application::AbortSpeaking(AbortReason reason) {
     ESP_LOGI(TAG, "Abort speaking");
-    protocol_->SendAbort();
+    protocol_->SendAbortSpeaking(reason);
 
     skip_to_end_ = true;
     auto codec = Board::GetInstance().GetAudioCodec();
@@ -386,7 +436,6 @@ void Application::SetChatState(ChatState state) {
         "connecting",
         "listening",
         "speaking",
-        "wake_word_detected",
         "upgrading",
         "invalid_state"
     };
@@ -394,12 +443,10 @@ void Application::SetChatState(ChatState state) {
         // No need to update the state
         return;
     }
-    chat_state_ = state;
-    ESP_LOGI(TAG, "STATE: %s", state_str[chat_state_]);
 
     auto display = Board::GetInstance().GetDisplay();
     auto builtin_led = Board::GetInstance().GetBuiltinLed();
-    switch (chat_state_) {
+    switch (state) {
         case kChatStateUnknown:
         case kChatStateIdle:
             builtin_led->TurnOff();
@@ -419,6 +466,7 @@ void Application::SetChatState(ChatState state) {
             builtin_led->TurnOn();
             display->SetStatus("聆听中...");
             display->SetEmotion("neutral");
+            opus_encoder_.ResetState();
 #ifdef CONFIG_USE_AFE_SR
             audio_processor_.Start();
 #endif
@@ -431,17 +479,17 @@ void Application::SetChatState(ChatState state) {
             audio_processor_.Stop();
 #endif
             break;
-        case kChatStateWakeWordDetected:
-            builtin_led->SetBlue();
-            builtin_led->TurnOn();
-            break;
         case kChatStateUpgrading:
             builtin_led->SetGreen();
             builtin_led->StartContinuousBlink(100);
             break;
+        default:
+            ESP_LOGE(TAG, "Invalid chat state: %d", chat_state_);
+            return;
     }
 
-    protocol_->SendState(state_str[chat_state_]);
+    chat_state_ = state;
+    ESP_LOGI(TAG, "STATE: %s", state_str[chat_state_]);
 }
 
 void Application::AudioEncodeTask() {

main/application.h

@@ -29,7 +29,6 @@ enum ChatState {
     kChatStateConnecting,
     kChatStateListening,
     kChatStateSpeaking,
-    kChatStateWakeWordDetected,
     kChatStateUpgrading
 };
 
@@ -41,17 +40,19 @@ public:
         static Application instance;
         return instance;
     }
+    // 删除拷贝构造函数和赋值运算符
+    Application(const Application&) = delete;
+    Application& operator=(const Application&) = delete;
 
     void Start();
     ChatState GetChatState() const { return chat_state_; }
     void Schedule(std::function<void()> callback);
     void SetChatState(ChatState state);
     void Alert(const std::string&& title, const std::string&& message);
-    void AbortSpeaking();
+    void AbortSpeaking(AbortReason reason);
     void ToggleChatState();
-    // 删除拷贝构造函数和赋值运算符
-    Application(const Application&) = delete;
-    Application& operator=(const Application&) = delete;
+    void StartListening();
+    void StopListening();
 
 private:
     Application();
@@ -68,6 +69,7 @@ private:
     Protocol* protocol_ = nullptr;
     EventGroupHandle_t event_group_;
     volatile ChatState chat_state_ = kChatStateUnknown;
+    bool keep_listening_ = false;
     bool skip_to_end_ = false;
 
     // Audio encode / decode

main/audio_codecs/no_audio_codec.cc

@@ -65,8 +65,6 @@ NoAudioCodec::NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_n
     };
     ESP_ERROR_CHECK(i2s_channel_init_std_mode(tx_handle_, &std_cfg));
     ESP_ERROR_CHECK(i2s_channel_init_std_mode(rx_handle_, &std_cfg));
-    ESP_ERROR_CHECK(i2s_channel_enable(tx_handle_));
-    ESP_ERROR_CHECK(i2s_channel_enable(rx_handle_));
     ESP_LOGI(TAG, "Duplex channels created");
 }
 

main/boards/bread-compact-wifi/compact_wifi_board.cc

@@ -7,6 +7,7 @@
 #include "led.h"
 #include "config.h"
 
+#include <wifi_station.h>
 #include <esp_log.h>
 #include <driver/i2c_master.h>
 
@@ -38,7 +39,11 @@ private:
 
     void InitializeButtons() {
         boot_button_.OnClick([this]() {
-            Application::GetInstance().ToggleChatState();
+            auto& app = Application::GetInstance();
+            if (app.GetChatState() == kChatStateUnknown && !WifiStation::GetInstance().IsConnected()) {
+                ResetWifiConfiguration();
+            }
+            app.ToggleChatState();
         });
 
         volume_up_button_.OnClick([this]() {

main/boards/common/button.cc

@@ -30,15 +30,28 @@ Button::~Button() {
     }
 }
 
-void Button::OnPress(std::function<void()> callback) {
+void Button::OnPressDown(std::function<void()> callback) {
     if (button_handle_ == nullptr) {
         return;
     }
-    on_press_ = callback;
+    on_press_down_ = callback;
     iot_button_register_cb(button_handle_, BUTTON_PRESS_DOWN, [](void* handle, void* usr_data) {
         Button* button = static_cast<Button*>(usr_data);
-        if (button->on_press_) {
-            button->on_press_();
+        if (button->on_press_down_) {
+            button->on_press_down_();
+        }
+    }, this);
+}
+
+void Button::OnPressUp(std::function<void()> callback) {
+    if (button_handle_ == nullptr) {
+        return;
+    }
+    on_press_up_ = callback;
+    iot_button_register_cb(button_handle_, BUTTON_PRESS_UP, [](void* handle, void* usr_data) {
+        Button* button = static_cast<Button*>(usr_data);
+        if (button->on_press_up_) {
+            button->on_press_up_();
         }
     }, this);
 }

main/boards/common/button.h

@@ -10,7 +10,8 @@ public:
     Button(gpio_num_t gpio_num);
     ~Button();
 
-    void OnPress(std::function<void()> callback);
+    void OnPressDown(std::function<void()> callback);
+    void OnPressUp(std::function<void()> callback);
     void OnLongPress(std::function<void()> callback);
     void OnClick(std::function<void()> callback);
     void OnDoubleClick(std::function<void()> callback);
@@ -19,7 +20,8 @@ private:
     button_handle_t button_handle_;
 
 
-    std::function<void()> on_press_;
+    std::function<void()> on_press_down_;
+    std::function<void()> on_press_up_;
     std::function<void()> on_long_press_;
     std::function<void()> on_click_;
     std::function<void()> on_double_click_;

main/boards/common/wifi_board.cc

@@ -2,6 +2,7 @@
 #include "application.h"
 #include "system_info.h"
 #include "font_awesome_symbols.h"
+#include "settings.h"
 
 #include <freertos/FreeRTOS.h>
 #include <freertos/task.h>
@@ -149,3 +150,15 @@ void WifiBoard::SetPowerSaveMode(bool enabled) {
     auto& wifi_station = WifiStation::GetInstance();
     wifi_station.SetPowerSaveMode(enabled);
 }
+
+void WifiBoard::ResetWifiConfiguration() {
+    // Reset the wifi station
+    {
+        Settings settings("wifi", true);
+        settings.EraseAll();
+    }
+    GetDisplay()->ShowNotification("已重置 WiFi...");
+    vTaskDelay(pdMS_TO_TICKS(1000));
+    // Reboot the device
+    esp_restart();
+}

main/boards/common/wifi_board.h

@@ -19,6 +19,7 @@ public:
     virtual bool GetNetworkState(std::string& network_name, int& signal_quality, std::string& signal_quality_text) override;
     virtual const char* GetNetworkStateIcon() override;
     virtual void SetPowerSaveMode(bool enabled) override;
+    virtual void ResetWifiConfiguration();
 };
 
 #endif // WIFI_BOARD_H

main/boards/kevin-box-2/axp2101.cc

@@ -1,46 +1,46 @@
 #include "axp2101.h"
+#include "board.h"
+#include "display.h"
 
 #include <esp_log.h>
 
-static const char *TAG = "AXP2101";
+#define TAG "Axp2101"
 
 Axp2101::Axp2101(i2c_master_bus_handle_t i2c_bus, uint8_t addr) : I2cDevice(i2c_bus, addr) {
+    // ** EFUSE defaults **
+    WriteReg(0x22, 0b110); // PWRON > OFFLEVEL as POWEROFF Source enable
+    WriteReg(0x27, 0x10);  // hold 4s to power off
 
-    WriteReg(0x93, 0x1c); // 配置aldo2输出为3.3v
+    WriteReg(0x93, 0x1C); // 配置 aldo2 输出为 3.3V
 
-    uint8_t value = ReadReg(0x90);                 // XPOWERS_AXP2101_LDO_ONOFF_CTRL0
+    uint8_t value = ReadReg(0x90); // XPOWERS_AXP2101_LDO_ONOFF_CTRL0
     value = value | 0x02; // set bit 1 (ALDO2)
     WriteReg(0x90, value);  // and power channels now enabled
 
-    WriteReg(0x64, 0x03); // CV charger voltage setting to 42V
-    value = ReadReg(0x62);
-    ESP_LOGI(TAG, "axp2101 read 0x62 get: 0x%X", value);
+    WriteReg(0x64, 0x03); // CV charger voltage setting to 4.2V
     
     WriteReg(0x61, 0x05); // set Main battery precharge current to 125mA
-    WriteReg(0x62, 0x10); // set Main battery charger current to 900mA ( 0x08-200mA, 0x09-300mA, 0x0A-400mA )
+    WriteReg(0x62, 0x0A); // set Main battery charger current to 400mA ( 0x08-200mA, 0x09-300mA, 0x0A-400mA )
     WriteReg(0x63, 0x15); // set Main battery term charge current to 125mA
-    value = ReadReg(0x62);
-    ESP_LOGI(TAG, "axp2101 read 0x62 get: 0x%X", value);
-
-    value = ReadReg(0x18);
-    ESP_LOGI(TAG, "axp2101 read 0x18 get: 0x%X", value);
-    value = value & 0b11100000;
-    value = value | 0b00001110;
-    WriteReg(0x18, value);
-    value = ReadReg(0x18);
-    ESP_LOGI(TAG, "axp2101 read 0x18 get: 0x%X", value);
 
     WriteReg(0x14, 0x00); // set minimum system voltage to 4.1V (default 4.7V), for poor USB cables
     WriteReg(0x15, 0x00); // set input voltage limit to 3.88v, for poor USB cables
-    WriteReg(0x16, 0x05); // set input voltage limit to 3.88v, for poor USB cables
+    WriteReg(0x16, 0x05); // set input current limit to 2000mA
 
     WriteReg(0x24, 0x01); // set Vsys for PWROFF threshold to 3.2V (default - 2.6V and kill battery)
     WriteReg(0x50, 0x14); // set TS pin to EXTERNAL input (not temperature)
 }
 
+int Axp2101::GetBatteryCurrentDirection() {
+    return (ReadReg(0x01) & 0b01100000) >> 5;
+}
+
 bool Axp2101::IsCharging() {
-    uint8_t value = ReadReg(0x01);
-    return (value & 0b01100000) == 0b00100000;
+    return GetBatteryCurrentDirection() == 1;
+}
+
+bool Axp2101::IsDischarging() {
+    return GetBatteryCurrentDirection() == 2;
 }
 
 bool Axp2101::IsChargingDone() {
@@ -49,8 +49,7 @@ bool Axp2101::IsChargingDone() {
 }
 
 int Axp2101::GetBatteryLevel() {
-    uint8_t value = ReadReg(0xA4);
-    return value;
+    return ReadReg(0xA4);
 }
 
 void Axp2101::PowerOff() {

main/boards/kevin-box-2/axp2101.h

@@ -7,9 +7,13 @@ class Axp2101 : public I2cDevice {
 public:
     Axp2101(i2c_master_bus_handle_t i2c_bus, uint8_t addr);
     bool IsCharging();
+    bool IsDischarging();
     bool IsChargingDone();
     int GetBatteryLevel();
     void PowerOff();
+
+private:
+    int GetBatteryCurrentDirection();
 };
 
 #endif

main/boards/kevin-box-2/kevin_box_board.cc

@@ -11,6 +11,7 @@
 #include <esp_spiffs.h>
 #include <driver/gpio.h>
 #include <driver/i2c_master.h>
+#include <esp_timer.h>
 
 static const char *TAG = "KevinBoxBoard";
 
@@ -23,6 +24,41 @@ private:
     Button volume_up_button_;
     Button volume_down_button_;
     uint8_t _data_buffer[2];
+    esp_timer_handle_t power_save_timer_ = nullptr;
+
+    void InitializePowerSaveTimer() {
+        esp_timer_create_args_t power_save_timer_args = {
+            .callback = [](void *arg) {
+                auto board = static_cast<KevinBoxBoard*>(arg);
+                board->PowerSaveCheck();
+            },
+            .arg = this,
+            .dispatch_method = ESP_TIMER_TASK,
+            .name = "Power Save Timer",
+            .skip_unhandled_events = false,
+        };
+        ESP_ERROR_CHECK(esp_timer_create(&power_save_timer_args, &power_save_timer_));
+        ESP_ERROR_CHECK(esp_timer_start_periodic(power_save_timer_, 1000000));
+    }
+
+    void PowerSaveCheck() {
+        // 电池放电模式下，如果待机超过一定时间，则自动关机
+        const int seconds_to_shutdown = 600;
+        static int seconds = 0;
+        if (Application::GetInstance().GetChatState() != kChatStateIdle) {
+            seconds = 0;
+            return;
+        }
+        if (!axp2101_->IsDischarging()) {
+            seconds = 0;
+            return;
+        }
+        
+        seconds++;
+        if (seconds >= seconds_to_shutdown) {
+            axp2101_->PowerOff();
+        }
+    }
 
     void MountStorage() {
         // Mount the storage partition
@@ -82,12 +118,15 @@ private:
     }
 
     void InitializeButtons() {
-        boot_button_.OnClick([this]() {
-            Application::GetInstance().ToggleChatState();
+        // 测试按住说话
+        // boot_button_.OnClick([this]() {
+        //     Application::GetInstance().ToggleChatState();
+        // });
+        boot_button_.OnPressDown([this]() {
+            Application::GetInstance().StartListening();
         });
-
-        boot_button_.OnLongPress([this]() {
-            axp2101_->PowerOff();
+        boot_button_.OnPressUp([this]() {
+            Application::GetInstance().StopListening();
         });
 
         volume_up_button_.OnClick([this]() {
@@ -140,6 +179,7 @@ public:
         Enable4GModule();
 
         InitializeButtons();
+        InitializePowerSaveTimer();
 
         Ml307Board::Initialize();
     }
@@ -162,8 +202,15 @@ public:
     }
 
     virtual bool GetBatteryLevel(int &level, bool& charging) override {
+        static int last_level = 0;
+        static bool last_charging = false;
         level = axp2101_->GetBatteryLevel();
         charging = axp2101_->IsCharging();
+        if (level != last_level || charging != last_charging) {
+            last_level = level;
+            last_charging = charging;
+            ESP_LOGI(TAG, "Battery level: %d, charging: %d", level, charging);
+        }
         return true;
     }
 };

main/boards/lichuang-dev/lichuang_dev_board.cc

@@ -11,6 +11,7 @@
 #include <esp_lcd_panel_vendor.h>
 #include <driver/i2c_master.h>
 #include <driver/spi_common.h>
+#include <wifi_station.h>
 
 #define TAG "LichuangDevBoard"
 
@@ -71,7 +72,11 @@ private:
 
     void InitializeButtons() {
         boot_button_.OnClick([this]() {
-            Application::GetInstance().ToggleChatState();
+            auto& app = Application::GetInstance();
+            if (app.GetChatState() == kChatStateUnknown && !WifiStation::GetInstance().IsConnected()) {
+                ResetWifiConfiguration();
+            }
+            app.ToggleChatState();
         });
     }
 

main/display/display.cc

@@ -120,7 +120,7 @@ void Display::Update() {
         }
     }
 
-    // 仅在聊天状态为空闲时，更新网络图标
+    // 仅在聊天状态为空闲时，读取网络状态（避免升级时占用 UART 资源）
     auto chat_state = Application::GetInstance().GetChatState();
     if (chat_state == kChatStateIdle || chat_state == kChatStateUnknown) {
         icon = board.GetNetworkStateIcon();

main/display/ssd1306_display.cc

@@ -184,7 +184,6 @@ void Ssd1306Display::SetupUI_128x64() {
 
     status_label_ = lv_label_create(status_bar_);
     lv_obj_set_flex_grow(status_label_, 1);
-    lv_label_set_long_mode(status_label_, LV_LABEL_LONG_SCROLL_CIRCULAR);
     lv_label_set_text(status_label_, "正在初始化");
     lv_obj_set_style_text_align(status_label_, LV_TEXT_ALIGN_CENTER, 0);
 
@@ -255,11 +254,14 @@ void Ssd1306Display::SetupUI_128x32() {
 
     status_label_ = lv_label_create(side_bar_);
     lv_obj_set_flex_grow(status_label_, 1);
+    lv_obj_set_width(status_label_, width_ - 32);
     lv_label_set_long_mode(status_label_, LV_LABEL_LONG_SCROLL_CIRCULAR);
     lv_label_set_text(status_label_, "正在初始化");
 
     notification_label_ = lv_label_create(side_bar_);
     lv_obj_set_flex_grow(notification_label_, 1);
+    lv_obj_set_width(notification_label_, width_ - 32);
+    lv_label_set_long_mode(notification_label_, LV_LABEL_LONG_SCROLL_CIRCULAR);
     lv_label_set_text(notification_label_, "通知");
     lv_obj_add_flag(notification_label_, LV_OBJ_FLAG_HIDDEN);
 }

main/display/st7789_display.cc

@@ -198,7 +198,6 @@ void St7789Display::SetupUI() {
 
     status_label_ = lv_label_create(status_bar_);
     lv_obj_set_flex_grow(status_label_, 1);
-    lv_label_set_long_mode(status_label_, LV_LABEL_LONG_SCROLL_CIRCULAR);
     lv_label_set_text(status_label_, "正在初始化");
     lv_obj_set_style_text_align(status_label_, LV_TEXT_ALIGN_CENTER, 0);
 

main/idf_component.yml

@@ -1,6 +1,6 @@
 ## IDF Component Manager Manifest File
 dependencies:
-  78/esp-wifi-connect: "~1.4.0"
+  78/esp-wifi-connect: "~1.4.1"
   78/esp-opus-encoder: "~1.1.0"
   78/esp-ml307: "~1.6.3"
   espressif/led_strip: "^2.4.1"

main/protocols/mqtt_protocol.cc

@@ -72,9 +72,9 @@ bool MqttProtocol::StartMqttClient() {
         } else if (strcmp(type->valuestring, "goodbye") == 0) {
             auto session_id = cJSON_GetObjectItem(root, "session_id");
             if (session_id == nullptr || session_id_ == session_id->valuestring) {
-                if (on_audio_channel_closed_ != nullptr) {
-                    on_audio_channel_closed_();
-                }
+                Application::GetInstance().Schedule([this]() {
+                    CloseAudioChannel();
+                });
             }
         } else if (on_incoming_json_ != nullptr) {
             on_incoming_json_(root);
@@ -85,6 +85,9 @@ bool MqttProtocol::StartMqttClient() {
     ESP_LOGI(TAG, "Connecting to endpoint %s", endpoint_.c_str());
     if (!mqtt_->Connect(endpoint_, 8883, client_id_, username_, password_)) {
         ESP_LOGE(TAG, "Failed to connect to endpoint");
+        if (on_network_error_ != nullptr) {
+            on_network_error_("无法连接服务");
+        }
         return false;
     }
 
@@ -126,23 +129,6 @@ void MqttProtocol::SendAudio(const std::string& data) {
     udp_->Send(encrypted);
 }
 
-void MqttProtocol::SendState(const std::string& state) {
-    std::string message = "{";
-    message += "\"session_id\":\"" + session_id_ + "\",";
-    message += "\"type\":\"state\",";
-    message += "\"state\":\"" + state + "\"";
-    message += "}";
-    SendText(message);
-}
-
-void MqttProtocol::SendAbort() {
-    std::string message = "{";
-    message += "\"session_id\":\"" + session_id_ + "\",";
-    message += "\"type\":\"abort\"";
-    message += "}";
-    SendText(message);
-}
-
 void MqttProtocol::CloseAudioChannel() {
     {
         std::lock_guard<std::mutex> lock(channel_mutex_);
@@ -167,7 +153,6 @@ bool MqttProtocol::OpenAudioChannel() {
     if (mqtt_ == nullptr || !mqtt_->IsConnected()) {
         ESP_LOGI(TAG, "MQTT is not connected, try to connect now");
         if (!StartMqttClient()) {
-            ESP_LOGE(TAG, "Failed to connect to MQTT");
             return false;
         }
     }
@@ -188,6 +173,9 @@ bool MqttProtocol::OpenAudioChannel() {
     EventBits_t bits = xEventGroupWaitBits(event_group_handle_, MQTT_PROTOCOL_SERVER_HELLO_EVENT, pdTRUE, pdFALSE, pdMS_TO_TICKS(10000));
     if (!(bits & MQTT_PROTOCOL_SERVER_HELLO_EVENT)) {
         ESP_LOGE(TAG, "Failed to receive server hello");
+        if (on_network_error_ != nullptr) {
+            on_network_error_("等待响应超时");
+        }
         return false;
     }
 

main/protocols/mqtt_protocol.h

@@ -26,9 +26,6 @@ public:
     ~MqttProtocol();
 
     void SendAudio(const std::string& data) override;
-    void SendText(const std::string& text) override;
-    void SendState(const std::string& state) override;
-    void SendAbort() override;
     bool OpenAudioChannel() override;
     void CloseAudioChannel() override;
     bool IsAudioChannelOpened() const override;
@@ -52,11 +49,12 @@ private:
     int udp_port_;
     uint32_t local_sequence_;
     uint32_t remote_sequence_;
-    std::string session_id_;
 
     bool StartMqttClient();
     void ParseServerHello(const cJSON* root);
     std::string DecodeHexString(const std::string& hex_string);
+
+    void SendText(const std::string& text) override;
 };
 
 

main/protocols/protocol.cc

@@ -19,3 +19,41 @@ void Protocol::OnAudioChannelOpened(std::function<void()> callback) {
 void Protocol::OnAudioChannelClosed(std::function<void()> callback) {
     on_audio_channel_closed_ = callback;
 }
+
+void Protocol::OnNetworkError(std::function<void(const std::string& message)> callback) {
+    on_network_error_ = callback;
+}
+
+void Protocol::SendAbortSpeaking(AbortReason reason) {
+    std::string message = "{\"session_id\":\"" + session_id_ + "\",\"type\":\"abort\"";
+    if (reason == kAbortReasonWakeWordDetected) {
+        message += ",\"reason\":\"wake_word_detected\"";
+    }
+    message += "}";
+    SendText(message);
+}
+
+void Protocol::SendWakeWordDetected(const std::string& wake_word) {
+    std::string json = "{\"session_id\":\"" + session_id_ + 
+                      "\",\"type\":\"listen\",\"state\":\"detect\",\"text\":\"" + wake_word + "\"}";
+    SendText(json);
+}
+
+void Protocol::SendStartListening(ListeningMode mode) {
+    std::string message = "{\"session_id\":\"" + session_id_ + "\"";
+    message += ",\"type\":\"listen\",\"state\":\"start\"";
+    if (mode == kListeningModeAlwaysOn) {
+        message += ",\"mode\":\"realtime\"";
+    } else if (mode == kListeningModeAutoStop) {
+        message += ",\"mode\":\"auto\"";
+    } else {
+        message += ",\"mode\":\"manual\"";
+    }
+    message += "}";
+    SendText(message);
+}
+
+void Protocol::SendStopListening() {
+    std::string message = "{\"session_id\":\"" + session_id_ + "\",\"type\":\"listen\",\"state\":\"stop\"}";
+    SendText(message);
+}

main/protocols/protocol.h

@@ -12,6 +12,16 @@ struct BinaryProtocol3 {
     uint8_t payload[];
 } __attribute__((packed));
 
+enum AbortReason {
+    kAbortReasonNone,
+    kAbortReasonWakeWordDetected
+};
+
+enum ListeningMode {
+    kListeningModeAutoStop,
+    kListeningModeManualStop,
+    kListeningModeAlwaysOn // 需要 AEC 支持
+};
 
 class Protocol {
 public:
@@ -25,22 +35,28 @@ public:
     void OnIncomingJson(std::function<void(const cJSON* root)> callback);
     void OnAudioChannelOpened(std::function<void()> callback);
     void OnAudioChannelClosed(std::function<void()> callback);
+    void OnNetworkError(std::function<void(const std::string& message)> callback);
 
-    virtual void SendAudio(const std::string& data) = 0;
-    virtual void SendText(const std::string& text) = 0;
-    virtual void SendState(const std::string& state) = 0;
-    virtual void SendAbort() = 0;
     virtual bool OpenAudioChannel() = 0;
     virtual void CloseAudioChannel() = 0;
     virtual bool IsAudioChannelOpened() const = 0;
+    virtual void SendAudio(const std::string& data) = 0;
+    virtual void SendWakeWordDetected(const std::string& wake_word);
+    virtual void SendStartListening(ListeningMode mode);
+    virtual void SendStopListening();
+    virtual void SendAbortSpeaking(AbortReason reason);
 
 protected:
     std::function<void(const cJSON* root)> on_incoming_json_;
     std::function<void(const std::string& data)> on_incoming_audio_;
     std::function<void()> on_audio_channel_opened_;
     std::function<void()> on_audio_channel_closed_;
+    std::function<void(const std::string& message)> on_network_error_;
 
     int server_sample_rate_ = 16000;
+    std::string session_id_;
+
+    virtual void SendText(const std::string& text) = 0;
 };
 
 #endif // PROTOCOL_H

main/protocols/websocket_protocol.cc

@@ -39,21 +39,6 @@ void WebsocketProtocol::SendText(const std::string& text) {
     websocket_->Send(text);
 }
 
-void WebsocketProtocol::SendState(const std::string& state) {
-    std::string message = "{";
-    message += "\"type\":\"state\",";
-    message += "\"state\":\"" + state + "\"";
-    message += "}";
-    SendText(message);
-}
-
-void WebsocketProtocol::SendAbort() {
-    std::string message = "{";
-    message += "\"type\":\"abort\"";
-    message += "}";
-    SendText(message);
-}
-
 bool WebsocketProtocol::IsAudioChannelOpened() const {
     return websocket_ != nullptr;
 }
@@ -110,6 +95,9 @@ bool WebsocketProtocol::OpenAudioChannel() {
 
     if (!websocket_->Connect(url.c_str())) {
         ESP_LOGE(TAG, "Failed to connect to websocket server");
+        if (on_network_error_ != nullptr) {
+            on_network_error_("无法连接服务");
+        }
         return false;
     }
 
@@ -128,6 +116,9 @@ bool WebsocketProtocol::OpenAudioChannel() {
     EventBits_t bits = xEventGroupWaitBits(event_group_handle_, WEBSOCKET_PROTOCOL_SERVER_HELLO_EVENT, pdTRUE, pdFALSE, pdMS_TO_TICKS(10000));
     if (!(bits & WEBSOCKET_PROTOCOL_SERVER_HELLO_EVENT)) {
         ESP_LOGE(TAG, "Failed to receive server hello");
+        if (on_network_error_ != nullptr) {
+            on_network_error_("等待响应超时");
+        }
         return false;
     }
 

main/protocols/websocket_protocol.h

@@ -16,9 +16,6 @@ public:
     ~WebsocketProtocol();
 
     void SendAudio(const std::string& data) override;
-    void SendText(const std::string& text) override;
-    void SendState(const std::string& state) override;
-    void SendAbort() override;
     bool OpenAudioChannel() override;
     void CloseAudioChannel() override;
     bool IsAudioChannelOpened() const override;
@@ -28,6 +25,7 @@ private:
     WebSocket* websocket_ = nullptr;
 
     void ParseServerHello(const cJSON* root);
+    void SendText(const std::string& text) override;
 };
 
 #endif

main/settings.cc

@@ -11,7 +11,7 @@ Settings::Settings(const std::string& ns, bool read_write) : ns_(ns), read_write
 
 Settings::~Settings() {
     if (nvs_handle_ != 0) {
-        if (read_write_) {
+        if (read_write_ && dirty_) {
             ESP_ERROR_CHECK(nvs_commit(nvs_handle_));
         }
         nvs_close(nvs_handle_);
@@ -37,6 +37,7 @@ std::string Settings::GetString(const std::string& key, const std::string& defau
 void Settings::SetString(const std::string& key, const std::string& value) {
     if (read_write_) {
         ESP_ERROR_CHECK(nvs_set_str(nvs_handle_, key.c_str(), value.c_str()));
+        dirty_ = true;
     } else {
         ESP_LOGW(TAG, "Namespace %s is not open for writing", ns_.c_str());
     }
@@ -57,6 +58,23 @@ int32_t Settings::GetInt(const std::string& key, int32_t default_value) {
 void Settings::SetInt(const std::string& key, int32_t value) {
     if (read_write_) {
         ESP_ERROR_CHECK(nvs_set_i32(nvs_handle_, key.c_str(), value));
+        dirty_ = true;
+    } else {
+        ESP_LOGW(TAG, "Namespace %s is not open for writing", ns_.c_str());
+    }
+}
+
+void Settings::EraseKey(const std::string& key) {
+    if (read_write_) {
+        ESP_ERROR_CHECK(nvs_erase_key(nvs_handle_, key.c_str()));
+    } else {
+        ESP_LOGW(TAG, "Namespace %s is not open for writing", ns_.c_str());
+    }
+}
+
+void Settings::EraseAll() {
+    if (read_write_) {
+        ESP_ERROR_CHECK(nvs_erase_all(nvs_handle_));
     } else {
         ESP_LOGW(TAG, "Namespace %s is not open for writing", ns_.c_str());
     }

main/settings.h

@@ -13,11 +13,14 @@ public:
     void SetString(const std::string& key, const std::string& value);
     int32_t GetInt(const std::string& key, int32_t default_value = 0);
     void SetInt(const std::string& key, int32_t value);
+    void EraseKey(const std::string& key);
+    void EraseAll();
 
 private:
     std::string ns_;
     nvs_handle_t nvs_handle_ = 0;
     bool read_write_ = false;
+    bool dirty_ = false;
 };
 
 #endif

main/wake_word_detect.cc

@@ -4,9 +4,9 @@
 #include <esp_log.h>
 #include <model_path.h>
 #include <arpa/inet.h>
+#include <sstream>
 
 #define DETECTION_RUNNING_EVENT 1
-#define WAKE_WORD_ENCODED_EVENT 2
 
 static const char* TAG = "WakeWordDetect";
 
@@ -40,6 +40,13 @@ void WakeWordDetect::Initialize(int channels, bool reference) {
         ESP_LOGI(TAG, "Model %d: %s", i, models->model_name[i]);
         if (strstr(models->model_name[i], ESP_WN_PREFIX) != NULL) {
             wakenet_model_ = models->model_name[i];
+            auto words = esp_srmodel_get_wake_words(models, wakenet_model_);
+            // split by ";" to get all wake words
+            std::stringstream ss(words);
+            std::string word;
+            while (std::getline(ss, word, ';')) {
+                wake_words_.push_back(word);
+            }
         }
     }
 
@@ -84,7 +91,7 @@ void WakeWordDetect::Initialize(int channels, bool reference) {
     }, "audio_detection", 4096 * 2, this, 1, nullptr);
 }
 
-void WakeWordDetect::OnWakeWordDetected(std::function<void()> callback) {
+void WakeWordDetect::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {
     wake_word_detected_callback_ = callback;
 }
 
@@ -144,11 +151,11 @@ void WakeWordDetect::AudioDetectionTask() {
         }
 
         if (res->wakeup_state == WAKENET_DETECTED) {
-            ESP_LOGI(TAG, "Wake word detected");
             StopDetection();
+            last_detected_wake_word_ = wake_words_[res->wake_word_index - 1];
 
             if (wake_word_detected_callback_) {
-                wake_word_detected_callback_();
+                wake_word_detected_callback_(last_detected_wake_word_);
             }
         }
     }
@@ -165,7 +172,6 @@ void WakeWordDetect::StoreWakeWordData(uint16_t* data, size_t samples) {
 }
 
 void WakeWordDetect::EncodeWakeWordData() {
-    xEventGroupClearBits(event_group_, WAKE_WORD_ENCODED_EVENT);
     wake_word_opus_.clear();
     if (wake_word_encode_task_stack_ == nullptr) {
         wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(4096 * 8, MALLOC_CAP_SPIRAM);
@@ -182,15 +188,18 @@ void WakeWordDetect::EncodeWakeWordData() {
             encoder->Encode(pcm, [this_](const uint8_t* opus, size_t opus_size) {
                 std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
                 this_->wake_word_opus_.emplace_back(std::string(reinterpret_cast<const char*>(opus), opus_size));
-                this_->wake_word_cv_.notify_one();
+                this_->wake_word_cv_.notify_all();
             });
         }
         this_->wake_word_pcm_.clear();
 
         auto end_time = esp_timer_get_time();
         ESP_LOGI(TAG, "Encode wake word opus %zu packets in %lld ms", this_->wake_word_opus_.size(), (end_time - start_time) / 1000);
-        xEventGroupSetBits(this_->event_group_, WAKE_WORD_ENCODED_EVENT);
-        this_->wake_word_cv_.notify_one();
+        {
+            std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
+            this_->wake_word_opus_.push_back("");
+            this_->wake_word_cv_.notify_all();
+        }
         delete encoder;
         vTaskDelete(NULL);
     }, "encode_detect_packets", 4096 * 8, this, 1, wake_word_encode_task_stack_, &wake_word_encode_task_buffer_);
@@ -199,12 +208,9 @@ void WakeWordDetect::EncodeWakeWordData() {
 bool WakeWordDetect::GetWakeWordOpus(std::string& opus) {
     std::unique_lock<std::mutex> lock(wake_word_mutex_);
     wake_word_cv_.wait(lock, [this]() {
-        return !wake_word_opus_.empty() || (xEventGroupGetBits(event_group_) & WAKE_WORD_ENCODED_EVENT);
+        return !wake_word_opus_.empty();
     });
-    if (wake_word_opus_.empty()) {
-        return false;
-    }
     opus.swap(wake_word_opus_.front());
     wake_word_opus_.pop_front();
-    return true;
+    return !opus.empty();
 }

main/wake_word_detect.h

@@ -23,24 +23,27 @@ public:
 
     void Initialize(int channels, bool reference);
     void Feed(std::vector<int16_t>& data);
-    void OnWakeWordDetected(std::function<void()> callback);
+    void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback);
     void OnVadStateChange(std::function<void(bool speaking)> callback);
     void StartDetection();
     void StopDetection();
     bool IsDetectionRunning();
     void EncodeWakeWordData();
     bool GetWakeWordOpus(std::string& opus);
+    const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; }
 
 private:
     esp_afe_sr_data_t* afe_detection_data_ = nullptr;
     char* wakenet_model_ = NULL;
+    std::vector<std::string> wake_words_;
     std::vector<int16_t> input_buffer_;
     EventGroupHandle_t event_group_;
-    std::function<void()> wake_word_detected_callback_;
+    std::function<void(const std::string& wake_word)> wake_word_detected_callback_;
     std::function<void(bool speaking)> vad_state_change_callback_;
     bool is_speaking_ = false;
     int channels_;
     bool reference_;
+    std::string last_detected_wake_word_;
 
     TaskHandle_t wake_word_encode_task_ = nullptr;
     StaticTask_t wake_word_encode_task_buffer_;

versions.py

@@ -64,7 +64,7 @@ def get_board_name(folder):
             return "bread-compact-wifi"
         elif "KevinBox1" in basename:
             return "kevin-box-1"
-    if basename.startswith("v0.7") or basename.startswith("v0.8"):
+    if basename.startswith("v0.7") or basename.startswith("v0.8") or basename.startswith("v0.9"):
         return basename.split("_")[1]
     raise Exception(f"Unknown board name: {basename}")