add iot speaker for boards

add pdm mic

CMakeLists.txt

@@ -4,7 +4,7 @@
 # CMakeLists in this exact order for cmake to work correctly
 cmake_minimum_required(VERSION 3.16)
 
-set(PROJECT_VER "0.9.2")
+set(PROJECT_VER "0.9.7")
 
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 project(xiaozhi)

main/CMakeLists.txt

@@ -1,6 +1,7 @@
 set(SOURCES "audio_codecs/audio_codec.cc"
             "audio_codecs/no_audio_codec.cc"
             "audio_codecs/box_audio_codec.cc"
+            "audio_codecs/es8311_audio_codec.cc"
             "display/display.cc"
             "display/no_display.cc"
             "display/st7789_display.cc"
@@ -8,14 +9,21 @@ set(SOURCES "audio_codecs/audio_codec.cc"
             "protocols/protocol.cc"
             "protocols/mqtt_protocol.cc"
             "protocols/websocket_protocol.cc"
+            "iot/thing.cc"
+            "iot/thing_manager.cc"
             "system_info.cc"
             "application.cc"
             "ota.cc"
             "settings.cc"
+            "background_task.cc"
             "main.cc"
             )
 
-set(INCLUDE_DIRS "." "display" "audio_codecs" "protocols")
+set(INCLUDE_DIRS "." "display" "audio_codecs" "protocols" "audio_processing")
+
+# 添加 IOT 相关文件
+file(GLOB IOT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/iot/things/*.cc)
+list(APPEND SOURCES ${IOT_SOURCES})
 
 # 字体
 file(GLOB FONT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/fonts/*.c)
@@ -38,19 +46,24 @@ elseif(CONFIG_BOARD_TYPE_KEVIN_BOX_1)
     set(BOARD_TYPE "kevin-box-1")
 elseif(CONFIG_BOARD_TYPE_KEVIN_BOX_2)
     set(BOARD_TYPE "kevin-box-2")
+elseif(CONFIG_BOARD_TYPE_KEVIN_C3)
+    set(BOARD_TYPE "kevin-c3")
 elseif(CONFIG_BOARD_TYPE_LICHUANG_DEV)
     set(BOARD_TYPE "lichuang-dev")
+elseif(CONFIG_BOARD_TYPE_TERRENCE_C3_DEV)
+    set(BOARD_TYPE "terrence-c3-dev")
 endif()
 file(GLOB BOARD_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/boards/${BOARD_TYPE}/*.cc)
 list(APPEND SOURCES ${BOARD_SOURCES})
 
-if(CONFIG_USE_AFE_SR)
-    list(APPEND SOURCES "audio_processor.cc" "wake_word_detect.cc")
+if(CONFIG_IDF_TARGET_ESP32S3)
+    list(APPEND SOURCES "audio_processing/audio_processor.cc" "audio_processing/wake_word_detect.cc")
 endif()
 
 idf_component_register(SRCS ${SOURCES}
                     EMBED_FILES "assets/err_reg.p3" "assets/err_pin.p3" "assets/err_wificonfig.p3"
                     INCLUDE_DIRS ${INCLUDE_DIRS}
+                    WHOLE_ARCHIVE
                     )
 
 # 使用 target_compile_definitions 来定义 BOARD_TYPE

main/Kconfig.projbuild

@@ -46,14 +46,10 @@ choice BOARD_TYPE
         bool "Kevin Box 1"
     config BOARD_TYPE_KEVIN_BOX_2
         bool "Kevin Box 2"
+    config BOARD_TYPE_KEVIN_C3
+        bool "Kevin C3"
     config BOARD_TYPE_LICHUANG_DEV
         bool "立创开发板"
 endchoice
 
-config USE_AFE_SR
-    bool "Use Espressif AFE SR"
-    default y
-    help
-        Use AFE SR for wake word detection.
-
 endmenu

main/application.cc

@@ -1,10 +1,13 @@
 #include "application.h"
+#include "board.h"
+#include "display.h"
 #include "system_info.h"
 #include "ml307_ssl_transport.h"
 #include "audio_codec.h"
 #include "mqtt_protocol.h"
 #include "websocket_protocol.h"
 #include "font_awesome_symbols.h"
+#include "iot/thing_manager.h"
 
 #include <cstring>
 #include <esp_log.h>
@@ -21,8 +24,17 @@ extern const char p3_err_pin_end[] asm("_binary_err_pin_p3_end");
 extern const char p3_err_wificonfig_start[] asm("_binary_err_wificonfig_p3_start");
 extern const char p3_err_wificonfig_end[] asm("_binary_err_wificonfig_p3_end");
 
+static const char* const STATE_STRINGS[] = {
+    "unknown",
+    "idle",
+    "connecting",
+    "listening",
+    "speaking",
+    "upgrading",
+    "invalid_state"
+};
 
-Application::Application() {
+Application::Application() : background_task_(4096 * 8) {
     event_group_ = xEventGroupCreate();
 
     ota_.SetCheckVersionUrl(CONFIG_OTA_VERSION_URL);
@@ -30,16 +42,6 @@ Application::Application() {
 }
 
 Application::~Application() {
-    if (protocol_ != nullptr) {
-        delete protocol_;
-    }
-    if (opus_decoder_ != nullptr) {
-        opus_decoder_destroy(opus_decoder_);
-    }
-    if (audio_encode_task_stack_ != nullptr) {
-        heap_caps_free(audio_encode_task_stack_);
-    }
-
     vEventGroupDelete(event_group_);
 }
 
@@ -86,7 +88,7 @@ void Application::CheckNewVersion() {
     }
 }
 
-void Application::Alert(const std::string&& title, const std::string&& message) {
+void Application::Alert(const std::string& title, const std::string& message) {
     ESP_LOGW(TAG, "Alert: %s, %s", title.c_str(), message.c_str());
     auto display = Board::GetInstance().GetDisplay();
     display->ShowNotification(message);
@@ -108,7 +110,7 @@ void Application::PlayLocalFile(const char* data, size_t size) {
         p += sizeof(BinaryProtocol3);
 
         auto payload_size = ntohs(p3->payload_size);
-        std::string opus;
+        std::vector<uint8_t> opus;
         opus.resize(payload_size);
         memcpy(opus.data(), p3->payload, payload_size);
         p += payload_size;
@@ -116,15 +118,19 @@ void Application::PlayLocalFile(const char* data, size_t size) {
         std::lock_guard<std::mutex> lock(mutex_);
         audio_decode_queue_.emplace_back(std::move(opus));
     }
-    cv_.notify_all();
 }
 
 void Application::ToggleChatState() {
     Schedule([this]() {
+        if (!protocol_) {
+            ESP_LOGE(TAG, "Protocol not initialized");
+            return;
+        }
+
         if (chat_state_ == kChatStateIdle) {
             SetChatState(kChatStateConnecting);
             if (!protocol_->OpenAudioChannel()) {
-                ESP_LOGE(TAG, "Failed to open audio channel");
+                Alert("Error", "Failed to open audio channel");
                 SetChatState(kChatStateIdle);
                 return;
             }
@@ -142,13 +148,18 @@ void Application::ToggleChatState() {
 
 void Application::StartListening() {
     Schedule([this]() {
+        if (!protocol_) {
+            ESP_LOGE(TAG, "Protocol not initialized");
+            return;
+        }
+        
         keep_listening_ = false;
         if (chat_state_ == kChatStateIdle) {
             if (!protocol_->IsAudioChannelOpened()) {
                 SetChatState(kChatStateConnecting);
                 if (!protocol_->OpenAudioChannel()) {
                     SetChatState(kChatStateIdle);
-                    ESP_LOGE(TAG, "Failed to open audio channel");
+                    Alert("Error", "Failed to open audio channel");
                     return;
                 }
             }
@@ -157,6 +168,8 @@ void Application::StartListening() {
         } else if (chat_state_ == kChatStateSpeaking) {
             AbortSpeaking(kAbortReasonNone);
             protocol_->SendStartListening(kListeningModeManualStop);
+            // FIXME: Wait for the speaker to empty the buffer
+            vTaskDelay(pdMS_TO_TICKS(120));
             SetChatState(kChatStateListening);
         }
     });
@@ -164,15 +177,15 @@ void Application::StartListening() {
 
 void Application::StopListening() {
     Schedule([this]() {
-        protocol_->SendStopListening();
-        SetChatState(kChatStateIdle);
+        if (chat_state_ == kChatStateListening) {
+            protocol_->SendStopListening();
+            SetChatState(kChatStateIdle);
+        }
     });
 }
 
 void Application::Start() {
     auto& board = Board::GetInstance();
-    board.Initialize();
-
     auto builtin_led = board.GetBuiltinLed();
     builtin_led->SetBlue();
     builtin_led->StartContinuousBlink(100);
@@ -183,72 +196,33 @@ void Application::Start() {
     /* Setup the audio codec */
     auto codec = board.GetAudioCodec();
     opus_decode_sample_rate_ = codec->output_sample_rate();
-    opus_decoder_ = opus_decoder_create(opus_decode_sample_rate_, 1, NULL);
-    opus_encoder_.Configure(16000, 1);
+    opus_decoder_ = std::make_unique<OpusDecoderWrapper>(opus_decode_sample_rate_, 1);
+    opus_encoder_ = std::make_unique<OpusEncoderWrapper>(16000, 1, OPUS_FRAME_DURATION_MS);
     if (codec->input_sample_rate() != 16000) {
         input_resampler_.Configure(codec->input_sample_rate(), 16000);
         reference_resampler_.Configure(codec->input_sample_rate(), 16000);
     }
-    codec->OnInputData([this, codec](std::vector<int16_t>&& data) {
-        if (codec->input_sample_rate() != 16000) {
-            if (codec->input_channels() == 2) {
-                auto mic_channel = std::vector<int16_t>(data.size() / 2);
-                auto reference_channel = std::vector<int16_t>(data.size() / 2);
-                for (size_t i = 0, j = 0; i < mic_channel.size(); ++i, j += 2) {
-                    mic_channel[i] = data[j];
-                    reference_channel[i] = data[j + 1];
-                }
-                auto resampled_mic = std::vector<int16_t>(input_resampler_.GetOutputSamples(mic_channel.size()));
-                auto resampled_reference = std::vector<int16_t>(reference_resampler_.GetOutputSamples(reference_channel.size()));
-                input_resampler_.Process(mic_channel.data(), mic_channel.size(), resampled_mic.data());
-                reference_resampler_.Process(reference_channel.data(), reference_channel.size(), resampled_reference.data());
-                data.resize(resampled_mic.size() + resampled_reference.size());
-                for (size_t i = 0, j = 0; i < resampled_mic.size(); ++i, j += 2) {
-                    data[j] = resampled_mic[i];
-                    data[j + 1] = resampled_reference[i];
-                }
-            } else {
-                auto resampled = std::vector<int16_t>(input_resampler_.GetOutputSamples(data.size()));
-                input_resampler_.Process(data.data(), data.size(), resampled.data());
-                data = std::move(resampled);
-            }
-        }
-#ifdef CONFIG_USE_AFE_SR
-        if (audio_processor_.IsRunning()) {
-            audio_processor_.Input(data);
-        }
-        if (wake_word_detect_.IsDetectionRunning()) {
-            wake_word_detect_.Feed(data);
-        }
-#else
-        Schedule([this, data = std::move(data)]() {
-            if (chat_state_ == kChatStateListening) {
-                std::lock_guard<std::mutex> lock(mutex_);
-                audio_encode_queue_.emplace_back(std::move(data));
-                cv_.notify_all();
-            }
-        });
-#endif
+    codec->OnInputReady([this, codec]() {
+        BaseType_t higher_priority_task_woken = pdFALSE;
+        xEventGroupSetBitsFromISR(event_group_, AUDIO_INPUT_READY_EVENT, &higher_priority_task_woken);
+        return higher_priority_task_woken == pdTRUE;
+    });
+    codec->OnOutputReady([this]() {
+        BaseType_t higher_priority_task_woken = pdFALSE;
+        xEventGroupSetBitsFromISR(event_group_, AUDIO_OUTPUT_READY_EVENT, &higher_priority_task_woken);
+        return higher_priority_task_woken == pdTRUE;
     });
-
-    const size_t opus_stack_size = 4096 * 8; // OPUS encoder / decoder use a lot of stack memory
-    audio_encode_task_stack_ = (StackType_t*)heap_caps_malloc(opus_stack_size, MALLOC_CAP_SPIRAM);
-    audio_encode_task_ = xTaskCreateStatic([](void* arg) {
-        Application* app = (Application*)arg;
-        app->AudioEncodeTask();
-        vTaskDelete(NULL);
-    }, "opus_encode", opus_stack_size, this, 1, audio_encode_task_stack_, &audio_encode_task_buffer_);
-
     codec->Start();
 
-    /* Wait for the network to be ready */
-    board.StartNetwork();
-
+    /* Start the main loop */
     xTaskCreate([](void* arg) {
         Application* app = (Application*)arg;
         app->MainLoop();
         vTaskDelete(NULL);
-    }, "main_loop", 4096 * 2, this, 1, nullptr);
+    }, "main_loop", 4096 * 2, this, 2, nullptr);
+
+    /* Wait for the network to be ready */
+    board.StartNetwork();
 
     // Check for new firmware version or get the MQTT broker address
     xTaskCreate([](void* arg) {
@@ -257,12 +231,16 @@ void Application::Start() {
         vTaskDelete(NULL);
     }, "check_new_version", 4096 * 2, this, 1, nullptr);
 
-#ifdef CONFIG_USE_AFE_SR
+#if CONFIG_IDF_TARGET_ESP32S3
     audio_processor_.Initialize(codec->input_channels(), codec->input_reference());
     audio_processor_.OnOutput([this](std::vector<int16_t>&& data) {
-        std::lock_guard<std::mutex> lock(mutex_);
-        audio_encode_queue_.emplace_back(std::move(data));
-        cv_.notify_all();
+        background_task_.Schedule([this, data = std::move(data)]() mutable {
+            opus_encoder_->Encode(std::move(data), [this](std::vector<uint8_t>&& opus) {
+                Schedule([this, opus = std::move(opus)]() {
+                    protocol_->SendAudio(opus);
+                });
+            });
+        });
     });
 
     wake_word_detect_.Initialize(codec->input_channels(), codec->input_reference());
@@ -293,7 +271,7 @@ void Application::Start() {
                     return;
                 }
                 
-                std::string opus;
+                std::vector<uint8_t> opus;
                 // Encode and send the wake word data to the server
                 while (wake_word_detect_.GetWakeWordOpus(opus)) {
                     protocol_->SendAudio(opus);
@@ -317,31 +295,36 @@ void Application::Start() {
     // Initialize the protocol
     display->SetStatus("初始化协议");
 #ifdef CONFIG_CONNECTION_TYPE_WEBSOCKET
-    protocol_ = new WebsocketProtocol();
+    protocol_ = std::make_unique<WebsocketProtocol>();
 #else
-    protocol_ = new MqttProtocol();
+    protocol_ = std::make_unique<MqttProtocol>();
 #endif
     protocol_->OnNetworkError([this](const std::string& message) {
         Alert("Error", std::move(message));
     });
-    protocol_->OnIncomingAudio([this](const std::string& data) {
+    protocol_->OnIncomingAudio([this](std::vector<uint8_t>&& data) {
         std::lock_guard<std::mutex> lock(mutex_);
-        audio_decode_queue_.emplace_back(std::move(data));
-        cv_.notify_all();
+        if (chat_state_ == kChatStateSpeaking) {
+            audio_decode_queue_.emplace_back(std::move(data));
+        }
     });
     protocol_->OnAudioChannelOpened([this, codec, &board]() {
+        board.SetPowerSaveMode(false);
         if (protocol_->server_sample_rate() != codec->output_sample_rate()) {
             ESP_LOGW(TAG, "服务器的音频采样率 %d 与设备输出的采样率 %d 不一致，重采样后可能会失真",
                 protocol_->server_sample_rate(), codec->output_sample_rate());
         }
         SetDecodeSampleRate(protocol_->server_sample_rate());
-        board.SetPowerSaveMode(false);
+        // 物联网设备描述符
+        last_iot_states_.clear();
+        auto& thing_manager = iot::ThingManager::GetInstance();
+        protocol_->SendIotDescriptors(thing_manager.GetDescriptorsJson());
     });
     protocol_->OnAudioChannelClosed([this, &board]() {
+        board.SetPowerSaveMode(true);
         Schedule([this]() {
             SetChatState(kChatStateIdle);
         });
-        board.SetPowerSaveMode(true);
     });
     protocol_->OnIncomingJson([this, display](const cJSON* root) {
         // Parse JSON data
@@ -350,17 +333,15 @@ void Application::Start() {
             auto state = cJSON_GetObjectItem(root, "state");
             if (strcmp(state->valuestring, "start") == 0) {
                 Schedule([this]() {
+                    aborted_ = false;
                     if (chat_state_ == kChatStateIdle || chat_state_ == kChatStateListening) {
-                        skip_to_end_ = false;
-                        opus_decoder_ctl(opus_decoder_, OPUS_RESET_STATE);
                         SetChatState(kChatStateSpeaking);
                     }
                 });
             } else if (strcmp(state->valuestring, "stop") == 0) {
                 Schedule([this]() {
-                    auto codec = Board::GetInstance().GetAudioCodec();
-                    codec->WaitForOutputDone();
                     if (chat_state_ == kChatStateSpeaking) {
+                        background_task_.WaitForCompletion();
                         if (keep_listening_) {
                             protocol_->SendStartListening(kListeningModeAutoStop);
                             SetChatState(kChatStateListening);
@@ -387,6 +368,15 @@ void Application::Start() {
             if (emotion != NULL) {
                 display->SetEmotion(emotion->valuestring);
             }
+        } else if (strcmp(type->valuestring, "iot") == 0) {
+            auto commands = cJSON_GetObjectItem(root, "commands");
+            if (commands != NULL) {
+                auto& thing_manager = iot::ThingManager::GetInstance();
+                for (int i = 0; i < cJSON_GetArraySize(commands); ++i) {
+                    auto command = cJSON_GetArrayItem(commands, i);
+                    thing_manager.Invoke(command);
+                }
+            }
         }
     });
 
@@ -400,8 +390,8 @@ void Application::Start() {
 
 void Application::Schedule(std::function<void()> callback) {
     std::lock_guard<std::mutex> lock(mutex_);
-    main_tasks_.push_back(callback);
-    cv_.notify_all();
+    main_tasks_.push_back(std::move(callback));
+    xEventGroupSetBits(event_group_, SCHEDULE_EVENT);
 }
 
 // The Main Loop controls the chat state and websocket connection
@@ -409,40 +399,150 @@ void Application::Schedule(std::function<void()> callback) {
 // they should use Schedule to call this function
 void Application::MainLoop() {
     while (true) {
-        std::unique_lock<std::mutex> lock(mutex_);
-        cv_.wait(lock, [this]() {
-            return !main_tasks_.empty();
-        });
-        auto task = std::move(main_tasks_.front());
-        main_tasks_.pop_front();
-        lock.unlock();
-        task();
+        auto bits = xEventGroupWaitBits(event_group_,
+            SCHEDULE_EVENT | AUDIO_INPUT_READY_EVENT | AUDIO_OUTPUT_READY_EVENT,
+            pdTRUE, pdFALSE, portMAX_DELAY);
+
+        if (bits & AUDIO_INPUT_READY_EVENT) {
+            InputAudio();
+        }
+        if (bits & AUDIO_OUTPUT_READY_EVENT) {
+            OutputAudio();
+        }
+        if (bits & SCHEDULE_EVENT) {
+            mutex_.lock();
+            std::list<std::function<void()>> tasks = std::move(main_tasks_);
+            mutex_.unlock();
+            for (auto& task : tasks) {
+                task();
+            }
+        }
     }
 }
 
+void Application::ResetDecoder() {
+    std::lock_guard<std::mutex> lock(mutex_);
+    opus_decoder_->ResetState();
+    audio_decode_queue_.clear();
+    last_output_time_ = std::chrono::steady_clock::now();
+    Board::GetInstance().GetAudioCodec()->EnableOutput(true);
+}
+
+void Application::OutputAudio() {
+    auto now = std::chrono::steady_clock::now();
+    auto codec = Board::GetInstance().GetAudioCodec();
+    const int max_silence_seconds = 10;
+
+    std::unique_lock<std::mutex> lock(mutex_);
+    if (audio_decode_queue_.empty()) {
+        // Disable the output if there is no audio data for a long time
+        if (chat_state_ == kChatStateIdle) {
+            auto duration = std::chrono::duration_cast<std::chrono::seconds>(now - last_output_time_).count();
+            if (duration > max_silence_seconds) {
+                codec->EnableOutput(false);
+            }
+        }
+        return;
+    }
+
+    if (chat_state_ == kChatStateListening) {
+        audio_decode_queue_.clear();
+        return;
+    }
+
+    last_output_time_ = now;
+    auto opus = std::move(audio_decode_queue_.front());
+    audio_decode_queue_.pop_front();
+    lock.unlock();
+
+    background_task_.Schedule([this, codec, opus = std::move(opus)]() mutable {
+        if (aborted_) {
+            return;
+        }
+
+        std::vector<int16_t> pcm;
+        if (!opus_decoder_->Decode(std::move(opus), pcm)) {
+            return;
+        }
+
+        // Resample if the sample rate is different
+        if (opus_decode_sample_rate_ != codec->output_sample_rate()) {
+            int target_size = output_resampler_.GetOutputSamples(pcm.size());
+            std::vector<int16_t> resampled(target_size);
+            output_resampler_.Process(pcm.data(), pcm.size(), resampled.data());
+            pcm = std::move(resampled);
+        }
+        
+        codec->OutputData(pcm);
+    });
+}
+
+void Application::InputAudio() {
+    auto codec = Board::GetInstance().GetAudioCodec();
+    std::vector<int16_t> data;
+    if (!codec->InputData(data)) {
+        return;
+    }
+
+    if (codec->input_sample_rate() != 16000) {
+        if (codec->input_channels() == 2) {
+            auto mic_channel = std::vector<int16_t>(data.size() / 2);
+            auto reference_channel = std::vector<int16_t>(data.size() / 2);
+            for (size_t i = 0, j = 0; i < mic_channel.size(); ++i, j += 2) {
+                mic_channel[i] = data[j];
+                reference_channel[i] = data[j + 1];
+            }
+            auto resampled_mic = std::vector<int16_t>(input_resampler_.GetOutputSamples(mic_channel.size()));
+            auto resampled_reference = std::vector<int16_t>(reference_resampler_.GetOutputSamples(reference_channel.size()));
+            input_resampler_.Process(mic_channel.data(), mic_channel.size(), resampled_mic.data());
+            reference_resampler_.Process(reference_channel.data(), reference_channel.size(), resampled_reference.data());
+            data.resize(resampled_mic.size() + resampled_reference.size());
+            for (size_t i = 0, j = 0; i < resampled_mic.size(); ++i, j += 2) {
+                data[j] = resampled_mic[i];
+                data[j + 1] = resampled_reference[i];
+            }
+        } else {
+            auto resampled = std::vector<int16_t>(input_resampler_.GetOutputSamples(data.size()));
+            input_resampler_.Process(data.data(), data.size(), resampled.data());
+            data = std::move(resampled);
+        }
+    }
+    
+#if CONFIG_IDF_TARGET_ESP32S3
+    if (audio_processor_.IsRunning()) {
+        audio_processor_.Input(data);
+    }
+    if (wake_word_detect_.IsDetectionRunning()) {
+        wake_word_detect_.Feed(data);
+    }
+#else
+    if (chat_state_ == kChatStateListening) {
+        background_task_.Schedule([this, data = std::move(data)]() mutable {
+            opus_encoder_->Encode(std::move(data), [this](std::vector<uint8_t>&& opus) {
+                Schedule([this, opus = std::move(opus)]() {
+                    protocol_->SendAudio(opus);
+                });
+            });
+        });
+    }
+#endif
+}
+
 void Application::AbortSpeaking(AbortReason reason) {
     ESP_LOGI(TAG, "Abort speaking");
+    aborted_ = true;
     protocol_->SendAbortSpeaking(reason);
-
-    skip_to_end_ = true;
-    auto codec = Board::GetInstance().GetAudioCodec();
-    codec->ClearOutputQueue();
 }
 
 void Application::SetChatState(ChatState state) {
-    const char* state_str[] = {
-        "unknown",
-        "idle",
-        "connecting",
-        "listening",
-        "speaking",
-        "upgrading",
-        "invalid_state"
-    };
     if (chat_state_ == state) {
-        // No need to update the state
         return;
     }
+    
+    chat_state_ = state;
+    ESP_LOGI(TAG, "STATE: %s", STATE_STRINGS[chat_state_]);
+    // The state is changed, wait for all background tasks to finish
+    background_task_.WaitForCompletion();
 
     auto display = Board::GetInstance().GetDisplay();
     auto builtin_led = Board::GetInstance().GetBuiltinLed();
@@ -452,7 +552,7 @@ void Application::SetChatState(ChatState state) {
             builtin_led->TurnOff();
             display->SetStatus("待命");
             display->SetEmotion("neutral");
-#ifdef CONFIG_USE_AFE_SR
+#ifdef CONFIG_IDF_TARGET_ESP32S3
             audio_processor_.Stop();
 #endif
             break;
@@ -466,16 +566,19 @@ void Application::SetChatState(ChatState state) {
             builtin_led->TurnOn();
             display->SetStatus("聆听中...");
             display->SetEmotion("neutral");
-            opus_encoder_.ResetState();
-#ifdef CONFIG_USE_AFE_SR
+            ResetDecoder();
+            opus_encoder_->ResetState();
+#if CONFIG_IDF_TARGET_ESP32S3
             audio_processor_.Start();
 #endif
+            UpdateIotStates();
             break;
         case kChatStateSpeaking:
             builtin_led->SetGreen();
             builtin_led->TurnOn();
             display->SetStatus("说话中...");
-#ifdef CONFIG_USE_AFE_SR
+            ResetDecoder();
+#if CONFIG_IDF_TARGET_ESP32S3
             audio_processor_.Stop();
 #endif
             break;
@@ -487,59 +590,6 @@ void Application::SetChatState(ChatState state) {
             ESP_LOGE(TAG, "Invalid chat state: %d", chat_state_);
             return;
     }
-
-    chat_state_ = state;
-    ESP_LOGI(TAG, "STATE: %s", state_str[chat_state_]);
-}
-
-void Application::AudioEncodeTask() {
-    ESP_LOGI(TAG, "Audio encode task started");
-    auto codec = Board::GetInstance().GetAudioCodec();
-
-    while (true) {
-        std::unique_lock<std::mutex> lock(mutex_);
-        cv_.wait(lock, [this]() {
-            return !audio_encode_queue_.empty() || !audio_decode_queue_.empty();
-        });
-
-        if (!audio_encode_queue_.empty()) {
-            auto pcm = std::move(audio_encode_queue_.front());
-            audio_encode_queue_.pop_front();
-            lock.unlock();
-
-            opus_encoder_.Encode(pcm, [this](const uint8_t* opus, size_t opus_size) {
-                Schedule([this, data = std::string(reinterpret_cast<const char*>(opus), opus_size)]() {
-                    protocol_->SendAudio(data);
-                });
-            });
-        } else if (!audio_decode_queue_.empty()) {
-            auto opus = std::move(audio_decode_queue_.front());
-            audio_decode_queue_.pop_front();
-            lock.unlock();
-
-            if (skip_to_end_) {
-                continue;
-            }
-
-            int frame_size = opus_decode_sample_rate_ * OPUS_FRAME_DURATION_MS / 1000;
-            std::vector<int16_t> pcm(frame_size);
-
-            int ret = opus_decode(opus_decoder_, (const unsigned char*)opus.data(), opus.size(), pcm.data(), frame_size, 0);
-            if (ret < 0) {
-                ESP_LOGE(TAG, "Failed to decode audio, error code: %d", ret);
-                continue;
-            }
-
-            // Resample if the sample rate is different
-            if (opus_decode_sample_rate_ != codec->output_sample_rate()) {
-                int target_size = output_resampler_.GetOutputSamples(frame_size);
-                std::vector<int16_t> resampled(target_size);
-                output_resampler_.Process(pcm.data(), frame_size, resampled.data());
-                pcm = std::move(resampled);
-            }
-            codec->OutputData(pcm);
-        }
-    }
 }
 
 void Application::SetDecodeSampleRate(int sample_rate) {
@@ -547,9 +597,8 @@ void Application::SetDecodeSampleRate(int sample_rate) {
         return;
     }
 
-    opus_decoder_destroy(opus_decoder_);
     opus_decode_sample_rate_ = sample_rate;
-    opus_decoder_ = opus_decoder_create(opus_decode_sample_rate_, 1, NULL);
+    opus_decoder_ = std::make_unique<OpusDecoderWrapper>(opus_decode_sample_rate_, 1);
 
     auto codec = Board::GetInstance().GetAudioCodec();
     if (opus_decode_sample_rate_ != codec->output_sample_rate()) {
@@ -557,3 +606,12 @@ void Application::SetDecodeSampleRate(int sample_rate) {
         output_resampler_.Configure(opus_decode_sample_rate_, codec->output_sample_rate());
     }
 }
+
+void Application::UpdateIotStates() {
+    auto& thing_manager = iot::ThingManager::GetInstance();
+    auto states = thing_manager.GetStatesJson();
+    if (states != last_iot_states_) {
+        last_iot_states_ = states;
+        protocol_->SendIotStates(states);
+    }
+}

main/application.h

@@ -4,24 +4,27 @@
 #include <freertos/FreeRTOS.h>
 #include <freertos/event_groups.h>
 #include <freertos/task.h>
-#include <opus.h>
+
+#include <string>
 #include <mutex>
 #include <list>
-#include <condition_variable>
 
-#include "opus_encoder.h"
-#include "opus_resampler.h"
+#include <opus_encoder.h>
+#include <opus_decoder.h>
+#include <opus_resampler.h>
 
 #include "protocol.h"
-#include "display.h"
-#include "board.h"
 #include "ota.h"
+#include "background_task.h"
 
-#ifdef CONFIG_USE_AFE_SR
+#if CONFIG_IDF_TARGET_ESP32S3
 #include "wake_word_detect.h"
 #include "audio_processor.h"
 #endif
 
+#define SCHEDULE_EVENT (1 << 0)
+#define AUDIO_INPUT_READY_EVENT (1 << 1)
+#define AUDIO_OUTPUT_READY_EVENT (1 << 2)
 
 enum ChatState {
     kChatStateUnknown,
@@ -48,39 +51,38 @@ public:
     ChatState GetChatState() const { return chat_state_; }
     void Schedule(std::function<void()> callback);
     void SetChatState(ChatState state);
-    void Alert(const std::string&& title, const std::string&& message);
+    void Alert(const std::string& title, const std::string& message);
     void AbortSpeaking(AbortReason reason);
     void ToggleChatState();
     void StartListening();
     void StopListening();
+    void UpdateIotStates();
 
 private:
     Application();
     ~Application();
 
-#ifdef CONFIG_USE_AFE_SR
+#if CONFIG_IDF_TARGET_ESP32S3
     WakeWordDetect wake_word_detect_;
     AudioProcessor audio_processor_;
 #endif
     Ota ota_;
     std::mutex mutex_;
-    std::condition_variable_any cv_;
     std::list<std::function<void()>> main_tasks_;
-    Protocol* protocol_ = nullptr;
+    std::unique_ptr<Protocol> protocol_;
     EventGroupHandle_t event_group_;
     volatile ChatState chat_state_ = kChatStateUnknown;
     bool keep_listening_ = false;
-    bool skip_to_end_ = false;
+    bool aborted_ = false;
+    std::string last_iot_states_;
 
     // Audio encode / decode
-    TaskHandle_t audio_encode_task_ = nullptr;
-    StaticTask_t audio_encode_task_buffer_;
-    StackType_t* audio_encode_task_stack_ = nullptr;
-    std::list<std::vector<int16_t>> audio_encode_queue_;
-    std::list<std::string> audio_decode_queue_;
+    BackgroundTask background_task_;
+    std::chrono::steady_clock::time_point last_output_time_;
+    std::list<std::vector<uint8_t>> audio_decode_queue_;
 
-    OpusEncoder opus_encoder_;
-    OpusDecoder* opus_decoder_ = nullptr;
+    std::unique_ptr<OpusEncoderWrapper> opus_encoder_;
+    std::unique_ptr<OpusDecoderWrapper> opus_decoder_;
 
     int opus_decode_sample_rate_ = -1;
     OpusResampler input_resampler_;
@@ -88,10 +90,12 @@ private:
     OpusResampler output_resampler_;
 
     void MainLoop();
+    void InputAudio();
+    void OutputAudio();
+    void ResetDecoder();
     void SetDecodeSampleRate(int sample_rate);
     void CheckNewVersion();
 
-    void AudioEncodeTask();
     void PlayLocalFile(const char* data, size_t size);
 };
 

main/audio_codecs/audio_codec.cc

@@ -9,34 +9,48 @@
 #define TAG "AudioCodec"
 
 AudioCodec::AudioCodec() {
-    audio_event_group_ = xEventGroupCreate();
 }
 
 AudioCodec::~AudioCodec() {
-    if (audio_input_task_ != nullptr) {
-        vTaskDelete(audio_input_task_);
-    }
-    if (audio_output_task_ != nullptr) {
-        vTaskDelete(audio_output_task_);
-    }
-    if (audio_event_group_ != nullptr) {
-        vEventGroupDelete(audio_event_group_);
-    }
 }
 
-void AudioCodec::OnInputData(std::function<void(std::vector<int16_t>&& data)> callback) {
-    on_input_data_ = callback;
+void AudioCodec::OnInputReady(std::function<bool()> callback) {
+    on_input_ready_ = callback;
+}
+
+void AudioCodec::OnOutputReady(std::function<bool()> callback) {
+    on_output_ready_ = callback;
 }
 
 void AudioCodec::OutputData(std::vector<int16_t>& data) {
-    std::lock_guard<std::mutex> lock(audio_output_queue_mutex_);
-    audio_output_queue_.emplace_back(std::move(data));
-    audio_output_queue_cv_.notify_one();
+    Write(data.data(), data.size());
+}
+
+bool AudioCodec::InputData(std::vector<int16_t>& data) {
+    int duration = 30;
+    int input_frame_size = input_sample_rate_ / 1000 * duration * input_channels_;
+
+    data.resize(input_frame_size);
+    int samples = Read(data.data(), data.size());
+    if (samples > 0) {
+        return true;
+    }
+    return false;
 }
 
 IRAM_ATTR bool AudioCodec::on_sent(i2s_chan_handle_t handle, i2s_event_data_t *event, void *user_ctx) {
     auto audio_codec = (AudioCodec*)user_ctx;
-    xEventGroupSetBits(audio_codec->audio_event_group_, AUDIO_EVENT_OUTPUT_DONE);
+    if (audio_codec->output_enabled_ && audio_codec->on_output_ready_) {
+        return audio_codec->on_output_ready_();
+    }
+    return false;
+}
+
+IRAM_ATTR bool AudioCodec::on_recv(i2s_chan_handle_t handle, i2s_event_data_t *event, void *user_ctx) {
+    auto audio_codec = (AudioCodec*)user_ctx;
+    if (audio_codec->input_enabled_ && audio_codec->on_input_ready_) {
+        return audio_codec->on_input_ready_();
+    }
     return false;
 }
 
@@ -44,85 +58,20 @@ void AudioCodec::Start() {
     Settings settings("audio", false);
     output_volume_ = settings.GetInt("output_volume", output_volume_);
 
-    // 注册音频输出回调
-    i2s_event_callbacks_t callbacks = {};
-    callbacks.on_sent = on_sent;
-    i2s_channel_register_event_callback(tx_handle_, &callbacks, this);
+    // 注册音频数据回调
+    i2s_event_callbacks_t rx_callbacks = {};
+    rx_callbacks.on_recv = on_recv;
+    i2s_channel_register_event_callback(rx_handle_, &rx_callbacks, this);
+
+    i2s_event_callbacks_t tx_callbacks = {};
+    tx_callbacks.on_sent = on_sent;
+    i2s_channel_register_event_callback(tx_handle_, &tx_callbacks, this);
 
     ESP_ERROR_CHECK(i2s_channel_enable(tx_handle_));
     ESP_ERROR_CHECK(i2s_channel_enable(rx_handle_));
 
     EnableInput(true);
     EnableOutput(true);
-
-    // 创建音频输入任务
-    if (audio_input_task_ == nullptr) {
-        xTaskCreate([](void* arg) {
-            auto audio_device = (AudioCodec*)arg;
-            audio_device->InputTask();
-        }, "audio_input", 4096 * 2, this, 3, &audio_input_task_);
-    }
-    // 创建音频输出任务
-    if (audio_output_task_ == nullptr) {
-        xTaskCreate([](void* arg) {
-            auto audio_device = (AudioCodec*)arg;
-            audio_device->OutputTask();
-        }, "audio_output", 4096 * 2, this, 3, &audio_output_task_);
-    }
-}
-
-void AudioCodec::InputTask() {
-    int duration = 30;
-    int input_frame_size = input_sample_rate_ / 1000 * duration * input_channels_;
-
-    while (true) {
-        std::vector<int16_t> input_data(input_frame_size);
-        int samples = Read(input_data.data(), input_data.size());
-        if (samples > 0) {
-            if (on_input_data_) {
-                on_input_data_(std::move(input_data));
-            }
-        }
-    }
-}
-
-void AudioCodec::OutputTask() {
-    while (true) {
-        std::unique_lock<std::mutex> lock(audio_output_queue_mutex_);
-        if (!audio_output_queue_cv_.wait_for(lock, std::chrono::seconds(30), [this]() {
-                return !audio_output_queue_.empty();
-            })) {
-            // If timeout, disable output
-            EnableOutput(false);
-            continue;
-        }
-        auto data = std::move(audio_output_queue_.front());
-        audio_output_queue_.pop_front();
-        lock.unlock();
-
-        if (!output_enabled_) {
-            EnableOutput(true);
-        }
-
-        xEventGroupClearBits(audio_event_group_, AUDIO_EVENT_OUTPUT_DONE);
-        Write(data.data(), data.size());
-        audio_output_queue_cv_.notify_all();
-    }
-}
-
-void AudioCodec::WaitForOutputDone() {
-    // Wait for the output queue to be empty and the output is done
-    std::unique_lock<std::mutex> lock(audio_output_queue_mutex_);
-    audio_output_queue_cv_.wait(lock, [this]() {
-        return audio_output_queue_.empty();
-    });
-    lock.unlock();
-    xEventGroupWaitBits(audio_event_group_, AUDIO_EVENT_OUTPUT_DONE, pdFALSE, pdFALSE, portMAX_DELAY);
-}
-
-void AudioCodec::ClearOutputQueue() {
-    std::lock_guard<std::mutex> lock(audio_output_queue_mutex_);
-    audio_output_queue_.clear();
 }
 
 void AudioCodec::SetOutputVolume(int volume) {

main/audio_codecs/audio_codec.h

@@ -2,21 +2,15 @@
 #define _AUDIO_CODEC_H
 
 #include <freertos/FreeRTOS.h>
-#include <freertos/task.h>
 #include <freertos/event_groups.h>
 #include <driver/i2s_std.h>
 
 #include <vector>
 #include <string>
 #include <functional>
-#include <list>
-#include <mutex>
-#include <condition_variable>
 
 #include "board.h"
 
-#define AUDIO_EVENT_OUTPUT_DONE (1 << 0)
-
 class AudioCodec {
 public:
     AudioCodec();
@@ -27,10 +21,10 @@ public:
     virtual void EnableOutput(bool enable);
 
     void Start();
-    void OnInputData(std::function<void(std::vector<int16_t>&& data)> callback);
     void OutputData(std::vector<int16_t>& data);
-    void WaitForOutputDone();
-    void ClearOutputQueue();
+    bool InputData(std::vector<int16_t>& data);
+    void OnOutputReady(std::function<bool()> callback);
+    void OnInputReady(std::function<bool()> callback);
 
     inline bool duplex() const { return duplex_; }
     inline bool input_reference() const { return input_reference_; }
@@ -41,18 +35,12 @@ public:
     inline int output_volume() const { return output_volume_; }
 
 private:
-    TaskHandle_t audio_input_task_ = nullptr;
-    TaskHandle_t audio_output_task_ = nullptr;
-    std::function<void(std::vector<int16_t>&& data)> on_input_data_; 
-    std::list<std::vector<int16_t>> audio_output_queue_;
-    std::mutex audio_output_queue_mutex_;
-    std::condition_variable audio_output_queue_cv_;
-    EventGroupHandle_t audio_event_group_ = nullptr;
+    std::function<bool()> on_input_ready_;
+    std::function<bool()> on_output_ready_;
+    
+    IRAM_ATTR static bool on_recv(i2s_chan_handle_t handle, i2s_event_data_t *event, void *user_ctx);
     IRAM_ATTR static bool on_sent(i2s_chan_handle_t handle, i2s_event_data_t *event, void *user_ctx);
 
-    void InputTask();
-    void OutputTask();
-
 protected:
     i2s_chan_handle_t tx_handle_ = nullptr;
     i2s_chan_handle_t rx_handle_ = nullptr;

main/audio_codecs/box_audio_codec.cc

@@ -28,7 +28,7 @@ BoxAudioCodec::BoxAudioCodec(void* i2c_master_handle, int input_sample_rate, int
 
     // Output
     audio_codec_i2c_cfg_t i2c_cfg = {
-        .port = I2C_NUM_1,
+        .port = (i2c_port_t)1,
         .addr = es8311_addr,
         .bus_handle = i2c_master_handle,
     };
@@ -96,8 +96,8 @@ void BoxAudioCodec::CreateDuplexChannels(gpio_num_t mclk, gpio_num_t bclk, gpio_
     i2s_chan_config_t chan_cfg = {
         .id = I2S_NUM_0,
         .role = I2S_ROLE_MASTER,
-        .dma_desc_num = 6,
-        .dma_frame_num = 240,
+        .dma_desc_num = 2,
+        .dma_frame_num = 240 * 3,
         .auto_clear_after_cb = true,
         .auto_clear_before_cb = false,
         .intr_priority = 0,

main/audio_codecs/es8311_audio_codec.cc

@@ -0,0 +1,186 @@
+#include "es8311_audio_codec.h"
+
+#include <esp_log.h>
+
+static const char TAG[] = "Es8311AudioCodec";
+
+Es8311AudioCodec::Es8311AudioCodec(void* i2c_master_handle, i2c_port_t i2c_port, int input_sample_rate, int output_sample_rate,
+    gpio_num_t mclk, gpio_num_t bclk, gpio_num_t ws, gpio_num_t dout, gpio_num_t din,
+    gpio_num_t pa_pin, uint8_t es8311_addr) {
+    duplex_ = true; // 是否双工
+    input_reference_ = false; // 是否使用参考输入，实现回声消除
+    input_channels_ = 1; // 输入通道数
+    input_sample_rate_ = input_sample_rate;
+    output_sample_rate_ = output_sample_rate;
+
+    CreateDuplexChannels(mclk, bclk, ws, dout, din);
+
+    // Do initialize of related interface: data_if, ctrl_if and gpio_if
+    audio_codec_i2s_cfg_t i2s_cfg = {
+        .port = I2S_NUM_0,
+        .rx_handle = rx_handle_,
+        .tx_handle = tx_handle_,
+    };
+    data_if_ = audio_codec_new_i2s_data(&i2s_cfg);
+    assert(data_if_ != NULL);
+
+    // Output
+    audio_codec_i2c_cfg_t i2c_cfg = {
+        .port = i2c_port,
+        .addr = es8311_addr,
+        .bus_handle = i2c_master_handle,
+    };
+    ctrl_if_ = audio_codec_new_i2c_ctrl(&i2c_cfg);
+    assert(ctrl_if_ != NULL);
+
+    gpio_if_ = audio_codec_new_gpio();
+    assert(gpio_if_ != NULL);
+
+    es8311_codec_cfg_t es8311_cfg = {};
+    es8311_cfg.ctrl_if = ctrl_if_;
+    es8311_cfg.gpio_if = gpio_if_;
+    es8311_cfg.codec_mode = ESP_CODEC_DEV_WORK_MODE_BOTH;
+    es8311_cfg.pa_pin = pa_pin;
+    es8311_cfg.use_mclk = true;
+    es8311_cfg.hw_gain.pa_voltage = 5.0;
+    es8311_cfg.hw_gain.codec_dac_voltage = 3.3;
+    codec_if_ = es8311_codec_new(&es8311_cfg);
+    assert(codec_if_ != NULL);
+
+    esp_codec_dev_cfg_t dev_cfg = {
+        .dev_type = ESP_CODEC_DEV_TYPE_OUT,
+        .codec_if = codec_if_,
+        .data_if = data_if_,
+    };
+    output_dev_ = esp_codec_dev_new(&dev_cfg);
+    assert(output_dev_ != NULL);
+    dev_cfg.dev_type = ESP_CODEC_DEV_TYPE_IN;
+    input_dev_ = esp_codec_dev_new(&dev_cfg);
+    assert(input_dev_ != NULL);
+
+    ESP_LOGI(TAG, "Es8311AudioCodec initialized");
+}
+
+Es8311AudioCodec::~Es8311AudioCodec() {
+    ESP_ERROR_CHECK(esp_codec_dev_close(output_dev_));
+    esp_codec_dev_delete(output_dev_);
+    ESP_ERROR_CHECK(esp_codec_dev_close(input_dev_));
+    esp_codec_dev_delete(input_dev_);
+
+    audio_codec_delete_codec_if(codec_if_);
+    audio_codec_delete_ctrl_if(ctrl_if_);
+    audio_codec_delete_gpio_if(gpio_if_);
+    audio_codec_delete_data_if(data_if_);
+}
+
+void Es8311AudioCodec::CreateDuplexChannels(gpio_num_t mclk, gpio_num_t bclk, gpio_num_t ws, gpio_num_t dout, gpio_num_t din) {
+    assert(input_sample_rate_ == output_sample_rate_);
+
+    i2s_chan_config_t chan_cfg = {
+        .id = I2S_NUM_0,
+        .role = I2S_ROLE_MASTER,
+        .dma_desc_num = 2,
+        .dma_frame_num = 240 * 3,
+        .auto_clear_after_cb = true,
+        .auto_clear_before_cb = false,
+        .intr_priority = 0,
+    };
+    ESP_ERROR_CHECK(i2s_new_channel(&chan_cfg, &tx_handle_, &rx_handle_));
+
+    i2s_std_config_t std_cfg = {
+        .clk_cfg = {
+            .sample_rate_hz = (uint32_t)output_sample_rate_,
+            .clk_src = I2S_CLK_SRC_DEFAULT,
+            .ext_clk_freq_hz = 0,
+            .mclk_multiple = I2S_MCLK_MULTIPLE_256
+        },
+        .slot_cfg = {
+            .data_bit_width = I2S_DATA_BIT_WIDTH_16BIT,
+            .slot_bit_width = I2S_SLOT_BIT_WIDTH_AUTO,
+            .slot_mode = I2S_SLOT_MODE_STEREO,
+            .slot_mask = I2S_STD_SLOT_BOTH,
+            .ws_width = I2S_DATA_BIT_WIDTH_16BIT,
+            .ws_pol = false,
+            .bit_shift = true,
+            .left_align = true,
+            .big_endian = false,
+            .bit_order_lsb = false
+        },
+        .gpio_cfg = {
+            .mclk = mclk,
+            .bclk = bclk,
+            .ws = ws,
+            .dout = dout,
+            .din = din,
+            .invert_flags = {
+                .mclk_inv = false,
+                .bclk_inv = false,
+                .ws_inv = false
+            }
+        }
+    };
+
+    ESP_ERROR_CHECK(i2s_channel_init_std_mode(tx_handle_, &std_cfg));
+    ESP_ERROR_CHECK(i2s_channel_init_std_mode(rx_handle_, &std_cfg));
+    ESP_LOGI(TAG, "Duplex channels created");
+}
+
+void Es8311AudioCodec::SetOutputVolume(int volume) {
+    ESP_ERROR_CHECK(esp_codec_dev_set_out_vol(output_dev_, volume));
+    AudioCodec::SetOutputVolume(volume);
+}
+
+void Es8311AudioCodec::EnableInput(bool enable) {
+    if (enable == input_enabled_) {
+        return;
+    }
+    if (enable) {
+        esp_codec_dev_sample_info_t fs = {
+            .bits_per_sample = 16,
+            .channel = 1,
+            .channel_mask = 0,
+            .sample_rate = (uint32_t)output_sample_rate_,
+            .mclk_multiple = 0,
+        };
+        ESP_ERROR_CHECK(esp_codec_dev_open(input_dev_, &fs));
+        ESP_ERROR_CHECK(esp_codec_dev_set_in_gain(input_dev_, 40.0));
+    } else {
+        ESP_ERROR_CHECK(esp_codec_dev_close(input_dev_));
+    }
+    AudioCodec::EnableInput(enable);
+}
+
+void Es8311AudioCodec::EnableOutput(bool enable) {
+    if (enable == output_enabled_) {
+        return;
+    }
+    if (enable) {
+        // Play 16bit 1 channel
+        esp_codec_dev_sample_info_t fs = {
+            .bits_per_sample = 16,
+            .channel = 1,
+            .channel_mask = 0,
+            .sample_rate = (uint32_t)output_sample_rate_,
+            .mclk_multiple = 0,
+        };
+        ESP_ERROR_CHECK(esp_codec_dev_open(output_dev_, &fs));
+        ESP_ERROR_CHECK(esp_codec_dev_set_out_vol(output_dev_, output_volume_));
+    } else {
+        ESP_ERROR_CHECK(esp_codec_dev_close(output_dev_));
+    }
+    AudioCodec::EnableOutput(enable);
+}
+
+int Es8311AudioCodec::Read(int16_t* dest, int samples) {
+    if (input_enabled_) {
+        ESP_ERROR_CHECK_WITHOUT_ABORT(esp_codec_dev_read(input_dev_, (void*)dest, samples * sizeof(int16_t)));
+    }
+    return samples;
+}
+
+int Es8311AudioCodec::Write(const int16_t* data, int samples) {
+    if (output_enabled_) {
+        ESP_ERROR_CHECK_WITHOUT_ABORT(esp_codec_dev_write(output_dev_, (void*)data, samples * sizeof(int16_t)));
+    }
+    return samples;
+}

main/audio_codecs/es8311_audio_codec.h

@@ -0,0 +1,36 @@
+#ifndef _ES8311_AUDIO_CODEC_H
+#define _ES8311_AUDIO_CODEC_H
+
+#include "audio_codec.h"
+
+#include <driver/i2c.h>
+#include <esp_codec_dev.h>
+#include <esp_codec_dev_defaults.h>
+
+class Es8311AudioCodec : public AudioCodec {
+private:
+    const audio_codec_data_if_t* data_if_ = nullptr;
+    const audio_codec_ctrl_if_t* ctrl_if_ = nullptr;
+    const audio_codec_if_t* codec_if_ = nullptr;
+    const audio_codec_gpio_if_t* gpio_if_ = nullptr;
+
+    esp_codec_dev_handle_t output_dev_ = nullptr;
+    esp_codec_dev_handle_t input_dev_ = nullptr;
+
+    void CreateDuplexChannels(gpio_num_t mclk, gpio_num_t bclk, gpio_num_t ws, gpio_num_t dout, gpio_num_t din);
+
+    virtual int Read(int16_t* dest, int samples) override;
+    virtual int Write(const int16_t* data, int samples) override;
+
+public:
+    Es8311AudioCodec(void* i2c_master_handle, i2c_port_t i2c_port, int input_sample_rate, int output_sample_rate,
+        gpio_num_t mclk, gpio_num_t bclk, gpio_num_t ws, gpio_num_t dout, gpio_num_t din,
+        gpio_num_t pa_pin, uint8_t es8311_addr);
+    virtual ~Es8311AudioCodec();
+
+    virtual void SetOutputVolume(int volume) override;
+    virtual void EnableInput(bool enable) override;
+    virtual void EnableOutput(bool enable) override;
+};
+
+#endif // _ES8311_AUDIO_CODEC_H

main/audio_codecs/no_audio_codec.cc

@@ -23,8 +23,8 @@ NoAudioCodec::NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_n
     i2s_chan_config_t chan_cfg = {
         .id = I2S_NUM_0,
         .role = I2S_ROLE_MASTER,
-        .dma_desc_num = 6,
-        .dma_frame_num = 240,
+        .dma_desc_num = 2,
+        .dma_frame_num = 240 * 3,
         .auto_clear_after_cb = false,
         .auto_clear_before_cb = false,
         .intr_priority = 0,
@@ -75,7 +75,7 @@ NoAudioCodec::NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_n
 
     // Create a new channel for speaker
     i2s_chan_config_t chan_cfg = {
-        .id = I2S_NUM_0,
+        .id = (i2s_port_t)0,
         .role = I2S_ROLE_MASTER,
         .dma_desc_num = 6,
         .dma_frame_num = 240,
@@ -120,7 +120,7 @@ NoAudioCodec::NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_n
     ESP_ERROR_CHECK(i2s_channel_init_std_mode(tx_handle_, &std_cfg));
 
     // Create a new channel for MIC
-    chan_cfg.id = I2S_NUM_1;
+    chan_cfg.id = (i2s_port_t)1;
     ESP_ERROR_CHECK(i2s_new_channel(&chan_cfg, nullptr, &rx_handle_));
     std_cfg.clk_cfg.sample_rate_hz = (uint32_t)input_sample_rate_;
     std_cfg.gpio_cfg.bclk = mic_sck;
@@ -131,6 +131,67 @@ NoAudioCodec::NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_n
     ESP_LOGI(TAG, "Simplex channels created");
 }
 
+NoAudioCodec::NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_num_t spk_bclk, gpio_num_t spk_ws, gpio_num_t spk_dout, gpio_num_t mic_sck, gpio_num_t mic_din) {
+    duplex_ = false;
+    input_sample_rate_ = input_sample_rate;
+    output_sample_rate_ = output_sample_rate;
+
+    // Create a new channel for speaker
+    i2s_chan_config_t tx_chan_cfg = I2S_CHANNEL_DEFAULT_CONFIG((i2s_port_t)1, I2S_ROLE_MASTER);
+    tx_chan_cfg.dma_desc_num = 6;
+    tx_chan_cfg.dma_frame_num = 240;
+    tx_chan_cfg.auto_clear_after_cb = true;
+    tx_chan_cfg.auto_clear_before_cb = false;
+    tx_chan_cfg.intr_priority = 0;
+    ESP_ERROR_CHECK(i2s_new_channel(&tx_chan_cfg, &tx_handle_, NULL));
+
+
+    i2s_std_config_t tx_std_cfg = {
+        .clk_cfg = {
+            .sample_rate_hz = (uint32_t)output_sample_rate_,
+            .clk_src = I2S_CLK_SRC_DEFAULT,
+            .ext_clk_freq_hz = 0,
+            .mclk_multiple = I2S_MCLK_MULTIPLE_256
+        },
+        .slot_cfg = I2S_STD_MSB_SLOT_DEFAULT_CONFIG(I2S_DATA_BIT_WIDTH_32BIT, I2S_SLOT_MODE_MONO),
+        .gpio_cfg = {
+            .mclk = I2S_GPIO_UNUSED,
+            .bclk = spk_bclk,
+            .ws = spk_ws,
+            .dout = spk_dout,
+            .din = I2S_GPIO_UNUSED,
+            .invert_flags = {
+                .mclk_inv = false,
+                .bclk_inv = false,
+                .ws_inv   = false,
+            },
+        },
+    };
+    ESP_ERROR_CHECK(i2s_channel_init_std_mode(tx_handle_, &tx_std_cfg));
+#if SOC_I2S_SUPPORTS_PDM_RX
+    // Create a new channel for MIC in PDM mode
+    i2s_chan_config_t rx_chan_cfg = I2S_CHANNEL_DEFAULT_CONFIG((i2s_port_t)0, I2S_ROLE_MASTER);
+    ESP_ERROR_CHECK(i2s_new_channel(&rx_chan_cfg, NULL, &rx_handle_));
+    i2s_pdm_rx_config_t pdm_rx_cfg = {
+        .clk_cfg = I2S_PDM_RX_CLK_DEFAULT_CONFIG((uint32_t)input_sample_rate_),
+        /* The data bit-width of PDM mode is fixed to 16 */
+        .slot_cfg = I2S_PDM_RX_SLOT_DEFAULT_CONFIG(I2S_DATA_BIT_WIDTH_16BIT, I2S_SLOT_MODE_MONO),
+        .gpio_cfg = {
+            .clk = mic_sck,
+            .din = mic_din,
+ 
+            .invert_flags = {
+                .clk_inv = false,
+            },
+        },
+    };
+    ESP_ERROR_CHECK(i2s_channel_init_pdm_rx_mode(rx_handle_, &pdm_rx_cfg));
+#else
+    ESP_LOGE(TAG, "PDM is not supported");
+#endif
+    ESP_LOGI(TAG, "Simplex channels created");
+}
+
 int NoAudioCodec::Write(const int16_t* data, int samples) {
     std::vector<int32_t> buffer(samples);
 

main/audio_codecs/no_audio_codec.h

@@ -4,7 +4,7 @@
 #include "audio_codec.h"
 
 #include <driver/gpio.h>
-
+#include <driver/i2s_pdm.h>
 class NoAudioCodec : public AudioCodec {
 private:
     virtual int Write(const int16_t* data, int samples) override;
@@ -15,6 +15,8 @@ public:
     NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_num_t bclk, gpio_num_t ws, gpio_num_t dout, gpio_num_t din);
     // Simplex
     NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_num_t spk_bclk, gpio_num_t spk_ws, gpio_num_t spk_dout, gpio_num_t mic_sck, gpio_num_t mic_ws, gpio_num_t mic_din);
+    // Simplex_PDM
+    NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_num_t spk_bclk, gpio_num_t spk_ws, gpio_num_t spk_dout, gpio_num_t mic_sck,  gpio_num_t mic_din);
     virtual ~NoAudioCodec();
 };
 

main/audio_processing/audio_processor.cc

@@ -63,7 +63,7 @@ AudioProcessor::~AudioProcessor() {
     vEventGroupDelete(event_group_);
 }
 
-void AudioProcessor::Input(std::vector<int16_t>& data) {
+void AudioProcessor::Input(const std::vector<int16_t>& data) {
     input_buffer_.insert(input_buffer_.end(), data.begin(), data.end());
 
     auto chunk_size = esp_afe_vc_v1.get_feed_chunksize(afe_communication_data_) * channels_;

main/audio_processing/audio_processor.h

@@ -16,7 +16,7 @@ public:
     ~AudioProcessor();
 
     void Initialize(int channels, bool reference);
-    void Input(std::vector<int16_t>& data);
+    void Input(const std::vector<int16_t>& data);
     void Start();
     void Stop();
     bool IsRunning();

main/audio_processing/wake_word_detect.cc

@@ -111,7 +111,7 @@ bool WakeWordDetect::IsDetectionRunning() {
     return xEventGroupGetBits(event_group_) & DETECTION_RUNNING_EVENT;
 }
 
-void WakeWordDetect::Feed(std::vector<int16_t>& data) {
+void WakeWordDetect::Feed(const std::vector<int16_t>& data) {
     input_buffer_.insert(input_buffer_.end(), data.begin(), data.end());
 
     auto chunk_size = esp_afe_sr_v1.get_feed_chunksize(afe_detection_data_) * channels_;
@@ -163,8 +163,7 @@ void WakeWordDetect::AudioDetectionTask() {
 
 void WakeWordDetect::StoreWakeWordData(uint16_t* data, size_t samples) {
     // store audio data to wake_word_pcm_
-    std::vector<int16_t> pcm(data, data + samples);
-    wake_word_pcm_.emplace_back(std::move(pcm));
+    wake_word_pcm_.emplace_back(std::vector<int16_t>(data, data + samples));
     // keep about 2 seconds of data, detect duration is 32ms (sample_rate == 16000, chunksize == 512)
     while (wake_word_pcm_.size() > 2000 / 32) {
         wake_word_pcm_.pop_front();
@@ -178,34 +177,33 @@ void WakeWordDetect::EncodeWakeWordData() {
     }
     wake_word_encode_task_ = xTaskCreateStatic([](void* arg) {
         auto this_ = (WakeWordDetect*)arg;
-        auto start_time = esp_timer_get_time();
-        // encode detect packets
-        OpusEncoder* encoder = new OpusEncoder();
-        encoder->Configure(16000, 1, 60);
-        encoder->SetComplexity(0);
-
-        for (auto& pcm: this_->wake_word_pcm_) {
-            encoder->Encode(pcm, [this_](const uint8_t* opus, size_t opus_size) {
-                std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
-                this_->wake_word_opus_.emplace_back(std::string(reinterpret_cast<const char*>(opus), opus_size));
-                this_->wake_word_cv_.notify_all();
-            });
-        }
-        this_->wake_word_pcm_.clear();
-
-        auto end_time = esp_timer_get_time();
-        ESP_LOGI(TAG, "Encode wake word opus %zu packets in %lld ms", this_->wake_word_opus_.size(), (end_time - start_time) / 1000);
         {
+            auto start_time = esp_timer_get_time();
+            auto encoder = std::make_unique<OpusEncoderWrapper>(16000, 1, OPUS_FRAME_DURATION_MS);
+            encoder->SetComplexity(0); // 0 is the fastest
+
+            for (auto& pcm: this_->wake_word_pcm_) {
+                encoder->Encode(std::move(pcm), [this_](std::vector<uint8_t>&& opus) {
+                    std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
+                    this_->wake_word_opus_.emplace_back(std::move(opus));
+                    this_->wake_word_cv_.notify_all();
+                });
+            }
+            this_->wake_word_pcm_.clear();
+
+            auto end_time = esp_timer_get_time();
+            ESP_LOGI(TAG, "Encode wake word opus %zu packets in %lld ms",
+                this_->wake_word_opus_.size(), (end_time - start_time) / 1000);
+
             std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
-            this_->wake_word_opus_.push_back("");
+            this_->wake_word_opus_.push_back(std::vector<uint8_t>());
             this_->wake_word_cv_.notify_all();
         }
-        delete encoder;
         vTaskDelete(NULL);
     }, "encode_detect_packets", 4096 * 8, this, 1, wake_word_encode_task_stack_, &wake_word_encode_task_buffer_);
 }
 
-bool WakeWordDetect::GetWakeWordOpus(std::string& opus) {
+bool WakeWordDetect::GetWakeWordOpus(std::vector<uint8_t>& opus) {
     std::unique_lock<std::mutex> lock(wake_word_mutex_);
     wake_word_cv_.wait(lock, [this]() {
         return !wake_word_opus_.empty();

main/audio_processing/wake_word_detect.h

@@ -22,14 +22,14 @@ public:
     ~WakeWordDetect();
 
     void Initialize(int channels, bool reference);
-    void Feed(std::vector<int16_t>& data);
+    void Feed(const std::vector<int16_t>& data);
     void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback);
     void OnVadStateChange(std::function<void(bool speaking)> callback);
     void StartDetection();
     void StopDetection();
     bool IsDetectionRunning();
     void EncodeWakeWordData();
-    bool GetWakeWordOpus(std::string& opus);
+    bool GetWakeWordOpus(std::vector<uint8_t>& opus);
     const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; }
 
 private:
@@ -49,7 +49,7 @@ private:
     StaticTask_t wake_word_encode_task_buffer_;
     StackType_t* wake_word_encode_task_stack_ = nullptr;
     std::list<std::vector<int16_t>> wake_word_pcm_;
-    std::list<std::string> wake_word_opus_;
+    std::list<std::vector<uint8_t>> wake_word_opus_;
     std::mutex wake_word_mutex_;
     std::condition_variable wake_word_cv_;
 

main/background_task.cc

@@ -0,0 +1,67 @@
+#include "background_task.h"
+
+#include <esp_log.h>
+
+#define TAG "BackgroundTask"
+
+BackgroundTask::BackgroundTask(uint32_t stack_size) {
+#if CONFIG_IDF_TARGET_ESP32S3
+    task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM);
+    background_task_handle_ = xTaskCreateStatic([](void* arg) {
+        BackgroundTask* task = (BackgroundTask*)arg;
+        task->BackgroundTaskLoop();
+    }, "background_task", stack_size, this, 1, task_stack_, &task_buffer_);
+#else
+    xTaskCreate([](void* arg) {
+        BackgroundTask* task = (BackgroundTask*)arg;
+        task->BackgroundTaskLoop();
+    }, "background_task", stack_size, this, 1, &background_task_handle_);
+#endif
+}
+
+BackgroundTask::~BackgroundTask() {
+    if (background_task_handle_ != nullptr) {
+        vTaskDelete(background_task_handle_);
+    }
+}
+
+void BackgroundTask::Schedule(std::function<void()> callback) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (active_tasks_ >= 30) {
+        ESP_LOGW(TAG, "active_tasks_ == %u", active_tasks_.load());
+    }
+    active_tasks_++;
+    main_tasks_.emplace_back([this, cb = std::move(callback)]() {
+        cb();
+        {
+            std::lock_guard<std::mutex> lock(mutex_);
+            active_tasks_--;
+            if (main_tasks_.empty() && active_tasks_ == 0) {
+                condition_variable_.notify_all();
+            }
+        }
+    });
+    condition_variable_.notify_all();
+}
+
+void BackgroundTask::WaitForCompletion() {
+    std::unique_lock<std::mutex> lock(mutex_);
+    condition_variable_.wait(lock, [this]() {
+        return main_tasks_.empty() && active_tasks_ == 0;
+    });
+}
+
+void BackgroundTask::BackgroundTaskLoop() {
+    ESP_LOGI(TAG, "background_task started");
+    while (true) {
+        std::unique_lock<std::mutex> lock(mutex_);
+        condition_variable_.wait(lock, [this]() { return !main_tasks_.empty(); });
+        
+        std::list<std::function<void()>> tasks = std::move(main_tasks_);
+        lock.unlock();
+
+        for (auto& task : tasks) {
+            task();
+        }
+    }
+}

main/background_task.h

@@ -0,0 +1,33 @@
+#ifndef BACKGROUND_TASK_H
+#define BACKGROUND_TASK_H
+
+#include <freertos/FreeRTOS.h>
+#include <freertos/task.h>
+#include <mutex>
+#include <list>
+#include <condition_variable>
+#include <atomic>
+
+class BackgroundTask {
+public:
+    BackgroundTask(uint32_t stack_size = 4096 * 2);
+    ~BackgroundTask();
+
+    void Schedule(std::function<void()> callback);
+    void WaitForCompletion();
+
+private:
+    std::mutex mutex_;
+    std::list<std::function<void()>> main_tasks_;
+    std::condition_variable condition_variable_;
+    TaskHandle_t background_task_handle_ = nullptr;
+    std::atomic<size_t> active_tasks_{0};
+
+    TaskHandle_t task_ = nullptr;
+    StaticTask_t task_buffer_;
+    StackType_t* task_stack_ = nullptr;
+
+    void BackgroundTaskLoop();
+};
+
+#endif

main/boards/bread-compact-ml307/compact_ml307_board.cc

@@ -6,6 +6,7 @@
 #include "button.h"
 #include "led.h"
 #include "config.h"
+#include "iot/thing_manager.h"
 
 #include <esp_log.h>
 #include <driver/i2c_master.h>
@@ -16,13 +17,14 @@ class CompactMl307Board : public Ml307Board {
 private:
     i2c_master_bus_handle_t display_i2c_bus_;
     Button boot_button_;
+    Button touch_button_;
     Button volume_up_button_;
     Button volume_down_button_;
     SystemReset system_reset_;
 
     void InitializeDisplayI2c() {
         i2c_master_bus_config_t bus_config = {
-            .i2c_port = I2C_NUM_0,
+            .i2c_port = (i2c_port_t)0,
             .sda_io_num = DISPLAY_SDA_PIN,
             .scl_io_num = DISPLAY_SCL_PIN,
             .clk_source = I2C_CLK_SRC_DEFAULT,
@@ -40,6 +42,12 @@ private:
         boot_button_.OnClick([this]() {
             Application::GetInstance().ToggleChatState();
         });
+        touch_button_.OnPressDown([this]() {
+            Application::GetInstance().StartListening();
+        });
+        touch_button_.OnPressUp([this]() {
+            Application::GetInstance().StopListening();
+        });
 
         volume_up_button_.OnClick([this]() {
             auto codec = GetAudioCodec();
@@ -52,8 +60,7 @@ private:
         });
 
         volume_up_button_.OnLongPress([this]() {
-            auto codec = GetAudioCodec();
-            codec->SetOutputVolume(100);
+            GetAudioCodec()->SetOutputVolume(100);
             GetDisplay()->ShowNotification("最大音量");
         });
 
@@ -68,29 +75,31 @@ private:
         });
 
         volume_down_button_.OnLongPress([this]() {
-            auto codec = GetAudioCodec();
-            codec->SetOutputVolume(0);
+            GetAudioCodec()->SetOutputVolume(0);
             GetDisplay()->ShowNotification("已静音");
         });
     }
 
+    // 物联网初始化，添加对 AI 可见设备
+    void InitializeIot() {
+        auto& thing_manager = iot::ThingManager::GetInstance();
+        thing_manager.AddThing(iot::CreateThing("Speaker"));
+        thing_manager.AddThing(iot::CreateThing("Lamp"));
+    }
+
 public:
     CompactMl307Board() : Ml307Board(ML307_TX_PIN, ML307_RX_PIN, 4096),
         boot_button_(BOOT_BUTTON_GPIO),
+        touch_button_(TOUCH_BUTTON_GPIO),
         volume_up_button_(VOLUME_UP_BUTTON_GPIO),
         volume_down_button_(VOLUME_DOWN_BUTTON_GPIO),
         system_reset_(RESET_NVS_BUTTON_GPIO, RESET_FACTORY_BUTTON_GPIO) {
-    }
-
-    virtual void Initialize() override {
-        ESP_LOGI(TAG, "Initializing CompactMl307Board");
         // Check if the reset button is pressed
         system_reset_.CheckButtons();
 
         InitializeDisplayI2c();
         InitializeButtons();
-
-        Ml307Board::Initialize();
+        InitializeIot();
     }
 
     virtual Led* GetBuiltinLed() override {

main/boards/bread-compact-ml307/config.h

@@ -29,6 +29,7 @@
 
 #define BUILTIN_LED_GPIO        GPIO_NUM_48
 #define BOOT_BUTTON_GPIO        GPIO_NUM_0
+#define TOUCH_BUTTON_GPIO       GPIO_NUM_47
 #define VOLUME_UP_BUTTON_GPIO   GPIO_NUM_40
 #define VOLUME_DOWN_BUTTON_GPIO GPIO_NUM_39
 #define RESET_NVS_BUTTON_GPIO     GPIO_NUM_1

main/boards/bread-compact-wifi/compact_wifi_board.cc

@@ -6,6 +6,7 @@
 #include "button.h"
 #include "led.h"
 #include "config.h"
+#include "iot/thing_manager.h"
 
 #include <wifi_station.h>
 #include <esp_log.h>
@@ -17,13 +18,14 @@ class CompactWifiBoard : public WifiBoard {
 private:
     i2c_master_bus_handle_t display_i2c_bus_;
     Button boot_button_;
+    Button touch_button_;
     Button volume_up_button_;
     Button volume_down_button_;
     SystemReset system_reset_;
 
     void InitializeDisplayI2c() {
         i2c_master_bus_config_t bus_config = {
-            .i2c_port = I2C_NUM_0,
+            .i2c_port = (i2c_port_t)0,
             .sda_io_num = DISPLAY_SDA_PIN,
             .scl_io_num = DISPLAY_SCL_PIN,
             .clk_source = I2C_CLK_SRC_DEFAULT,
@@ -45,6 +47,12 @@ private:
             }
             app.ToggleChatState();
         });
+        touch_button_.OnPressDown([this]() {
+            Application::GetInstance().StartListening();
+        });
+        touch_button_.OnPressUp([this]() {
+            Application::GetInstance().StopListening();
+        });
 
         volume_up_button_.OnClick([this]() {
             auto codec = GetAudioCodec();
@@ -57,8 +65,7 @@ private:
         });
 
         volume_up_button_.OnLongPress([this]() {
-            auto codec = GetAudioCodec();
-            codec->SetOutputVolume(100);
+            GetAudioCodec()->SetOutputVolume(100);
             GetDisplay()->ShowNotification("最大音量");
         });
 
@@ -73,29 +80,31 @@ private:
         });
 
         volume_down_button_.OnLongPress([this]() {
-            auto codec = GetAudioCodec();
-            codec->SetOutputVolume(0);
+            GetAudioCodec()->SetOutputVolume(0);
             GetDisplay()->ShowNotification("已静音");
         });
     }
 
+    // 物联网初始化，添加对 AI 可见设备
+    void InitializeIot() {
+        auto& thing_manager = iot::ThingManager::GetInstance();
+        thing_manager.AddThing(iot::CreateThing("Speaker"));
+        thing_manager.AddThing(iot::CreateThing("Lamp"));
+    }
+
 public:
     CompactWifiBoard() :
         boot_button_(BOOT_BUTTON_GPIO),
+        touch_button_(TOUCH_BUTTON_GPIO),
         volume_up_button_(VOLUME_UP_BUTTON_GPIO),
         volume_down_button_(VOLUME_DOWN_BUTTON_GPIO),
         system_reset_(RESET_NVS_BUTTON_GPIO, RESET_FACTORY_BUTTON_GPIO) {
-    }
-
-    virtual void Initialize() override {
-        ESP_LOGI(TAG, "Initializing CompactWifiBoard");
         // Check if the reset button is pressed
         system_reset_.CheckButtons();
 
         InitializeDisplayI2c();
         InitializeButtons();
-
-        WifiBoard::Initialize();
+        InitializeIot();
     }
 
     virtual Led* GetBuiltinLed() override {

main/boards/bread-compact-wifi/config.h

@@ -30,6 +30,7 @@
 
 #define BUILTIN_LED_GPIO        GPIO_NUM_48
 #define BOOT_BUTTON_GPIO        GPIO_NUM_0
+#define TOUCH_BUTTON_GPIO       GPIO_NUM_47
 #define VOLUME_UP_BUTTON_GPIO   GPIO_NUM_40
 #define VOLUME_DOWN_BUTTON_GPIO GPIO_NUM_39
 #define RESET_NVS_BUTTON_GPIO     GPIO_NUM_1

main/boards/common/board.cc

@@ -1,16 +1,26 @@
 #include "board.h"
 #include "system_info.h"
+#include "display/no_display.h"
 
 #include <esp_log.h>
 #include <esp_ota_ops.h>
 #include <esp_chip_info.h>
 
-// static const char *TAG = "Board";
+#define TAG "Board"
+
+Board::Board() {
+}
 
 bool Board::GetBatteryLevel(int &level, bool& charging) {
     return false;
 }
 
+Display* Board::GetDisplay() {
+    static NoDisplay display;
+    return &display;
+}
+
+
 std::string Board::GetJson() {
     /* 
         {
@@ -101,4 +111,4 @@ std::string Board::GetJson() {
     // Close the JSON object
     json += "}";
     return json;
-}
+}

main/boards/common/board.h

@@ -19,7 +19,7 @@ private:
     virtual std::string GetBoardJson() = 0;
 
 protected:
-    Board() = default;
+    Board();
 
 public:
     static Board& GetInstance() {
@@ -30,12 +30,11 @@ public:
         return *instance;
     }
 
-    virtual void Initialize() = 0;
     virtual void StartNetwork() = 0;
     virtual ~Board() = default;
     virtual Led* GetBuiltinLed() = 0;
     virtual AudioCodec* GetAudioCodec() = 0;
-    virtual Display* GetDisplay() = 0;
+    virtual Display* GetDisplay();
     virtual Http* CreateHttp() = 0;
     virtual WebSocket* CreateWebSocket() = 0;
     virtual Mqtt* CreateMqtt() = 0;

main/boards/common/button.cc

@@ -4,7 +4,7 @@
 
 static const char* TAG = "Button";
 
-Button::Button(gpio_num_t gpio_num) : gpio_num_(gpio_num) {
+Button::Button(gpio_num_t gpio_num, bool active_high) : gpio_num_(gpio_num) {
     if (gpio_num == GPIO_NUM_NC) {
         return;
     }
@@ -14,7 +14,7 @@ Button::Button(gpio_num_t gpio_num) : gpio_num_(gpio_num) {
         .short_press_time = 50,
         .gpio_button_config = {
             .gpio_num = gpio_num,
-            .active_level = 0
+            .active_level = static_cast<uint8_t>(active_high ? 1 : 0)
         }
     };
     button_handle_ = iot_button_create(&button_config);

main/boards/common/button.h

@@ -7,7 +7,7 @@
 
 class Button {
 public:
-    Button(gpio_num_t gpio_num);
+    Button(gpio_num_t gpio_num, bool active_high = false);
     ~Button();
 
     void OnPressDown(std::function<void()> callback);

main/boards/common/led.cc

@@ -7,10 +7,6 @@
 #define TAG "Led"
 
 Led::Led(gpio_num_t gpio) {
-    mutex_ = xSemaphoreCreateMutex();
-    blink_event_group_ = xEventGroupCreate();
-    xEventGroupSetBits(blink_event_group_, BLINK_TASK_STOPPED_BIT);
-
     if (gpio == GPIO_NUM_NC) {
         ESP_LOGI(TAG, "Builtin LED not connected");
         return;
@@ -29,19 +25,25 @@ Led::Led(gpio_num_t gpio) {
     led_strip_clear(led_strip_);
 
     SetGrey();
+
+    esp_timer_create_args_t blink_timer_args = {
+        .callback = [](void *arg) {
+            auto led = static_cast<Led*>(arg);
+            led->OnBlinkTimer();
+        },
+        .arg = this,
+        .dispatch_method = ESP_TIMER_TASK,
+        .name = "Blink Timer",
+        .skip_unhandled_events = false,
+    };
+    ESP_ERROR_CHECK(esp_timer_create(&blink_timer_args, &blink_timer_));
 }
 
 Led::~Led() {
-    StopBlinkInternal();
+    esp_timer_stop(blink_timer_);
     if (led_strip_ != nullptr) {
         led_strip_del(led_strip_);
     }
-    if (mutex_ != nullptr) {
-        vSemaphoreDelete(mutex_);
-    }
-    if (blink_event_group_ != nullptr) {
-        vEventGroupDelete(blink_event_group_);
-    }
 }
 
 void Led::SetColor(uint8_t r, uint8_t g, uint8_t b) {
@@ -54,21 +56,21 @@ void Led::TurnOn() {
     if (led_strip_ == nullptr) {
         return;
     }
-    StopBlinkInternal();
-    xSemaphoreTake(mutex_, portMAX_DELAY);
+    
+    std::lock_guard<std::mutex> lock(mutex_);
+    esp_timer_stop(blink_timer_);
     led_strip_set_pixel(led_strip_, 0, r_, g_, b_);
     led_strip_refresh(led_strip_);
-    xSemaphoreGive(mutex_);
 }
 
 void Led::TurnOff() {
     if (led_strip_ == nullptr) {
         return;
     }
-    StopBlinkInternal();
-    xSemaphoreTake(mutex_, portMAX_DELAY);
+
+    std::lock_guard<std::mutex> lock(mutex_);
+    esp_timer_stop(blink_timer_);
     led_strip_clear(led_strip_);
-    xSemaphoreGive(mutex_);
 }
 
 void Led::BlinkOnce() {
@@ -87,45 +89,27 @@ void Led::StartBlinkTask(int times, int interval_ms) {
     if (led_strip_ == nullptr) {
         return;
     }
-    StopBlinkInternal();
-    xSemaphoreTake(mutex_, portMAX_DELAY);
+
+    std::lock_guard<std::mutex> lock(mutex_);
+    esp_timer_stop(blink_timer_);
     
-    blink_times_ = times;
+    led_strip_clear(led_strip_);
+    blink_counter_ = times * 2;
     blink_interval_ms_ = interval_ms;
-    should_blink_ = true;
+    esp_timer_start_periodic(blink_timer_, interval_ms * 1000);
+}
 
-    xEventGroupClearBits(blink_event_group_, BLINK_TASK_STOPPED_BIT);
-    xEventGroupSetBits(blink_event_group_, BLINK_TASK_RUNNING_BIT);
+void Led::OnBlinkTimer() {
+    std::lock_guard<std::mutex> lock(mutex_);
+    blink_counter_--;
+    if (blink_counter_ & 1) {
+        led_strip_set_pixel(led_strip_, 0, r_, g_, b_);
+        led_strip_refresh(led_strip_);
+    } else {
+        led_strip_clear(led_strip_);
 
-    xTaskCreate([](void* obj) {
-        auto this_ = static_cast<Led*>(obj);
-        int count = 0;
-        while (this_->should_blink_ && (this_->blink_times_ == BLINK_INFINITE || count < this_->blink_times_)) {
-            xSemaphoreTake(this_->mutex_, portMAX_DELAY);
-            led_strip_set_pixel(this_->led_strip_, 0, this_->r_, this_->g_, this_->b_);
-            led_strip_refresh(this_->led_strip_);
-            xSemaphoreGive(this_->mutex_);
-
-            vTaskDelay(this_->blink_interval_ms_ / portTICK_PERIOD_MS);
-            if (!this_->should_blink_) break;
-
-            xSemaphoreTake(this_->mutex_, portMAX_DELAY);
-            led_strip_clear(this_->led_strip_);
-            xSemaphoreGive(this_->mutex_);
-
-            vTaskDelay(this_->blink_interval_ms_ / portTICK_PERIOD_MS);
-            if (this_->blink_times_ != BLINK_INFINITE) count++;
+        if (blink_counter_ == 0) {
+            esp_timer_stop(blink_timer_);
         }
-        this_->blink_task_ = nullptr;
-        xEventGroupClearBits(this_->blink_event_group_, BLINK_TASK_RUNNING_BIT);
-        xEventGroupSetBits(this_->blink_event_group_, BLINK_TASK_STOPPED_BIT);
-        vTaskDelete(NULL);
-    }, "blink", 2048, this, tskIDLE_PRIORITY, &blink_task_);
-
-    xSemaphoreGive(mutex_);
-}
-
-void Led::StopBlinkInternal() {
-    should_blink_ = false;
-    xEventGroupWaitBits(blink_event_group_, BLINK_TASK_STOPPED_BIT, pdFALSE, pdTRUE, portMAX_DELAY);
+    }
 }

main/boards/common/led.h

@@ -2,10 +2,9 @@
 #define _LED_H_
 
 #include <led_strip.h>
-#include <freertos/semphr.h>
-#include <freertos/task.h>
-#include <freertos/event_groups.h>
+#include <esp_timer.h>
 #include <atomic>
+#include <mutex>
 
 #define BLINK_INFINITE -1
 #define BLINK_TASK_STOPPED_BIT BIT0
@@ -33,17 +32,16 @@ public:
     void SetBlue(uint8_t brightness = DEFAULT_BRIGHTNESS) { SetColor(0, 0, brightness); }
 
 private:
-    SemaphoreHandle_t mutex_;
-    EventGroupHandle_t blink_event_group_;
+    std::mutex mutex_;
     TaskHandle_t blink_task_ = nullptr;
     led_strip_handle_t led_strip_ = nullptr;
     uint8_t r_ = 0, g_ = 0, b_ = 0;
-    int blink_times_ = 0;
+    int blink_counter_ = 0;
     int blink_interval_ms_ = 0;
-    std::atomic<bool> should_blink_{false};
+    esp_timer_handle_t blink_timer_ = nullptr;
 
     void StartBlinkTask(int times, int interval_ms);
-    void StopBlinkInternal();
+    void OnBlinkTimer();
 };
 
 #endif // _LED_H_

main/boards/common/ml307_board.cc

@@ -1,5 +1,7 @@
 #include "ml307_board.h"
+
 #include "application.h"
+#include "display.h"
 #include "font_awesome_symbols.h"
 
 #include <esp_log.h>
@@ -72,10 +74,9 @@ void Ml307Board::WaitForNetworkReady() {
     ESP_LOGI(TAG, "ML307 Module: %s", module_name.c_str());
     ESP_LOGI(TAG, "ML307 IMEI: %s", imei.c_str());
     ESP_LOGI(TAG, "ML307 ICCID: %s", iccid.c_str());
-}
 
-void Ml307Board::Initialize() {
-    ESP_LOGI(TAG, "Initializing Ml307Board");
+    // Close all previous connections
+    modem_.ResetConnections();
 }
 
 Http* Ml307Board::CreateHttp() {

main/boards/common/ml307_board.h

@@ -13,7 +13,6 @@ protected:
 
 public:
     Ml307Board(gpio_num_t tx_pin, gpio_num_t rx_pin, size_t rx_buffer_size = 4096);
-    virtual void Initialize() override;
     virtual void StartNetwork() override;
     virtual Http* CreateHttp() override;
     virtual WebSocket* CreateWebSocket() override;

main/boards/common/wifi_board.cc

@@ -1,4 +1,6 @@
 #include "wifi_board.h"
+
+#include "display.h"
 #include "application.h"
 #include "system_info.h"
 #include "font_awesome_symbols.h"
@@ -62,15 +64,14 @@ void WifiBoard::StartNetwork() {
         
         // Wait forever until reset after configuration
         while (true) {
-            vTaskDelay(pdMS_TO_TICKS(1000));
+            int free_sram = heap_caps_get_free_size(MALLOC_CAP_INTERNAL);
+            int min_free_sram = heap_caps_get_minimum_free_size(MALLOC_CAP_INTERNAL);
+            ESP_LOGI(TAG, "Free internal: %u minimal internal: %u", free_sram, min_free_sram);
+            vTaskDelay(pdMS_TO_TICKS(10000));
         }
     }
 }
 
-void WifiBoard::Initialize() {
-    ESP_LOGI(TAG, "Initializing WifiBoard");
-}
-
 Http* WifiBoard::CreateHttp() {
     return new EspHttp();
 }

main/boards/common/wifi_board.h

@@ -10,7 +10,6 @@ protected:
     virtual std::string GetBoardJson() override;
 
 public:
-    virtual void Initialize() override;
     virtual void StartNetwork() override;
     virtual Http* CreateHttp() override;
     virtual WebSocket* CreateWebSocket() override;

main/boards/esp-box-3/esp_box3_board.cc

@@ -5,6 +5,7 @@
 #include "button.h"
 #include "led.h"
 #include "config.h"
+#include "iot/thing_manager.h"
 
 #include <esp_log.h>
 #include <driver/i2c_master.h>
@@ -19,7 +20,7 @@ private:
     void InitializeI2c() {
         // Initialize I2C peripheral
         i2c_master_bus_config_t i2c_bus_cfg = {
-            .i2c_port = I2C_NUM_1,
+            .i2c_port = (i2c_port_t)1,
             .sda_io_num = AUDIO_CODEC_I2C_SDA_PIN,
             .scl_io_num = AUDIO_CODEC_I2C_SCL_PIN,
             .clk_source = I2C_CLK_SRC_DEFAULT,
@@ -39,15 +40,17 @@ private:
         });
     }
 
-public:
-    EspBox3Board() : boot_button_(BOOT_BUTTON_GPIO) {
+    // 物联网初始化，添加对 AI 可见设备
+    void InitializeIot() {
+        auto& thing_manager = iot::ThingManager::GetInstance();
+        thing_manager.AddThing(iot::CreateThing("Speaker"));
     }
 
-    virtual void Initialize() override {
-        ESP_LOGI(TAG, "Initializing EspBox3Board");
+public:
+    EspBox3Board() : boot_button_(BOOT_BUTTON_GPIO) {
         InitializeI2c();
         InitializeButtons();
-        WifiBoard::Initialize();
+        InitializeIot();
     }
 
     virtual Led* GetBuiltinLed() override {

main/boards/kevin-box-1/kevin_box_board.cc

@@ -5,13 +5,14 @@
 #include "button.h"
 #include "led.h"
 #include "config.h"
+#include "iot/thing_manager.h"
 
 #include <esp_log.h>
 #include <esp_spiffs.h>
 #include <driver/gpio.h>
 #include <driver/i2c_master.h>
 
-static const char *TAG = "KevinBoxBoard";
+#define TAG "KevinBoxBoard"
 
 class KevinBoxBoard : public Ml307Board {
 private:
@@ -48,7 +49,7 @@ private:
 
     void InitializeDisplayI2c() {
         i2c_master_bus_config_t bus_config = {
-            .i2c_port = I2C_NUM_0,
+            .i2c_port = (i2c_port_t)0,
             .sda_io_num = DISPLAY_SDA_PIN,
             .scl_io_num = DISPLAY_SCL_PIN,
             .clk_source = I2C_CLK_SRC_DEFAULT,
@@ -65,7 +66,7 @@ private:
     void InitializeCodecI2c() {
         // Initialize I2C peripheral
         i2c_master_bus_config_t i2c_bus_cfg = {
-            .i2c_port = I2C_NUM_1,
+            .i2c_port = (i2c_port_t)1,
             .sda_io_num = AUDIO_CODEC_I2C_SDA_PIN,
             .scl_io_num = AUDIO_CODEC_I2C_SCL_PIN,
             .clk_source = I2C_CLK_SRC_DEFAULT,
@@ -95,8 +96,7 @@ private:
         });
 
         volume_up_button_.OnLongPress([this]() {
-            auto codec = GetAudioCodec();
-            codec->SetOutputVolume(100);
+            GetAudioCodec()->SetOutputVolume(100);
             GetDisplay()->ShowNotification("最大音量");
         });
 
@@ -111,29 +111,29 @@ private:
         });
 
         volume_down_button_.OnLongPress([this]() {
-            auto codec = GetAudioCodec();
-            codec->SetOutputVolume(0);
+            GetAudioCodec()->SetOutputVolume(0);
             GetDisplay()->ShowNotification("已静音");
         });
     }
 
+    // 物联网初始化，添加对 AI 可见设备
+    void InitializeIot() {
+        auto& thing_manager = iot::ThingManager::GetInstance();
+        thing_manager.AddThing(iot::CreateThing("Speaker"));
+    }
+
 public:
     KevinBoxBoard() : Ml307Board(ML307_TX_PIN, ML307_RX_PIN, 4096),
         boot_button_(BOOT_BUTTON_GPIO),
         volume_up_button_(VOLUME_UP_BUTTON_GPIO),
         volume_down_button_(VOLUME_DOWN_BUTTON_GPIO) {
-    }
-
-    virtual void Initialize() override {
-        ESP_LOGI(TAG, "Initializing KevinBoxBoard");
         InitializeDisplayI2c();
         InitializeCodecI2c();
         MountStorage();
         Enable4GModule();
 
         InitializeButtons();
-
-        Ml307Board::Initialize();
+        InitializeIot();
     }
 
     virtual Led* GetBuiltinLed() override {

main/boards/kevin-box-2/kevin_box_board.cc

@@ -6,6 +6,7 @@
 #include "led.h"
 #include "config.h"
 #include "axp2101.h"
+#include "iot/thing_manager.h"
 
 #include <esp_log.h>
 #include <esp_spiffs.h>
@@ -13,7 +14,7 @@
 #include <driver/i2c_master.h>
 #include <esp_timer.h>
 
-static const char *TAG = "KevinBoxBoard";
+#define TAG "KevinBoxBoard"
 
 class KevinBoxBoard : public Ml307Board {
 private:
@@ -86,7 +87,7 @@ private:
 
     void InitializeDisplayI2c() {
         i2c_master_bus_config_t bus_config = {
-            .i2c_port = I2C_NUM_0,
+            .i2c_port = (i2c_port_t)0,
             .sda_io_num = DISPLAY_SDA_PIN,
             .scl_io_num = DISPLAY_SCL_PIN,
             .clk_source = I2C_CLK_SRC_DEFAULT,
@@ -103,7 +104,7 @@ private:
     void InitializeCodecI2c() {
         // Initialize I2C peripheral
         i2c_master_bus_config_t i2c_bus_cfg = {
-            .i2c_port = I2C_NUM_1,
+            .i2c_port = (i2c_port_t)1,
             .sda_io_num = AUDIO_CODEC_I2C_SDA_PIN,
             .scl_io_num = AUDIO_CODEC_I2C_SCL_PIN,
             .clk_source = I2C_CLK_SRC_DEFAULT,
@@ -118,10 +119,6 @@ private:
     }
 
     void InitializeButtons() {
-        // 测试按住说话
-        // boot_button_.OnClick([this]() {
-        //     Application::GetInstance().ToggleChatState();
-        // });
         boot_button_.OnPressDown([this]() {
             Application::GetInstance().StartListening();
         });
@@ -140,8 +137,7 @@ private:
         });
 
         volume_up_button_.OnLongPress([this]() {
-            auto codec = GetAudioCodec();
-            codec->SetOutputVolume(100);
+            GetAudioCodec()->SetOutputVolume(100);
             GetDisplay()->ShowNotification("最大音量");
         });
 
@@ -156,21 +152,22 @@ private:
         });
 
         volume_down_button_.OnLongPress([this]() {
-            auto codec = GetAudioCodec();
-            codec->SetOutputVolume(0);
+            GetAudioCodec()->SetOutputVolume(0);
             GetDisplay()->ShowNotification("已静音");
         });
     }
 
+    // 物联网初始化，添加对 AI 可见设备
+    void InitializeIot() {
+        auto& thing_manager = iot::ThingManager::GetInstance();
+        thing_manager.AddThing(iot::CreateThing("Speaker"));
+    }
+
 public:
     KevinBoxBoard() : Ml307Board(ML307_TX_PIN, ML307_RX_PIN, 4096),
         boot_button_(BOOT_BUTTON_GPIO),
         volume_up_button_(VOLUME_UP_BUTTON_GPIO),
         volume_down_button_(VOLUME_DOWN_BUTTON_GPIO) {
-    }
-
-    virtual void Initialize() override {
-        ESP_LOGI(TAG, "Initializing KevinBoxBoard");
         InitializeDisplayI2c();
         InitializeCodecI2c();
         axp2101_ = new Axp2101(codec_i2c_bus_, AXP2101_I2C_ADDR);
@@ -180,8 +177,7 @@ public:
 
         InitializeButtons();
         InitializePowerSaveTimer();
-
-        Ml307Board::Initialize();
+        InitializeIot();
     }
 
     virtual Led* GetBuiltinLed() override {

main/boards/kevin-c3/config.h

@@ -0,0 +1,24 @@
+#ifndef _BOARD_CONFIG_H_
+#define _BOARD_CONFIG_H_
+
+#include <driver/gpio.h>
+
+#define AUDIO_INPUT_SAMPLE_RATE  24000
+#define AUDIO_OUTPUT_SAMPLE_RATE 24000
+
+#define AUDIO_I2S_GPIO_MCLK GPIO_NUM_10
+#define AUDIO_I2S_GPIO_WS GPIO_NUM_12
+#define AUDIO_I2S_GPIO_BCLK GPIO_NUM_8
+#define AUDIO_I2S_GPIO_DIN  GPIO_NUM_7
+#define AUDIO_I2S_GPIO_DOUT GPIO_NUM_11
+
+#define AUDIO_CODEC_PA_PIN       GPIO_NUM_13
+#define AUDIO_CODEC_I2C_SDA_PIN  GPIO_NUM_0
+#define AUDIO_CODEC_I2C_SCL_PIN  GPIO_NUM_1
+#define AUDIO_CODEC_ES8311_ADDR  ES8311_CODEC_DEFAULT_ADDR
+
+#define BUILTIN_LED_GPIO        GPIO_NUM_2
+#define BOOT_BUTTON_GPIO        GPIO_NUM_9
+
+
+#endif // _BOARD_CONFIG_H_

main/boards/kevin-c3/kevin_box_board.cc

@@ -0,0 +1,78 @@
+#include "wifi_board.h"
+#include "audio_codecs/es8311_audio_codec.h"
+#include "application.h"
+#include "button.h"
+#include "led.h"
+#include "config.h"
+#include "iot/thing_manager.h"
+
+#include <wifi_station.h>
+#include <esp_log.h>
+#include <driver/i2c_master.h>
+
+#define TAG "KevinBoxBoard"
+
+class KevinBoxBoard : public WifiBoard {
+private:
+    i2c_master_bus_handle_t codec_i2c_bus_;
+    Button boot_button_;
+
+    void InitializeCodecI2c() {
+        // Initialize I2C peripheral
+        i2c_master_bus_config_t i2c_bus_cfg = {
+            .i2c_port = I2C_NUM_0,
+            .sda_io_num = AUDIO_CODEC_I2C_SDA_PIN,
+            .scl_io_num = AUDIO_CODEC_I2C_SCL_PIN,
+            .clk_source = I2C_CLK_SRC_DEFAULT,
+            .glitch_ignore_cnt = 7,
+            .intr_priority = 0,
+            .trans_queue_depth = 0,
+            .flags = {
+                .enable_internal_pullup = 1,
+            },
+        };
+        ESP_ERROR_CHECK(i2c_new_master_bus(&i2c_bus_cfg, &codec_i2c_bus_));
+    }
+
+    void InitializeButtons() {
+        boot_button_.OnClick([this]() {
+            auto& app = Application::GetInstance();
+            if (app.GetChatState() == kChatStateUnknown && !WifiStation::GetInstance().IsConnected()) {
+                ResetWifiConfiguration();
+            }
+        });
+        boot_button_.OnPressDown([this]() {
+            Application::GetInstance().StartListening();
+        });
+        boot_button_.OnPressUp([this]() {
+            Application::GetInstance().StopListening();
+        });
+    }
+
+    // 物联网初始化，添加对 AI 可见设备
+    void InitializeIot() {
+        auto& thing_manager = iot::ThingManager::GetInstance();
+        thing_manager.AddThing(iot::CreateThing("Speaker"));
+    }
+
+public:
+    KevinBoxBoard() : boot_button_(BOOT_BUTTON_GPIO) {
+        InitializeCodecI2c();
+        InitializeButtons();
+        InitializeIot();
+    }
+
+    virtual Led* GetBuiltinLed() override {
+        static Led led(BUILTIN_LED_GPIO);
+        return &led;
+    }
+
+    virtual AudioCodec* GetAudioCodec() override {
+        static Es8311AudioCodec audio_codec(codec_i2c_bus_, I2C_NUM_0, AUDIO_INPUT_SAMPLE_RATE, AUDIO_OUTPUT_SAMPLE_RATE,
+            AUDIO_I2S_GPIO_MCLK, AUDIO_I2S_GPIO_BCLK, AUDIO_I2S_GPIO_WS, AUDIO_I2S_GPIO_DOUT, AUDIO_I2S_GPIO_DIN,
+            AUDIO_CODEC_PA_PIN, AUDIO_CODEC_ES8311_ADDR);
+        return &audio_codec;
+    }
+};
+
+DECLARE_BOARD(KevinBoxBoard);

main/boards/lichuang-dev/config.h

@@ -32,6 +32,9 @@
 #define DISPLAY_MIRROR_Y false
 #define DISPLAY_SWAP_XY true
 
+#define DISPLAY_OFFSET_X  0
+#define DISPLAY_OFFSET_Y  0
+
 #define DISPLAY_BACKLIGHT_PIN GPIO_NUM_42
 #define DISPLAY_BACKLIGHT_OUTPUT_INVERT true
 

main/boards/lichuang-dev/lichuang_dev_board.cc

@@ -6,6 +6,7 @@
 #include "led.h"
 #include "config.h"
 #include "i2c_device.h"
+#include "iot/thing_manager.h"
 
 #include <esp_log.h>
 #include <esp_lcd_panel_vendor.h>
@@ -42,7 +43,7 @@ private:
     void InitializeI2c() {
         // Initialize I2C peripheral
         i2c_master_bus_config_t i2c_bus_cfg = {
-            .i2c_port = I2C_NUM_1,
+            .i2c_port = (i2c_port_t)1,
             .sda_io_num = AUDIO_CODEC_I2C_SDA_PIN,
             .scl_io_num = AUDIO_CODEC_I2C_SCL_PIN,
             .clk_source = I2C_CLK_SRC_DEFAULT,
@@ -76,7 +77,12 @@ private:
             if (app.GetChatState() == kChatStateUnknown && !WifiStation::GetInstance().IsConnected()) {
                 ResetWifiConfiguration();
             }
-            app.ToggleChatState();
+        });
+        boot_button_.OnPressDown([this]() {
+            Application::GetInstance().StartListening();
+        });
+        boot_button_.OnPressUp([this]() {
+            Application::GetInstance().StopListening();
         });
     }
 
@@ -111,20 +117,22 @@ private:
         esp_lcd_panel_swap_xy(panel, DISPLAY_SWAP_XY);
         esp_lcd_panel_mirror(panel, DISPLAY_MIRROR_X, DISPLAY_MIRROR_Y);
         display_ = new St7789Display(panel_io, panel, DISPLAY_BACKLIGHT_PIN, DISPLAY_BACKLIGHT_OUTPUT_INVERT,
-                                    DISPLAY_WIDTH, DISPLAY_HEIGHT, DISPLAY_MIRROR_X, DISPLAY_MIRROR_Y, DISPLAY_SWAP_XY);
+                                    DISPLAY_WIDTH, DISPLAY_HEIGHT, DISPLAY_OFFSET_X, DISPLAY_OFFSET_Y, DISPLAY_MIRROR_X, DISPLAY_MIRROR_Y, DISPLAY_SWAP_XY);
+    }
+
+    // 物联网初始化，添加对 AI 可见设备
+    void InitializeIot() {
+        auto& thing_manager = iot::ThingManager::GetInstance();
+        thing_manager.AddThing(iot::CreateThing("Speaker"));
     }
 
 public:
     LichuangDevBoard() : boot_button_(BOOT_BUTTON_GPIO) {
-    }
-
-    virtual void Initialize() override {
-        ESP_LOGI(TAG, "Initializing LichuangDevBoard");
         InitializeI2c();
         InitializeSpi();
         InitializeSt7789Display();
         InitializeButtons();
-        WifiBoard::Initialize();
+        InitializeIot();
     }
 
     virtual Led* GetBuiltinLed() override {

main/display/display.h

@@ -41,7 +41,7 @@ protected:
     esp_timer_handle_t update_timer_ = nullptr;
 
     friend class DisplayLockGuard;
-    virtual void Lock() = 0;
+    virtual bool Lock(int timeout_ms = 0) = 0;
     virtual void Unlock() = 0;
 
     virtual void Update();

main/display/no_display.cc

@@ -4,6 +4,8 @@ NoDisplay::NoDisplay() {}
 
 NoDisplay::~NoDisplay() {}
 
-void NoDisplay::Lock() {}
+bool NoDisplay::Lock(int timeout_ms) {
+    return true;
+}
 
 void NoDisplay::Unlock() {}

main/display/no_display.h

@@ -5,7 +5,7 @@
 
 class NoDisplay : public Display {
 private:
-    virtual void Lock() override;
+    virtual bool Lock(int timeout_ms = 0) override;
     virtual void Unlock() override;
 
 public:

main/display/ssd1306_display.cc

@@ -126,8 +126,8 @@ Ssd1306Display::~Ssd1306Display() {
     lvgl_port_deinit();
 }
 
-void Ssd1306Display::Lock() {
-    lvgl_port_lock(0);
+bool Ssd1306Display::Lock(int timeout_ms) {
+    return lvgl_port_lock(timeout_ms);
 }
 
 void Ssd1306Display::Unlock() {

main/display/ssd1306_display.h

@@ -18,7 +18,7 @@ private:
     lv_obj_t* container_ = nullptr;
     lv_obj_t* side_bar_ = nullptr;
 
-    virtual void Lock() override;
+    virtual bool Lock(int timeout_ms = 0) override;
     virtual void Unlock() override;
 
     void SetupUI_128x64();

main/display/st7789_display.cc

@@ -3,29 +3,120 @@
 
 #include <esp_log.h>
 #include <esp_err.h>
-#include <esp_lvgl_port.h>
 #include <driver/ledc.h>
 #include <vector>
 
 #define TAG "St7789Display"
 #define LCD_LEDC_CH LEDC_CHANNEL_0
 
+#define ST7789_LVGL_TICK_PERIOD_MS 2
+#define ST7789_LVGL_TASK_MAX_DELAY_MS 20
+#define ST7789_LVGL_TASK_MIN_DELAY_MS 1
+#define ST7789_LVGL_TASK_STACK_SIZE (4 * 1024)
+#define ST7789_LVGL_TASK_PRIORITY 10
+
 LV_FONT_DECLARE(font_puhui_14_1);
 LV_FONT_DECLARE(font_awesome_30_1);
 LV_FONT_DECLARE(font_awesome_14_1);
 
+static lv_disp_drv_t disp_drv;
+static void st7789_lvgl_flush_cb(lv_disp_drv_t *drv, const lv_area_t *area, lv_color_t *color_map)
+{
+    esp_lcd_panel_handle_t panel_handle = (esp_lcd_panel_handle_t)drv->user_data;
+    int offsetx1 = area->x1;
+    int offsetx2 = area->x2;
+    int offsety1 = area->y1;
+    int offsety2 = area->y2;
+    // copy a buffer's content to a specific area of the display
+    esp_lcd_panel_draw_bitmap(panel_handle, offsetx1, offsety1, offsetx2 + 1, offsety2 + 1, color_map);
+    lv_disp_flush_ready(&disp_drv);
+}
+
+/* Rotate display and touch, when rotated screen in LVGL. Called when driver parameters are updated. */
+static void st7789_lvgl_port_update_callback(lv_disp_drv_t *drv)
+{
+    esp_lcd_panel_handle_t panel_handle = (esp_lcd_panel_handle_t)drv->user_data;
+
+    switch (drv->rotated)
+    {
+    case LV_DISP_ROT_NONE:
+        // Rotate LCD display
+        esp_lcd_panel_swap_xy(panel_handle, false);
+        esp_lcd_panel_mirror(panel_handle, true, false);
+#if CONFIG_ST7789_LCD_TOUCH_ENABLED
+        // Rotate LCD touch
+        esp_lcd_touch_set_mirror_y(tp, false);
+        esp_lcd_touch_set_mirror_x(tp, false);
+#endif
+        break;
+    case LV_DISP_ROT_90:
+        // Rotate LCD display
+        esp_lcd_panel_swap_xy(panel_handle, true);
+        esp_lcd_panel_mirror(panel_handle, true, true);
+#if CONFIG_ST7789_LCD_TOUCH_ENABLED
+        // Rotate LCD touch
+        esp_lcd_touch_set_mirror_y(tp, false);
+        esp_lcd_touch_set_mirror_x(tp, false);
+#endif
+        break;
+    case LV_DISP_ROT_180:
+        // Rotate LCD display
+        esp_lcd_panel_swap_xy(panel_handle, false);
+        esp_lcd_panel_mirror(panel_handle, false, true);
+#if CONFIG_ST7789_LCD_TOUCH_ENABLED
+        // Rotate LCD touch
+        esp_lcd_touch_set_mirror_y(tp, false);
+        esp_lcd_touch_set_mirror_x(tp, false);
+#endif
+        break;
+    case LV_DISP_ROT_270:
+        // Rotate LCD display
+        esp_lcd_panel_swap_xy(panel_handle, true);
+        esp_lcd_panel_mirror(panel_handle, false, false);
+#if CONFIG_ST7789_LCD_TOUCH_ENABLED
+        // Rotate LCD touch
+        esp_lcd_touch_set_mirror_y(tp, false);
+        esp_lcd_touch_set_mirror_x(tp, false);
+#endif
+        break;
+    }
+}
+
+void St7789Display::LvglTask() {
+    ESP_LOGI(TAG, "Starting LVGL task");
+    uint32_t task_delay_ms = ST7789_LVGL_TASK_MAX_DELAY_MS;
+    while (1)
+    {
+        // Lock the mutex due to the LVGL APIs are not thread-safe
+        if (Lock())
+        {
+            task_delay_ms = lv_timer_handler();
+            Unlock();
+        }
+        if (task_delay_ms > ST7789_LVGL_TASK_MAX_DELAY_MS)
+        {
+            task_delay_ms = ST7789_LVGL_TASK_MAX_DELAY_MS;
+        }
+        else if (task_delay_ms < ST7789_LVGL_TASK_MIN_DELAY_MS)
+        {
+            task_delay_ms = ST7789_LVGL_TASK_MIN_DELAY_MS;
+        }
+        vTaskDelay(pdMS_TO_TICKS(task_delay_ms));
+    }
+}
+
+
 St7789Display::St7789Display(esp_lcd_panel_io_handle_t panel_io, esp_lcd_panel_handle_t panel,
                            gpio_num_t backlight_pin, bool backlight_output_invert,
-                           int width, int height, bool mirror_x, bool mirror_y, bool swap_xy)
+                           int width, int height, int offset_x, int offset_y, bool mirror_x, bool mirror_y, bool swap_xy)
     : panel_io_(panel_io), panel_(panel), backlight_pin_(backlight_pin), backlight_output_invert_(backlight_output_invert),
       mirror_x_(mirror_x), mirror_y_(mirror_y), swap_xy_(swap_xy) {
     width_ = width;
     height_ = height;
+    offset_x_ = offset_x;
+    offset_y_ = offset_y;
 
-    ESP_LOGI(TAG, "Initialize LVGL");
-    lvgl_port_cfg_t port_cfg = ESP_LVGL_PORT_INIT_CONFIG();
-    lvgl_port_init(&port_cfg);
-
+    
     InitializeBacklight(backlight_pin);
 
     // draw white
@@ -38,32 +129,52 @@ St7789Display::St7789Display(esp_lcd_panel_io_handle_t panel_io, esp_lcd_panel_h
     ESP_LOGI(TAG, "Turning display on");
     ESP_ERROR_CHECK(esp_lcd_panel_disp_on_off(panel_, true));
 
-    ESP_LOGI(TAG, "Adding LCD screen");
-    const lvgl_port_display_cfg_t display_cfg = {
-        .io_handle = panel_io_,
-        .panel_handle = panel_,
-        .control_handle = nullptr,
-        .buffer_size = static_cast<uint32_t>(width_ * 20),
-        .double_buffer = false,
-        .trans_size = 0,
-        .hres = static_cast<uint32_t>(width_),
-        .vres = static_cast<uint32_t>(height_),
-        .monochrome = false,
-        .rotation = {
-            .swap_xy = swap_xy_,
-            .mirror_x = mirror_x_,
-            .mirror_y = mirror_y_,
-        },
-        .flags = {
-            .buff_dma = 1,
-            .buff_spiram = 0,
-            .sw_rotate = 0,
-            .full_refresh = 0,
-            .direct_mode = 0,
-        },
-    };
+    ESP_LOGI(TAG, "Initialize LVGL library");
+    lv_init();
+    // alloc draw buffers used by LVGL
+    static lv_disp_draw_buf_t disp_buf; // contains internal graphic buffer(s) called draw buffer(s)
+    // it's recommended to choose the size of the draw buffer(s) to be at least 1/10 screen sized
+    lv_color_t *buf1 = (lv_color_t *)heap_caps_malloc(width_ * 10 * sizeof(lv_color_t), MALLOC_CAP_DMA);
+    assert(buf1);
+    lv_color_t *buf2 = (lv_color_t *)heap_caps_malloc(width_ * 10 * sizeof(lv_color_t), MALLOC_CAP_DMA);
+    assert(buf2);
+    // initialize LVGL draw buffers
+    lv_disp_draw_buf_init(&disp_buf, buf1, buf2, width_ * 10);
 
-    disp_ = lvgl_port_add_disp(&display_cfg);
+    ESP_LOGI(TAG, "Register display driver to LVGL");
+    lv_disp_drv_init(&disp_drv);
+    disp_drv.hor_res = width_;
+    disp_drv.ver_res = height_;
+    disp_drv.offset_x = offset_x_;
+    disp_drv.offset_y = offset_y_;
+    disp_drv.flush_cb = st7789_lvgl_flush_cb;
+    disp_drv.drv_update_cb = st7789_lvgl_port_update_callback;
+    disp_drv.draw_buf = &disp_buf;
+    disp_drv.user_data = panel_;
+
+    lv_disp_drv_register(&disp_drv);
+
+    ESP_LOGI(TAG, "Install LVGL tick timer");
+    // Tick interface for LVGL (using esp_timer to generate 2ms periodic event)
+    const esp_timer_create_args_t lvgl_tick_timer_args = {
+        .callback = [](void* arg) {
+            lv_tick_inc(ST7789_LVGL_TICK_PERIOD_MS);
+        },
+        .arg = NULL,
+        .dispatch_method = ESP_TIMER_TASK,
+        .name = "LVGL Tick Timer",
+        .skip_unhandled_events = false
+    };
+    ESP_ERROR_CHECK(esp_timer_create(&lvgl_tick_timer_args, &lvgl_tick_timer_));
+    ESP_ERROR_CHECK(esp_timer_start_periodic(lvgl_tick_timer_, ST7789_LVGL_TICK_PERIOD_MS * 1000));
+
+    lvgl_mutex_ = xSemaphoreCreateRecursiveMutex();
+    assert(lvgl_mutex_ != nullptr);
+    ESP_LOGI(TAG, "Create LVGL task");
+    xTaskCreate([](void *arg) {
+        static_cast<St7789Display*>(arg)->LvglTask();
+        vTaskDelete(NULL);
+    }, "LVGL", ST7789_LVGL_TASK_STACK_SIZE, this, ST7789_LVGL_TASK_PRIORITY, NULL);
 
     SetBacklight(100);
 
@@ -71,6 +182,9 @@ St7789Display::St7789Display(esp_lcd_panel_io_handle_t panel_io, esp_lcd_panel_h
 }
 
 St7789Display::~St7789Display() {
+    ESP_ERROR_CHECK(esp_timer_stop(lvgl_tick_timer_));
+    ESP_ERROR_CHECK(esp_timer_delete(lvgl_tick_timer_));
+
     if (content_ != nullptr) {
         lv_obj_del(content_);
     }
@@ -90,7 +204,7 @@ St7789Display::~St7789Display() {
     if (panel_io_ != nullptr) {
         esp_lcd_panel_io_del(panel_io_);
     }
-    lvgl_port_deinit();
+    vSemaphoreDelete(lvgl_mutex_);
 }
 
 void St7789Display::InitializeBacklight(gpio_num_t backlight_pin) {
@@ -140,18 +254,21 @@ void St7789Display::SetBacklight(uint8_t brightness) {
     ESP_ERROR_CHECK(ledc_update_duty(LEDC_LOW_SPEED_MODE, LCD_LEDC_CH));
 }
 
-void St7789Display::Lock() {
-    lvgl_port_lock(0);
+bool St7789Display::Lock(int timeout_ms) {
+    // Convert timeout in milliseconds to FreeRTOS ticks
+    // If `timeout_ms` is set to 0, the program will block until the condition is met
+    const TickType_t timeout_ticks = (timeout_ms == 0) ? portMAX_DELAY : pdMS_TO_TICKS(timeout_ms);
+    return xSemaphoreTakeRecursive(lvgl_mutex_, timeout_ticks) == pdTRUE;
 }
 
 void St7789Display::Unlock() {
-    lvgl_port_unlock();
+    xSemaphoreGiveRecursive(lvgl_mutex_);
 }
 
 void St7789Display::SetupUI() {
     DisplayLockGuard lock(this);
 
-    auto screen = lv_disp_get_scr_act(disp_);
+    auto screen = lv_disp_get_scr_act(lv_disp_get_default());
     lv_obj_set_style_text_font(screen, &font_puhui_14_1, 0);
     lv_obj_set_style_text_color(screen, lv_color_black(), 0);
 

main/display/st7789_display.h

@@ -3,9 +3,12 @@
 
 #include "display.h"
 
+#include <freertos/FreeRTOS.h>
+#include <freertos/semphr.h>
 #include <driver/gpio.h>
 #include <esp_lcd_panel_io.h>
 #include <esp_lcd_panel_ops.h>
+#include <esp_timer.h>
 
 class St7789Display : public Display {
 private:
@@ -16,6 +19,10 @@ private:
     bool mirror_x_ = false;
     bool mirror_y_ = false;
     bool swap_xy_ = false;
+    int offset_x_ = 0;
+    int offset_y_ = 0;
+    SemaphoreHandle_t lvgl_mutex_ = nullptr;
+    esp_timer_handle_t lvgl_tick_timer_ = nullptr;
     
     lv_obj_t* status_bar_ = nullptr;
     lv_obj_t* content_ = nullptr;
@@ -25,14 +32,15 @@ private:
     void InitializeBacklight(gpio_num_t backlight_pin);
     void SetBacklight(uint8_t brightness);
     void SetupUI();
+    void LvglTask();
 
-    virtual void Lock() override;
+    virtual bool Lock(int timeout_ms = 0) override;
     virtual void Unlock() override;
 
 public:
     St7789Display(esp_lcd_panel_io_handle_t panel_io, esp_lcd_panel_handle_t panel,
                   gpio_num_t backlight_pin, bool backlight_output_invert,
-                  int width, int height, bool mirror_x, bool mirror_y, bool swap_xy);
+                  int width, int height,  int offset_x, int offset_y, bool mirror_x, bool mirror_y, bool swap_xy);
     ~St7789Display();
 };
 

main/idf_component.yml

@@ -1,8 +1,8 @@
 ## IDF Component Manager Manifest File
 dependencies:
   78/esp-wifi-connect: "~1.4.1"
-  78/esp-opus-encoder: "~1.1.0"
-  78/esp-ml307: "~1.6.3"
+  78/esp-opus-encoder: "~2.0.0"
+  78/esp-ml307: "~1.7.0"
   espressif/led_strip: "^2.4.1"
   espressif/esp_codec_dev: "^1.3.1"
   espressif/esp-sr: "^1.9.0"

main/iot/sample_interface.json

@@ -0,0 +1,38 @@
+[
+    {
+        "name": "lamp",
+        "description": "A lamp",
+        "properties": {
+            "power": {
+                "type": "boolean",
+                "description": "Whether the lamp is on or off"
+            }
+        },
+        "methods": {
+            "TurnOn": {
+                "description": "Turns the lamp on"
+            }
+        }
+    },
+    {
+        "name": "speaker",
+        "description": "当前 AI 机器人的扬声器",
+        "properties": {
+            "volume": {
+                "type": "number",
+                "description": "当前扬声器的音量（0-100）"
+            }
+        },
+        "methods": {
+            "SetVolume": {
+                "description": "设置当前扬声器的音量",
+                "parameters": {
+                    "volume": {
+                        "type": "number",
+                        "description": "The volume of the speaker (0-100)"
+                    }
+                }
+            }
+        }
+    }
+]

main/iot/thing.cc

@@ -0,0 +1,77 @@
+#include "thing.h"
+#include "application.h"
+
+#include <esp_log.h>
+
+#define TAG "Thing"
+
+
+namespace iot {
+
+static std::map<std::string, std::function<Thing*()>>* thing_creators = nullptr;
+
+void RegisterThing(const std::string& type, std::function<Thing*()> creator) {
+    if (thing_creators == nullptr) {
+        thing_creators = new std::map<std::string, std::function<Thing*()>>();
+    }
+    (*thing_creators)[type] = creator;
+}
+
+Thing* CreateThing(const std::string& type) {
+    auto creator = thing_creators->find(type);
+    if (creator == thing_creators->end()) {
+        ESP_LOGE(TAG, "Thing type not found: %s", type.c_str());
+        return nullptr;
+    }
+    return creator->second();
+}
+
+std::string Thing::GetDescriptorJson() {
+    std::string json_str = "{";
+    json_str += "\"name\":\"" + name_ + "\",";
+    json_str += "\"description\":\"" + description_ + "\",";
+    json_str += "\"properties\":" + properties_.GetDescriptorJson() + ",";
+    json_str += "\"methods\":" + methods_.GetDescriptorJson();
+    json_str += "}";
+    return json_str;
+}
+
+std::string Thing::GetStateJson() {
+    std::string json_str = "{";
+    json_str += "\"name\":\"" + name_ + "\",";
+    json_str += "\"state\":" + properties_.GetStateJson();
+    json_str += "}";
+    return json_str;
+}
+
+void Thing::Invoke(const cJSON* command) {
+    auto method_name = cJSON_GetObjectItem(command, "method");
+    auto input_params = cJSON_GetObjectItem(command, "parameters");
+
+    try {
+        auto& method = methods_[method_name->valuestring];
+        for (auto& param : method.parameters()) {
+            auto input_param = cJSON_GetObjectItem(input_params, param.name().c_str());
+            if (param.required() && input_param == nullptr) {
+                throw std::runtime_error("Parameter " + param.name() + " is required");
+            }
+            if (param.type() == kValueTypeNumber) {
+                param.set_number(input_param->valueint);
+            } else if (param.type() == kValueTypeString) {
+                param.set_string(input_param->valuestring);
+            } else if (param.type() == kValueTypeBoolean) {
+                param.set_boolean(input_param->valueint == 1);
+            }
+        }
+
+        Application::GetInstance().Schedule([&method]() {
+            method.Invoke();
+        });
+    } catch (const std::runtime_error& e) {
+        ESP_LOGE(TAG, "Method not found: %s", method_name->valuestring);
+        return;
+    }
+}
+
+
+} // namespace iot

main/iot/thing.h

@@ -0,0 +1,300 @@
+#ifndef THING_H
+#define THING_H
+
+#include <string>
+#include <map>
+#include <functional>
+#include <vector>
+#include <stdexcept>
+#include <cJSON.h>
+
+namespace iot {
+
+enum ValueType {
+    kValueTypeBoolean,
+    kValueTypeNumber,
+    kValueTypeString
+};
+
+class Property {
+private:
+    std::string name_;
+    std::string description_;
+    ValueType type_;
+    std::function<bool()> boolean_getter_;
+    std::function<int()> number_getter_;
+    std::function<std::string()> string_getter_;
+
+public:
+    Property(const std::string& name, const std::string& description, std::function<bool()> getter) :
+        name_(name), description_(description), type_(kValueTypeBoolean), boolean_getter_(getter) {}
+    Property(const std::string& name, const std::string& description, std::function<int()> getter) :
+        name_(name), description_(description), type_(kValueTypeNumber), number_getter_(getter) {}
+    Property(const std::string& name, const std::string& description, std::function<std::string()> getter) :
+        name_(name), description_(description), type_(kValueTypeString), string_getter_(getter) {}
+
+    const std::string& name() const { return name_; }
+    const std::string& description() const { return description_; }
+    ValueType type() const { return type_; }
+
+    bool boolean() const { return boolean_getter_(); }
+    int number() const { return number_getter_(); }
+    std::string string() const { return string_getter_(); }
+
+    std::string GetDescriptorJson() {
+        std::string json_str = "{";
+        json_str += "\"description\":\"" + description_ + "\",";
+        if (type_ == kValueTypeBoolean) {
+            json_str += "\"type\":\"boolean\"";
+        } else if (type_ == kValueTypeNumber) {
+            json_str += "\"type\":\"number\"";
+        } else if (type_ == kValueTypeString) {
+            json_str += "\"type\":\"string\"";
+        }
+        json_str += "}";
+        return json_str;
+    }
+
+    std::string GetStateJson() {
+        if (type_ == kValueTypeBoolean) {
+            return boolean_getter_() ? "true" : "false";
+        } else if (type_ == kValueTypeNumber) {
+            return std::to_string(number_getter_());
+        } else if (type_ == kValueTypeString) {
+            return "\"" + string_getter_() + "\"";
+        }
+        return "null";
+    }
+};
+
+class PropertyList {
+private:
+    std::vector<Property> properties_;
+
+public:
+    PropertyList() = default;
+    PropertyList(const std::vector<Property>& properties) : properties_(properties) {}
+
+    void AddBooleanProperty(const std::string& name, const std::string& description, std::function<bool()> getter) {
+        properties_.push_back(Property(name, description, getter));
+    }
+    void AddNumberProperty(const std::string& name, const std::string& description, std::function<int()> getter) {
+        properties_.push_back(Property(name, description, getter));
+    }
+    void AddStringProperty(const std::string& name, const std::string& description, std::function<std::string()> getter) {
+        properties_.push_back(Property(name, description, getter));
+    }
+
+    const Property& operator[](const std::string& name) const {
+        for (auto& property : properties_) {
+            if (property.name() == name) {
+                return property;
+            }
+        }
+        throw std::runtime_error("Property not found: " + name);
+    }
+
+    std::string GetDescriptorJson() {
+        std::string json_str = "{";
+        for (auto& property : properties_) {
+            json_str += "\"" + property.name() + "\":" + property.GetDescriptorJson() + ",";
+        }
+        if (json_str.back() == ',') {
+            json_str.pop_back();
+        }
+        json_str += "}";
+        return json_str;
+    }
+
+    std::string GetStateJson() {
+        std::string json_str = "{";
+        for (auto& property : properties_) {
+            json_str += "\"" + property.name() + "\":" + property.GetStateJson() + ",";
+        }
+        if (json_str.back() == ',') {
+            json_str.pop_back();
+        }
+        json_str += "}";
+        return json_str;
+    }
+};
+
+class Parameter {
+private:
+    std::string name_;
+    std::string description_;
+    ValueType type_;
+    bool required_;
+    bool boolean_;
+    int number_;
+    std::string string_;
+
+public:
+    Parameter(const std::string& name, const std::string& description, ValueType type, bool required = true) :
+        name_(name), description_(description), type_(type), required_(required) {}
+
+    const std::string& name() const { return name_; }
+    const std::string& description() const { return description_; }
+    ValueType type() const { return type_; }
+    bool required() const { return required_; }
+
+    bool boolean() const { return boolean_; }
+    int number() const { return number_; }
+    const std::string& string() const { return string_; }
+
+    void set_boolean(bool value) { boolean_ = value; }
+    void set_number(int value) { number_ = value; }
+    void set_string(const std::string& value) { string_ = value; }
+
+    std::string GetDescriptorJson() {
+        std::string json_str = "{";
+        json_str += "\"description\":\"" + description_ + "\",";
+        if (type_ == kValueTypeBoolean) {
+            json_str += "\"type\":\"boolean\"";
+        } else if (type_ == kValueTypeNumber) {
+            json_str += "\"type\":\"number\"";
+        } else if (type_ == kValueTypeString) {
+            json_str += "\"type\":\"string\"";
+        }
+        json_str += "}";
+        return json_str;
+    }
+};
+
+class ParameterList {
+private:
+    std::vector<Parameter> parameters_;
+
+public:
+    ParameterList() = default;
+    ParameterList(const std::vector<Parameter>& parameters) : parameters_(parameters) {}
+    void AddParameter(const Parameter& parameter) {
+        parameters_.push_back(parameter);
+    }
+
+    const Parameter& operator[](const std::string& name) const {
+        for (auto& parameter : parameters_) {
+            if (parameter.name() == name) {
+                return parameter;
+            }
+        }
+        throw std::runtime_error("Parameter not found: " + name);
+    }
+
+    // iterator
+    auto begin() { return parameters_.begin(); }
+    auto end() { return parameters_.end(); }
+
+    std::string GetDescriptorJson() {
+        std::string json_str = "{";
+        for (auto& parameter : parameters_) {
+            json_str += "\"" + parameter.name() + "\":" + parameter.GetDescriptorJson() + ",";
+        }
+        if (json_str.back() == ',') {
+            json_str.pop_back();
+        }
+        json_str += "}";
+        return json_str;
+    }
+};
+
+class Method {
+private:
+    std::string name_;
+    std::string description_;
+    ParameterList parameters_;
+    std::function<void(const ParameterList&)> callback_;
+
+public:
+    Method(const std::string& name, const std::string& description, const ParameterList& parameters, std::function<void(const ParameterList&)> callback) :
+        name_(name), description_(description), parameters_(parameters), callback_(callback) {}
+
+    const std::string& name() const { return name_; }
+    const std::string& description() const { return description_; }
+    ParameterList& parameters() { return parameters_; }
+
+    std::string GetDescriptorJson() {
+        std::string json_str = "{";
+        json_str += "\"description\":\"" + description_ + "\",";
+        json_str += "\"parameters\":" + parameters_.GetDescriptorJson();
+        json_str += "}";
+        return json_str;
+    }
+
+    void Invoke() {
+        callback_(parameters_);
+    }
+};
+
+class MethodList {
+private:
+    std::vector<Method> methods_;
+
+public:
+    MethodList() = default;
+    MethodList(const std::vector<Method>& methods) : methods_(methods) {}
+
+    void AddMethod(const std::string& name, const std::string& description, const ParameterList& parameters, std::function<void(const ParameterList&)> callback) {
+        methods_.push_back(Method(name, description, parameters, callback));
+    }
+
+    Method& operator[](const std::string& name) {
+        for (auto& method : methods_) {
+            if (method.name() == name) {
+                return method;
+            }
+        }
+        throw std::runtime_error("Method not found: " + name);
+    }
+
+    std::string GetDescriptorJson() {
+        std::string json_str = "{";
+        for (auto& method : methods_) {
+            json_str += "\"" + method.name() + "\":" + method.GetDescriptorJson() + ",";
+        }
+        if (json_str.back() == ',') {
+            json_str.pop_back();
+        }
+        json_str += "}";
+        return json_str;
+    }
+};
+
+class Thing {
+public:
+    Thing(const std::string& name, const std::string& description) :
+        name_(name), description_(description) {}
+    virtual ~Thing() = default;
+
+    virtual std::string GetDescriptorJson();
+    virtual std::string GetStateJson();
+    virtual void Invoke(const cJSON* command);
+
+    const std::string& name() const { return name_; }
+    const std::string& description() const { return description_; }
+
+protected:
+    PropertyList properties_;
+    MethodList methods_;
+
+private:
+    std::string name_;
+    std::string description_;
+};
+
+
+void RegisterThing(const std::string& type, std::function<Thing*()> creator);
+Thing* CreateThing(const std::string& type);
+
+#define DECLARE_THING(TypeName) \
+    static iot::Thing* Create##TypeName() { \
+        return new iot::TypeName(); \
+    } \
+    static bool Register##TypeNameHelper = []() { \
+        RegisterThing(#TypeName, Create##TypeName); \
+        return true; \
+    }();
+
+} // namespace iot
+
+#endif // THING_H

main/iot/thing_manager.cc

@@ -0,0 +1,47 @@
+#include "thing_manager.h"
+
+#include <esp_log.h>
+
+#define TAG "ThingManager"
+
+namespace iot {
+
+void ThingManager::AddThing(Thing* thing) {
+    things_.push_back(thing);
+}
+
+std::string ThingManager::GetDescriptorsJson() {
+    std::string json_str = "[";
+    for (auto& thing : things_) {
+        json_str += thing->GetDescriptorJson() + ",";
+    }
+    if (json_str.back() == ',') {
+        json_str.pop_back();
+    }
+    json_str += "]";
+    return json_str;
+}
+
+std::string ThingManager::GetStatesJson() {
+    std::string json_str = "[";
+    for (auto& thing : things_) {
+        json_str += thing->GetStateJson() + ",";
+    }
+    if (json_str.back() == ',') {
+        json_str.pop_back();
+    }
+    json_str += "]";
+    return json_str;
+}
+
+void ThingManager::Invoke(const cJSON* command) {
+    auto name = cJSON_GetObjectItem(command, "name");
+    for (auto& thing : things_) {
+        if (thing->name() == name->valuestring) {
+            thing->Invoke(command);
+            return;
+        }
+    }
+}
+
+} // namespace iot

main/iot/thing_manager.h

@@ -0,0 +1,41 @@
+#ifndef THING_MANAGER_H
+#define THING_MANAGER_H
+
+
+#include "thing.h"
+
+#include <cJSON.h>
+
+#include <vector>
+#include <memory>
+#include <functional>
+#include <map>
+
+namespace iot {
+
+class ThingManager {
+public:
+    static ThingManager& GetInstance() {
+        static ThingManager instance;
+        return instance;
+    }
+    ThingManager(const ThingManager&) = delete;
+    ThingManager& operator=(const ThingManager&) = delete;
+
+    void AddThing(Thing* thing);
+
+    std::string GetDescriptorsJson();
+    std::string GetStatesJson();
+    void Invoke(const cJSON* command);
+
+private:
+    ThingManager() = default;
+    ~ThingManager() = default;
+
+    std::vector<Thing*> things_;
+};
+
+
+} // namespace iot
+
+#endif // THING_MANAGER_H

main/iot/things/lamp.cc

@@ -0,0 +1,54 @@
+#include "iot/thing.h"
+#include "board.h"
+#include "audio_codec.h"
+
+#include <driver/gpio.h>
+#include <esp_log.h>
+
+#define TAG "Lamp"
+
+namespace iot {
+
+// 这里仅定义 Lamp 的属性和方法，不包含具体的实现
+class Lamp : public Thing {
+private:
+    gpio_num_t gpio_num_ = GPIO_NUM_18;
+    bool power_ = false;
+
+    void InitializeGpio() {
+        gpio_config_t config = {
+            .pin_bit_mask = (1ULL << gpio_num_),
+            .mode = GPIO_MODE_OUTPUT,
+            .pull_up_en = GPIO_PULLUP_DISABLE,
+            .pull_down_en = GPIO_PULLDOWN_DISABLE,
+            .intr_type = GPIO_INTR_DISABLE,
+        };
+        ESP_ERROR_CHECK(gpio_config(&config));
+        gpio_set_level(gpio_num_, 0);
+    }
+
+public:
+    Lamp() : Thing("Lamp", "一个测试用的灯"), power_(false) {
+        InitializeGpio();
+
+        // 定义设备的属性
+        properties_.AddBooleanProperty("power", "灯是否打开", [this]() -> bool {
+            return power_;
+        });
+
+        // 定义设备可以被远程执行的指令
+        methods_.AddMethod("TurnOn", "打开灯", ParameterList(), [this](const ParameterList& parameters) {
+            power_ = true;
+            gpio_set_level(gpio_num_, 1);
+        });
+
+        methods_.AddMethod("TurnOff", "关闭灯", ParameterList(), [this](const ParameterList& parameters) {
+            power_ = false;
+            gpio_set_level(gpio_num_, 0);
+        });
+    }
+};
+
+} // namespace iot
+
+DECLARE_THING(Lamp);

main/iot/things/speaker.cc

@@ -0,0 +1,33 @@
+#include "iot/thing.h"
+#include "board.h"
+#include "audio_codec.h"
+
+#include <esp_log.h>
+
+#define TAG "Speaker"
+
+namespace iot {
+
+// 这里仅定义 Speaker 的属性和方法，不包含具体的实现
+class Speaker : public Thing {
+public:
+    Speaker() : Thing("Speaker", "当前 AI 机器人的扬声器") {
+        // 定义设备的属性
+        properties_.AddNumberProperty("volume", "当前音量值", [this]() -> int {
+            auto codec = Board::GetInstance().GetAudioCodec();
+            return codec->output_volume();
+        });
+
+        // 定义设备可以被远程执行的指令
+        methods_.AddMethod("SetVolume", "设置音量", ParameterList({
+            Parameter("volume", "0到100之间的整数", kValueTypeNumber, true)
+        }), [this](const ParameterList& parameters) {
+            auto codec = Board::GetInstance().GetAudioCodec();
+            codec->SetOutputVolume(static_cast<uint8_t>(parameters["volume"].number()));
+        });
+    }
+};
+
+} // namespace iot
+
+DECLARE_THING(Speaker);

main/ota.cc

@@ -50,11 +50,11 @@ bool Ota::CheckVersion() {
     }
 
     http->SetHeader("Content-Type", "application/json");
-    if (post_data_.length() > 0) {
-        http->SetContent(post_data_);
-        http->Open("POST", check_version_url_);
-    } else {
-        http->Open("GET", check_version_url_);
+    std::string method = post_data_.length() > 0 ? "POST" : "GET";
+    if (!http->Open(method, check_version_url_, post_data_)) {
+        ESP_LOGE(TAG, "Failed to open HTTP connection");
+        delete http;
+        return false;
     }
 
     auto response = http->GetBody();

main/protocols/mqtt_protocol.cc

@@ -105,7 +105,7 @@ void MqttProtocol::SendText(const std::string& text) {
     mqtt_->Publish(publish_topic_, text);
 }
 
-void MqttProtocol::SendAudio(const std::string& data) {
+void MqttProtocol::SendAudio(const std::vector<uint8_t>& data) {
     std::lock_guard<std::mutex> lock(channel_mutex_);
     if (udp_ == nullptr) {
         return;
@@ -202,7 +202,7 @@ bool MqttProtocol::OpenAudioChannel() {
             ESP_LOGW(TAG, "Received audio packet with wrong sequence: %lu, expected: %lu", sequence, remote_sequence_ + 1);
         }
 
-        std::string decrypted;
+        std::vector<uint8_t> decrypted;
         size_t decrypted_size = data.size() - aes_nonce_.size();
         size_t nc_off = 0;
         uint8_t stream_block[16] = {0};
@@ -215,7 +215,7 @@ bool MqttProtocol::OpenAudioChannel() {
             return;
         }
         if (on_incoming_audio_ != nullptr) {
-            on_incoming_audio_(decrypted);
+            on_incoming_audio_(std::move(decrypted));
         }
         remote_sequence_ = sequence;
     });

main/protocols/mqtt_protocol.h

@@ -25,7 +25,7 @@ public:
     MqttProtocol();
     ~MqttProtocol();
 
-    void SendAudio(const std::string& data) override;
+    void SendAudio(const std::vector<uint8_t>& data) override;
     bool OpenAudioChannel() override;
     void CloseAudioChannel() override;
     bool IsAudioChannelOpened() const override;

main/protocols/protocol.cc

@@ -8,7 +8,7 @@ void Protocol::OnIncomingJson(std::function<void(const cJSON* root)> callback) {
     on_incoming_json_ = callback;
 }
 
-void Protocol::OnIncomingAudio(std::function<void(const std::string& data)> callback) {
+void Protocol::OnIncomingAudio(std::function<void(std::vector<uint8_t>&& data)> callback) {
     on_incoming_audio_ = callback;
 }
 
@@ -57,3 +57,14 @@ void Protocol::SendStopListening() {
     std::string message = "{\"session_id\":\"" + session_id_ + "\",\"type\":\"listen\",\"state\":\"stop\"}";
     SendText(message);
 }
+
+void Protocol::SendIotDescriptors(const std::string& descriptors) {
+    std::string message = "{\"session_id\":\"" + session_id_ + "\",\"type\":\"iot\",\"descriptors\":" + descriptors + "}";
+    SendText(message);
+}
+
+void Protocol::SendIotStates(const std::string& states) {
+    std::string message = "{\"session_id\":\"" + session_id_ + "\",\"type\":\"iot\",\"states\":" + states + "}";
+    SendText(message);
+}
+

main/protocols/protocol.h

@@ -31,7 +31,7 @@ public:
         return server_sample_rate_;
     }
 
-    void OnIncomingAudio(std::function<void(const std::string& data)> callback);
+    void OnIncomingAudio(std::function<void(std::vector<uint8_t>&& data)> callback);
     void OnIncomingJson(std::function<void(const cJSON* root)> callback);
     void OnAudioChannelOpened(std::function<void()> callback);
     void OnAudioChannelClosed(std::function<void()> callback);
@@ -40,15 +40,17 @@ public:
     virtual bool OpenAudioChannel() = 0;
     virtual void CloseAudioChannel() = 0;
     virtual bool IsAudioChannelOpened() const = 0;
-    virtual void SendAudio(const std::string& data) = 0;
+    virtual void SendAudio(const std::vector<uint8_t>& data) = 0;
     virtual void SendWakeWordDetected(const std::string& wake_word);
     virtual void SendStartListening(ListeningMode mode);
     virtual void SendStopListening();
     virtual void SendAbortSpeaking(AbortReason reason);
+    virtual void SendIotDescriptors(const std::string& descriptors);
+    virtual void SendIotStates(const std::string& states);
 
 protected:
     std::function<void(const cJSON* root)> on_incoming_json_;
-    std::function<void(const std::string& data)> on_incoming_audio_;
+    std::function<void(std::vector<uint8_t>&& data)> on_incoming_audio_;
     std::function<void()> on_audio_channel_opened_;
     std::function<void()> on_audio_channel_closed_;
     std::function<void(const std::string& message)> on_network_error_;

main/protocols/websocket_protocol.cc

@@ -23,7 +23,7 @@ WebsocketProtocol::~WebsocketProtocol() {
     vEventGroupDelete(event_group_handle_);
 }
 
-void WebsocketProtocol::SendAudio(const std::string& data) {
+void WebsocketProtocol::SendAudio(const std::vector<uint8_t>& data) {
     if (websocket_ == nullptr) {
         return;
     }
@@ -65,7 +65,7 @@ bool WebsocketProtocol::OpenAudioChannel() {
     websocket_->OnData([this](const char* data, size_t len, bool binary) {
         if (binary) {
             if (on_incoming_audio_ != nullptr) {
-                on_incoming_audio_(std::string(data, len));
+                on_incoming_audio_(std::vector<uint8_t>((uint8_t*)data, (uint8_t*)data + len));
             }
         } else {
             // Parse JSON data

main/protocols/websocket_protocol.h

@@ -15,7 +15,7 @@ public:
     WebsocketProtocol();
     ~WebsocketProtocol();
 
-    void SendAudio(const std::string& data) override;
+    void SendAudio(const std::vector<uint8_t>& data) override;
     bool OpenAudioChannel() override;
     void CloseAudioChannel() override;
     bool IsAudioChannelOpened() const override;

sdkconfig.defaults

@@ -1,3 +1,6 @@
+CONFIG_COMPILER_CXX_EXCEPTIONS=y
+CONFIG_COMPILER_CXX_EXCEPTIONS_EMG_POOL_SIZE=1024
+
 CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_PERF=y
 CONFIG_BOOTLOADER_LOG_LEVEL_NONE=y
 CONFIG_BOOTLOADER_SKIP_VALIDATE_ALWAYS=y
@@ -10,13 +13,13 @@ CONFIG_PARTITION_TABLE_CUSTOM=y
 CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv"
 CONFIG_PARTITION_TABLE_OFFSET=0x8000
 
-CONFIG_USE_WAKENET=y
-CONFIG_SR_WN_WN9_NIHAOXIAOZHI_TTS=y
-CONFIG_USE_MULTINET=n
-
 ESP_TASK_WDT_TIMEOUT_S=10
 CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS=y
 CONFIG_FREERTOS_USE_STATS_FORMATTING_FUNCTIONS=y
 
 CONFIG_LV_COLOR_16_SWAP=y
 CONFIG_LV_MEM_CUSTOM=y
+
+CONFIG_MBEDTLS_DYNAMIC_BUFFER=y
+CONFIG_ESP_WIFI_IRAM_OPT=n
+CONFIG_ESP_WIFI_RX_IRAM_OPT=n

sdkconfig.defaults.esp32c3

@@ -1,6 +1,2 @@
 
 CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y
-
-CONFIG_PARTITION_TABLE_CUSTOM=y
-CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_4M.csv"
-CONFIG_PARTITION_TABLE_OFFSET=0x8000

sdkconfig.defaults.esp32s3

@@ -16,3 +16,7 @@ CONFIG_MBEDTLS_EXTERNAL_MEM_ALLOC=y
 CONFIG_ESP32S3_INSTRUCTION_CACHE_32KB=y
 CONFIG_ESP32S3_DATA_CACHE_64KB=y
 CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y
+
+CONFIG_USE_WAKENET=y
+CONFIG_SR_WN_WN9_NIHAOXIAOZHI_TTS=y
+CONFIG_USE_MULTINET=n