add iot speaker for boards

add pdm mic

CMakeLists.txt

@@ -4,7 +4,7 @@
 # CMakeLists in this exact order for cmake to work correctly
 cmake_minimum_required(VERSION 3.16)
 
-set(PROJECT_VER "0.9.5")
+set(PROJECT_VER "0.9.7")
 
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 project(xiaozhi)

main/CMakeLists.txt

@@ -9,6 +9,8 @@ set(SOURCES "audio_codecs/audio_codec.cc"
             "protocols/protocol.cc"
             "protocols/mqtt_protocol.cc"
             "protocols/websocket_protocol.cc"
+            "iot/thing.cc"
+            "iot/thing_manager.cc"
             "system_info.cc"
             "application.cc"
             "ota.cc"
@@ -19,6 +21,10 @@ set(SOURCES "audio_codecs/audio_codec.cc"
 
 set(INCLUDE_DIRS "." "display" "audio_codecs" "protocols" "audio_processing")
 
+# 添加 IOT 相关文件
+file(GLOB IOT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/iot/things/*.cc)
+list(APPEND SOURCES ${IOT_SOURCES})
+
 # 字体
 file(GLOB FONT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/fonts/*.c)
 list(APPEND SOURCES ${FONT_SOURCES})
@@ -44,6 +50,8 @@ elseif(CONFIG_BOARD_TYPE_KEVIN_C3)
     set(BOARD_TYPE "kevin-c3")
 elseif(CONFIG_BOARD_TYPE_LICHUANG_DEV)
     set(BOARD_TYPE "lichuang-dev")
+elseif(CONFIG_BOARD_TYPE_TERRENCE_C3_DEV)
+    set(BOARD_TYPE "terrence-c3-dev")
 endif()
 file(GLOB BOARD_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/boards/${BOARD_TYPE}/*.cc)
 list(APPEND SOURCES ${BOARD_SOURCES})
@@ -55,6 +63,7 @@ endif()
 idf_component_register(SRCS ${SOURCES}
                     EMBED_FILES "assets/err_reg.p3" "assets/err_pin.p3" "assets/err_wificonfig.p3"
                     INCLUDE_DIRS ${INCLUDE_DIRS}
+                    WHOLE_ARCHIVE
                     )
 
 # 使用 target_compile_definitions 来定义 BOARD_TYPE

main/application.cc

@@ -1,10 +1,13 @@
 #include "application.h"
+#include "board.h"
+#include "display.h"
 #include "system_info.h"
 #include "ml307_ssl_transport.h"
 #include "audio_codec.h"
 #include "mqtt_protocol.h"
 #include "websocket_protocol.h"
 #include "font_awesome_symbols.h"
+#include "iot/thing_manager.h"
 
 #include <cstring>
 #include <esp_log.h>
@@ -21,6 +24,15 @@ extern const char p3_err_pin_end[] asm("_binary_err_pin_p3_end");
 extern const char p3_err_wificonfig_start[] asm("_binary_err_wificonfig_p3_start");
 extern const char p3_err_wificonfig_end[] asm("_binary_err_wificonfig_p3_end");
 
+static const char* const STATE_STRINGS[] = {
+    "unknown",
+    "idle",
+    "connecting",
+    "listening",
+    "speaking",
+    "upgrading",
+    "invalid_state"
+};
 
 Application::Application() : background_task_(4096 * 8) {
     event_group_ = xEventGroupCreate();
@@ -30,13 +42,6 @@ Application::Application() : background_task_(4096 * 8) {
 }
 
 Application::~Application() {
-    if (protocol_ != nullptr) {
-        delete protocol_;
-    }
-    if (opus_decoder_ != nullptr) {
-        opus_decoder_destroy(opus_decoder_);
-    }
-
     vEventGroupDelete(event_group_);
 }
 
@@ -83,7 +88,7 @@ void Application::CheckNewVersion() {
     }
 }
 
-void Application::Alert(const std::string&& title, const std::string&& message) {
+void Application::Alert(const std::string& title, const std::string& message) {
     ESP_LOGW(TAG, "Alert: %s, %s", title.c_str(), message.c_str());
     auto display = Board::GetInstance().GetDisplay();
     display->ShowNotification(message);
@@ -105,7 +110,7 @@ void Application::PlayLocalFile(const char* data, size_t size) {
         p += sizeof(BinaryProtocol3);
 
         auto payload_size = ntohs(p3->payload_size);
-        std::string opus;
+        std::vector<uint8_t> opus;
         opus.resize(payload_size);
         memcpy(opus.data(), p3->payload, payload_size);
         p += payload_size;
@@ -117,10 +122,15 @@ void Application::PlayLocalFile(const char* data, size_t size) {
 
 void Application::ToggleChatState() {
     Schedule([this]() {
+        if (!protocol_) {
+            ESP_LOGE(TAG, "Protocol not initialized");
+            return;
+        }
+
         if (chat_state_ == kChatStateIdle) {
             SetChatState(kChatStateConnecting);
             if (!protocol_->OpenAudioChannel()) {
-                ESP_LOGE(TAG, "Failed to open audio channel");
+                Alert("Error", "Failed to open audio channel");
                 SetChatState(kChatStateIdle);
                 return;
             }
@@ -138,13 +148,18 @@ void Application::ToggleChatState() {
 
 void Application::StartListening() {
     Schedule([this]() {
+        if (!protocol_) {
+            ESP_LOGE(TAG, "Protocol not initialized");
+            return;
+        }
+        
         keep_listening_ = false;
         if (chat_state_ == kChatStateIdle) {
             if (!protocol_->IsAudioChannelOpened()) {
                 SetChatState(kChatStateConnecting);
                 if (!protocol_->OpenAudioChannel()) {
                     SetChatState(kChatStateIdle);
-                    ESP_LOGE(TAG, "Failed to open audio channel");
+                    Alert("Error", "Failed to open audio channel");
                     return;
                 }
             }
@@ -171,8 +186,6 @@ void Application::StopListening() {
 
 void Application::Start() {
     auto& board = Board::GetInstance();
-    board.Initialize();
-
     auto builtin_led = board.GetBuiltinLed();
     builtin_led->SetBlue();
     builtin_led->StartContinuousBlink(100);
@@ -183,8 +196,8 @@ void Application::Start() {
     /* Setup the audio codec */
     auto codec = board.GetAudioCodec();
     opus_decode_sample_rate_ = codec->output_sample_rate();
-    opus_decoder_ = opus_decoder_create(opus_decode_sample_rate_, 1, NULL);
-    opus_encoder_.Configure(16000, 1, OPUS_FRAME_DURATION_MS);
+    opus_decoder_ = std::make_unique<OpusDecoderWrapper>(opus_decode_sample_rate_, 1);
+    opus_encoder_ = std::make_unique<OpusEncoderWrapper>(16000, 1, OPUS_FRAME_DURATION_MS);
     if (codec->input_sample_rate() != 16000) {
         input_resampler_.Configure(codec->input_sample_rate(), 16000);
         reference_resampler_.Configure(codec->input_sample_rate(), 16000);
@@ -221,9 +234,9 @@ void Application::Start() {
 #if CONFIG_IDF_TARGET_ESP32S3
     audio_processor_.Initialize(codec->input_channels(), codec->input_reference());
     audio_processor_.OnOutput([this](std::vector<int16_t>&& data) {
-        background_task_.Schedule([this, data = std::move(data)]() {
-            opus_encoder_.Encode(data, [this](const uint8_t* opus, size_t opus_size) {
-                Schedule([this, opus = std::string(reinterpret_cast<const char*>(opus), opus_size)]() {
+        background_task_.Schedule([this, data = std::move(data)]() mutable {
+            opus_encoder_->Encode(std::move(data), [this](std::vector<uint8_t>&& opus) {
+                Schedule([this, opus = std::move(opus)]() {
                     protocol_->SendAudio(opus);
                 });
             });
@@ -258,7 +271,7 @@ void Application::Start() {
                     return;
                 }
                 
-                std::string opus;
+                std::vector<uint8_t> opus;
                 // Encode and send the wake word data to the server
                 while (wake_word_detect_.GetWakeWordOpus(opus)) {
                     protocol_->SendAudio(opus);
@@ -282,32 +295,36 @@ void Application::Start() {
     // Initialize the protocol
     display->SetStatus("初始化协议");
 #ifdef CONFIG_CONNECTION_TYPE_WEBSOCKET
-    protocol_ = new WebsocketProtocol();
+    protocol_ = std::make_unique<WebsocketProtocol>();
 #else
-    protocol_ = new MqttProtocol();
+    protocol_ = std::make_unique<MqttProtocol>();
 #endif
     protocol_->OnNetworkError([this](const std::string& message) {
         Alert("Error", std::move(message));
     });
-    protocol_->OnIncomingAudio([this](const std::string& data) {
+    protocol_->OnIncomingAudio([this](std::vector<uint8_t>&& data) {
         std::lock_guard<std::mutex> lock(mutex_);
         if (chat_state_ == kChatStateSpeaking) {
             audio_decode_queue_.emplace_back(std::move(data));
         }
     });
     protocol_->OnAudioChannelOpened([this, codec, &board]() {
+        board.SetPowerSaveMode(false);
         if (protocol_->server_sample_rate() != codec->output_sample_rate()) {
             ESP_LOGW(TAG, "服务器的音频采样率 %d 与设备输出的采样率 %d 不一致，重采样后可能会失真",
                 protocol_->server_sample_rate(), codec->output_sample_rate());
         }
         SetDecodeSampleRate(protocol_->server_sample_rate());
-        board.SetPowerSaveMode(false);
+        // 物联网设备描述符
+        last_iot_states_.clear();
+        auto& thing_manager = iot::ThingManager::GetInstance();
+        protocol_->SendIotDescriptors(thing_manager.GetDescriptorsJson());
     });
     protocol_->OnAudioChannelClosed([this, &board]() {
+        board.SetPowerSaveMode(true);
         Schedule([this]() {
             SetChatState(kChatStateIdle);
         });
-        board.SetPowerSaveMode(true);
     });
     protocol_->OnIncomingJson([this, display](const cJSON* root) {
         // Parse JSON data
@@ -351,6 +368,15 @@ void Application::Start() {
             if (emotion != NULL) {
                 display->SetEmotion(emotion->valuestring);
             }
+        } else if (strcmp(type->valuestring, "iot") == 0) {
+            auto commands = cJSON_GetObjectItem(root, "commands");
+            if (commands != NULL) {
+                auto& thing_manager = iot::ThingManager::GetInstance();
+                for (int i = 0; i < cJSON_GetArraySize(commands); ++i) {
+                    auto command = cJSON_GetArrayItem(commands, i);
+                    thing_manager.Invoke(command);
+                }
+            }
         }
     });
 
@@ -363,9 +389,8 @@ void Application::Start() {
 }
 
 void Application::Schedule(std::function<void()> callback) {
-    mutex_.lock();
-    main_tasks_.push_back(callback);
-    mutex_.unlock();
+    std::lock_guard<std::mutex> lock(mutex_);
+    main_tasks_.push_back(std::move(callback));
     xEventGroupSetBits(event_group_, SCHEDULE_EVENT);
 }
 
@@ -397,7 +422,7 @@ void Application::MainLoop() {
 
 void Application::ResetDecoder() {
     std::lock_guard<std::mutex> lock(mutex_);
-    opus_decoder_ctl(opus_decoder_, OPUS_RESET_STATE);
+    opus_decoder_->ResetState();
     audio_decode_queue_.clear();
     last_output_time_ = std::chrono::steady_clock::now();
     Board::GetInstance().GetAudioCodec()->EnableOutput(true);
@@ -430,24 +455,21 @@ void Application::OutputAudio() {
     audio_decode_queue_.pop_front();
     lock.unlock();
 
-    background_task_.Schedule([this, codec, opus = std::move(opus)]() {
+    background_task_.Schedule([this, codec, opus = std::move(opus)]() mutable {
         if (aborted_) {
             return;
         }
-        int frame_size = opus_decode_sample_rate_ * OPUS_FRAME_DURATION_MS / 1000;
-        std::vector<int16_t> pcm(frame_size);
 
-        int ret = opus_decode(opus_decoder_, (const unsigned char*)opus.data(), opus.size(), pcm.data(), frame_size, 0);
-        if (ret < 0) {
-            ESP_LOGE(TAG, "Failed to decode audio, error code: %d", ret);
+        std::vector<int16_t> pcm;
+        if (!opus_decoder_->Decode(std::move(opus), pcm)) {
             return;
         }
 
         // Resample if the sample rate is different
         if (opus_decode_sample_rate_ != codec->output_sample_rate()) {
-            int target_size = output_resampler_.GetOutputSamples(frame_size);
+            int target_size = output_resampler_.GetOutputSamples(pcm.size());
             std::vector<int16_t> resampled(target_size);
-            output_resampler_.Process(pcm.data(), frame_size, resampled.data());
+            output_resampler_.Process(pcm.data(), pcm.size(), resampled.data());
             pcm = std::move(resampled);
         }
         
@@ -495,9 +517,9 @@ void Application::InputAudio() {
     }
 #else
     if (chat_state_ == kChatStateListening) {
-        background_task_.Schedule([this, data = std::move(data)]() {
-            opus_encoder_.Encode(data, [this](const uint8_t* opus, size_t opus_size) {
-                Schedule([this, opus = std::string(reinterpret_cast<const char*>(opus), opus_size)]() {
+        background_task_.Schedule([this, data = std::move(data)]() mutable {
+            opus_encoder_->Encode(std::move(data), [this](std::vector<uint8_t>&& opus) {
+                Schedule([this, opus = std::move(opus)]() {
                     protocol_->SendAudio(opus);
                 });
             });
@@ -513,22 +535,12 @@ void Application::AbortSpeaking(AbortReason reason) {
 }
 
 void Application::SetChatState(ChatState state) {
-    const char* state_str[] = {
-        "unknown",
-        "idle",
-        "connecting",
-        "listening",
-        "speaking",
-        "upgrading",
-        "invalid_state"
-    };
     if (chat_state_ == state) {
-        // No need to update the state
         return;
     }
-
+    
     chat_state_ = state;
-    ESP_LOGI(TAG, "STATE: %s", state_str[chat_state_]);
+    ESP_LOGI(TAG, "STATE: %s", STATE_STRINGS[chat_state_]);
     // The state is changed, wait for all background tasks to finish
     background_task_.WaitForCompletion();
 
@@ -555,10 +567,11 @@ void Application::SetChatState(ChatState state) {
             display->SetStatus("聆听中...");
             display->SetEmotion("neutral");
             ResetDecoder();
-            opus_encoder_.ResetState();
+            opus_encoder_->ResetState();
 #if CONFIG_IDF_TARGET_ESP32S3
             audio_processor_.Start();
 #endif
+            UpdateIotStates();
             break;
         case kChatStateSpeaking:
             builtin_led->SetGreen();
@@ -584,9 +597,8 @@ void Application::SetDecodeSampleRate(int sample_rate) {
         return;
     }
 
-    opus_decoder_destroy(opus_decoder_);
     opus_decode_sample_rate_ = sample_rate;
-    opus_decoder_ = opus_decoder_create(opus_decode_sample_rate_, 1, NULL);
+    opus_decoder_ = std::make_unique<OpusDecoderWrapper>(opus_decode_sample_rate_, 1);
 
     auto codec = Board::GetInstance().GetAudioCodec();
     if (opus_decode_sample_rate_ != codec->output_sample_rate()) {
@@ -594,3 +606,12 @@ void Application::SetDecodeSampleRate(int sample_rate) {
         output_resampler_.Configure(opus_decode_sample_rate_, codec->output_sample_rate());
     }
 }
+
+void Application::UpdateIotStates() {
+    auto& thing_manager = iot::ThingManager::GetInstance();
+    auto states = thing_manager.GetStatesJson();
+    if (states != last_iot_states_) {
+        last_iot_states_ = states;
+        protocol_->SendIotStates(states);
+    }
+}

main/application.h

@@ -4,17 +4,16 @@
 #include <freertos/FreeRTOS.h>
 #include <freertos/event_groups.h>
 #include <freertos/task.h>
-#include <opus.h>
+
+#include <string>
 #include <mutex>
 #include <list>
-#include <condition_variable>
 
-#include "opus_encoder.h"
-#include "opus_resampler.h"
+#include <opus_encoder.h>
+#include <opus_decoder.h>
+#include <opus_resampler.h>
 
 #include "protocol.h"
-#include "display.h"
-#include "board.h"
 #include "ota.h"
 #include "background_task.h"
 
@@ -52,11 +51,12 @@ public:
     ChatState GetChatState() const { return chat_state_; }
     void Schedule(std::function<void()> callback);
     void SetChatState(ChatState state);
-    void Alert(const std::string&& title, const std::string&& message);
+    void Alert(const std::string& title, const std::string& message);
     void AbortSpeaking(AbortReason reason);
     void ToggleChatState();
     void StartListening();
     void StopListening();
+    void UpdateIotStates();
 
 private:
     Application();
@@ -69,19 +69,20 @@ private:
     Ota ota_;
     std::mutex mutex_;
     std::list<std::function<void()>> main_tasks_;
-    Protocol* protocol_ = nullptr;
+    std::unique_ptr<Protocol> protocol_;
     EventGroupHandle_t event_group_;
     volatile ChatState chat_state_ = kChatStateUnknown;
     bool keep_listening_ = false;
     bool aborted_ = false;
+    std::string last_iot_states_;
 
     // Audio encode / decode
     BackgroundTask background_task_;
     std::chrono::steady_clock::time_point last_output_time_;
-    std::list<std::string> audio_decode_queue_;
+    std::list<std::vector<uint8_t>> audio_decode_queue_;
 
-    OpusEncoder opus_encoder_;
-    OpusDecoder* opus_decoder_ = nullptr;
+    std::unique_ptr<OpusEncoderWrapper> opus_encoder_;
+    std::unique_ptr<OpusDecoderWrapper> opus_decoder_;
 
     int opus_decode_sample_rate_ = -1;
     OpusResampler input_resampler_;

main/audio_codecs/no_audio_codec.cc

@@ -131,6 +131,67 @@ NoAudioCodec::NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_n
     ESP_LOGI(TAG, "Simplex channels created");
 }
 
+NoAudioCodec::NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_num_t spk_bclk, gpio_num_t spk_ws, gpio_num_t spk_dout, gpio_num_t mic_sck, gpio_num_t mic_din) {
+    duplex_ = false;
+    input_sample_rate_ = input_sample_rate;
+    output_sample_rate_ = output_sample_rate;
+
+    // Create a new channel for speaker
+    i2s_chan_config_t tx_chan_cfg = I2S_CHANNEL_DEFAULT_CONFIG((i2s_port_t)1, I2S_ROLE_MASTER);
+    tx_chan_cfg.dma_desc_num = 6;
+    tx_chan_cfg.dma_frame_num = 240;
+    tx_chan_cfg.auto_clear_after_cb = true;
+    tx_chan_cfg.auto_clear_before_cb = false;
+    tx_chan_cfg.intr_priority = 0;
+    ESP_ERROR_CHECK(i2s_new_channel(&tx_chan_cfg, &tx_handle_, NULL));
+
+
+    i2s_std_config_t tx_std_cfg = {
+        .clk_cfg = {
+            .sample_rate_hz = (uint32_t)output_sample_rate_,
+            .clk_src = I2S_CLK_SRC_DEFAULT,
+            .ext_clk_freq_hz = 0,
+            .mclk_multiple = I2S_MCLK_MULTIPLE_256
+        },
+        .slot_cfg = I2S_STD_MSB_SLOT_DEFAULT_CONFIG(I2S_DATA_BIT_WIDTH_32BIT, I2S_SLOT_MODE_MONO),
+        .gpio_cfg = {
+            .mclk = I2S_GPIO_UNUSED,
+            .bclk = spk_bclk,
+            .ws = spk_ws,
+            .dout = spk_dout,
+            .din = I2S_GPIO_UNUSED,
+            .invert_flags = {
+                .mclk_inv = false,
+                .bclk_inv = false,
+                .ws_inv   = false,
+            },
+        },
+    };
+    ESP_ERROR_CHECK(i2s_channel_init_std_mode(tx_handle_, &tx_std_cfg));
+#if SOC_I2S_SUPPORTS_PDM_RX
+    // Create a new channel for MIC in PDM mode
+    i2s_chan_config_t rx_chan_cfg = I2S_CHANNEL_DEFAULT_CONFIG((i2s_port_t)0, I2S_ROLE_MASTER);
+    ESP_ERROR_CHECK(i2s_new_channel(&rx_chan_cfg, NULL, &rx_handle_));
+    i2s_pdm_rx_config_t pdm_rx_cfg = {
+        .clk_cfg = I2S_PDM_RX_CLK_DEFAULT_CONFIG((uint32_t)input_sample_rate_),
+        /* The data bit-width of PDM mode is fixed to 16 */
+        .slot_cfg = I2S_PDM_RX_SLOT_DEFAULT_CONFIG(I2S_DATA_BIT_WIDTH_16BIT, I2S_SLOT_MODE_MONO),
+        .gpio_cfg = {
+            .clk = mic_sck,
+            .din = mic_din,
+ 
+            .invert_flags = {
+                .clk_inv = false,
+            },
+        },
+    };
+    ESP_ERROR_CHECK(i2s_channel_init_pdm_rx_mode(rx_handle_, &pdm_rx_cfg));
+#else
+    ESP_LOGE(TAG, "PDM is not supported");
+#endif
+    ESP_LOGI(TAG, "Simplex channels created");
+}
+
 int NoAudioCodec::Write(const int16_t* data, int samples) {
     std::vector<int32_t> buffer(samples);
 

main/audio_codecs/no_audio_codec.h

@@ -4,7 +4,7 @@
 #include "audio_codec.h"
 
 #include <driver/gpio.h>
-
+#include <driver/i2s_pdm.h>
 class NoAudioCodec : public AudioCodec {
 private:
     virtual int Write(const int16_t* data, int samples) override;
@@ -15,6 +15,8 @@ public:
     NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_num_t bclk, gpio_num_t ws, gpio_num_t dout, gpio_num_t din);
     // Simplex
     NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_num_t spk_bclk, gpio_num_t spk_ws, gpio_num_t spk_dout, gpio_num_t mic_sck, gpio_num_t mic_ws, gpio_num_t mic_din);
+    // Simplex_PDM
+    NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_num_t spk_bclk, gpio_num_t spk_ws, gpio_num_t spk_dout, gpio_num_t mic_sck,  gpio_num_t mic_din);
     virtual ~NoAudioCodec();
 };
 

main/audio_processing/audio_processor.cc

@@ -63,7 +63,7 @@ AudioProcessor::~AudioProcessor() {
     vEventGroupDelete(event_group_);
 }
 
-void AudioProcessor::Input(std::vector<int16_t>& data) {
+void AudioProcessor::Input(const std::vector<int16_t>& data) {
     input_buffer_.insert(input_buffer_.end(), data.begin(), data.end());
 
     auto chunk_size = esp_afe_vc_v1.get_feed_chunksize(afe_communication_data_) * channels_;

main/audio_processing/audio_processor.h

@@ -16,7 +16,7 @@ public:
     ~AudioProcessor();
 
     void Initialize(int channels, bool reference);
-    void Input(std::vector<int16_t>& data);
+    void Input(const std::vector<int16_t>& data);
     void Start();
     void Stop();
     bool IsRunning();

main/audio_processing/wake_word_detect.cc

@@ -111,7 +111,7 @@ bool WakeWordDetect::IsDetectionRunning() {
     return xEventGroupGetBits(event_group_) & DETECTION_RUNNING_EVENT;
 }
 
-void WakeWordDetect::Feed(std::vector<int16_t>& data) {
+void WakeWordDetect::Feed(const std::vector<int16_t>& data) {
     input_buffer_.insert(input_buffer_.end(), data.begin(), data.end());
 
     auto chunk_size = esp_afe_sr_v1.get_feed_chunksize(afe_detection_data_) * channels_;
@@ -163,8 +163,7 @@ void WakeWordDetect::AudioDetectionTask() {
 
 void WakeWordDetect::StoreWakeWordData(uint16_t* data, size_t samples) {
     // store audio data to wake_word_pcm_
-    std::vector<int16_t> pcm(data, data + samples);
-    wake_word_pcm_.emplace_back(std::move(pcm));
+    wake_word_pcm_.emplace_back(std::vector<int16_t>(data, data + samples));
     // keep about 2 seconds of data, detect duration is 32ms (sample_rate == 16000, chunksize == 512)
     while (wake_word_pcm_.size() > 2000 / 32) {
         wake_word_pcm_.pop_front();
@@ -178,34 +177,33 @@ void WakeWordDetect::EncodeWakeWordData() {
     }
     wake_word_encode_task_ = xTaskCreateStatic([](void* arg) {
         auto this_ = (WakeWordDetect*)arg;
-        auto start_time = esp_timer_get_time();
-        // encode detect packets
-        OpusEncoder* encoder = new OpusEncoder();
-        encoder->Configure(16000, 1, 60);
-        encoder->SetComplexity(0);
-
-        for (auto& pcm: this_->wake_word_pcm_) {
-            encoder->Encode(pcm, [this_](const uint8_t* opus, size_t opus_size) {
-                std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
-                this_->wake_word_opus_.emplace_back(std::string(reinterpret_cast<const char*>(opus), opus_size));
-                this_->wake_word_cv_.notify_all();
-            });
-        }
-        this_->wake_word_pcm_.clear();
-
-        auto end_time = esp_timer_get_time();
-        ESP_LOGI(TAG, "Encode wake word opus %zu packets in %lld ms", this_->wake_word_opus_.size(), (end_time - start_time) / 1000);
         {
+            auto start_time = esp_timer_get_time();
+            auto encoder = std::make_unique<OpusEncoderWrapper>(16000, 1, OPUS_FRAME_DURATION_MS);
+            encoder->SetComplexity(0); // 0 is the fastest
+
+            for (auto& pcm: this_->wake_word_pcm_) {
+                encoder->Encode(std::move(pcm), [this_](std::vector<uint8_t>&& opus) {
+                    std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
+                    this_->wake_word_opus_.emplace_back(std::move(opus));
+                    this_->wake_word_cv_.notify_all();
+                });
+            }
+            this_->wake_word_pcm_.clear();
+
+            auto end_time = esp_timer_get_time();
+            ESP_LOGI(TAG, "Encode wake word opus %zu packets in %lld ms",
+                this_->wake_word_opus_.size(), (end_time - start_time) / 1000);
+
             std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
-            this_->wake_word_opus_.push_back("");
+            this_->wake_word_opus_.push_back(std::vector<uint8_t>());
             this_->wake_word_cv_.notify_all();
         }
-        delete encoder;
         vTaskDelete(NULL);
     }, "encode_detect_packets", 4096 * 8, this, 1, wake_word_encode_task_stack_, &wake_word_encode_task_buffer_);
 }
 
-bool WakeWordDetect::GetWakeWordOpus(std::string& opus) {
+bool WakeWordDetect::GetWakeWordOpus(std::vector<uint8_t>& opus) {
     std::unique_lock<std::mutex> lock(wake_word_mutex_);
     wake_word_cv_.wait(lock, [this]() {
         return !wake_word_opus_.empty();

main/audio_processing/wake_word_detect.h

@@ -22,14 +22,14 @@ public:
     ~WakeWordDetect();
 
     void Initialize(int channels, bool reference);
-    void Feed(std::vector<int16_t>& data);
+    void Feed(const std::vector<int16_t>& data);
     void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback);
     void OnVadStateChange(std::function<void(bool speaking)> callback);
     void StartDetection();
     void StopDetection();
     bool IsDetectionRunning();
     void EncodeWakeWordData();
-    bool GetWakeWordOpus(std::string& opus);
+    bool GetWakeWordOpus(std::vector<uint8_t>& opus);
     const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; }
 
 private:
@@ -49,7 +49,7 @@ private:
     StaticTask_t wake_word_encode_task_buffer_;
     StackType_t* wake_word_encode_task_stack_ = nullptr;
     std::list<std::vector<int16_t>> wake_word_pcm_;
-    std::list<std::string> wake_word_opus_;
+    std::list<std::vector<uint8_t>> wake_word_opus_;
     std::mutex wake_word_mutex_;
     std::condition_variable wake_word_cv_;
 

main/background_task.cc

@@ -31,12 +31,16 @@ void BackgroundTask::Schedule(std::function<void()> callback) {
         ESP_LOGW(TAG, "active_tasks_ == %u", active_tasks_.load());
     }
     active_tasks_++;
-    auto wrapped_callback = [this, callback]() {
-        callback();
-        active_tasks_--;
-        condition_variable_.notify_all();
-    };
-    main_tasks_.push_back(wrapped_callback);
+    main_tasks_.emplace_back([this, cb = std::move(callback)]() {
+        cb();
+        {
+            std::lock_guard<std::mutex> lock(mutex_);
+            active_tasks_--;
+            if (main_tasks_.empty() && active_tasks_ == 0) {
+                condition_variable_.notify_all();
+            }
+        }
+    });
     condition_variable_.notify_all();
 }
 

main/boards/bread-compact-ml307/compact_ml307_board.cc

@@ -6,6 +6,7 @@
 #include "button.h"
 #include "led.h"
 #include "config.h"
+#include "iot/thing_manager.h"
 
 #include <esp_log.h>
 #include <driver/i2c_master.h>
@@ -59,8 +60,7 @@ private:
         });
 
         volume_up_button_.OnLongPress([this]() {
-            auto codec = GetAudioCodec();
-            codec->SetOutputVolume(100);
+            GetAudioCodec()->SetOutputVolume(100);
             GetDisplay()->ShowNotification("最大音量");
         });
 
@@ -75,30 +75,31 @@ private:
         });
 
         volume_down_button_.OnLongPress([this]() {
-            auto codec = GetAudioCodec();
-            codec->SetOutputVolume(0);
+            GetAudioCodec()->SetOutputVolume(0);
             GetDisplay()->ShowNotification("已静音");
         });
     }
 
+    // 物联网初始化，添加对 AI 可见设备
+    void InitializeIot() {
+        auto& thing_manager = iot::ThingManager::GetInstance();
+        thing_manager.AddThing(iot::CreateThing("Speaker"));
+        thing_manager.AddThing(iot::CreateThing("Lamp"));
+    }
+
 public:
     CompactMl307Board() : Ml307Board(ML307_TX_PIN, ML307_RX_PIN, 4096),
         boot_button_(BOOT_BUTTON_GPIO),
-        touch_button_(TOUCH_BUTTON_GPIO, 1),
+        touch_button_(TOUCH_BUTTON_GPIO),
         volume_up_button_(VOLUME_UP_BUTTON_GPIO),
         volume_down_button_(VOLUME_DOWN_BUTTON_GPIO),
         system_reset_(RESET_NVS_BUTTON_GPIO, RESET_FACTORY_BUTTON_GPIO) {
-    }
-
-    virtual void Initialize() override {
-        ESP_LOGI(TAG, "Initializing CompactMl307Board");
         // Check if the reset button is pressed
         system_reset_.CheckButtons();
 
         InitializeDisplayI2c();
         InitializeButtons();
-
-        Ml307Board::Initialize();
+        InitializeIot();
     }
 
     virtual Led* GetBuiltinLed() override {

main/boards/bread-compact-ml307/config.h

@@ -29,7 +29,7 @@
 
 #define BUILTIN_LED_GPIO        GPIO_NUM_48
 #define BOOT_BUTTON_GPIO        GPIO_NUM_0
-#define TOUCH_BUTTON_GPIO       GPIO_NUM_45
+#define TOUCH_BUTTON_GPIO       GPIO_NUM_47
 #define VOLUME_UP_BUTTON_GPIO   GPIO_NUM_40
 #define VOLUME_DOWN_BUTTON_GPIO GPIO_NUM_39
 #define RESET_NVS_BUTTON_GPIO     GPIO_NUM_1

main/boards/bread-compact-wifi/compact_wifi_board.cc

@@ -6,6 +6,7 @@
 #include "button.h"
 #include "led.h"
 #include "config.h"
+#include "iot/thing_manager.h"
 
 #include <wifi_station.h>
 #include <esp_log.h>
@@ -64,8 +65,7 @@ private:
         });
 
         volume_up_button_.OnLongPress([this]() {
-            auto codec = GetAudioCodec();
-            codec->SetOutputVolume(100);
+            GetAudioCodec()->SetOutputVolume(100);
             GetDisplay()->ShowNotification("最大音量");
         });
 
@@ -80,30 +80,31 @@ private:
         });
 
         volume_down_button_.OnLongPress([this]() {
-            auto codec = GetAudioCodec();
-            codec->SetOutputVolume(0);
+            GetAudioCodec()->SetOutputVolume(0);
             GetDisplay()->ShowNotification("已静音");
         });
     }
 
+    // 物联网初始化，添加对 AI 可见设备
+    void InitializeIot() {
+        auto& thing_manager = iot::ThingManager::GetInstance();
+        thing_manager.AddThing(iot::CreateThing("Speaker"));
+        thing_manager.AddThing(iot::CreateThing("Lamp"));
+    }
+
 public:
     CompactWifiBoard() :
         boot_button_(BOOT_BUTTON_GPIO),
-        touch_button_(TOUCH_BUTTON_GPIO, 1),
+        touch_button_(TOUCH_BUTTON_GPIO),
         volume_up_button_(VOLUME_UP_BUTTON_GPIO),
         volume_down_button_(VOLUME_DOWN_BUTTON_GPIO),
         system_reset_(RESET_NVS_BUTTON_GPIO, RESET_FACTORY_BUTTON_GPIO) {
-    }
-
-    virtual void Initialize() override {
-        ESP_LOGI(TAG, "Initializing CompactWifiBoard");
         // Check if the reset button is pressed
         system_reset_.CheckButtons();
 
         InitializeDisplayI2c();
         InitializeButtons();
-
-        WifiBoard::Initialize();
+        InitializeIot();
     }
 
     virtual Led* GetBuiltinLed() override {

main/boards/bread-compact-wifi/config.h

@@ -30,7 +30,7 @@
 
 #define BUILTIN_LED_GPIO        GPIO_NUM_48
 #define BOOT_BUTTON_GPIO        GPIO_NUM_0
-#define TOUCH_BUTTON_GPIO       GPIO_NUM_45
+#define TOUCH_BUTTON_GPIO       GPIO_NUM_47
 #define VOLUME_UP_BUTTON_GPIO   GPIO_NUM_40
 #define VOLUME_DOWN_BUTTON_GPIO GPIO_NUM_39
 #define RESET_NVS_BUTTON_GPIO     GPIO_NUM_1

main/boards/common/board.cc

@@ -6,7 +6,10 @@
 #include <esp_ota_ops.h>
 #include <esp_chip_info.h>
 
-// static const char *TAG = "Board";
+#define TAG "Board"
+
+Board::Board() {
+}
 
 bool Board::GetBatteryLevel(int &level, bool& charging) {
     return false;

main/boards/common/board.h

@@ -19,7 +19,7 @@ private:
     virtual std::string GetBoardJson() = 0;
 
 protected:
-    Board() = default;
+    Board();
 
 public:
     static Board& GetInstance() {
@@ -30,7 +30,6 @@ public:
         return *instance;
     }
 
-    virtual void Initialize() = 0;
     virtual void StartNetwork() = 0;
     virtual ~Board() = default;
     virtual Led* GetBuiltinLed() = 0;

main/boards/common/ml307_board.cc

@@ -1,5 +1,7 @@
 #include "ml307_board.h"
+
 #include "application.h"
+#include "display.h"
 #include "font_awesome_symbols.h"
 
 #include <esp_log.h>
@@ -72,10 +74,9 @@ void Ml307Board::WaitForNetworkReady() {
     ESP_LOGI(TAG, "ML307 Module: %s", module_name.c_str());
     ESP_LOGI(TAG, "ML307 IMEI: %s", imei.c_str());
     ESP_LOGI(TAG, "ML307 ICCID: %s", iccid.c_str());
-}
 
-void Ml307Board::Initialize() {
-    ESP_LOGI(TAG, "Initializing Ml307Board");
+    // Close all previous connections
+    modem_.ResetConnections();
 }
 
 Http* Ml307Board::CreateHttp() {

main/boards/common/ml307_board.h

@@ -13,7 +13,6 @@ protected:
 
 public:
     Ml307Board(gpio_num_t tx_pin, gpio_num_t rx_pin, size_t rx_buffer_size = 4096);
-    virtual void Initialize() override;
     virtual void StartNetwork() override;
     virtual Http* CreateHttp() override;
     virtual WebSocket* CreateWebSocket() override;

main/boards/common/wifi_board.cc

@@ -1,4 +1,6 @@
 #include "wifi_board.h"
+
+#include "display.h"
 #include "application.h"
 #include "system_info.h"
 #include "font_awesome_symbols.h"
@@ -70,10 +72,6 @@ void WifiBoard::StartNetwork() {
     }
 }
 
-void WifiBoard::Initialize() {
-    ESP_LOGI(TAG, "Initializing WifiBoard");
-}
-
 Http* WifiBoard::CreateHttp() {
     return new EspHttp();
 }

main/boards/common/wifi_board.h

@@ -10,7 +10,6 @@ protected:
     virtual std::string GetBoardJson() override;
 
 public:
-    virtual void Initialize() override;
     virtual void StartNetwork() override;
     virtual Http* CreateHttp() override;
     virtual WebSocket* CreateWebSocket() override;

main/boards/esp-box-3/esp_box3_board.cc

@@ -5,6 +5,7 @@
 #include "button.h"
 #include "led.h"
 #include "config.h"
+#include "iot/thing_manager.h"
 
 #include <esp_log.h>
 #include <driver/i2c_master.h>
@@ -39,15 +40,17 @@ private:
         });
     }
 
-public:
-    EspBox3Board() : boot_button_(BOOT_BUTTON_GPIO) {
+    // 物联网初始化，添加对 AI 可见设备
+    void InitializeIot() {
+        auto& thing_manager = iot::ThingManager::GetInstance();
+        thing_manager.AddThing(iot::CreateThing("Speaker"));
     }
 
-    virtual void Initialize() override {
-        ESP_LOGI(TAG, "Initializing EspBox3Board");
+public:
+    EspBox3Board() : boot_button_(BOOT_BUTTON_GPIO) {
         InitializeI2c();
         InitializeButtons();
-        WifiBoard::Initialize();
+        InitializeIot();
     }
 
     virtual Led* GetBuiltinLed() override {

main/boards/kevin-box-1/kevin_box_board.cc

@@ -5,13 +5,14 @@
 #include "button.h"
 #include "led.h"
 #include "config.h"
+#include "iot/thing_manager.h"
 
 #include <esp_log.h>
 #include <esp_spiffs.h>
 #include <driver/gpio.h>
 #include <driver/i2c_master.h>
 
-static const char *TAG = "KevinBoxBoard";
+#define TAG "KevinBoxBoard"
 
 class KevinBoxBoard : public Ml307Board {
 private:
@@ -95,8 +96,7 @@ private:
         });
 
         volume_up_button_.OnLongPress([this]() {
-            auto codec = GetAudioCodec();
-            codec->SetOutputVolume(100);
+            GetAudioCodec()->SetOutputVolume(100);
             GetDisplay()->ShowNotification("最大音量");
         });
 
@@ -111,29 +111,29 @@ private:
         });
 
         volume_down_button_.OnLongPress([this]() {
-            auto codec = GetAudioCodec();
-            codec->SetOutputVolume(0);
+            GetAudioCodec()->SetOutputVolume(0);
             GetDisplay()->ShowNotification("已静音");
         });
     }
 
+    // 物联网初始化，添加对 AI 可见设备
+    void InitializeIot() {
+        auto& thing_manager = iot::ThingManager::GetInstance();
+        thing_manager.AddThing(iot::CreateThing("Speaker"));
+    }
+
 public:
     KevinBoxBoard() : Ml307Board(ML307_TX_PIN, ML307_RX_PIN, 4096),
         boot_button_(BOOT_BUTTON_GPIO),
         volume_up_button_(VOLUME_UP_BUTTON_GPIO),
         volume_down_button_(VOLUME_DOWN_BUTTON_GPIO) {
-    }
-
-    virtual void Initialize() override {
-        ESP_LOGI(TAG, "Initializing KevinBoxBoard");
         InitializeDisplayI2c();
         InitializeCodecI2c();
         MountStorage();
         Enable4GModule();
 
         InitializeButtons();
-
-        Ml307Board::Initialize();
+        InitializeIot();
     }
 
     virtual Led* GetBuiltinLed() override {

main/boards/kevin-box-2/kevin_box_board.cc

@@ -6,6 +6,7 @@
 #include "led.h"
 #include "config.h"
 #include "axp2101.h"
+#include "iot/thing_manager.h"
 
 #include <esp_log.h>
 #include <esp_spiffs.h>
@@ -13,7 +14,7 @@
 #include <driver/i2c_master.h>
 #include <esp_timer.h>
 
-static const char *TAG = "KevinBoxBoard";
+#define TAG "KevinBoxBoard"
 
 class KevinBoxBoard : public Ml307Board {
 private:
@@ -136,8 +137,7 @@ private:
         });
 
         volume_up_button_.OnLongPress([this]() {
-            auto codec = GetAudioCodec();
-            codec->SetOutputVolume(100);
+            GetAudioCodec()->SetOutputVolume(100);
             GetDisplay()->ShowNotification("最大音量");
         });
 
@@ -152,21 +152,22 @@ private:
         });
 
         volume_down_button_.OnLongPress([this]() {
-            auto codec = GetAudioCodec();
-            codec->SetOutputVolume(0);
+            GetAudioCodec()->SetOutputVolume(0);
             GetDisplay()->ShowNotification("已静音");
         });
     }
 
+    // 物联网初始化，添加对 AI 可见设备
+    void InitializeIot() {
+        auto& thing_manager = iot::ThingManager::GetInstance();
+        thing_manager.AddThing(iot::CreateThing("Speaker"));
+    }
+
 public:
     KevinBoxBoard() : Ml307Board(ML307_TX_PIN, ML307_RX_PIN, 4096),
         boot_button_(BOOT_BUTTON_GPIO),
         volume_up_button_(VOLUME_UP_BUTTON_GPIO),
         volume_down_button_(VOLUME_DOWN_BUTTON_GPIO) {
-    }
-
-    virtual void Initialize() override {
-        ESP_LOGI(TAG, "Initializing KevinBoxBoard");
         InitializeDisplayI2c();
         InitializeCodecI2c();
         axp2101_ = new Axp2101(codec_i2c_bus_, AXP2101_I2C_ADDR);
@@ -176,8 +177,7 @@ public:
 
         InitializeButtons();
         InitializePowerSaveTimer();
-
-        Ml307Board::Initialize();
+        InitializeIot();
     }
 
     virtual Led* GetBuiltinLed() override {

main/boards/kevin-c3/kevin_box_board.cc

@@ -4,6 +4,7 @@
 #include "button.h"
 #include "led.h"
 #include "config.h"
+#include "iot/thing_manager.h"
 
 #include <wifi_station.h>
 #include <esp_log.h>
@@ -48,18 +49,17 @@ private:
         });
     }
 
-public:
-    KevinBoxBoard() :
-        boot_button_(BOOT_BUTTON_GPIO) {
+    // 物联网初始化，添加对 AI 可见设备
+    void InitializeIot() {
+        auto& thing_manager = iot::ThingManager::GetInstance();
+        thing_manager.AddThing(iot::CreateThing("Speaker"));
     }
 
-    virtual void Initialize() override {
-        ESP_LOGI(TAG, "Initializing KevinBoxBoard");
-
+public:
+    KevinBoxBoard() : boot_button_(BOOT_BUTTON_GPIO) {
         InitializeCodecI2c();
         InitializeButtons();
-
-        WifiBoard::Initialize();
+        InitializeIot();
     }
 
     virtual Led* GetBuiltinLed() override {

main/boards/lichuang-dev/lichuang_dev_board.cc

@@ -6,6 +6,7 @@
 #include "led.h"
 #include "config.h"
 #include "i2c_device.h"
+#include "iot/thing_manager.h"
 
 #include <esp_log.h>
 #include <esp_lcd_panel_vendor.h>
@@ -119,17 +120,19 @@ private:
                                     DISPLAY_WIDTH, DISPLAY_HEIGHT, DISPLAY_OFFSET_X, DISPLAY_OFFSET_Y, DISPLAY_MIRROR_X, DISPLAY_MIRROR_Y, DISPLAY_SWAP_XY);
     }
 
-public:
-    LichuangDevBoard() : boot_button_(BOOT_BUTTON_GPIO) {
+    // 物联网初始化，添加对 AI 可见设备
+    void InitializeIot() {
+        auto& thing_manager = iot::ThingManager::GetInstance();
+        thing_manager.AddThing(iot::CreateThing("Speaker"));
     }
 
-    virtual void Initialize() override {
-        ESP_LOGI(TAG, "Initializing LichuangDevBoard");
+public:
+    LichuangDevBoard() : boot_button_(BOOT_BUTTON_GPIO) {
         InitializeI2c();
         InitializeSpi();
         InitializeSt7789Display();
         InitializeButtons();
-        WifiBoard::Initialize();
+        InitializeIot();
     }
 
     virtual Led* GetBuiltinLed() override {

main/idf_component.yml

@@ -1,7 +1,7 @@
 ## IDF Component Manager Manifest File
 dependencies:
   78/esp-wifi-connect: "~1.4.1"
-  78/esp-opus-encoder: "~1.1.0"
+  78/esp-opus-encoder: "~2.0.0"
   78/esp-ml307: "~1.7.0"
   espressif/led_strip: "^2.4.1"
   espressif/esp_codec_dev: "^1.3.1"

main/iot/sample_interface.json

@@ -0,0 +1,38 @@
+[
+    {
+        "name": "lamp",
+        "description": "A lamp",
+        "properties": {
+            "power": {
+                "type": "boolean",
+                "description": "Whether the lamp is on or off"
+            }
+        },
+        "methods": {
+            "TurnOn": {
+                "description": "Turns the lamp on"
+            }
+        }
+    },
+    {
+        "name": "speaker",
+        "description": "当前 AI 机器人的扬声器",
+        "properties": {
+            "volume": {
+                "type": "number",
+                "description": "当前扬声器的音量（0-100）"
+            }
+        },
+        "methods": {
+            "SetVolume": {
+                "description": "设置当前扬声器的音量",
+                "parameters": {
+                    "volume": {
+                        "type": "number",
+                        "description": "The volume of the speaker (0-100)"
+                    }
+                }
+            }
+        }
+    }
+]

main/iot/thing.cc

@@ -0,0 +1,77 @@
+#include "thing.h"
+#include "application.h"
+
+#include <esp_log.h>
+
+#define TAG "Thing"
+
+
+namespace iot {
+
+static std::map<std::string, std::function<Thing*()>>* thing_creators = nullptr;
+
+void RegisterThing(const std::string& type, std::function<Thing*()> creator) {
+    if (thing_creators == nullptr) {
+        thing_creators = new std::map<std::string, std::function<Thing*()>>();
+    }
+    (*thing_creators)[type] = creator;
+}
+
+Thing* CreateThing(const std::string& type) {
+    auto creator = thing_creators->find(type);
+    if (creator == thing_creators->end()) {
+        ESP_LOGE(TAG, "Thing type not found: %s", type.c_str());
+        return nullptr;
+    }
+    return creator->second();
+}
+
+std::string Thing::GetDescriptorJson() {
+    std::string json_str = "{";
+    json_str += "\"name\":\"" + name_ + "\",";
+    json_str += "\"description\":\"" + description_ + "\",";
+    json_str += "\"properties\":" + properties_.GetDescriptorJson() + ",";
+    json_str += "\"methods\":" + methods_.GetDescriptorJson();
+    json_str += "}";
+    return json_str;
+}
+
+std::string Thing::GetStateJson() {
+    std::string json_str = "{";
+    json_str += "\"name\":\"" + name_ + "\",";
+    json_str += "\"state\":" + properties_.GetStateJson();
+    json_str += "}";
+    return json_str;
+}
+
+void Thing::Invoke(const cJSON* command) {
+    auto method_name = cJSON_GetObjectItem(command, "method");
+    auto input_params = cJSON_GetObjectItem(command, "parameters");
+
+    try {
+        auto& method = methods_[method_name->valuestring];
+        for (auto& param : method.parameters()) {
+            auto input_param = cJSON_GetObjectItem(input_params, param.name().c_str());
+            if (param.required() && input_param == nullptr) {
+                throw std::runtime_error("Parameter " + param.name() + " is required");
+            }
+            if (param.type() == kValueTypeNumber) {
+                param.set_number(input_param->valueint);
+            } else if (param.type() == kValueTypeString) {
+                param.set_string(input_param->valuestring);
+            } else if (param.type() == kValueTypeBoolean) {
+                param.set_boolean(input_param->valueint == 1);
+            }
+        }
+
+        Application::GetInstance().Schedule([&method]() {
+            method.Invoke();
+        });
+    } catch (const std::runtime_error& e) {
+        ESP_LOGE(TAG, "Method not found: %s", method_name->valuestring);
+        return;
+    }
+}
+
+
+} // namespace iot

main/iot/thing.h

@@ -0,0 +1,300 @@
+#ifndef THING_H
+#define THING_H
+
+#include <string>
+#include <map>
+#include <functional>
+#include <vector>
+#include <stdexcept>
+#include <cJSON.h>
+
+namespace iot {
+
+enum ValueType {
+    kValueTypeBoolean,
+    kValueTypeNumber,
+    kValueTypeString
+};
+
+class Property {
+private:
+    std::string name_;
+    std::string description_;
+    ValueType type_;
+    std::function<bool()> boolean_getter_;
+    std::function<int()> number_getter_;
+    std::function<std::string()> string_getter_;
+
+public:
+    Property(const std::string& name, const std::string& description, std::function<bool()> getter) :
+        name_(name), description_(description), type_(kValueTypeBoolean), boolean_getter_(getter) {}
+    Property(const std::string& name, const std::string& description, std::function<int()> getter) :
+        name_(name), description_(description), type_(kValueTypeNumber), number_getter_(getter) {}
+    Property(const std::string& name, const std::string& description, std::function<std::string()> getter) :
+        name_(name), description_(description), type_(kValueTypeString), string_getter_(getter) {}
+
+    const std::string& name() const { return name_; }
+    const std::string& description() const { return description_; }
+    ValueType type() const { return type_; }
+
+    bool boolean() const { return boolean_getter_(); }
+    int number() const { return number_getter_(); }
+    std::string string() const { return string_getter_(); }
+
+    std::string GetDescriptorJson() {
+        std::string json_str = "{";
+        json_str += "\"description\":\"" + description_ + "\",";
+        if (type_ == kValueTypeBoolean) {
+            json_str += "\"type\":\"boolean\"";
+        } else if (type_ == kValueTypeNumber) {
+            json_str += "\"type\":\"number\"";
+        } else if (type_ == kValueTypeString) {
+            json_str += "\"type\":\"string\"";
+        }
+        json_str += "}";
+        return json_str;
+    }
+
+    std::string GetStateJson() {
+        if (type_ == kValueTypeBoolean) {
+            return boolean_getter_() ? "true" : "false";
+        } else if (type_ == kValueTypeNumber) {
+            return std::to_string(number_getter_());
+        } else if (type_ == kValueTypeString) {
+            return "\"" + string_getter_() + "\"";
+        }
+        return "null";
+    }
+};
+
+class PropertyList {
+private:
+    std::vector<Property> properties_;
+
+public:
+    PropertyList() = default;
+    PropertyList(const std::vector<Property>& properties) : properties_(properties) {}
+
+    void AddBooleanProperty(const std::string& name, const std::string& description, std::function<bool()> getter) {
+        properties_.push_back(Property(name, description, getter));
+    }
+    void AddNumberProperty(const std::string& name, const std::string& description, std::function<int()> getter) {
+        properties_.push_back(Property(name, description, getter));
+    }
+    void AddStringProperty(const std::string& name, const std::string& description, std::function<std::string()> getter) {
+        properties_.push_back(Property(name, description, getter));
+    }
+
+    const Property& operator[](const std::string& name) const {
+        for (auto& property : properties_) {
+            if (property.name() == name) {
+                return property;
+            }
+        }
+        throw std::runtime_error("Property not found: " + name);
+    }
+
+    std::string GetDescriptorJson() {
+        std::string json_str = "{";
+        for (auto& property : properties_) {
+            json_str += "\"" + property.name() + "\":" + property.GetDescriptorJson() + ",";
+        }
+        if (json_str.back() == ',') {
+            json_str.pop_back();
+        }
+        json_str += "}";
+        return json_str;
+    }
+
+    std::string GetStateJson() {
+        std::string json_str = "{";
+        for (auto& property : properties_) {
+            json_str += "\"" + property.name() + "\":" + property.GetStateJson() + ",";
+        }
+        if (json_str.back() == ',') {
+            json_str.pop_back();
+        }
+        json_str += "}";
+        return json_str;
+    }
+};
+
+class Parameter {
+private:
+    std::string name_;
+    std::string description_;
+    ValueType type_;
+    bool required_;
+    bool boolean_;
+    int number_;
+    std::string string_;
+
+public:
+    Parameter(const std::string& name, const std::string& description, ValueType type, bool required = true) :
+        name_(name), description_(description), type_(type), required_(required) {}
+
+    const std::string& name() const { return name_; }
+    const std::string& description() const { return description_; }
+    ValueType type() const { return type_; }
+    bool required() const { return required_; }
+
+    bool boolean() const { return boolean_; }
+    int number() const { return number_; }
+    const std::string& string() const { return string_; }
+
+    void set_boolean(bool value) { boolean_ = value; }
+    void set_number(int value) { number_ = value; }
+    void set_string(const std::string& value) { string_ = value; }
+
+    std::string GetDescriptorJson() {
+        std::string json_str = "{";
+        json_str += "\"description\":\"" + description_ + "\",";
+        if (type_ == kValueTypeBoolean) {
+            json_str += "\"type\":\"boolean\"";
+        } else if (type_ == kValueTypeNumber) {
+            json_str += "\"type\":\"number\"";
+        } else if (type_ == kValueTypeString) {
+            json_str += "\"type\":\"string\"";
+        }
+        json_str += "}";
+        return json_str;
+    }
+};
+
+class ParameterList {
+private:
+    std::vector<Parameter> parameters_;
+
+public:
+    ParameterList() = default;
+    ParameterList(const std::vector<Parameter>& parameters) : parameters_(parameters) {}
+    void AddParameter(const Parameter& parameter) {
+        parameters_.push_back(parameter);
+    }
+
+    const Parameter& operator[](const std::string& name) const {
+        for (auto& parameter : parameters_) {
+            if (parameter.name() == name) {
+                return parameter;
+            }
+        }
+        throw std::runtime_error("Parameter not found: " + name);
+    }
+
+    // iterator
+    auto begin() { return parameters_.begin(); }
+    auto end() { return parameters_.end(); }
+
+    std::string GetDescriptorJson() {
+        std::string json_str = "{";
+        for (auto& parameter : parameters_) {
+            json_str += "\"" + parameter.name() + "\":" + parameter.GetDescriptorJson() + ",";
+        }
+        if (json_str.back() == ',') {
+            json_str.pop_back();
+        }
+        json_str += "}";
+        return json_str;
+    }
+};
+
+class Method {
+private:
+    std::string name_;
+    std::string description_;
+    ParameterList parameters_;
+    std::function<void(const ParameterList&)> callback_;
+
+public:
+    Method(const std::string& name, const std::string& description, const ParameterList& parameters, std::function<void(const ParameterList&)> callback) :
+        name_(name), description_(description), parameters_(parameters), callback_(callback) {}
+
+    const std::string& name() const { return name_; }
+    const std::string& description() const { return description_; }
+    ParameterList& parameters() { return parameters_; }
+
+    std::string GetDescriptorJson() {
+        std::string json_str = "{";
+        json_str += "\"description\":\"" + description_ + "\",";
+        json_str += "\"parameters\":" + parameters_.GetDescriptorJson();
+        json_str += "}";
+        return json_str;
+    }
+
+    void Invoke() {
+        callback_(parameters_);
+    }
+};
+
+class MethodList {
+private:
+    std::vector<Method> methods_;
+
+public:
+    MethodList() = default;
+    MethodList(const std::vector<Method>& methods) : methods_(methods) {}
+
+    void AddMethod(const std::string& name, const std::string& description, const ParameterList& parameters, std::function<void(const ParameterList&)> callback) {
+        methods_.push_back(Method(name, description, parameters, callback));
+    }
+
+    Method& operator[](const std::string& name) {
+        for (auto& method : methods_) {
+            if (method.name() == name) {
+                return method;
+            }
+        }
+        throw std::runtime_error("Method not found: " + name);
+    }
+
+    std::string GetDescriptorJson() {
+        std::string json_str = "{";
+        for (auto& method : methods_) {
+            json_str += "\"" + method.name() + "\":" + method.GetDescriptorJson() + ",";
+        }
+        if (json_str.back() == ',') {
+            json_str.pop_back();
+        }
+        json_str += "}";
+        return json_str;
+    }
+};
+
+class Thing {
+public:
+    Thing(const std::string& name, const std::string& description) :
+        name_(name), description_(description) {}
+    virtual ~Thing() = default;
+
+    virtual std::string GetDescriptorJson();
+    virtual std::string GetStateJson();
+    virtual void Invoke(const cJSON* command);
+
+    const std::string& name() const { return name_; }
+    const std::string& description() const { return description_; }
+
+protected:
+    PropertyList properties_;
+    MethodList methods_;
+
+private:
+    std::string name_;
+    std::string description_;
+};
+
+
+void RegisterThing(const std::string& type, std::function<Thing*()> creator);
+Thing* CreateThing(const std::string& type);
+
+#define DECLARE_THING(TypeName) \
+    static iot::Thing* Create##TypeName() { \
+        return new iot::TypeName(); \
+    } \
+    static bool Register##TypeNameHelper = []() { \
+        RegisterThing(#TypeName, Create##TypeName); \
+        return true; \
+    }();
+
+} // namespace iot
+
+#endif // THING_H

main/iot/thing_manager.cc

@@ -0,0 +1,47 @@
+#include "thing_manager.h"
+
+#include <esp_log.h>
+
+#define TAG "ThingManager"
+
+namespace iot {
+
+void ThingManager::AddThing(Thing* thing) {
+    things_.push_back(thing);
+}
+
+std::string ThingManager::GetDescriptorsJson() {
+    std::string json_str = "[";
+    for (auto& thing : things_) {
+        json_str += thing->GetDescriptorJson() + ",";
+    }
+    if (json_str.back() == ',') {
+        json_str.pop_back();
+    }
+    json_str += "]";
+    return json_str;
+}
+
+std::string ThingManager::GetStatesJson() {
+    std::string json_str = "[";
+    for (auto& thing : things_) {
+        json_str += thing->GetStateJson() + ",";
+    }
+    if (json_str.back() == ',') {
+        json_str.pop_back();
+    }
+    json_str += "]";
+    return json_str;
+}
+
+void ThingManager::Invoke(const cJSON* command) {
+    auto name = cJSON_GetObjectItem(command, "name");
+    for (auto& thing : things_) {
+        if (thing->name() == name->valuestring) {
+            thing->Invoke(command);
+            return;
+        }
+    }
+}
+
+} // namespace iot

main/iot/thing_manager.h

@@ -0,0 +1,41 @@
+#ifndef THING_MANAGER_H
+#define THING_MANAGER_H
+
+
+#include "thing.h"
+
+#include <cJSON.h>
+
+#include <vector>
+#include <memory>
+#include <functional>
+#include <map>
+
+namespace iot {
+
+class ThingManager {
+public:
+    static ThingManager& GetInstance() {
+        static ThingManager instance;
+        return instance;
+    }
+    ThingManager(const ThingManager&) = delete;
+    ThingManager& operator=(const ThingManager&) = delete;
+
+    void AddThing(Thing* thing);
+
+    std::string GetDescriptorsJson();
+    std::string GetStatesJson();
+    void Invoke(const cJSON* command);
+
+private:
+    ThingManager() = default;
+    ~ThingManager() = default;
+
+    std::vector<Thing*> things_;
+};
+
+
+} // namespace iot
+
+#endif // THING_MANAGER_H

main/iot/things/lamp.cc

@@ -0,0 +1,54 @@
+#include "iot/thing.h"
+#include "board.h"
+#include "audio_codec.h"
+
+#include <driver/gpio.h>
+#include <esp_log.h>
+
+#define TAG "Lamp"
+
+namespace iot {
+
+// 这里仅定义 Lamp 的属性和方法，不包含具体的实现
+class Lamp : public Thing {
+private:
+    gpio_num_t gpio_num_ = GPIO_NUM_18;
+    bool power_ = false;
+
+    void InitializeGpio() {
+        gpio_config_t config = {
+            .pin_bit_mask = (1ULL << gpio_num_),
+            .mode = GPIO_MODE_OUTPUT,
+            .pull_up_en = GPIO_PULLUP_DISABLE,
+            .pull_down_en = GPIO_PULLDOWN_DISABLE,
+            .intr_type = GPIO_INTR_DISABLE,
+        };
+        ESP_ERROR_CHECK(gpio_config(&config));
+        gpio_set_level(gpio_num_, 0);
+    }
+
+public:
+    Lamp() : Thing("Lamp", "一个测试用的灯"), power_(false) {
+        InitializeGpio();
+
+        // 定义设备的属性
+        properties_.AddBooleanProperty("power", "灯是否打开", [this]() -> bool {
+            return power_;
+        });
+
+        // 定义设备可以被远程执行的指令
+        methods_.AddMethod("TurnOn", "打开灯", ParameterList(), [this](const ParameterList& parameters) {
+            power_ = true;
+            gpio_set_level(gpio_num_, 1);
+        });
+
+        methods_.AddMethod("TurnOff", "关闭灯", ParameterList(), [this](const ParameterList& parameters) {
+            power_ = false;
+            gpio_set_level(gpio_num_, 0);
+        });
+    }
+};
+
+} // namespace iot
+
+DECLARE_THING(Lamp);

main/iot/things/speaker.cc

@@ -0,0 +1,33 @@
+#include "iot/thing.h"
+#include "board.h"
+#include "audio_codec.h"
+
+#include <esp_log.h>
+
+#define TAG "Speaker"
+
+namespace iot {
+
+// 这里仅定义 Speaker 的属性和方法，不包含具体的实现
+class Speaker : public Thing {
+public:
+    Speaker() : Thing("Speaker", "当前 AI 机器人的扬声器") {
+        // 定义设备的属性
+        properties_.AddNumberProperty("volume", "当前音量值", [this]() -> int {
+            auto codec = Board::GetInstance().GetAudioCodec();
+            return codec->output_volume();
+        });
+
+        // 定义设备可以被远程执行的指令
+        methods_.AddMethod("SetVolume", "设置音量", ParameterList({
+            Parameter("volume", "0到100之间的整数", kValueTypeNumber, true)
+        }), [this](const ParameterList& parameters) {
+            auto codec = Board::GetInstance().GetAudioCodec();
+            codec->SetOutputVolume(static_cast<uint8_t>(parameters["volume"].number()));
+        });
+    }
+};
+
+} // namespace iot
+
+DECLARE_THING(Speaker);

main/protocols/mqtt_protocol.cc

@@ -105,7 +105,7 @@ void MqttProtocol::SendText(const std::string& text) {
     mqtt_->Publish(publish_topic_, text);
 }
 
-void MqttProtocol::SendAudio(const std::string& data) {
+void MqttProtocol::SendAudio(const std::vector<uint8_t>& data) {
     std::lock_guard<std::mutex> lock(channel_mutex_);
     if (udp_ == nullptr) {
         return;
@@ -202,7 +202,7 @@ bool MqttProtocol::OpenAudioChannel() {
             ESP_LOGW(TAG, "Received audio packet with wrong sequence: %lu, expected: %lu", sequence, remote_sequence_ + 1);
         }
 
-        std::string decrypted;
+        std::vector<uint8_t> decrypted;
         size_t decrypted_size = data.size() - aes_nonce_.size();
         size_t nc_off = 0;
         uint8_t stream_block[16] = {0};
@@ -215,7 +215,7 @@ bool MqttProtocol::OpenAudioChannel() {
             return;
         }
         if (on_incoming_audio_ != nullptr) {
-            on_incoming_audio_(decrypted);
+            on_incoming_audio_(std::move(decrypted));
         }
         remote_sequence_ = sequence;
     });

main/protocols/mqtt_protocol.h

@@ -25,7 +25,7 @@ public:
     MqttProtocol();
     ~MqttProtocol();
 
-    void SendAudio(const std::string& data) override;
+    void SendAudio(const std::vector<uint8_t>& data) override;
     bool OpenAudioChannel() override;
     void CloseAudioChannel() override;
     bool IsAudioChannelOpened() const override;

main/protocols/protocol.cc

@@ -8,7 +8,7 @@ void Protocol::OnIncomingJson(std::function<void(const cJSON* root)> callback) {
     on_incoming_json_ = callback;
 }
 
-void Protocol::OnIncomingAudio(std::function<void(const std::string& data)> callback) {
+void Protocol::OnIncomingAudio(std::function<void(std::vector<uint8_t>&& data)> callback) {
     on_incoming_audio_ = callback;
 }
 
@@ -57,3 +57,14 @@ void Protocol::SendStopListening() {
     std::string message = "{\"session_id\":\"" + session_id_ + "\",\"type\":\"listen\",\"state\":\"stop\"}";
     SendText(message);
 }
+
+void Protocol::SendIotDescriptors(const std::string& descriptors) {
+    std::string message = "{\"session_id\":\"" + session_id_ + "\",\"type\":\"iot\",\"descriptors\":" + descriptors + "}";
+    SendText(message);
+}
+
+void Protocol::SendIotStates(const std::string& states) {
+    std::string message = "{\"session_id\":\"" + session_id_ + "\",\"type\":\"iot\",\"states\":" + states + "}";
+    SendText(message);
+}
+

main/protocols/protocol.h

@@ -31,7 +31,7 @@ public:
         return server_sample_rate_;
     }
 
-    void OnIncomingAudio(std::function<void(const std::string& data)> callback);
+    void OnIncomingAudio(std::function<void(std::vector<uint8_t>&& data)> callback);
     void OnIncomingJson(std::function<void(const cJSON* root)> callback);
     void OnAudioChannelOpened(std::function<void()> callback);
     void OnAudioChannelClosed(std::function<void()> callback);
@@ -40,15 +40,17 @@ public:
     virtual bool OpenAudioChannel() = 0;
     virtual void CloseAudioChannel() = 0;
     virtual bool IsAudioChannelOpened() const = 0;
-    virtual void SendAudio(const std::string& data) = 0;
+    virtual void SendAudio(const std::vector<uint8_t>& data) = 0;
     virtual void SendWakeWordDetected(const std::string& wake_word);
     virtual void SendStartListening(ListeningMode mode);
     virtual void SendStopListening();
     virtual void SendAbortSpeaking(AbortReason reason);
+    virtual void SendIotDescriptors(const std::string& descriptors);
+    virtual void SendIotStates(const std::string& states);
 
 protected:
     std::function<void(const cJSON* root)> on_incoming_json_;
-    std::function<void(const std::string& data)> on_incoming_audio_;
+    std::function<void(std::vector<uint8_t>&& data)> on_incoming_audio_;
     std::function<void()> on_audio_channel_opened_;
     std::function<void()> on_audio_channel_closed_;
     std::function<void(const std::string& message)> on_network_error_;

main/protocols/websocket_protocol.cc

@@ -23,7 +23,7 @@ WebsocketProtocol::~WebsocketProtocol() {
     vEventGroupDelete(event_group_handle_);
 }
 
-void WebsocketProtocol::SendAudio(const std::string& data) {
+void WebsocketProtocol::SendAudio(const std::vector<uint8_t>& data) {
     if (websocket_ == nullptr) {
         return;
     }
@@ -65,7 +65,7 @@ bool WebsocketProtocol::OpenAudioChannel() {
     websocket_->OnData([this](const char* data, size_t len, bool binary) {
         if (binary) {
             if (on_incoming_audio_ != nullptr) {
-                on_incoming_audio_(std::string(data, len));
+                on_incoming_audio_(std::vector<uint8_t>((uint8_t*)data, (uint8_t*)data + len));
             }
         } else {
             // Parse JSON data

main/protocols/websocket_protocol.h

@@ -15,7 +15,7 @@ public:
     WebsocketProtocol();
     ~WebsocketProtocol();
 
-    void SendAudio(const std::string& data) override;
+    void SendAudio(const std::vector<uint8_t>& data) override;
     bool OpenAudioChannel() override;
     void CloseAudioChannel() override;
     bool IsAudioChannelOpened() const override;

sdkconfig.defaults

@@ -1,3 +1,6 @@
+CONFIG_COMPILER_CXX_EXCEPTIONS=y
+CONFIG_COMPILER_CXX_EXCEPTIONS_EMG_POOL_SIZE=1024
+
 CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_PERF=y
 CONFIG_BOOTLOADER_LOG_LEVEL_NONE=y
 CONFIG_BOOTLOADER_SKIP_VALIDATE_ALWAYS=y