bump to 0.3.3

CMakeLists.txt

@@ -4,7 +4,7 @@
 # CMakeLists in this exact order for cmake to work correctly
 cmake_minimum_required(VERSION 3.16)
 
-set(PROJECT_VER "0.3.1")
+set(PROJECT_VER "0.3.3")
 
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 project(xiaozhi)

main/Application.cc

@@ -1,4 +1,5 @@
 #include <BuiltinLed.h>
+#include <TcpTransport.h>
 #include <TlsTransport.h>
 #include <Ml307SslTransport.h>
 #include <WifiConfigurationAp.h>
@@ -16,7 +17,8 @@
 
 
 Application::Application()
-    : button_((gpio_num_t)CONFIG_BOOT_BUTTON_GPIO)
+    : boot_button_((gpio_num_t)CONFIG_BOOT_BUTTON_GPIO),
+      volume_up_button_((gpio_num_t)CONFIG_VOLUME_UP_BUTTON_GPIO)
 #ifdef CONFIG_USE_ML307
     , ml307_at_modem_(CONFIG_ML307_TX_PIN, CONFIG_ML307_RX_PIN, 4096),
       http_(ml307_at_modem_),
@@ -153,6 +155,14 @@ void Application::Start() {
 
     ESP_LOGI(TAG, "ML307 IMEI: %s", ml307_at_modem_.GetImei().c_str());
     ESP_LOGI(TAG, "ML307 ICCID: %s", ml307_at_modem_.GetIccid().c_str());
+
+    // If low power, the material ready event will be triggered by the modem because of a reset
+    ml307_at_modem_.OnMaterialReady([this]() {
+        ESP_LOGI(TAG, "ML307 material ready");
+        Schedule([this]() {
+            SetChatState(kChatStateIdle);
+        });
+    });
 #else
     // Try to connect to WiFi, if failed, launch the WiFi configuration AP
     auto& wifi_station = WifiStation::GetInstance();    
@@ -211,7 +221,7 @@ void Application::Start() {
         Application* app = (Application*)arg;
         app->AudioPlayTask();
         vTaskDelete(NULL);
-    }, "play_audio", 4096 * 2, this, 5, NULL);
+    }, "play_audio", 4096 * 4, this, 5, NULL);
 
 #ifdef CONFIG_USE_AFE_SR
     wake_word_detect_.OnVadStateChange([this](bool speaking) {
@@ -277,7 +287,7 @@ void Application::Start() {
     builtin_led.SetGreen();
     builtin_led.BlinkOnce();
 
-    button_.OnClick([this]() {
+    boot_button_.OnClick([this]() {
         Schedule([this]() {
             if (chat_state_ == kChatStateIdle) {
                 SetChatState(kChatStateConnecting);
@@ -303,6 +313,28 @@ void Application::Start() {
         });
     });
 
+    volume_up_button_.OnClick([this]() {
+        Schedule([this]() {
+            auto volume = audio_device_.output_volume() + 10;
+            if (volume > 100) {
+                volume = 0;
+            }
+            audio_device_.SetOutputVolume(volume);
+#ifdef CONFIG_USE_DISPLAY
+            display_.ShowNotification("Volume\n" + std::to_string(volume));
+#endif
+        });
+    });
+
+    volume_up_button_.OnLongPress([this]() {
+        Schedule([this]() {
+            audio_device_.SetOutputVolume(0);
+#ifdef CONFIG_USE_DISPLAY
+            display_.ShowNotification("Volume\n0");
+#endif
+        });
+    });
+
     xTaskCreate([](void* arg) {
         Application* app = (Application*)arg;
         app->MainLoop();
@@ -433,7 +465,9 @@ void Application::AudioEncodeTask() {
                 auto protocol = AllocateBinaryProtocol(opus, opus_size);
                 Schedule([this, protocol, opus_size]() {
                     if (ws_client_ && ws_client_->IsConnected()) {
-                        ws_client_->Send(protocol, sizeof(BinaryProtocol) + opus_size, true);
+                        if (!ws_client_->Send(protocol, sizeof(BinaryProtocol) + opus_size, true)) {
+                            ESP_LOGE(TAG, "Failed to send audio data");
+                        }
                     }
                     heap_caps_free(protocol);
                 });
@@ -443,7 +477,7 @@ void Application::AudioEncodeTask() {
             audio_decode_queue_.pop_front();
             lock.unlock();
 
-            int frame_size = opus_decode_sample_rate_ / 1000 * opus_duration_ms_;
+            int frame_size = opus_decode_sample_rate_ * opus_duration_ms_ / 1000;
             packet->pcm.resize(frame_size);
 
             int ret = opus_decode(opus_decoder_, packet->opus.data(), packet->opus.size(), packet->pcm.data(), frame_size, 0);
@@ -539,6 +573,7 @@ void Application::SetDecodeSampleRate(int sample_rate) {
     opus_decode_sample_rate_ = sample_rate;
     opus_decoder_ = opus_decoder_create(opus_decode_sample_rate_, 1, NULL);
     if (opus_decode_sample_rate_ != CONFIG_AUDIO_OUTPUT_SAMPLE_RATE) {
+        ESP_LOGI(TAG, "Resampling audio from %d to %d", opus_decode_sample_rate_, CONFIG_AUDIO_OUTPUT_SAMPLE_RATE);
         opus_resampler_.Configure(opus_decode_sample_rate_, CONFIG_AUDIO_OUTPUT_SAMPLE_RATE);
     }
 }
@@ -549,11 +584,16 @@ void Application::StartWebSocketClient() {
         delete ws_client_;
     }
 
+    std::string url = CONFIG_WEBSOCKET_URL;
     std::string token = "Bearer " + std::string(CONFIG_WEBSOCKET_ACCESS_TOKEN);
 #ifdef CONFIG_USE_ML307
     ws_client_ = new WebSocket(new Ml307SslTransport(ml307_at_modem_, 0));
 #else
-    ws_client_ = new WebSocket(new TlsTransport());
+    if (url.find("wss://") == 0) {
+        ws_client_ = new WebSocket(new TlsTransport());
+    } else {
+        ws_client_ = new WebSocket(new TcpTransport());
+    }
 #endif
     ws_client_->SetHeader("Authorization", token.c_str());
     ws_client_->SetHeader("Device-Id", SystemInfo::GetMacAddress().c_str());
@@ -617,7 +657,16 @@ void Application::StartWebSocketClient() {
                     if (text != NULL) {
                         ESP_LOGI(TAG, ">> %s", text->valuestring);
                     }
+                } else if (strcmp(type->valuestring, "llm") == 0) {
+                    auto emotion = cJSON_GetObjectItem(root, "emotion");
+                    if (emotion != NULL) {
+                        ESP_LOGD(TAG, "EMOTION: %s", emotion->valuestring);
+                    }
+                } else {
+                    ESP_LOGW(TAG, "Unknown message type: %s", type->valuestring);
                 }
+            } else {
+                ESP_LOGE(TAG, "Missing message type, data: %s", data);
             }
             cJSON_Delete(root);
         }
@@ -639,7 +688,7 @@ void Application::StartWebSocketClient() {
         });
     });
 
-    if (!ws_client_->Connect(CONFIG_WEBSOCKET_URL)) {
+    if (!ws_client_->Connect(url.c_str())) {
         ESP_LOGE(TAG, "Failed to connect to websocket server");
         return;
     }

main/Application.h

@@ -84,7 +84,8 @@ private:
     Application();
     ~Application();
 
-    Button button_;
+    Button boot_button_;
+    Button volume_up_button_;
     AudioDevice audio_device_;
 #ifdef CONFIG_USE_AFE_SR
     WakeWordDetect wake_word_detect_;

main/AudioDevice.cc

@@ -1,7 +1,7 @@
 #include "AudioDevice.h"
 #include <esp_log.h>
 #include <cstring>
-
+#include <cmath>
 #define TAG "AudioDevice"
 
 AudioDevice::AudioDevice() {
@@ -152,8 +152,12 @@ void AudioDevice::CreateSimplexChannels() {
 
 void AudioDevice::Write(const int16_t* data, int samples) {
     int32_t buffer[samples];
+
+    // output_volume_: 0-100
+    // volume_factor_: 0-65536
+    int32_t volume_factor = pow(double(output_volume_) / 100.0, 2) * 65536;
     for (int i = 0; i < samples; i++) {
-        buffer[i] = int32_t(data[i]) << 15;
+        buffer[i] = int32_t(data[i]) * volume_factor;
     }
 
     size_t bytes_written;
@@ -196,3 +200,8 @@ void AudioDevice::InputTask() {
         }
     }
 }
+
+void AudioDevice::SetOutputVolume(int volume) {
+    output_volume_ = volume;
+    ESP_LOGI(TAG, "Set output volume to %d", output_volume_);
+}

main/AudioDevice.h

@@ -17,15 +17,17 @@ public:
     void Start(int input_sample_rate, int output_sample_rate);
     void OnInputData(std::function<void(const int16_t*, int)> callback);
     void OutputData(std::vector<int16_t>& data);
+    void SetOutputVolume(int volume);
 
     int input_sample_rate() const { return input_sample_rate_; }
     int output_sample_rate() const { return output_sample_rate_; }
     bool duplex() const { return duplex_; }
-
+    int output_volume() const { return output_volume_; }
 private:
     bool duplex_ = false;
     int input_sample_rate_ = 0;
     int output_sample_rate_ = 0;
+    int output_volume_ = 80;
     i2s_chan_handle_t tx_handle_ = nullptr;
     i2s_chan_handle_t rx_handle_ = nullptr;
 

main/Button.cc

@@ -6,8 +6,8 @@ static const char* TAG = "Button";
 Button::Button(gpio_num_t gpio_num) : gpio_num_(gpio_num) {
     button_config_t button_config = {
         .type = BUTTON_TYPE_GPIO,
-        .long_press_time = 3000,
-        .short_press_time = 100,
+        .long_press_time = 1000,
+        .short_press_time = 50,
         .gpio_button_config = {
             .gpio_num = gpio_num,
             .active_level = 0

main/Display.cc

@@ -104,20 +104,29 @@ Display::Display(int sda_pin, int scl_pin) : sda_pin_(sda_pin), scl_pin_(scl_pin
         lv_label_set_text(label_, "Initializing...");
         lv_obj_set_width(label_, disp_->driver->hor_res);
         lv_obj_set_height(label_, disp_->driver->ver_res);
-        lv_obj_set_style_text_line_space(label_, 0, 0);
-        lv_obj_set_style_pad_all(label_, 0, 0);
-        lv_obj_set_style_outline_pad(label_, 0, 0);
+
+        notification_ = lv_label_create(lv_disp_get_scr_act(disp_));
+        lv_label_set_text(notification_, "Notification\nTest");
+        lv_obj_set_width(notification_, disp_->driver->hor_res);
+        lv_obj_set_height(notification_, disp_->driver->ver_res);
+        lv_obj_set_style_opa(notification_, LV_OPA_MIN, 0);
         lvgl_port_unlock();
     }
 }
 
 Display::~Display() {
-    if (label_ != nullptr) {
-        lvgl_port_lock(0);
-        lv_obj_del(label_);
-        lvgl_port_unlock();
+    if (notification_timer_ != nullptr) {
+        esp_timer_stop(notification_timer_);
+        esp_timer_delete(notification_timer_);
     }
 
+    lvgl_port_lock(0);
+    if (label_ != nullptr) {
+        lv_obj_del(label_);
+        lv_obj_del(notification_);
+    }
+    lvgl_port_unlock();
+
     if (disp_ != nullptr) {
         lvgl_port_deinit();
         esp_lcd_panel_del(panel_);
@@ -136,4 +145,35 @@ void Display::SetText(const std::string &text) {
     }
 }
 
+void Display::ShowNotification(const std::string &text) {
+    if (notification_ != nullptr) {
+        lvgl_port_lock(0);
+        lv_label_set_text(notification_, text.c_str());
+        lv_obj_set_style_opa(notification_, LV_OPA_MAX, 0);
+        lv_obj_set_style_opa(label_, LV_OPA_MIN, 0);
+        lvgl_port_unlock();
+
+        if (notification_timer_ != nullptr) {
+            esp_timer_stop(notification_timer_);
+            esp_timer_delete(notification_timer_);
+        }
+
+        esp_timer_create_args_t timer_args = {
+            .callback = [](void *arg) {
+                Display *display = static_cast<Display*>(arg);
+                lvgl_port_lock(0);
+                lv_obj_set_style_opa(display->notification_, LV_OPA_MIN, 0);
+                lv_obj_set_style_opa(display->label_, LV_OPA_MAX, 0);
+                lvgl_port_unlock();
+            },
+            .arg = this,
+            .dispatch_method = ESP_TIMER_TASK,
+            .name = "Notification Timer",
+            .skip_unhandled_events = false,
+        };
+        ESP_ERROR_CHECK(esp_timer_create(&timer_args, &notification_timer_));
+        ESP_ERROR_CHECK(esp_timer_start_once(notification_timer_, 3000000));
+    }
+}
+
 #endif

main/Display.h

@@ -5,6 +5,7 @@
 #include <esp_lcd_panel_io.h>
 #include <esp_lcd_panel_ops.h>
 #include <lvgl.h>
+#include <esp_timer.h>
 
 #include <string>
 
@@ -14,6 +15,7 @@ public:
     ~Display();
 
     void SetText(const std::string &text);
+    void ShowNotification(const std::string &text);
 
 private:
     int sda_pin_;
@@ -25,6 +27,8 @@ private:
     esp_lcd_panel_handle_t panel_ = nullptr;
     lv_disp_t *disp_ = nullptr;
     lv_obj_t *label_ = nullptr;
+    lv_obj_t *notification_ = nullptr;
+    esp_timer_handle_t notification_timer_ = nullptr;
 
     std::string text_;
 };

main/Kconfig.projbuild

@@ -80,6 +80,12 @@ config BOOT_BUTTON_GPIO
     help
         GPIO number of the boot button.
 
+config VOLUME_UP_BUTTON_GPIO
+    int "Volume Up Button GPIO"
+    default 40
+    help
+        GPIO number of the volume up button.
+
 config USE_AFE_SR
     bool "Use Espressif AFE SR"
     default y

main/idf_component.yml

@@ -3,7 +3,7 @@ dependencies:
   78/esp-builtin-led: "^1.0.2"
   78/esp-wifi-connect: "^1.1.0"
   78/esp-opus-encoder: "^1.0.2"
-  78/esp-ml307: "^1.1.1"
+  78/esp-ml307: "^1.2.1"
   espressif/esp-sr: "^1.9.0"
   espressif/button: "^3.3.1"
   lvgl/lvgl: "^8.4.0"