Skip to content

Commit b4529ec

Browse files
author
LittleMouse
committed
[perf] Optimize the initialization process
1 parent 4928281 commit b4529ec

File tree

1 file changed

+186
-96
lines changed

1 file changed

+186
-96
lines changed

‎ESP32/cpp/src/main.cpp

Lines changed: 186 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,8 @@ struct VlmData {
9797
bool finish;
9898
};
9999

100-
const int BlueButtonPin = 8;
101-
const int RedButtonPin = 9;
100+
const int BlueButtonPin = 1;
101+
const int RedButtonPin = 2;
102102
static constexpr std::size_t box_count = 7;
103103
static box_t box_list[box_count];
104104
static yolo_box_t yolo_box;
@@ -198,7 +198,7 @@ void setup_comm(void)
198198
void setup_speaker(void)
199199
{
200200
CoreS3.Speaker.begin();
201-
CoreS3.Speaker.setVolume(100);
201+
CoreS3.Speaker.setVolume(10);
202202
}
203203

204204
void setup_llm(void)
@@ -237,14 +237,22 @@ void setup_yolo_detect(void)
237237
yolo_config.model = "yolo11n";
238238
yolo_box.model = "yolo11n";
239239
yolo_work_id = module_llm.yolo.setup(yolo_config, "yolo_setup");
240-
while (yolo_work_id == nullptr) vTaskDelay(100);
241-
if (lvgl_port_lock()) {
242-
lv_label_set_text(initialization_state, "Initialization yolo detection completed");
243-
lvgl_port_unlock();
240+
while (yolo_work_id.isEmpty()) vTaskDelay(100);
241+
if (yolo_work_id == "yolo") {
242+
if (lvgl_port_lock()) {
243+
lv_label_set_text(initialization_state, "Initialization yolo detection failed");
244+
lvgl_port_unlock();
245+
}
246+
while (1);
247+
} else {
248+
if (lvgl_port_lock()) {
249+
lv_label_set_text(initialization_state, "Initialization yolo detection completed");
250+
lvgl_port_unlock();
251+
}
252+
vTaskDelay(200);
253+
lv_obj_add_flag(initialization_state, LV_OBJ_FLAG_HIDDEN);
254+
lv_obj_add_flag(background, LV_OBJ_FLAG_HIDDEN);
244255
}
245-
vTaskDelay(200);
246-
lv_obj_add_flag(initialization_state, LV_OBJ_FLAG_HIDDEN);
247-
lv_obj_add_flag(background, LV_OBJ_FLAG_HIDDEN);
248256
}
249257

250258
void setup_yolo_pose(void)
@@ -258,14 +266,22 @@ void setup_yolo_pose(void)
258266
yolo_config.model = "yolo11n-pose";
259267
yolo_box.model = "yolo11n-pose";
260268
yolo_work_id = module_llm.yolo.setup(yolo_config, "yolo_setup");
261-
while (yolo_work_id == nullptr) vTaskDelay(100);
262-
if (lvgl_port_lock()) {
263-
lv_label_set_text(initialization_state, "Initialization yolo pose detection completed");
264-
lvgl_port_unlock();
269+
while (yolo_work_id.isEmpty()) vTaskDelay(100);
270+
if (yolo_work_id == "yolo") {
271+
if (lvgl_port_lock()) {
272+
lv_label_set_text(initialization_state, "Initialization yolo pose failed");
273+
lvgl_port_unlock();
274+
}
275+
while (1);
276+
} else {
277+
if (lvgl_port_lock()) {
278+
lv_label_set_text(initialization_state, "Initialization yolo pose completed");
279+
lvgl_port_unlock();
280+
}
281+
vTaskDelay(200);
282+
lv_obj_add_flag(initialization_state, LV_OBJ_FLAG_HIDDEN);
283+
lv_obj_add_flag(background, LV_OBJ_FLAG_HIDDEN);
265284
}
266-
vTaskDelay(200);
267-
lv_obj_add_flag(initialization_state, LV_OBJ_FLAG_HIDDEN);
268-
lv_obj_add_flag(background, LV_OBJ_FLAG_HIDDEN);
269285
}
270286

271287
void setup_yolo_seg(void)
@@ -279,14 +295,23 @@ void setup_yolo_seg(void)
279295
yolo_config.model = "yolo11n-seg";
280296
yolo_box.model = "yolo11n-seg";
281297
yolo_work_id = module_llm.yolo.setup(yolo_config, "yolo_setup");
282-
while (yolo_work_id == nullptr) vTaskDelay(100);
283-
if (lvgl_port_lock()) {
284-
lv_label_set_text(initialization_state, "Initialization yolo segmentation completed");
285-
lvgl_port_unlock();
298+
yolo_work_id = module_llm.yolo.setup(yolo_config, "yolo_setup");
299+
while (yolo_work_id.isEmpty()) vTaskDelay(100);
300+
if (yolo_work_id == "yolo") {
301+
if (lvgl_port_lock()) {
302+
lv_label_set_text(initialization_state, "Initialization yolo segmentation failed");
303+
lvgl_port_unlock();
304+
}
305+
while (1);
306+
} else {
307+
if (lvgl_port_lock()) {
308+
lv_label_set_text(initialization_state, "Initialization yolo segmentation completed");
309+
lvgl_port_unlock();
310+
}
311+
vTaskDelay(200);
312+
lv_obj_add_flag(initialization_state, LV_OBJ_FLAG_HIDDEN);
313+
lv_obj_add_flag(background, LV_OBJ_FLAG_HIDDEN);
286314
}
287-
vTaskDelay(200);
288-
lv_obj_add_flag(initialization_state, LV_OBJ_FLAG_HIDDEN);
289-
lv_obj_add_flag(background, LV_OBJ_FLAG_HIDDEN);
290315
}
291316

292317
void setup_yolo_hand(void)
@@ -300,14 +325,22 @@ void setup_yolo_hand(void)
300325
yolo_config.model = "yolo11n-hand-pose";
301326
yolo_box.model = "yolo11n-hand-pose";
302327
yolo_work_id = module_llm.yolo.setup(yolo_config, "yolo_setup");
303-
while (yolo_work_id == nullptr) vTaskDelay(100);
304-
if (lvgl_port_lock()) {
305-
lv_label_set_text(initialization_state, "Initialization yolo hand detection completed");
306-
lvgl_port_unlock();
328+
while (yolo_work_id.isEmpty()) vTaskDelay(100);
329+
if (yolo_work_id == "yolo") {
330+
if (lvgl_port_lock()) {
331+
lv_label_set_text(initialization_state, "Initialization yolo hand failed");
332+
lvgl_port_unlock();
333+
}
334+
while (1);
335+
} else {
336+
if (lvgl_port_lock()) {
337+
lv_label_set_text(initialization_state, "Initialization yolo hand completed");
338+
lvgl_port_unlock();
339+
}
340+
vTaskDelay(200);
341+
lv_obj_add_flag(initialization_state, LV_OBJ_FLAG_HIDDEN);
342+
lv_obj_add_flag(background, LV_OBJ_FLAG_HIDDEN);
307343
}
308-
vTaskDelay(200);
309-
lv_obj_add_flag(initialization_state, LV_OBJ_FLAG_HIDDEN);
310-
lv_obj_add_flag(background, LV_OBJ_FLAG_HIDDEN);
311344
}
312345

313346
void setup_vlm(void)
@@ -317,15 +350,24 @@ void setup_vlm(void)
317350
lvgl_port_unlock();
318351
}
319352
m5_module_llm::ApiVlmSetupConfig_t vlm_config;
320-
vlm_work_id = module_llm.vlm.setup(vlm_config, "vlm_setup");
321-
while (vlm_work_id == nullptr) vTaskDelay(100);
322-
if (lvgl_port_lock()) {
323-
lv_label_set_text(initialization_state, "Initialization InternVL2_5-1B-MPO completed");
324-
lvgl_port_unlock();
353+
vlm_config.model = "internvl2.5-1B-364-ax630c";
354+
vlm_work_id = module_llm.vlm.setup(vlm_config, "vlm_setup");
355+
while (vlm_work_id.isEmpty()) vTaskDelay(100);
356+
if (vlm_work_id == "vlm") {
357+
if (lvgl_port_lock()) {
358+
lv_label_set_text(initialization_state, "Initialization VLLM failed");
359+
lvgl_port_unlock();
360+
}
361+
while (1);
362+
} else {
363+
if (lvgl_port_lock()) {
364+
lv_label_set_text(initialization_state, "Initialization VLLM completed");
365+
lvgl_port_unlock();
366+
}
367+
vTaskDelay(200);
368+
// lv_obj_add_flag(initialization_state, LV_OBJ_FLAG_HIDDEN);
369+
// lv_obj_add_flag(background, LV_OBJ_FLAG_HIDDEN);
325370
}
326-
inference = false;
327-
button_count = 0;
328-
vTaskDelay(200);
329371
}
330372

331373
void setup_melotts(void)
@@ -337,14 +379,22 @@ void setup_melotts(void)
337379
m5_module_llm::ApiMelottsSetupConfig_t melotts_config;
338380
melotts_config.input = {vlm_work_id};
339381
melotts_work_id = module_llm.melotts.setup(melotts_config, "melotts_setup");
340-
while (melotts_work_id == nullptr) vTaskDelay(100);
341-
if (lvgl_port_lock()) {
342-
lv_label_set_text(initialization_state, "Initialization meloTTS completed");
343-
lvgl_port_unlock();
382+
while (melotts_work_id.isEmpty()) vTaskDelay(100);
383+
if (melotts_work_id == "melotts") {
384+
if (lvgl_port_lock()) {
385+
lv_label_set_text(initialization_state, "Initialization meloTTS failed");
386+
lvgl_port_unlock();
387+
}
388+
while (1);
389+
} else {
390+
if (lvgl_port_lock()) {
391+
lv_label_set_text(initialization_state, "Initialization meloTTS completed");
392+
lvgl_port_unlock();
393+
}
394+
vTaskDelay(200);
395+
lv_obj_add_flag(initialization_state, LV_OBJ_FLAG_HIDDEN);
396+
lv_obj_add_flag(background, LV_OBJ_FLAG_HIDDEN);
344397
}
345-
vTaskDelay(200);
346-
lv_obj_add_flag(initialization_state, LV_OBJ_FLAG_HIDDEN);
347-
lv_obj_add_flag(background, LV_OBJ_FLAG_HIDDEN);
348398
}
349399

350400
void setup_yolo(int j)
@@ -440,10 +490,19 @@ void menuBackTask(void* pvParameters)
440490
play_wav();
441491
box_list[5].touch_id = t.id;
442492
state = 0;
443-
vTaskDelay(100);
444-
if (!yolo_work_id.isEmpty()) module_llm.yolo.exit(yolo_work_id);
445-
if (!vlm_work_id.isEmpty()) module_llm.vlm.exit(vlm_work_id);
446-
if (!melotts_work_id.isEmpty()) module_llm.melotts.exit(melotts_work_id);
493+
vTaskDelay(200);
494+
if (!yolo_work_id.isEmpty()) {
495+
module_llm.yolo.exit(yolo_work_id);
496+
yolo_work_id.clear();
497+
}
498+
if (!vlm_work_id.isEmpty()) {
499+
module_llm.vlm.exit(vlm_work_id);
500+
vlm_work_id.clear();
501+
}
502+
if (!melotts_work_id.isEmpty()) {
503+
module_llm.melotts.exit(melotts_work_id);
504+
melotts_work_id.clear();
505+
}
447506
setup_menu();
448507
}
449508

@@ -592,9 +651,9 @@ void vllm_inference()
592651
send_camera_data(out_jpg, out_jpg_len, vlm_work_id);
593652
free(out_jpg);
594653
vTaskDelay(100);
595-
module_llm.vlm.inference(vlm_work_id,
596-
"Please describe this picture in a humorous way with a word limit of 60 words.");
597-
// module_llm.vlm.inference(vlm_work_id, "请用幽默的方式描述图片，字数不超过60个"); # 中文输出
654+
// module_llm.vlm.inference(vlm_work_id,
655+
// "Please describe this picture in a humorous way with a word limit of 60 words.");
656+
module_llm.vlm.inference(vlm_work_id, "请用幽默的方式描述图片，字数不超过60个"); // 中文输出
598657
}
599658

600659
void action_task(void* pvParameters)
@@ -631,22 +690,51 @@ void recvTask(void* pvParameters)
631690

632691
void button_task(void* pvParameters)
633692
{
693+
static uint32_t last_blue_press = 0;
694+
static uint32_t last_red_press = 0;
695+
634696
while (true) {
697+
uint32_t now = xTaskGetTickCount() * portTICK_PERIOD_MS;
698+
635699
if (!digitalRead(BlueButtonPin)) {
636-
// play_camera_wav();
637-
button_count = 4;
638-
vTaskDelay(1000);
700+
if ((state == 2) && (inference != true)) {
701+
vTaskDelay(20 / portTICK_PERIOD_MS);
702+
if (!digitalRead(BlueButtonPin)) {
703+
if (now - last_blue_press > 1000) {
704+
button_count = 4;
705+
last_blue_press = now;
706+
}
707+
}
708+
}
639709
}
710+
640711
if (!digitalRead(RedButtonPin)) {
641-
play_wav();
642-
state = 0;
643-
vTaskDelay(100);
644-
if (!yolo_work_id.isEmpty()) module_llm.yolo.exit(yolo_work_id);
645-
if (!vlm_work_id.isEmpty()) module_llm.vlm.exit(vlm_work_id);
646-
if (!melotts_work_id.isEmpty()) module_llm.melotts.exit(melotts_work_id);
647-
setup_menu();
712+
vTaskDelay(20 / portTICK_PERIOD_MS);
713+
if (!digitalRead(RedButtonPin)) {
714+
if (state != 0)
715+
if (now - last_red_press > 1000) {
716+
state = 0;
717+
play_wav();
718+
vTaskDelay(200 / portTICK_PERIOD_MS);
719+
if (!yolo_work_id.isEmpty()) {
720+
module_llm.yolo.exit(yolo_work_id);
721+
yolo_work_id.clear();
722+
}
723+
if (!vlm_work_id.isEmpty()) {
724+
module_llm.vlm.exit(vlm_work_id);
725+
vlm_work_id.clear();
726+
}
727+
if (!melotts_work_id.isEmpty()) {
728+
module_llm.melotts.exit(melotts_work_id);
729+
melotts_work_id.clear();
730+
}
731+
setup_menu();
732+
last_red_press = now;
733+
}
734+
}
648735
}
649-
vTaskDelay(100);
736+
737+
vTaskDelay(20 / portTICK_PERIOD_MS);
650738
}
651739
}
652740

@@ -714,36 +802,36 @@ void cameraTask(void* pvParameters)
714802
(uint16_t*)CoreS3.Camera.fb->buf);
715803
if (button_count > 0) {
716804
int edgeLength = 40;
717-
canvas.fillRect(0, 0, edgeLength, 5, RED); // 上边
718-
canvas.fillRect(0, 0, 5, edgeLength, RED); // 左边
719-
canvas.fillRect(320 - edgeLength, 0, edgeLength, 5, RED); // 上边
720-
canvas.fillRect(320 - 5, 0, 5, edgeLength, RED); // 右边
721-
canvas.fillRect(0, 240 - edgeLength, 5, edgeLength, RED); // 左边
722-
canvas.fillRect(0, 240 - 5, edgeLength, 5, RED); // 下边
723-
canvas.fillRect(320 - edgeLength, 240 - 5, edgeLength, 5, RED); // 下边
724-
canvas.fillRect(320 - 5, 240 - edgeLength, 5, edgeLength, RED); // 右边
725-
726-
canvas.setTextSize(2);
727-
canvas.setTextDatum(CC_DATUM);
728-
if (button_count == 4) {
729-
canvas.setTextColor(GREEN, GREEN);
730-
canvas.drawString("3", 160, 120);
731-
} else if (button_count == 3) {
732-
canvas.setTextColor(YELLOW, YELLOW);
733-
canvas.drawString("2", 160, 120);
734-
} else if (button_count == 2) {
735-
canvas.setTextColor(RED, RED);
736-
canvas.drawString("1", 160, 120);
737-
} else if (button_count == 1) {
738-
CoreS3.Display.fillScreen(BLACK);
739-
canvas.fillRect(0, 0, 320, 10, SKYBLUE);
740-
canvas.fillRect(0, 230, 320, 10, SKYBLUE);
741-
canvas.fillRect(0, 0, 10, 240, SKYBLUE);
742-
canvas.fillRect(310, 0, 10, 240, SKYBLUE);
743-
canvas.pushRotateZoomWithAA(160, 120, random(-15, 15), 0.8, 0.8);
744-
button_count--;
745-
vllm_inference();
746-
}
805+
// canvas.fillRect(0, 0, edgeLength, 5, RED); // 上边
806+
// canvas.fillRect(0, 0, 5, edgeLength, RED); // 左边
807+
// canvas.fillRect(320 - edgeLength, 0, edgeLength, 5, RED); // 上边
808+
// canvas.fillRect(320 - 5, 0, 5, edgeLength, RED); // 右边
809+
// canvas.fillRect(0, 240 - edgeLength, 5, edgeLength, RED); // 左边
810+
// canvas.fillRect(0, 240 - 5, edgeLength, 5, RED); // 下边
811+
// canvas.fillRect(320 - edgeLength, 240 - 5, edgeLength, 5, RED); // 下边
812+
// canvas.fillRect(320 - 5, 240 - edgeLength, 5, edgeLength, RED); // 右边
813+
814+
// canvas.setTextSize(2);
815+
// canvas.setTextDatum(CC_DATUM);
816+
// if (button_count == 4) {
817+
// canvas.setTextColor(GREEN, GREEN);
818+
// canvas.drawString("3", 160, 120);
819+
// } else if (button_count == 3) {
820+
// canvas.setTextColor(YELLOW, YELLOW);
821+
// canvas.drawString("2", 160, 120);
822+
// } else if (button_count == 2) {
823+
// canvas.setTextColor(RED, RED);
824+
// canvas.drawString("1", 160, 120);
825+
// } else if (button_count == 1) {
826+
CoreS3.Display.fillScreen(BLACK);
827+
canvas.fillRect(0, 0, 320, 10, SKYBLUE);
828+
canvas.fillRect(0, 230, 320, 10, SKYBLUE);
829+
canvas.fillRect(0, 0, 10, 240, SKYBLUE);
830+
canvas.fillRect(310, 0, 10, 240, SKYBLUE);
831+
canvas.pushRotateZoomWithAA(160, 120, random(-15, 15), 0.8, 0.8);
832+
button_count = 0;
833+
vllm_inference();
834+
// }
747835
}
748836
canvas.pushSprite(&M5.Display, 0, 0);
749837
CoreS3.Camera.free();
@@ -767,12 +855,14 @@ void setup_task(void)
767855
xTaskCreatePinnedToCore(menuTask, "Menu Task", 8192, NULL, 1, NULL, 1);
768856
xTaskCreatePinnedToCore(menuBackTask, "Menu Back Task", 8192, NULL, 1, NULL, 0);
769857
xTaskCreatePinnedToCore(button_task, "Button Task", 8192, NULL, 3, NULL, 1);
770-
xTaskCreatePinnedToCore(action_task, "Action Task", 8192, NULL, 3, NULL, 1);
858+
// xTaskCreatePinnedToCore(action_task, "Action Task", 8192, NULL, 3, NULL, 1);
771859
}
772860

773861
void setup()
774862
{
775-
CoreS3.begin();
863+
auto cfg = M5.config();
864+
cfg.output_power = true;
865+
CoreS3.begin(cfg);
776866
lvgl_port_init(M5.Lcd);
777867
vllm_display();
778868
setup_bsp();

0 commit comments

Comments
 (0)