import inference app

Author: sandeepmistry

.gitignore

@@ -0,0 +1,2 @@
+inference-app/build/
+

inference-app/CMakeLists.txt

@@ -0,0 +1,54 @@
+cmake_minimum_required(VERSION 3.12)
+
+# initialize pico_sdk from GIT
+# (note this can come from environment, CMake cache etc)
+# set(PICO_SDK_FETCH_FROM_GIT on)
+
+# pico_sdk_import.cmake is a single file copied from this SDK
+# note: this must happen before project()
+include(pico_sdk_import.cmake)
+
+project(pico_inference_app)
+
+# initialize the Pico SDK
+pico_sdk_init()
+
+# Define ARM_CPU, CMSIS ROOT and DSP to use CMSIS-DSP
+set(ARM_CPU "cortex-m0plus")
+set(ROOT ${CMAKE_CURRENT_LIST_DIR}/lib/CMSIS_5)
+set(DSP ${ROOT}/CMSIS/DSP)
+
+set(CONFIGTABLE ON)
+set(RFFT_Q15_256 ON)
+set(ALLFAST ON)
+
+# include CMSIS-DSP .cmake for GCC Toolchain
+include(${DSP}/Toolchain/GCC.cmake)
+
+# add CMSIS-DSP Source directory as subdirectory
+add_subdirectory(${DSP}/Source EXCLUDE_FROM_ALL)
+
+# rest of your project
+add_executable(pico_inference_app
+    ${CMAKE_CURRENT_LIST_DIR}/src/main.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/dsp_pipeline.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/ml_model.cpp
+)
+
+target_link_libraries(pico_inference_app
+    pico_stdlib
+    hardware_pwm
+    pico-tflmicro
+    pico_pdm_microphone
+    CMSISDSPTransform CMSISDSPSupport CMSISDSPCommon CMSISDSPComplexMath CMSISDSPFastMath CMSISDSPBasicMath
+)
+
+# enable usb output, disable uart output
+pico_enable_stdio_usb(pico_inference_app 1)
+pico_enable_stdio_uart(pico_inference_app 0)
+
+# create map/bin/hex/uf2 file in addition to ELF.
+pico_add_extra_outputs(pico_inference_app)
+
+add_subdirectory("lib/microphone-library-for-pico" EXCLUDE_FROM_ALL)
+add_subdirectory("lib/pico-tflmicro" EXCLUDE_FROM_ALL)

inference-app/src/dsp_pipeline.cpp

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2021 Arm Limited and Contributors. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * 
+ */
+
+#include "dsp_pipeline.h"
+
+DSPPipeline::DSPPipeline(int fft_size) :
+    _fft_size(fft_size),
+    _hanning_window(NULL)
+{
+}
+
+DSPPipeline::~DSPPipeline()
+{
+    if (_hanning_window != NULL) {
+        delete [] _hanning_window;
+
+        _hanning_window = NULL;
+    }
+}
+
+int DSPPipeline::init()
+{
+    _hanning_window = new int16_t[_fft_size];
+    if (_hanning_window == NULL) {
+        return 0;
+    }
+    
+    for (size_t i = 0; i < _fft_size; i++) {
+       float32_t f = 0.5 * (1.0 - arm_cos_f32(2 * PI * i / _fft_size));
+
+       arm_float_to_q15(&f, &_hanning_window[i], 1);
+    }
+
+    if (arm_rfft_init_q15(&_S_q15, _fft_size, 0, 1) != ARM_MATH_SUCCESS) {
+        return 0;
+    }
+
+    return 1;
+}
+
+void DSPPipeline::calculate_spectrum(const int16_t* input, int8_t* output, int32_t scale_divider, float scale_zero_point)
+{
+    int16_t windowed_input[_fft_size];
+    int16_t fft_q15[_fft_size * 2];
+    int16_t fft_mag_q15[_fft_size / 2 + 1];
+
+    // apply the DSP pipeline: Hanning Window + FFT
+    arm_mult_q15(_hanning_window, input, windowed_input, _fft_size);
+    arm_rfft_q15(&_S_q15, windowed_input, fft_q15);
+    arm_cmplx_mag_q15(fft_q15, fft_mag_q15, (_fft_size / 2) + 1);
+
+    int8_t* dst = output;
+
+    for (int j = 0; j < ((_fft_size / 2) + 1); j++) {
+        *dst++ = __SSAT((fft_mag_q15[j] / scale_divider) + scale_zero_point, 8);
+    }
+}
+
+void DSPPipeline::shift_spectrogram(int8_t* spectrogram, int shift_amount, int spectrogram_width)
+{
+    int spectrogram_height = _fft_size / 2 + 1;
+
+    memmove(spectrogram, spectrogram + (spectrogram_height * shift_amount), spectrogram_height * (spectrogram_width - shift_amount) * sizeof(spectrogram[0]));
+}

inference-app/src/dsp_pipeline.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2021 Arm Limited and Contributors. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * 
+ */
+
+#ifndef _DSP_PIPELINE_H_
+#define _DSP_PIPELINE_H_
+
+#include "arm_math.h"
+
+class DSPPipeline {
+    public:
+        DSPPipeline(int fft_size);
+        virtual ~DSPPipeline();
+
+        int init();
+        void calculate_spectrum(const int16_t* input, int8_t* output, int32_t scale_divider, float scale_zero_point);
+        void shift_spectrogram(int8_t* spectrogram, int shift_amount, int spectrogram_width);
+
+    private:
+        int _fft_size;
+        int16_t* _hanning_window;
+        arm_rfft_instance_q15 _S_q15;
+};
+
+#endif

inference-app/src/main.cpp

@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) 2021 Arm Limited and Contributors. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * 
+ */
+
+#include <stdio.h>
+
+#include "pico/stdlib.h"
+#include "hardware/pwm.h"
+
+extern "C" {
+#include "pico/pdm_microphone.h"
+}
+
+#include "tflite_model.h"
+
+#include "dsp_pipeline.h"
+#include "ml_model.h"
+
+// constants
+#define SAMPLE_RATE       16000
+#define FFT_SIZE          256
+#define SPECTRUM_SHIFT    4
+#define INPUT_BUFFER_SIZE ((FFT_SIZE / 2) * SPECTRUM_SHIFT)
+#define INPUT_SHIFT       0
+
+// microphone configuration
+const struct pdm_microphone_config pdm_config = {
+    // GPIO pin for the PDM DAT signal
+    .gpio_data = 2,
+
+    // GPIO pin for the PDM CLK signal
+    .gpio_clk = 3,
+
+    // PIO instance to use
+    .pio = pio0,
+
+    // PIO State Machine instance to use
+    .pio_sm = 0,
+
+    // sample rate in Hz
+    .sample_rate = SAMPLE_RATE,
+
+    // number of samples to buffer
+    .sample_buffer_size = INPUT_BUFFER_SIZE,
+};
+
+q15_t capture_buffer_q15[INPUT_BUFFER_SIZE];
+volatile int new_samples_captured = 0;
+
+q15_t input_q15[INPUT_BUFFER_SIZE + (FFT_SIZE / 2)];
+
+DSPPipeline dsp_pipeline(FFT_SIZE);
+MLModel ml_model(tflite_model, 128 * 1024);
+
+int8_t* scaled_spectrum = nullptr;
+int32_t spectogram_divider;
+float spectrogram_zero_point;
+
+void on_pdm_samples_ready();
+
+int main( void )
+{
+    // initialize stdio
+    stdio_init_all();
+
+    printf("hello pico fire alarm detection\n");
+
+    gpio_set_function(PICO_DEFAULT_LED_PIN, GPIO_FUNC_PWM);
+    
+    uint pwm_slice_num = pwm_gpio_to_slice_num(PICO_DEFAULT_LED_PIN);
+    uint pwm_chan_num = pwm_gpio_to_channel(PICO_DEFAULT_LED_PIN);
+
+    // Set period of 256 cycles (0 to 255 inclusive)
+    pwm_set_wrap(pwm_slice_num, 256);
+
+    // Set the PWM running
+    pwm_set_enabled(pwm_slice_num, true);
+
+    if (!ml_model.init()) {
+        printf("Failed to initialize ML model!\n");
+        while (1) { tight_loop_contents(); }
+    }
+
+    if (!dsp_pipeline.init()) {
+        printf("Failed to initialize DSP Pipeline!\n");
+        while (1) { tight_loop_contents(); }
+    }
+
+    scaled_spectrum = (int8_t*)ml_model.input_data();
+    spectogram_divider = 64 * ml_model.input_scale(); 
+    spectrogram_zero_point = ml_model.input_zero_point();
+
+    // initialize the PDM microphone
+    if (pdm_microphone_init(&pdm_config) < 0) {
+        printf("PDM microphone initialization failed!\n");
+        while (1) { tight_loop_contents(); }
+    }
+
+    // set callback that is called when all the samples in the library
+    // internal sample buffer are ready for reading
+    pdm_microphone_set_samples_ready_handler(on_pdm_samples_ready);
+
+    // start capturing data from the PDM microphone
+    if (pdm_microphone_start() < 0) {
+        printf("PDM microphone start failed!\n");
+        while (1) { tight_loop_contents(); }
+    }
+
+    while (1) {
+        // wait for new samples
+        while (new_samples_captured == 0) {
+            tight_loop_contents();
+        }
+        new_samples_captured = 0;
+
+        dsp_pipeline.shift_spectrogram(scaled_spectrum, SPECTRUM_SHIFT, 124);
+
+        // move input buffer values over by INPUT_BUFFER_SIZE samples
+        memmove(input_q15, &input_q15[INPUT_BUFFER_SIZE], (FFT_SIZE / 2));
+
+        // copy new samples to end of the input buffer with a bit shift of INPUT_SHIFT
+        arm_shift_q15(capture_buffer_q15, INPUT_SHIFT, input_q15 + (FFT_SIZE / 2), INPUT_BUFFER_SIZE);
+    
+        for (int i = 0; i < SPECTRUM_SHIFT; i++) {
+            dsp_pipeline.calculate_spectrum(
+                input_q15 + i * ((FFT_SIZE / 2)),
+                scaled_spectrum + (129 * (124 - SPECTRUM_SHIFT + i)),
+                spectogram_divider, spectrogram_zero_point
+            );
+        }
+
+        float y = ml_model.predict();
+
+        printf("y = %f\n", y);
+
+        pwm_set_chan_level(pwm_slice_num, pwm_chan_num, y * 255);
+    }
+
+    return 0;
+}
+
+void on_pdm_samples_ready()
+{
+    // callback from library when all the samples in the library
+    // internal sample buffer are ready for reading 
+
+    // read in the new samples
+    new_samples_captured = pdm_microphone_read(capture_buffer_q15, INPUT_BUFFER_SIZE);
+}