audio hear

Author: previ

audio.py

@@ -26,6 +26,7 @@
 
 import pyaudio
 import wave
+import audioop
 import logging
 
 try:
@@ -51,6 +52,21 @@ def get_instance(cls):
       cls._instance = Audio()
     return cls._instance
 
+  def __init__(self):
+    self.pyaudio = pyaudio.PyAudio()
+    try:
+      self.stream_in = self.pyaudio.open(format=FORMAT, channels=1, input_device_index=0, rate=RATE,
+        input=True,
+        frames_per_buffer=CHUNK_SIZE)
+      self.stream_in.start_stream()
+    except:
+      logging.info("Audio: input stream not available")
+
+  def exit(self):
+    # cleanup stuff.
+    self.stream_in.close()  
+    self.pyaudio.terminate()
+
   def say(self, what, locale='en'):
     if what and "$" in what:
       os.system ('omxplayer sounds/' + what[1:])
@@ -68,11 +84,6 @@ def normalize(self, snd_data):
     return r
 
   def record(self, elapse):
-    p = pyaudio.PyAudio()
-    stream = p.open(format=FORMAT, channels=1, input_device_index=0, rate=RATE,
-        input=True,
-        frames_per_buffer=CHUNK_SIZE)
-
     num_silent = 0
     snd_started = False
     c = 0
@@ -82,15 +93,12 @@ def record(self, elapse):
     while (c * 2.0 * 8192 / RATE) < elapse:
       c += 1
       # little endian, signed short
-      snd_data = array('h', stream.read(CHUNK_SIZE))
+      snd_data = array('h', self.stream_in.read(CHUNK_SIZE))
       if byteorder == 'big':
         snd_data.byteswap()
       r.extend(snd_data)
 
     sample_width = p.get_sample_size(FORMAT)
-    stream.stop_stream()
-    stream.close()
-    p.terminate()
 
     r = self.normalize(r)
 
@@ -111,11 +119,8 @@ def play(self, filename):
     # open the file for reading.
     wf = wave.open(SOUNDDIR + filename, 'rb')
 
-    # create an audio object
-    p = pyaudio.PyAudio()
-
     # open stream based on the wave object which has been input.
-    stream = p.open(format =
+    stream = self.pyaudio.open(format =
                 p.get_format_from_width(wf.getsampwidth()),
                 channels = wf.getnchannels(),
                 rate = wf.getframerate(),
@@ -132,7 +137,32 @@ def play(self, filename):
 
     # cleanup stuff.
     stream.close()    
-    p.terminate()
+
+  def hear(self, level, elapse=1.0):
+    sig_hear = False
+    ts_total = time.time()
+    ts_signal = None
+
+    while time.time() - ts_total < elapse:
+      try:
+        snd_data = self.stream_in.read(CHUNK_SIZE)
+        snd_rms = audioop.rms(snd_data, 2)
+        logging.info("snd.rms: " + str(snd_rms))
+        if snd_rms > level:
+          sig_hear = True
+          break
+      
+      except IOError as ex:
+        if ex[1] != pyaudio.paInputOverflowed:
+          raise
+        buf = '\x00' * CHUNK_SIZE #white noise
+        logging.info("white noise")
+      except AttributeError:
+        pass
+
+
+    return sig_hear
+  
 
   def speech_recog(self, model):
 
@@ -147,20 +177,13 @@ def speech_recog(self, model):
     config.set_string('-dict', MODELDIR + model + '.dict')
     decoder = Decoder(config)
 
-    p = pyaudio.PyAudio()
-    logging.info("device info: " + str(p.get_device_info_by_index(0)))
-    #stream = p.open(format=pyaudio.paInt16, channels=1, input_device_index=0, rate=16000, input=True, frames_per_buffer=1024)
-    stream = p.open(format=FORMAT, channels=1, input_device_index=0, rate=RATE,
-                    input=True,
-                    frames_per_buffer=CHUNK_SIZE*256)
-    stream.start_stream()
     decoder.start_utt()
     tstamp = time.time()
     recog_text = ''
 
     while len(recog_text) < 1:
       try:
-        buf = stream.read(CHUNK_SIZE)
+        buf = self.stream_in.read(CHUNK_SIZE)
         logging.info("actual voice")
         decoder.process_raw(buf, False, False)
         if decoder.hyp().hypstr != '':
@@ -176,8 +199,6 @@ def speech_recog(self, model):
         pass
 
     decoder.end_utt()
-    stream.close()
-    p.terminate()
 
     logging.info("recog text: " + recog_text)
     return recog_text

coderbot.cfg

@@ -1 +1 @@
-{"move_tr_speed": "80", "move_fw_elapse": "1", "camera_path_object_size_min": "4000", "load_at_start": "", "move_tr_elapse": "0.5", "sound_stop": "$shutdown.mp3", "camera_color_object_size_min": "4000", "prog_level": "adv", "prog_scrollbars": "true", "move_fw_speed": "100", "camera_color_object_size_max": "160000", "sound_shutter": "$shutter.mp3", "show_page_prefs": "true", "cv_image_factor": "2", "ctrl_hud_image": "", "button_func": "none", "ctrl_fw_elapse": "-1", "ctrl_tr_elapse": "-1", "move_power_angle_2": "20", "move_power_angle_3": "20", "move_power_angle_1": "15", "move_motor_trim": "0.9", "show_page_program": "true", "sound_start": "$startup.mp3", "camera_exposure_mode": "auto", "ctrl_tr_speed": "80", "ctrl_fw_speed": "100", "camera_refresh_timeout": "0.1", "camera_jpeg_quality": "20", "prog_maxblocks": "-1", "move_motor_mode": "dc", "camera_path_object_size_max": "160000", "show_page_control": "true"}
+{"move_tr_speed": "80", "move_fw_elapse": "1", "camera_path_object_size_min": "4000", "load_at_start": "", "move_tr_elapse": "0.5", "sound_stop": "$shutdown.mp3", "camera_color_object_size_min": "4000", "prog_level": "adv", "prog_scrollbars": "true", "move_fw_speed": "100", "camera_color_object_size_max": "160000", "sound_shutter": "$shutter.mp3", "show_page_prefs": "true", "cv_image_factor": "1", "ctrl_hud_image": "", "button_func": "none", "ctrl_fw_elapse": "-1", "ctrl_tr_elapse": "-1", "move_power_angle_2": "20", "move_power_angle_3": "20", "move_power_angle_1": "15", "move_motor_trim": "0.9", "show_page_program": "true", "sound_start": "$startup.mp3", "camera_exposure_mode": "auto", "ctrl_tr_speed": "80", "ctrl_fw_speed": "100", "camera_refresh_timeout": "0.1", "camera_jpeg_quality": "20", "prog_maxblocks": "-1", "move_motor_mode": "dc", "camera_path_object_size_max": "160000", "show_page_control": "true"}

data/program_find_text.data

@@ -1 +1 @@
-{"dom_code": "<xml xmlns=\"http://www.w3.org/1999/xhtml\"><block type=\"controls_whileUntil\" x=\"7\" y=\"60\"><field name=\"MODE\">WHILE</field><value name=\"BOOL\"><block type=\"logic_boolean\"><field name=\"BOOL\">TRUE</field></block></value><statement name=\"DO\"><block type=\"text_print\"><value name=\"TEXT\"><block type=\"coderbot_adv_findText\"><field name=\"ACCEPT\">alpha</field><value name=\"COLOR\"><block type=\"text\"><field name=\"TEXT\">#00f05d</field></block></value></block></value></block></statement></block></xml>", "code": "while True:\n  get_prog_eng().check_end()\n  get_cam().set_text(get_cam().find_text(accept=\"alpha\", back_color='#00f05d'))\n", "name": "find_text"}
+{"dom_code": "<xml xmlns=\"http://www.w3.org/1999/xhtml\"><block type=\"controls_whileUntil\" x=\"7\" y=\"60\"><field name=\"MODE\">WHILE</field><value name=\"BOOL\"><block type=\"logic_boolean\"><field name=\"BOOL\">TRUE</field></block></value><statement name=\"DO\"><block type=\"text_print\"><value name=\"TEXT\"><block type=\"coderbot_adv_findText\"><field name=\"ACCEPT\">alpha</field><value name=\"COLOR\"><block type=\"text\"><field name=\"TEXT\">#20554b</field></block></value></block></value></block></statement></block></xml>", "code": "while True:\n  get_prog_eng().check_end()\n  get_cam().set_text(get_cam().find_text(accept=\"alpha\", back_color='#20554b'))\n", "name": "find_text"}

data/program_hear_test.data

@@ -0,0 +1 @@
+{"dom_code": "<xml xmlns=\"http://www.w3.org/1999/xhtml\"><block type=\"controls_whileUntil\" x=\"-68\" y=\"108\"><field name=\"MODE\">WHILE</field><value name=\"BOOL\"><block type=\"logic_boolean\"><field name=\"BOOL\">TRUE</field></block></value><statement name=\"DO\"><block type=\"text_print\"><value name=\"TEXT\"><block type=\"coderbot_audio_hear\"><value name=\"LEVEL\"><block type=\"math_number\"><field name=\"NUM\">100</field></block></value><value name=\"ELAPSE\"><block type=\"math_number\"><field name=\"NUM\">1.0</field></block></value></block></value></block></statement></block></xml>", "code": "while True:\n  get_prog_eng().check_end()\n  get_cam().set_text(get_audio().hear(level=100, elapse=1))\n", "name": "hear_test"}

main.py

@@ -49,11 +49,12 @@
 app = Flask(__name__,static_url_path="")
 #app.config.from_pyfile('coderbot.cfg')
 babel = Babel(app)
-app.debug = True
+app.debug = False
 #sockets = Sockets(app)
 
 app.prog_engine = ProgramEngine.get_instance()
 app.prog = None
+app.shutdown_requested = False
 
 @babel.localeselector
 def get_locale():
@@ -139,7 +140,7 @@ def handle_bot_status():
 
 def video_stream(cam):
     refresh_timeout = float(app.bot_config.get("camera_refresh_timeout", "0.1")) 
-    while True:
+    while not app.shutdown_requested:
         last_refresh_time = time.time()
         frame = cam.get_image_jpeg()
         yield ("--BOUNDARYSTRING\r\n" +
@@ -289,3 +290,4 @@ def run_server():
       cam.exit()
     if bot:
       bot.exit()
+    app.shutdown_requested = True

static/js/blockly/blocks.js

@@ -704,6 +704,34 @@ Blockly.Python['coderbot_audio_play'] = function(block) {
   return 'get_audio().play(' + filename + ')\n';
 };
 
+Blockly.Blocks['coderbot_audio_hear'] = {
+  /**
+   * Block for audio hear function.
+   * @this Blockly.Block
+   */
+  init: function() {
+    this.setHelpUrl(Blockly.Msg.LOGIC_BOOLEAN_HELPURL);
+    this.setColour(290);
+    this.appendValueInput('LEVEL')
+        .setCheck(["Number"])
+        .appendField(Blockly.Msg.CODERBOT_AUDIO_HEAR + Blockly.Msg.CODERBOT_AUDIO_HEAR_LEVEL);
+    this.appendValueInput('ELAPSE')
+        .setCheck(["Number"])
+        .appendField(Blockly.Msg.CODERBOT_AUDIO_HEAR_ELAPSE);
+    this.setInputsInline(true);
+    this.setOutput(true, ['Number']);
+    this.setTooltip(Blockly.Msg.LOGIC_BOOLEAN_TOOLTIP);
+  }
+};
+
+Blockly.Python['coderbot_audio_hear'] = function(block) {
+  // Boolean values true and false.
+  var level = Blockly.Python.valueToCode(block, 'LEVEL', Blockly.Python.ORDER_NONE) || '\'\'';
+  var elapse = Blockly.Python.valueToCode(block, 'ELAPSE', Blockly.Python.ORDER_NONE) || '\'\'';
+  var code = 'get_audio().hear(level=' + level + ', elapse=' + elapse  + ')';
+  return [code, Blockly.Python.ORDER_ATOMIC];
+};
+
 Blockly.Blocks['coderbot_audio_listen'] = {
   /**
    * Block for findText function.

static/js/blockly/bot_en.js

@@ -55,6 +55,9 @@ Blockly.Msg.CODERBOT_SENSOR_FINDCODE = "read Bar-QRCode";
 Blockly.Msg.CODERBOT_AUDIO_RECORD_FILE_NAME = "record as file";
 Blockly.Msg.CODERBOT_AUDIO_RECORD_FILE_ELAPSE = " of seconds";
 Blockly.Msg.CODERBOT_AUDIO_PLAY_FILE = "play file";
+Blockly.Msg.CODERBOT_AUDIO_HEAR = "hear sound";
+Blockly.Msg.CODERBOT_AUDIO_HEAR_LEVEL = " of level";
+Blockly.Msg.CODERBOT_AUDIO_HEAR_ELAPSE = "for up to seconds";
 Blockly.Msg.CODERBOT_AUDIO_LISTEN = "listen";
 Blockly.Msg.CODERBOT_AUDIO_LISTEN_MODEL_SIMPLE = "simple commands";
 Blockly.Msg.CODERBOT_AUDIO_LISTEN_MODEL_MEDIUM = "medium commands";
@@ -63,4 +66,3 @@ Blockly.Msg.CODERBOT_SONAR_GET_DISTANCE = "get distance with";
 Blockly.Msg.CODERBOT_SONAR_SENSOR_1 = "sonar 1";
 Blockly.Msg.CODERBOT_SONAR_SENSOR_2 = "sonar 2";
 Blockly.Msg.CODERBOT_SONAR_SENSOR_3 = "sonar 3";
-

static/js/blockly/bot_fr.js

@@ -45,4 +45,24 @@ Blockly.Msg.CODERBOT_SENSOR_FINDFACE_Y = "ordonnée";
 Blockly.Msg.CODERBOT_SENSOR_FINDFACE_SIZE = "taille";
 Blockly.Msg.CODERBOT_SENSOR_FINDFACE_ALL = "x, y, taille (sous forme de liste)";
 Blockly.Msg.CODERBOT_SENSOR_FINDLOGO = "trouve le logo";
+Blockly.Msg.CODERBOT_SENSOR_FINDTEXT_FIND = "find text of kind";
+Blockly.Msg.CODERBOT_SENSOR_FINDTEXT_ACCEPT_ALPHA = "Alpha (A..Z)";
+Blockly.Msg.CODERBOT_SENSOR_FINDTEXT_ACCEPT_NUM = "Numeric (0..9)";
+Blockly.Msg.CODERBOT_SENSOR_FINDTEXT_ACCEPT_ALPHANUM = "Alphanumeric (A..Z;0..9)";
+Blockly.Msg.CODERBOT_SENSOR_FINDTEXT_ACCEPT_UNSPEC = "Any";
+Blockly.Msg.CODERBOT_SENSOR_FINDTEXT_COLOR = "background color";
 Blockly.Msg.CODERBOT_SENSOR_FINDCODE = "trouve le Bar/QRCode";
+Blockly.Msg.CODERBOT_AUDIO_RECORD_FILE_NAME = "record as file";
+Blockly.Msg.CODERBOT_AUDIO_RECORD_FILE_ELAPSE = " of seconds";
+Blockly.Msg.CODERBOT_AUDIO_PLAY_FILE = "play file";
+Blockly.Msg.CODERBOT_AUDIO_HEAR = "hear sound";
+Blockly.Msg.CODERBOT_AUDIO_HEAR_LEVEL = " of level";
+Blockly.Msg.CODERBOT_AUDIO_HEAR_ELAPSE = "for up to seconds";
+Blockly.Msg.CODERBOT_AUDIO_LISTEN = "listen";
+Blockly.Msg.CODERBOT_AUDIO_LISTEN_MODEL_SIMPLE = "simple commands";
+Blockly.Msg.CODERBOT_AUDIO_LISTEN_MODEL_MEDIUM = "medium commands";
+Blockly.Msg.CODERBOT_AUDIO_LISTEN_MODEL_ADV = "advance commands";
+Blockly.Msg.CODERBOT_SONAR_GET_DISTANCE = "get distance with";
+Blockly.Msg.CODERBOT_SONAR_SENSOR_1 = "sonar 1";
+Blockly.Msg.CODERBOT_SONAR_SENSOR_2 = "sonar 2";
+Blockly.Msg.CODERBOT_SONAR_SENSOR_3 = "sonar 3";

static/js/blockly/bot_it.js

@@ -55,6 +55,9 @@ Blockly.Msg.CODERBOT_SENSOR_FINDCODE = "leggi Bar/QRCode";
 Blockly.Msg.CODERBOT_AUDIO_RECORD_FILE_NAME = "registra audio su file";
 Blockly.Msg.CODERBOT_AUDIO_RECORD_FILE_ELAPSE = " per secondi";
 Blockly.Msg.CODERBOT_AUDIO_PLAY_FILE = "riproduci file";
+Blockly.Msg.CODERBOT_AUDIO_HEAR = "senti suono";
+Blockly.Msg.CODERBOT_AUDIO_HEAR_LEVEL = " di volume";
+Blockly.Msg.CODERBOT_AUDIO_HEAR_ELAPSE = "per secondi";
 Blockly.Msg.CODERBOT_AUDIO_LISTEN = "ascolta";
 Blockly.Msg.CODERBOT_AUDIO_LISTEN_MODEL_SIMPLE = "commandi semplici";
 Blockly.Msg.CODERBOT_AUDIO_LISTEN_MODEL_MEDIUM = "commandi medi";

templates/blocks_adv.xml

@@ -255,6 +255,18 @@
       <block type="coderbot_audio_say"></block>
       <block type="coderbot_audio_record"></block>
       <block type="coderbot_audio_play"></block>
+      <block type="coderbot_audio_hear">
+        <value name="LEVEL">
+          <block type="math_number">
+            <field name="NUM">100</field>
+          </block>
+        </value>
+        <value name="ELAPSE">
+          <block type="math_number">
+            <field name="NUM">1.0</field>
+          </block>
+        </value>
+      </block>
       <block type="coderbot_audio_listen"></block>
     </category>      
   </xml>

viz/image.py

@@ -22,6 +22,7 @@
 import colorsys
 import copy
 import blob
+import time
 import logging
 
 tesseract_whitelists = {
@@ -32,7 +33,7 @@
 }
 
 try:
-  ocr = cv2.text.OCRTesseract_create(".", "eng", tesseract_whitelist['unspec'], 0, cv2.text.OCR_LEVEL_TEXTLINE)
+  ocr = cv2.text.OCRTesseract_create(".", "eng", tesseract_whitelists['unspec'], 0, cv2.text.OCR_LEVEL_TEXTLINE)
 except:
   logging.info("tesseract not availabe")
 
@@ -206,6 +207,7 @@ def find_rect(self, color):
       rect_image = None
       filtered_image = self.filter_color(color)
       blobs = filtered_image.find_blobs(minsize=1000)
+      image_size = self.size()
       logging.info("blobs: " + str(blobs))
       if len(blobs):
         blob = blobs[0]
@@ -224,9 +226,9 @@ def find_rect(self, color):
 
         rot_matrix = cv2.getRotationMatrix2D(center, angle, 1)
         logging.info("center: " + str(center) + " size: " + str(size) + " angle: " + str(angle))
-        rect_image = Image(cv2.warpAffine(self._data, rot_matrix, (160, 120)))
+        rect_image = Image(cv2.warpAffine(self._data, rot_matrix, (image_size[1], image_size[0])))
         border = 5
-        rect_image = rect_image.crop(int(max(0,border+center[0]-(size[0])/2)), int(max(0,border+center[1]-(size[1]+5)/2)), int(min(160,-border+center[0]+(size[0])/2)), int(min(120,-border+center[1]+(size[1]-5)/2))) 
+        rect_image = rect_image.crop(int(max(0,border+center[0]-(size[0])/2)), int(max(0,border+center[1]-(size[1]+5)/2)), int(min(image_size[1],-border+center[0]+(size[0])/2)), int(min(image_size[0],-border+center[1]+(size[1]-5)/2))) 
       return rect_image 
 
     def find_text(self, accept):