1.1 --- a/org/hearing.org	Sat Feb 11 07:08:38 2012 -0700
     1.2 +++ b/org/hearing.org	Sat Feb 11 12:15:07 2012 -0700
     1.3 @@ -788,7 +788,7 @@
     1.4  rarefaction of the air while 1.0 represents maximum compression of the
     1.5  air at a given instant.
     1.6  
     1.7 -#+name: ears
     1.8 +#+name: hearing-pipeline
     1.9  #+begin_src clojure
    1.10  (in-ns 'cortex.hearing)
    1.11  
    1.12 @@ -832,6 +832,7 @@
    1.13  listeners, =(update-listener-velocity!)= ensures that this velocity
    1.14  information is always up-to-date.
    1.15  
    1.16 +#+name: hearing-ears
    1.17  #+begin_src clojure
    1.18  (defvar 
    1.19    ^{:arglists '([creature])}
    1.20 @@ -874,6 +875,7 @@
    1.21  
    1.22  ** Ear Creation
    1.23  
    1.24 +#+name: hearing-kernel
    1.25  #+begin_src clojure
    1.26  (defn hearing-kernel
    1.27    "Returns a functon which returns auditory sensory data when called
    1.28 @@ -920,7 +922,10 @@
    1.29  [-1.0, 1.0] to the range [0 255], converts to integer, and displays
    1.30  the number as a greyscale pixel.
    1.31  
    1.32 +#+name: hearing-display
    1.33  #+begin_src clojure
    1.34 +(in-ns 'cortex.hearing)
    1.35 +
    1.36  (defn view-hearing
    1.37    "Creates a function which accepts a list of auditory data and
    1.38     display each element of the list to the screen as an image."
    1.39 @@ -933,7 +938,7 @@
    1.40               #(rem (int (* 255 (/ (+ 1 %) 2))) 256)
    1.41               sensor-data))
    1.42             height 50
    1.43 -           image (BufferedImage. (count coords) height
    1.44 +           image (BufferedImage. (max 1 (count coords)) height
    1.45                                   BufferedImage/TYPE_INT_RGB)]
    1.46         (dorun
    1.47          (for [x (range (count coords))]
    1.48 @@ -968,7 +973,6 @@
    1.49    detects sound which is louder than a certain threshold. As the blue
    1.50    sphere travels along the path, it excites each of the cubes in turn.</p>
    1.51  </div>
    1.52 -
    1.53  #+end_html
    1.54  
    1.55  #+include "../../jmeCapture/src/com/aurellem/capture/examples/Advanced.java" src java  
    1.56 @@ -976,7 +980,7 @@
    1.57  Here is a small clojure program to drive the java program and make it
    1.58  available as part of my test suite.
    1.59  
    1.60 -#+name: test-hearing
    1.61 +#+name: test-hearing-1
    1.62  #+begin_src clojure
    1.63  (in-ns 'cortex.test.hearing)
    1.64  
    1.65 @@ -994,9 +998,139 @@
    1.66      (.setPauseOnLostFocus false)))
    1.67  #+end_src
    1.68  
    1.69 +
    1.70 +
    1.71 +
    1.72  ** Adding Hearing to the Worm
    1.73  
    1.74 +To the worm, I add a new node called "ears" with one child which
    1.75 +represents the worm's single ear.
    1.76  
    1.77 +#+attr_html: width=755
    1.78 +#+caption: The Worm with a newly added nodes describing an ear.
    1.79 +[[../images/worm-with-ear.png]]
    1.80 +
    1.81 +The node highlighted in yellow it the top-level "ears" node. It's
    1.82 +child, highlighted in orange, represents a the single ear the creature
    1.83 +has. The ear will be localized right above the curved part of the
    1.84 +worm's lower hemispherical region opposite the eye.  
    1.85 +
    1.86 +The other empty nodes represent the worm's single joint and eye and are
    1.87 +described in [[./body.org][body]] and [[./vision.org][vision]].
    1.88 +
    1.89 +#+name: test-hearing-2
    1.90 +#+begin_src clojure 
    1.91 +(in-ns 'cortex.test.hearing)
    1.92 +
    1.93 +(cortex.import/mega-import-jme3)
    1.94 +(import java.io.File)
    1.95 +
    1.96 +(use 'cortex.body)
    1.97 +
    1.98 +(defn test-worm-hearing []
    1.99 +  (let [the-worm (doto (worm) (body!))
   1.100 +        hearing (hearing! the-worm)
   1.101 +        hearing-display (view-hearing)
   1.102 +        
   1.103 +        tone (AudioNode. (asset-manager)
   1.104 +                         "Sounds/pure.wav" false)
   1.105 +        
   1.106 +        hymn (AudioNode. (asset-manager) 
   1.107 +                         "Sounds/ear-and-eye.wav" false)]
   1.108 +    (world
   1.109 +     (nodify [the-worm (floor)])
   1.110 +     (merge standard-debug-controls
   1.111 +            {"key-return"
   1.112 +             (fn [_ value]
   1.113 +               (if value (.play tone)))
   1.114 +             "key-l"
   1.115 +             (fn [_ value]
   1.116 +               (if value (.play hymn)))})
   1.117 +     (fn [world]
   1.118 +       (light-up-everything world)
   1.119 +       (com.aurellem.capture.Capture/captureVideo
   1.120 +        world
   1.121 +        (File."/home/r/proj/cortex/render/worm-audio/frames"))
   1.122 +       (com.aurellem.capture.Capture/captureAudio
   1.123 +        world
   1.124 +        (File."/home/r/proj/cortex/render/worm-audio/audio.wav")))
   1.125 +     
   1.126 +     (fn [world tpf]
   1.127 +       (hearing-display
   1.128 +        (map #(% world) hearing)
   1.129 +        (File. "/home/r/proj/cortex/render/worm-audio/hearing-data"))))))
   1.130 +#+end_src
   1.131 +
   1.132 +In this test, I load the worm with its newly formed ear and let it
   1.133 +hear sounds. The sound the worm is hearing is localized to the origin
   1.134 +of the world, and you can see that as the worm moves farther away from
   1.135 +the origin when it is hit by balls, it hears the sound less intensely.
   1.136 +
   1.137 +The sound you hear in the video is from the worm's perspective. Notice
   1.138 +how the pure tone becomes fainter and the visual display of the
   1.139 +auditory data becomes less pronounced as the worm falls farther away
   1.140 +from the source of the sound.
   1.141 +
   1.142 +#+begin_html
   1.143 +<div class="figure">
   1.144 +<center>
   1.145 +<video controls="controls" width="600">
   1.146 +  <source src="../video/worm-hearing.ogg" type="video/ogg"
   1.147 +	  preload="none" poster="../images/aurellem-1280x480.png" />
   1.148 +</video>
   1.149 +</center>
   1.150 +<p>The worm can now hear the sound pulses produced from the
   1.151 +  hymn. Notice the strikingly different pattern that human speech
   1.152 +  makes compared to the insturments. Once the worm is pushed off the
   1.153 +  floor, the sound it hears is attenuated, and the display of the
   1.154 +  sound it hears beomes fainter. This shows the 3D localization of
   1.155 +  sound in this world.</p>
   1.156 +</div>
   1.157 +
   1.158 +#+end_html
   1.159 +
   1.160 +*** Creating the Ear Video
   1.161 +#+name: magick-3
   1.162 +#+begin_src clojure
   1.163 +(ns cortex.video.magick3
   1.164 +  (:import java.io.File)
   1.165 +  (:use clojure.contrib.shell-out))
   1.166 +
   1.167 +(defn images [path]
   1.168 +  (sort (rest (file-seq (File. path)))))
   1.169 +
   1.170 +(def base "/home/r/proj/cortex/render/worm-audio/")
   1.171 +
   1.172 +(defn pics [file]
   1.173 +  (images (str base file)))
   1.174 +
   1.175 +(defn combine-images []
   1.176 +  (let [main-view (pics "frames")
   1.177 +        hearing (pics "hearing-data")
   1.178 +        background (repeat 9001 (File. (str base "background.png")))
   1.179 +        targets (map
   1.180 +                 #(File. (str base "out/" (format "%07d.png" %)))
   1.181 +                 (range 0 (count main-view)))]
   1.182 +    (dorun
   1.183 +     (pmap
   1.184 +      (comp
   1.185 +       (fn [[background main-view hearing target]]
   1.186 +         (println target)
   1.187 +         (sh "convert"
   1.188 +             background
   1.189 +             main-view "-geometry" "+66+21"   "-composite"
   1.190 +             hearing   "-geometry" "+21+526"  "-composite"
   1.191 +             target))
   1.192 +       (fn [& args] (map #(.getCanonicalPath %) args)))
   1.193 +      background main-view hearing targets))))
   1.194 +#+end_src
   1.195 +
   1.196 +#+begin_src sh
   1.197 +cd /home/r/proj/cortex/render/worm-audio
   1.198 +ffmpeg -r 60 -i out/%07d.png -i audio.wav \
   1.199 +       -b:a 128k  -b:v 9001k \
   1.200 +       -acodec libvorbis -vcodec libtheora worm-hearing.ogg 
   1.201 +#+end_src
   1.202  
   1.203  * Headers
   1.204  
   1.205 @@ -1006,7 +1140,7 @@
   1.206    "Simulate the sense of hearing in jMonkeyEngine3. Enables multiple
   1.207    listeners at different positions in the same world. Automatically
   1.208    reads ear-nodes from specially prepared blender files and
   1.209 -  instantiates them in the world as actual ears."
   1.210 +  instantiates them in the world as simulated ears."
   1.211    {:author "Robert McIntyre"}
   1.212    (:use (cortex world util sense))
   1.213    (:use clojure.contrib.def)
   1.214 @@ -1022,9 +1156,11 @@
   1.215    (:import com.jme3.scene.control.AbstractControl))
   1.216  #+end_src
   1.217  
   1.218 +#+name: test-header
   1.219  #+begin_src clojure
   1.220  (ns cortex.test.hearing
   1.221    (:use (cortex world util hearing))
   1.222 +  (:use cortex.test.body)
   1.223    (:import (com.jme3.audio AudioNode Listener))
   1.224    (:import com.jme3.scene.Node
   1.225  	   com.jme3.system.AppSettings))
   1.226 @@ -1042,11 +1178,20 @@
   1.227  
   1.228  #+begin_src clojure :tangle ../src/cortex/hearing.clj
   1.229  <<hearing-header>>
   1.230 -<<ears>>
   1.231 +<<hearing-pipeline>>
   1.232 +<<hearing-ears>>
   1.233 +<<hearing-kernel>>
   1.234 +<<hearing-display>>
   1.235  #+end_src
   1.236  
   1.237  #+begin_src clojure :tangle ../src/cortex/test/hearing.clj
   1.238 -<<test-hearing>>
   1.239 +<<test-header>>
   1.240 +<<test-hearing-1>>
   1.241 +<<test-hearing-2>>
   1.242 +#+end_src
   1.243 +
   1.244 +#+begin_src clojure :tangle ../src/cortex/video/magick3.clj
   1.245 +<<magick-3>>
   1.246  #+end_src
   1.247  
   1.248  #+begin_src C :tangle ../../audio-send/Alc/backends/send.c