1.1 --- a/org/ear.org	Thu Nov 03 13:32:27 2011 -0700
     1.2 +++ b/org/ear.org	Thu Nov 03 14:54:45 2011 -0700
     1.3 @@ -46,6 +46,7 @@
     1.4  world from their own perspective, it is necessary to go all the way
     1.5  back to =OpenAL= and implement support for simulated hearing there.
     1.6  
     1.7 +* Extending =OpenAL=
     1.8  ** =OpenAL= Devices
     1.9     
    1.10  =OpenAL= goes to great lengths to support many different systems, all
    1.11 @@ -75,7 +76,6 @@
    1.12  data in a form that the AIs can use, it is necessary to create a new
    1.13  Device, which supports this features.
    1.14  
    1.15 -
    1.16  ** The Send Device
    1.17  Adding a device to OpenAL is rather tricky -- there are five separate
    1.18  files in the =OpenAL= source tree that must be modified to do so. I've
    1.19 @@ -331,7 +331,7 @@
    1.20  about is automoatically creating new sources whenever a slave context
    1.21  does not have the same number of sources as the master context.
    1.22  
    1.23 -* Context Creation
    1.24 +** Context Creation
    1.25  #+begin_src C
    1.26  static void addContext(ALCdevice *Device, ALCcontext *context){
    1.27    send_data *data = (send_data*)Device->ExtraData;
    1.28 @@ -359,7 +359,7 @@
    1.29  here is where the rendered sound samples for this slave context will
    1.30  eventually go.
    1.31  
    1.32 -* Context Switching
    1.33 +** Context Switching
    1.34  #+begin_src C
    1.35  ////////////////////  Context Switching 
    1.36  
    1.37 @@ -451,7 +451,17 @@
    1.38  contexts, then walks each context, rendering just that context to it's
    1.39  audio-sample storage buffer.
    1.40  
    1.41 -* JNI Methods
    1.42 +** JNI Methods
    1.43 +
    1.44 +At this point, we have the ability to create multiple listeners by
    1.45 +using the master/slave context trick, and the rendered audio data is
    1.46 +waiting patiently in internal buffers, one for each listener.  We need
    1.47 +a way to transport this information to Java, and also a way to drive
    1.48 +this device from Java.  The following JNI interface code is inspired
    1.49 +by the way LWJGL interfaces with =OpenAL=.
    1.50 +
    1.51 +*** step
    1.52 +
    1.53  #+begin_src C
    1.54  ////////////////////   JNI Methods
    1.55  
    1.56 @@ -467,7 +477,18 @@
    1.57    UNUSED(env);UNUSED(clazz);UNUSED(device);
    1.58    renderData((ALCdevice*)((intptr_t)device), samples);
    1.59  }
    1.60 +#+end_src
    1.61 +This device, unlike most of the other devices in =OpenAL=, does not
    1.62 +render sound unless asked. This enables the system to slow down or
    1.63 +speed up depending on the needs of the AIs who are using it to
    1.64 +listen.  If the device tried to render samples in real-time, a
    1.65 +complicated AI whose mind takes 100 seconds of computer time to
    1.66 +simulate 1 second of AI-time would miss almost all of the sound in
    1.67 +its environment.
    1.68  
    1.69 +
    1.70 +*** getSamples
    1.71 +#+begin_src C
    1.72  /*
    1.73   * Class:     com_aurellem_send_AudioSend
    1.74   * Method:    ngetSamples
    1.75 @@ -486,7 +507,19 @@
    1.76    memcpy(buffer_address, data->contexts[n]->renderBuffer, 
    1.77  	 BytesFromDevFmt(recorder->FmtType) * recorder->NumChan * samples);
    1.78  }
    1.79 +#+end_src
    1.80  
    1.81 +This is the transport layer between C and Java that will eventually
    1.82 +allow us to access rendered sound data from clojure.
    1.83 +
    1.84 +*** Listener Management
    1.85 +
    1.86 +=addListener=, =setNthListenerf=, and =setNthListener3f= are
    1.87 +necessary to change the properties of any listener other than the
    1.88 +master one, since only the listener of the current active context is
    1.89 +affected by the normal =OpenAL= listener calls.
    1.90 +
    1.91 +#+begin_src C
    1.92  /*
    1.93   * Class:     com_aurellem_send_AudioSend
    1.94   * Method:    naddListener
    1.95 @@ -541,7 +574,19 @@
    1.96    alListenerf(param, v1);
    1.97    alcMakeContextCurrent(current);
    1.98  }
    1.99 +#+end_src
   1.100  
   1.101 +*** Initilazation
   1.102 +=initDevice= is called from the Java side after LWJGL has created its
   1.103 +context, and before any calls to =addListener=. It establishes the
   1.104 +LWJGL context as the master context.
   1.105 +
   1.106 +=getAudioFormat= is a convienence function that uses JNI to build up a
   1.107 +=javax.sound.sampled.AudioFormat= object from data in the Device. This
   1.108 +way, there is no ambiguity about what the bits created by =step= and
   1.109 +returned by =getSamples= mean.
   1.110 +
   1.111 +#+begin_src C
   1.112  /*
   1.113   * Class:     com_aurellem_send_AudioSend
   1.114   * Method:    ninitDevice
   1.115 @@ -550,13 +595,10 @@
   1.116  JNIEXPORT void JNICALL Java_com_aurellem_send_AudioSend_ninitDevice
   1.117  (JNIEnv *env, jclass clazz, jlong device){
   1.118    UNUSED(env);UNUSED(clazz);
   1.119 -  
   1.120    ALCdevice *Device = (ALCdevice*) ((intptr_t)device);
   1.121    init(Device);
   1.122 -
   1.123  }
   1.124  
   1.125 -
   1.126  /*
   1.127   * Class:     com_aurellem_send_AudioSend
   1.128   * Method:    ngetAudioFormat
   1.129 @@ -571,9 +613,6 @@
   1.130      (*env)->GetMethodID(env, AudioFormatClass, "<init>", "(FIIZZ)V");
   1.131    
   1.132    ALCdevice *Device = (ALCdevice*) ((intptr_t)device);
   1.133 -
   1.134 -  //float frequency
   1.135 -
   1.136    int isSigned;
   1.137    switch (Device->FmtType)
   1.138      {
   1.139 @@ -584,11 +623,6 @@
   1.140    float frequency = Device->Frequency;
   1.141    int bitsPerFrame = (8 * BytesFromDevFmt(Device->FmtType));
   1.142    int channels = Device->NumChan;
   1.143 -
   1.144 -
   1.145 -  //printf("freq = %f, bpf = %d, channels = %d, signed? = %d\n",
   1.146 -  //	 frequency, bitsPerFrame, channels, isSigned);
   1.147 -
   1.148    jobject format = (*env)->
   1.149      NewObject(
   1.150  	      env,AudioFormatClass,AudioFormatConstructor,
   1.151 @@ -599,7 +633,13 @@
   1.152  	      0);
   1.153    return format;
   1.154  }
   1.155 +#+end_src
   1.156  
   1.157 +*** Boring Device management stuff
   1.158 +This code is more-or-less copied verbatim from the other =OpenAL=
   1.159 +backends. It's the basis for =OpenAL='s primitive object system.
   1.160 +
   1.161 +#+begin_src C
   1.162  ////////////////////   Device Initilization / Management
   1.163  
   1.164  static const ALCchar sendDevice[] = "Multiple Audio Send";
   1.165 @@ -685,57 +725,60 @@
   1.166  }
   1.167  #+end_src
   1.168  
   1.169 +* The Java interface, =AudioSend=
   1.170  
   1.171 +The Java interface to the Send Device follows naturally from the JNI
   1.172 +definitions. It is included here for completeness. The only thing here
   1.173 +of note is the =deviceID=. This is available from LWJGL, but to only
   1.174 +way to get it is reflection. Unfornatuently, there is no other way to
   1.175 +control the Send device than to obtain a pointer to it.
   1.176  
   1.177 +#+include: "../java/src/com/aurellem/send/AudioSend.java" src java :exports code
   1.178  
   1.179 +* Finally, Ears in clojure! 
   1.180  
   1.181 +Now that the infastructure is complete (modulo a few patches to
   1.182 +jMonkeyEngine3 to support accessing this modified version of =OpenAL=
   1.183 +that are not worth discussing), the clojure ear abstraction is rather
   1.184 +simple.  Just as there were =SceneProcessors= for vision, there are
   1.185 +now =SoundProcessors= for hearing.
   1.186  
   1.187 -
   1.188 +#+include "../../jmeCapture/src/com/aurellem/capture/audio/SoundProcessor.java" src java 
   1.189  
   1.190  #+srcname: ears
   1.191  #+begin_src clojure
   1.192 -(ns cortex.hearing)
   1.193 -(use 'cortex.world)
   1.194 -(use 'cortex.import)
   1.195 -(use 'clojure.contrib.def)
   1.196 -(cortex.import/mega-import-jme3)
   1.197 -(rlm.rlm-commands/help)
   1.198 -(import java.nio.ByteBuffer)
   1.199 -(import java.awt.image.BufferedImage)
   1.200 -(import java.awt.Color)
   1.201 -(import java.awt.Dimension)
   1.202 -(import java.awt.Graphics)
   1.203 -(import java.awt.Graphics2D)
   1.204 -(import java.awt.event.WindowAdapter)
   1.205 -(import java.awt.event.WindowEvent)
   1.206 -(import java.awt.image.BufferedImage)
   1.207 -(import java.nio.ByteBuffer)
   1.208 -(import javax.swing.JFrame)
   1.209 -(import javax.swing.JPanel)
   1.210 -(import javax.swing.SwingUtilities)
   1.211 -(import javax.swing.ImageIcon)
   1.212 -(import javax.swing.JOptionPane)
   1.213 -(import java.awt.image.ImageObserver)
   1.214 -
   1.215 -(import 'com.jme3.capture.SoundProcessor)
   1.216 -
   1.217 -
   1.218 +(ns cortex.hearing
   1.219 +  "Simulate the sense of hearing in jMonkeyEngine3. Enables multiple
   1.220 +  listeners at different positions in the same world. Passes vectors
   1.221 +  of floats in the range [-1.0 -- 1.0] in PCM format to any arbitray
   1.222 +  function."
   1.223 +  {:author "Robert McIntyre"}
   1.224 +  (:use (cortex world util))
   1.225 +  (:import java.nio.ByteBuffer)
   1.226 +  (:import org.tritonus.share.sampled.FloatSampleTools)
   1.227 +  (:import com.aurellem.capture.audio.SoundProcessor)
   1.228 +  (:import javax.sound.sampled.AudioFormat))
   1.229 +  
   1.230  (defn sound-processor
   1.231 -  "deals with converting ByteBuffers into Arrays of bytes so that the
   1.232 -  continuation functions can be defined in terms of immutable stuff."
   1.233 +  "Deals with converting ByteBuffers into Vectors of floats so that
   1.234 +  the continuation functions can be defined in terms of immutable
   1.235 +  stuff."
   1.236    [continuation]
   1.237    (proxy [SoundProcessor] []
   1.238      (cleanup [])
   1.239      (process
   1.240 -      [#^ByteBuffer audioSamples numSamples]
   1.241 -      (no-exceptions
   1.242 -       (let [byte-array (byte-array numSamples)]
   1.243 -	 (.get audioSamples byte-array 0 numSamples)
   1.244 -	 (continuation
   1.245 -	  (vec byte-array)))))))
   1.246 +      [#^ByteBuffer audioSamples numSamples #^AudioFormat audioFormat]
   1.247 +      (let [bytes  (byte-array numSamples)
   1.248 +            floats (float-array numSamples)]
   1.249 +        (.get audioSamples bytes 0 numSamples)
   1.250 +        (FloatSampleTools/byte2floatInterleaved
   1.251 +         bytes 0 floats 0
   1.252 +         (/ numSamples (.getFrameSize audioFormat)) audioFormat)
   1.253 +        (continuation
   1.254 +         (vec floats))))))
   1.255  
   1.256  (defn add-ear 
   1.257 -  "add an ear to the world.  The continuation function will be called
   1.258 +  "Add an ear to the world.  The continuation function will be called
   1.259    on the FFT or the sounds which the ear hears in the given
   1.260    timeframe. Sound is 3D."
   1.261    [world listener continuation]
   1.262 @@ -744,45 +787,39 @@
   1.263      (.registerSoundProcessor renderer listener
   1.264  			     (sound-processor continuation))
   1.265      listener))
   1.266 -
   1.267  #+end_src
   1.268  
   1.269 -
   1.270 +* Example
   1.271  
   1.272  #+srcname: test-hearing
   1.273  #+begin_src clojure :results silent
   1.274 -(ns test.hearing)
   1.275 -(use 'cortex.world)
   1.276 -(use 'cortex.import)
   1.277 -(use 'clojure.contrib.def)
   1.278 -(use 'body.ear)
   1.279 -(cortex.import/mega-import-jme3)
   1.280 -(rlm.rlm-commands/help)
   1.281 +(ns test.hearing
   1.282 +  (:use (cortex world util hearing))
   1.283 +  (:import (com.jme3.audio AudioNode Listener))
   1.284 +  (:import com.jme3.scene.Node))
   1.285  
   1.286  (defn setup-fn [world]
   1.287    (let [listener (Listener.)]
   1.288 -    (add-ear world listener #(println (nth % 0)))))
   1.289 +    (add-ear world listener #(println-repl (nth % 0)))))
   1.290    
   1.291  (defn play-sound [node world value]
   1.292    (if (not value)
   1.293      (do
   1.294        (.playSource (.getAudioRenderer world) node))))
   1.295  
   1.296 -(defn test-world []
   1.297 -  (let [node1 (AudioNode. (asset-manager) "Sounds/pure.wav" false false)]
   1.298 -    (world
   1.299 -     (Node.)
   1.300 -     {"key-space" (partial play-sound node1)}
   1.301 -     setup-fn
   1.302 -     no-op
   1.303 -     )))
   1.304 -    
   1.305 -
   1.306 +(defn test-basic-hearing []
   1.307 +  (.start
   1.308 +   (let [node1 (AudioNode. (asset-manager) "Sounds/pure.wav" false false)]
   1.309 +     (world
   1.310 +      (Node.)
   1.311 +      {"key-space" (partial play-sound node1)}
   1.312 +      setup-fn
   1.313 +      no-op))))  
   1.314  #+end_src
   1.315  
   1.316 -
   1.317 -
   1.318 -* Example
   1.319 +This extremely basic program prints out the first sample it encounters
   1.320 +at every time stamp. You can see the rendered sound begin printed at
   1.321 +the REPL.
   1.322  
   1.323  * COMMENT Code Generation
   1.324  
   1.325 @@ -798,3 +835,5 @@
   1.326  #+begin_src C :tangle ../Alc/backends/send.c
   1.327  <<send>>
   1.328  #+end_src
   1.329 +
   1.330 +