1.1 --- a/java/src/com/aurellem/send/AudioSend.java	Thu Nov 03 13:32:27 2011 -0700
     1.2 +++ b/java/src/com/aurellem/send/AudioSend.java	Thu Nov 03 14:54:45 2011 -0700
     1.3 @@ -6,86 +6,102 @@
     1.4  
     1.5  public class AudioSend {
     1.6  
     1.7 -	private final long deviceID;
     1.8 +    private final long deviceID;
     1.9  	
    1.10 -	public AudioSend(long deviceID){
    1.11 -		this.deviceID = deviceID;
    1.12 -	}
    1.13 +    public AudioSend(long deviceID){
    1.14 +	this.deviceID = deviceID;
    1.15 +    }
    1.16  	
    1.17 -	/** This establishes the LWJGL context as the context which will be copies to all 
    1.18 -	 *  other contexts.  It must be called before any calls to <code>addListener();</code>
    1.19 -	 */
    1.20 -	public void initDevice(){
    1.21 -		ninitDevice(this.deviceID);}
    1.22 -	public static native void ninitDevice(long device);
    1.23 +    /** This establishes the LWJGL context as the context which
    1.24 +     *  will be copies to all other contexts.  It must be called
    1.25 +     *  before any calls to <code>addListener();</code>
    1.26 +     */
    1.27 +    public void initDevice(){
    1.28 +	ninitDevice(this.deviceID);}
    1.29 +    public static native void ninitDevice(long device);
    1.30  	
    1.31 -	/**
    1.32 -	 * The send device does not automatically process sound.  This step function will cause 
    1.33 -	 * the desired number of samples to be processed for each listener.  The results will then 
    1.34 -	 * be available via calls to <code>getSamples()</code> for each listener.
    1.35 -	 * @param samples
    1.36 -	 */
    1.37 -	public void step(int samples){
    1.38 -		nstep(this.deviceID, samples);}
    1.39 -	public static native void nstep(long device, int samples);
    1.40 +    /**
    1.41 +     * The send device does not automatically process sound.  This
    1.42 +     * step function will cause the desired number of samples to
    1.43 +     * be processed for each listener.  The results will then be
    1.44 +     * available via calls to <code>getSamples()</code> for each
    1.45 +     * listener.
    1.46 +     * @param samples
    1.47 +     */
    1.48 +    public void step(int samples){
    1.49 +	nstep(this.deviceID, samples);}
    1.50 +    public static native void nstep(long device, int samples);
    1.51  
    1.52 -	/**
    1.53 -	 * Retrieve the final rendered sound for a particular listener.  <code>contextNum == 0</code>
    1.54 -	 * is the main LWJGL context.
    1.55 -	 * @param buffer
    1.56 -	 * @param samples
    1.57 -	 * @param contextNum
    1.58 -	 */
    1.59 -	public void getSamples(ByteBuffer buffer, int samples, int contextNum){
    1.60 -		ngetSamples(this.deviceID, buffer, buffer.position(), samples, contextNum);}
    1.61 -	public static native void ngetSamples(
    1.62 -			long device, ByteBuffer buffer, int position, int samples, int contextNum);
    1.63 +    /**
    1.64 +     * Retrieve the final rendered sound for a particular
    1.65 +     * listener.  <code>contextNum == 0</code> is the main LWJGL
    1.66 +     * context.
    1.67 +     * @param buffer
    1.68 +     * @param samples
    1.69 +     * @param contextNum
    1.70 +     */
    1.71 +    public void getSamples(ByteBuffer buffer, 
    1.72 +			   int samples, int contextNum){
    1.73 +	ngetSamples(this.deviceID, buffer, 
    1.74 +		    buffer.position(), samples, contextNum);}
    1.75 +    public static native void 
    1.76 +	ngetSamples(long device, ByteBuffer buffer, 
    1.77 +		    int position, int samples, int contextNum);
    1.78  	
    1.79 -	/**
    1.80 -	 * Create an additional listener on the recorder device.  The device itself will manage 
    1.81 -	 * this listener and synchronize it with the main LWJGL context. Processed sound samples
    1.82 -	 * for this listener will be available via a call to <code>getSamples()</code> with 
    1.83 -	 * <code>contextNum</code> equal to the number of times this method has been called. 
    1.84 -	 */
    1.85 -	public void addListener(){naddListener(this.deviceID);}
    1.86 -	public static native void naddListener(long device);
    1.87 +    /**
    1.88 +     * Create an additional listener on the recorder device.  The
    1.89 +     * device itself will manage this listener and synchronize it
    1.90 +     * with the main LWJGL context. Processed sound samples for
    1.91 +     * this listener will be available via a call to
    1.92 +     * <code>getSamples()</code> with <code>contextNum</code>
    1.93 +     * equal to the number of times this method has been called.
    1.94 +     */
    1.95 +    public void addListener(){naddListener(this.deviceID);}
    1.96 +    public static native void naddListener(long device);
    1.97  	
    1.98 -	/**
    1.99 -	 * This will internally call <code>alListener3f<code> in the appropriate slave context and update
   1.100 -	 * that context's listener's parameters. Calling this for a number greater than the current 
   1.101 -	 * number of slave contexts will have no effect.
   1.102 -	 * @param pname
   1.103 -	 * @param v1
   1.104 -	 * @param v2
   1.105 -	 * @param v3
   1.106 -	 * @param contextNum
   1.107 -	 */
   1.108 -	public void setNthListener3f(int pname, float v1, float v2, float v3, int contextNum){
   1.109 -		nsetNthListener3f(pname, v1, v2, v3, this.deviceID, contextNum);}
   1.110 -	public static native void 
   1.111 -	nsetNthListener3f(int pname, float v1, float v2, float v3, long device, int contextNum);
   1.112 -	
   1.113 -	/**
   1.114 -	 * This will internally call <code>alListenerf<code> in the appropriate slave context and update
   1.115 -	 * that context's listener's parameters. Calling this for a number greater than the current 
   1.116 -	 * number of slave contexts will have no effect.
   1.117 -	 * @param pname
   1.118 -	 * @param v1
   1.119 -	 * @param contextNum
   1.120 -	 */
   1.121 -	public void setNthListenerf(int pname, float v1, int contextNum){
   1.122 -		nsetNthListenerf(pname, v1, this.deviceID, contextNum);}
   1.123 -	public static native void nsetNthListenerf(int pname, float v1, long device, int contextNum);
   1.124 -	
   1.125 -	
   1.126 -	/**
   1.127 -	 * Retrieve the AudioFormat which the device is using.  This format is itself derived
   1.128 -	 * from the OpenAL config file under the "format" variable.
   1.129 -	 */
   1.130 -	public AudioFormat getAudioFormat(){
   1.131 -		return ngetAudioFormat(this.deviceID);}
   1.132 -	public static native AudioFormat ngetAudioFormat(long device);
   1.133 -	
   1.134 -	
   1.135 -	
   1.136 +    /**
   1.137 +     * This will internally call <code>alListener3f<code> in the
   1.138 +     * appropriate slave context and update that context's
   1.139 +     * listener's parameters. Calling this for a number greater
   1.140 +     * than the current number of slave contexts will have no
   1.141 +     * effect.
   1.142 +     * @param pname
   1.143 +     * @param v1
   1.144 +     * @param v2
   1.145 +     * @param v3
   1.146 +     * @param contextNum
   1.147 +     */
   1.148 +    public void 
   1.149 +	setNthListener3f(int pname, float v1, 
   1.150 +			 float v2, float v3, int contextNum){
   1.151 +	nsetNthListener3f(pname, v1, v2, v3, 
   1.152 +			  this.deviceID, contextNum);}
   1.153 +    public static native void 
   1.154 +	nsetNthListener3f(int pname, float v1, float v2, 
   1.155 +			  float v3, long device, int contextNum);
   1.156 +    
   1.157 +    /**
   1.158 +     * This will internally call <code>alListenerf<code> in the
   1.159 +     * appropriate slave context and update that context's
   1.160 +     * listener's parameters. Calling this for a number greater
   1.161 +     * than the current number of slave contexts will have no
   1.162 +     * effect.
   1.163 +     * @param pname
   1.164 +     * @param v1
   1.165 +     * @param contextNum
   1.166 +     */
   1.167 +    public void setNthListenerf(int pname, float v1, int contextNum){
   1.168 +	nsetNthListenerf(pname, v1, this.deviceID, contextNum);}
   1.169 +    public static native void 
   1.170 +	nsetNthListenerf(int pname, float v1, 
   1.171 +			 long device, int contextNum);
   1.172 +		
   1.173 +    /**
   1.174 +     * Retrieve the AudioFormat which the device is using.  This
   1.175 +     * format is itself derived from the OpenAL config file under
   1.176 +     * the "format" variable.
   1.177 +     */
   1.178 +    public AudioFormat getAudioFormat(){
   1.179 +	return ngetAudioFormat(this.deviceID);}
   1.180 +    public static native AudioFormat ngetAudioFormat(long device);	
   1.181  }

     2.1 --- a/org/ear.org	Thu Nov 03 13:32:27 2011 -0700
     2.2 +++ b/org/ear.org	Thu Nov 03 14:54:45 2011 -0700
     2.3 @@ -46,6 +46,7 @@
     2.4  world from their own perspective, it is necessary to go all the way
     2.5  back to =OpenAL= and implement support for simulated hearing there.
     2.6  
     2.7 +* Extending =OpenAL=
     2.8  ** =OpenAL= Devices
     2.9     
    2.10  =OpenAL= goes to great lengths to support many different systems, all
    2.11 @@ -75,7 +76,6 @@
    2.12  data in a form that the AIs can use, it is necessary to create a new
    2.13  Device, which supports this features.
    2.14  
    2.15 -
    2.16  ** The Send Device
    2.17  Adding a device to OpenAL is rather tricky -- there are five separate
    2.18  files in the =OpenAL= source tree that must be modified to do so. I've
    2.19 @@ -331,7 +331,7 @@
    2.20  about is automoatically creating new sources whenever a slave context
    2.21  does not have the same number of sources as the master context.
    2.22  
    2.23 -* Context Creation
    2.24 +** Context Creation
    2.25  #+begin_src C
    2.26  static void addContext(ALCdevice *Device, ALCcontext *context){
    2.27    send_data *data = (send_data*)Device->ExtraData;
    2.28 @@ -359,7 +359,7 @@
    2.29  here is where the rendered sound samples for this slave context will
    2.30  eventually go.
    2.31  
    2.32 -* Context Switching
    2.33 +** Context Switching
    2.34  #+begin_src C
    2.35  ////////////////////  Context Switching 
    2.36  
    2.37 @@ -451,7 +451,17 @@
    2.38  contexts, then walks each context, rendering just that context to it's
    2.39  audio-sample storage buffer.
    2.40  
    2.41 -* JNI Methods
    2.42 +** JNI Methods
    2.43 +
    2.44 +At this point, we have the ability to create multiple listeners by
    2.45 +using the master/slave context trick, and the rendered audio data is
    2.46 +waiting patiently in internal buffers, one for each listener.  We need
    2.47 +a way to transport this information to Java, and also a way to drive
    2.48 +this device from Java.  The following JNI interface code is inspired
    2.49 +by the way LWJGL interfaces with =OpenAL=.
    2.50 +
    2.51 +*** step
    2.52 +
    2.53  #+begin_src C
    2.54  ////////////////////   JNI Methods
    2.55  
    2.56 @@ -467,7 +477,18 @@
    2.57    UNUSED(env);UNUSED(clazz);UNUSED(device);
    2.58    renderData((ALCdevice*)((intptr_t)device), samples);
    2.59  }
    2.60 +#+end_src
    2.61 +This device, unlike most of the other devices in =OpenAL=, does not
    2.62 +render sound unless asked. This enables the system to slow down or
    2.63 +speed up depending on the needs of the AIs who are using it to
    2.64 +listen.  If the device tried to render samples in real-time, a
    2.65 +complicated AI whose mind takes 100 seconds of computer time to
    2.66 +simulate 1 second of AI-time would miss almost all of the sound in
    2.67 +its environment.
    2.68  
    2.69 +
    2.70 +*** getSamples
    2.71 +#+begin_src C
    2.72  /*
    2.73   * Class:     com_aurellem_send_AudioSend
    2.74   * Method:    ngetSamples
    2.75 @@ -486,7 +507,19 @@
    2.76    memcpy(buffer_address, data->contexts[n]->renderBuffer, 
    2.77  	 BytesFromDevFmt(recorder->FmtType) * recorder->NumChan * samples);
    2.78  }
    2.79 +#+end_src
    2.80  
    2.81 +This is the transport layer between C and Java that will eventually
    2.82 +allow us to access rendered sound data from clojure.
    2.83 +
    2.84 +*** Listener Management
    2.85 +
    2.86 +=addListener=, =setNthListenerf=, and =setNthListener3f= are
    2.87 +necessary to change the properties of any listener other than the
    2.88 +master one, since only the listener of the current active context is
    2.89 +affected by the normal =OpenAL= listener calls.
    2.90 +
    2.91 +#+begin_src C
    2.92  /*
    2.93   * Class:     com_aurellem_send_AudioSend
    2.94   * Method:    naddListener
    2.95 @@ -541,7 +574,19 @@
    2.96    alListenerf(param, v1);
    2.97    alcMakeContextCurrent(current);
    2.98  }
    2.99 +#+end_src
   2.100  
   2.101 +*** Initilazation
   2.102 +=initDevice= is called from the Java side after LWJGL has created its
   2.103 +context, and before any calls to =addListener=. It establishes the
   2.104 +LWJGL context as the master context.
   2.105 +
   2.106 +=getAudioFormat= is a convienence function that uses JNI to build up a
   2.107 +=javax.sound.sampled.AudioFormat= object from data in the Device. This
   2.108 +way, there is no ambiguity about what the bits created by =step= and
   2.109 +returned by =getSamples= mean.
   2.110 +
   2.111 +#+begin_src C
   2.112  /*
   2.113   * Class:     com_aurellem_send_AudioSend
   2.114   * Method:    ninitDevice
   2.115 @@ -550,13 +595,10 @@
   2.116  JNIEXPORT void JNICALL Java_com_aurellem_send_AudioSend_ninitDevice
   2.117  (JNIEnv *env, jclass clazz, jlong device){
   2.118    UNUSED(env);UNUSED(clazz);
   2.119 -  
   2.120    ALCdevice *Device = (ALCdevice*) ((intptr_t)device);
   2.121    init(Device);
   2.122 -
   2.123  }
   2.124  
   2.125 -
   2.126  /*
   2.127   * Class:     com_aurellem_send_AudioSend
   2.128   * Method:    ngetAudioFormat
   2.129 @@ -571,9 +613,6 @@
   2.130      (*env)->GetMethodID(env, AudioFormatClass, "<init>", "(FIIZZ)V");
   2.131    
   2.132    ALCdevice *Device = (ALCdevice*) ((intptr_t)device);
   2.133 -
   2.134 -  //float frequency
   2.135 -
   2.136    int isSigned;
   2.137    switch (Device->FmtType)
   2.138      {
   2.139 @@ -584,11 +623,6 @@
   2.140    float frequency = Device->Frequency;
   2.141    int bitsPerFrame = (8 * BytesFromDevFmt(Device->FmtType));
   2.142    int channels = Device->NumChan;
   2.143 -
   2.144 -
   2.145 -  //printf("freq = %f, bpf = %d, channels = %d, signed? = %d\n",
   2.146 -  //	 frequency, bitsPerFrame, channels, isSigned);
   2.147 -
   2.148    jobject format = (*env)->
   2.149      NewObject(
   2.150  	      env,AudioFormatClass,AudioFormatConstructor,
   2.151 @@ -599,7 +633,13 @@
   2.152  	      0);
   2.153    return format;
   2.154  }
   2.155 +#+end_src
   2.156  
   2.157 +*** Boring Device management stuff
   2.158 +This code is more-or-less copied verbatim from the other =OpenAL=
   2.159 +backends. It's the basis for =OpenAL='s primitive object system.
   2.160 +
   2.161 +#+begin_src C
   2.162  ////////////////////   Device Initilization / Management
   2.163  
   2.164  static const ALCchar sendDevice[] = "Multiple Audio Send";
   2.165 @@ -685,57 +725,60 @@
   2.166  }
   2.167  #+end_src
   2.168  
   2.169 +* The Java interface, =AudioSend=
   2.170  
   2.171 +The Java interface to the Send Device follows naturally from the JNI
   2.172 +definitions. It is included here for completeness. The only thing here
   2.173 +of note is the =deviceID=. This is available from LWJGL, but to only
   2.174 +way to get it is reflection. Unfornatuently, there is no other way to
   2.175 +control the Send device than to obtain a pointer to it.
   2.176  
   2.177 +#+include: "../java/src/com/aurellem/send/AudioSend.java" src java :exports code
   2.178  
   2.179 +* Finally, Ears in clojure! 
   2.180  
   2.181 +Now that the infastructure is complete (modulo a few patches to
   2.182 +jMonkeyEngine3 to support accessing this modified version of =OpenAL=
   2.183 +that are not worth discussing), the clojure ear abstraction is rather
   2.184 +simple.  Just as there were =SceneProcessors= for vision, there are
   2.185 +now =SoundProcessors= for hearing.
   2.186  
   2.187 -
   2.188 +#+include "../../jmeCapture/src/com/aurellem/capture/audio/SoundProcessor.java" src java 
   2.189  
   2.190  #+srcname: ears
   2.191  #+begin_src clojure
   2.192 -(ns cortex.hearing)
   2.193 -(use 'cortex.world)
   2.194 -(use 'cortex.import)
   2.195 -(use 'clojure.contrib.def)
   2.196 -(cortex.import/mega-import-jme3)
   2.197 -(rlm.rlm-commands/help)
   2.198 -(import java.nio.ByteBuffer)
   2.199 -(import java.awt.image.BufferedImage)
   2.200 -(import java.awt.Color)
   2.201 -(import java.awt.Dimension)
   2.202 -(import java.awt.Graphics)
   2.203 -(import java.awt.Graphics2D)
   2.204 -(import java.awt.event.WindowAdapter)
   2.205 -(import java.awt.event.WindowEvent)
   2.206 -(import java.awt.image.BufferedImage)
   2.207 -(import java.nio.ByteBuffer)
   2.208 -(import javax.swing.JFrame)
   2.209 -(import javax.swing.JPanel)
   2.210 -(import javax.swing.SwingUtilities)
   2.211 -(import javax.swing.ImageIcon)
   2.212 -(import javax.swing.JOptionPane)
   2.213 -(import java.awt.image.ImageObserver)
   2.214 -
   2.215 -(import 'com.jme3.capture.SoundProcessor)
   2.216 -
   2.217 -
   2.218 +(ns cortex.hearing
   2.219 +  "Simulate the sense of hearing in jMonkeyEngine3. Enables multiple
   2.220 +  listeners at different positions in the same world. Passes vectors
   2.221 +  of floats in the range [-1.0 -- 1.0] in PCM format to any arbitray
   2.222 +  function."
   2.223 +  {:author "Robert McIntyre"}
   2.224 +  (:use (cortex world util))
   2.225 +  (:import java.nio.ByteBuffer)
   2.226 +  (:import org.tritonus.share.sampled.FloatSampleTools)
   2.227 +  (:import com.aurellem.capture.audio.SoundProcessor)
   2.228 +  (:import javax.sound.sampled.AudioFormat))
   2.229 +  
   2.230  (defn sound-processor
   2.231 -  "deals with converting ByteBuffers into Arrays of bytes so that the
   2.232 -  continuation functions can be defined in terms of immutable stuff."
   2.233 +  "Deals with converting ByteBuffers into Vectors of floats so that
   2.234 +  the continuation functions can be defined in terms of immutable
   2.235 +  stuff."
   2.236    [continuation]
   2.237    (proxy [SoundProcessor] []
   2.238      (cleanup [])
   2.239      (process
   2.240 -      [#^ByteBuffer audioSamples numSamples]
   2.241 -      (no-exceptions
   2.242 -       (let [byte-array (byte-array numSamples)]
   2.243 -	 (.get audioSamples byte-array 0 numSamples)
   2.244 -	 (continuation
   2.245 -	  (vec byte-array)))))))
   2.246 +      [#^ByteBuffer audioSamples numSamples #^AudioFormat audioFormat]
   2.247 +      (let [bytes  (byte-array numSamples)
   2.248 +            floats (float-array numSamples)]
   2.249 +        (.get audioSamples bytes 0 numSamples)
   2.250 +        (FloatSampleTools/byte2floatInterleaved
   2.251 +         bytes 0 floats 0
   2.252 +         (/ numSamples (.getFrameSize audioFormat)) audioFormat)
   2.253 +        (continuation
   2.254 +         (vec floats))))))
   2.255  
   2.256  (defn add-ear 
   2.257 -  "add an ear to the world.  The continuation function will be called
   2.258 +  "Add an ear to the world.  The continuation function will be called
   2.259    on the FFT or the sounds which the ear hears in the given
   2.260    timeframe. Sound is 3D."
   2.261    [world listener continuation]
   2.262 @@ -744,45 +787,39 @@
   2.263      (.registerSoundProcessor renderer listener
   2.264  			     (sound-processor continuation))
   2.265      listener))
   2.266 -
   2.267  #+end_src
   2.268  
   2.269 -
   2.270 +* Example
   2.271  
   2.272  #+srcname: test-hearing
   2.273  #+begin_src clojure :results silent
   2.274 -(ns test.hearing)
   2.275 -(use 'cortex.world)
   2.276 -(use 'cortex.import)
   2.277 -(use 'clojure.contrib.def)
   2.278 -(use 'body.ear)
   2.279 -(cortex.import/mega-import-jme3)
   2.280 -(rlm.rlm-commands/help)
   2.281 +(ns test.hearing
   2.282 +  (:use (cortex world util hearing))
   2.283 +  (:import (com.jme3.audio AudioNode Listener))
   2.284 +  (:import com.jme3.scene.Node))
   2.285  
   2.286  (defn setup-fn [world]
   2.287    (let [listener (Listener.)]
   2.288 -    (add-ear world listener #(println (nth % 0)))))
   2.289 +    (add-ear world listener #(println-repl (nth % 0)))))
   2.290    
   2.291  (defn play-sound [node world value]
   2.292    (if (not value)
   2.293      (do
   2.294        (.playSource (.getAudioRenderer world) node))))
   2.295  
   2.296 -(defn test-world []
   2.297 -  (let [node1 (AudioNode. (asset-manager) "Sounds/pure.wav" false false)]
   2.298 -    (world
   2.299 -     (Node.)
   2.300 -     {"key-space" (partial play-sound node1)}
   2.301 -     setup-fn
   2.302 -     no-op
   2.303 -     )))
   2.304 -    
   2.305 -
   2.306 +(defn test-basic-hearing []
   2.307 +  (.start
   2.308 +   (let [node1 (AudioNode. (asset-manager) "Sounds/pure.wav" false false)]
   2.309 +     (world
   2.310 +      (Node.)
   2.311 +      {"key-space" (partial play-sound node1)}
   2.312 +      setup-fn
   2.313 +      no-op))))  
   2.314  #+end_src
   2.315  
   2.316 -
   2.317 -
   2.318 -* Example
   2.319 +This extremely basic program prints out the first sample it encounters
   2.320 +at every time stamp. You can see the rendered sound begin printed at
   2.321 +the REPL.
   2.322  
   2.323  * COMMENT Code Generation
   2.324  
   2.325 @@ -798,3 +835,5 @@
   2.326  #+begin_src C :tangle ../Alc/backends/send.c
   2.327  <<send>>
   2.328  #+end_src
   2.329 +
   2.330 +