# HG changeset patch # User Robert McIntyre # Date 1320357285 25200 # Node ID 22ac5a0367cdd51466ae8db5bddf839fdb8d6648 # Parent 1e201037f6669afdd88da7178b70d309adcbd326 finally, a first pass at ear.org diff -r 1e201037f666 -r 22ac5a0367cd java/src/com/aurellem/send/AudioSend.java --- a/java/src/com/aurellem/send/AudioSend.java Thu Nov 03 13:32:27 2011 -0700 +++ b/java/src/com/aurellem/send/AudioSend.java Thu Nov 03 14:54:45 2011 -0700 @@ -6,86 +6,102 @@ public class AudioSend { - private final long deviceID; + private final long deviceID; - public AudioSend(long deviceID){ - this.deviceID = deviceID; - } + public AudioSend(long deviceID){ + this.deviceID = deviceID; + } - /** This establishes the LWJGL context as the context which will be copies to all - * other contexts. It must be called before any calls to addListener(); - */ - public void initDevice(){ - ninitDevice(this.deviceID);} - public static native void ninitDevice(long device); + /** This establishes the LWJGL context as the context which + * will be copies to all other contexts. It must be called + * before any calls to addListener(); + */ + public void initDevice(){ + ninitDevice(this.deviceID);} + public static native void ninitDevice(long device); - /** - * The send device does not automatically process sound. This step function will cause - * the desired number of samples to be processed for each listener. The results will then - * be available via calls to getSamples() for each listener. - * @param samples - */ - public void step(int samples){ - nstep(this.deviceID, samples);} - public static native void nstep(long device, int samples); + /** + * The send device does not automatically process sound. This + * step function will cause the desired number of samples to + * be processed for each listener. The results will then be + * available via calls to getSamples() for each + * listener. + * @param samples + */ + public void step(int samples){ + nstep(this.deviceID, samples);} + public static native void nstep(long device, int samples); - /** - * Retrieve the final rendered sound for a particular listener. contextNum == 0 - * is the main LWJGL context. - * @param buffer - * @param samples - * @param contextNum - */ - public void getSamples(ByteBuffer buffer, int samples, int contextNum){ - ngetSamples(this.deviceID, buffer, buffer.position(), samples, contextNum);} - public static native void ngetSamples( - long device, ByteBuffer buffer, int position, int samples, int contextNum); + /** + * Retrieve the final rendered sound for a particular + * listener. contextNum == 0 is the main LWJGL + * context. + * @param buffer + * @param samples + * @param contextNum + */ + public void getSamples(ByteBuffer buffer, + int samples, int contextNum){ + ngetSamples(this.deviceID, buffer, + buffer.position(), samples, contextNum);} + public static native void + ngetSamples(long device, ByteBuffer buffer, + int position, int samples, int contextNum); - /** - * Create an additional listener on the recorder device. The device itself will manage - * this listener and synchronize it with the main LWJGL context. Processed sound samples - * for this listener will be available via a call to getSamples() with - * contextNum equal to the number of times this method has been called. - */ - public void addListener(){naddListener(this.deviceID);} - public static native void naddListener(long device); + /** + * Create an additional listener on the recorder device. The + * device itself will manage this listener and synchronize it + * with the main LWJGL context. Processed sound samples for + * this listener will be available via a call to + * getSamples() with contextNum + * equal to the number of times this method has been called. + */ + public void addListener(){naddListener(this.deviceID);} + public static native void naddListener(long device); - /** - * This will internally call alListener3f in the appropriate slave context and update - * that context's listener's parameters. Calling this for a number greater than the current - * number of slave contexts will have no effect. - * @param pname - * @param v1 - * @param v2 - * @param v3 - * @param contextNum - */ - public void setNthListener3f(int pname, float v1, float v2, float v3, int contextNum){ - nsetNthListener3f(pname, v1, v2, v3, this.deviceID, contextNum);} - public static native void - nsetNthListener3f(int pname, float v1, float v2, float v3, long device, int contextNum); - - /** - * This will internally call alListenerf in the appropriate slave context and update - * that context's listener's parameters. Calling this for a number greater than the current - * number of slave contexts will have no effect. - * @param pname - * @param v1 - * @param contextNum - */ - public void setNthListenerf(int pname, float v1, int contextNum){ - nsetNthListenerf(pname, v1, this.deviceID, contextNum);} - public static native void nsetNthListenerf(int pname, float v1, long device, int contextNum); - - - /** - * Retrieve the AudioFormat which the device is using. This format is itself derived - * from the OpenAL config file under the "format" variable. - */ - public AudioFormat getAudioFormat(){ - return ngetAudioFormat(this.deviceID);} - public static native AudioFormat ngetAudioFormat(long device); - - - + /** + * This will internally call alListener3f in the + * appropriate slave context and update that context's + * listener's parameters. Calling this for a number greater + * than the current number of slave contexts will have no + * effect. + * @param pname + * @param v1 + * @param v2 + * @param v3 + * @param contextNum + */ + public void + setNthListener3f(int pname, float v1, + float v2, float v3, int contextNum){ + nsetNthListener3f(pname, v1, v2, v3, + this.deviceID, contextNum);} + public static native void + nsetNthListener3f(int pname, float v1, float v2, + float v3, long device, int contextNum); + + /** + * This will internally call alListenerf in the + * appropriate slave context and update that context's + * listener's parameters. Calling this for a number greater + * than the current number of slave contexts will have no + * effect. + * @param pname + * @param v1 + * @param contextNum + */ + public void setNthListenerf(int pname, float v1, int contextNum){ + nsetNthListenerf(pname, v1, this.deviceID, contextNum);} + public static native void + nsetNthListenerf(int pname, float v1, + long device, int contextNum); + + /** + * Retrieve the AudioFormat which the device is using. This + * format is itself derived from the OpenAL config file under + * the "format" variable. + */ + public AudioFormat getAudioFormat(){ + return ngetAudioFormat(this.deviceID);} + public static native AudioFormat ngetAudioFormat(long device); } diff -r 1e201037f666 -r 22ac5a0367cd org/ear.org --- a/org/ear.org Thu Nov 03 13:32:27 2011 -0700 +++ b/org/ear.org Thu Nov 03 14:54:45 2011 -0700 @@ -46,6 +46,7 @@ world from their own perspective, it is necessary to go all the way back to =OpenAL= and implement support for simulated hearing there. +* Extending =OpenAL= ** =OpenAL= Devices =OpenAL= goes to great lengths to support many different systems, all @@ -75,7 +76,6 @@ data in a form that the AIs can use, it is necessary to create a new Device, which supports this features. - ** The Send Device Adding a device to OpenAL is rather tricky -- there are five separate files in the =OpenAL= source tree that must be modified to do so. I've @@ -331,7 +331,7 @@ about is automoatically creating new sources whenever a slave context does not have the same number of sources as the master context. -* Context Creation +** Context Creation #+begin_src C static void addContext(ALCdevice *Device, ALCcontext *context){ send_data *data = (send_data*)Device->ExtraData; @@ -359,7 +359,7 @@ here is where the rendered sound samples for this slave context will eventually go. -* Context Switching +** Context Switching #+begin_src C //////////////////// Context Switching @@ -451,7 +451,17 @@ contexts, then walks each context, rendering just that context to it's audio-sample storage buffer. -* JNI Methods +** JNI Methods + +At this point, we have the ability to create multiple listeners by +using the master/slave context trick, and the rendered audio data is +waiting patiently in internal buffers, one for each listener. We need +a way to transport this information to Java, and also a way to drive +this device from Java. The following JNI interface code is inspired +by the way LWJGL interfaces with =OpenAL=. + +*** step + #+begin_src C //////////////////// JNI Methods @@ -467,7 +477,18 @@ UNUSED(env);UNUSED(clazz);UNUSED(device); renderData((ALCdevice*)((intptr_t)device), samples); } +#+end_src +This device, unlike most of the other devices in =OpenAL=, does not +render sound unless asked. This enables the system to slow down or +speed up depending on the needs of the AIs who are using it to +listen. If the device tried to render samples in real-time, a +complicated AI whose mind takes 100 seconds of computer time to +simulate 1 second of AI-time would miss almost all of the sound in +its environment. + +*** getSamples +#+begin_src C /* * Class: com_aurellem_send_AudioSend * Method: ngetSamples @@ -486,7 +507,19 @@ memcpy(buffer_address, data->contexts[n]->renderBuffer, BytesFromDevFmt(recorder->FmtType) * recorder->NumChan * samples); } +#+end_src +This is the transport layer between C and Java that will eventually +allow us to access rendered sound data from clojure. + +*** Listener Management + +=addListener=, =setNthListenerf=, and =setNthListener3f= are +necessary to change the properties of any listener other than the +master one, since only the listener of the current active context is +affected by the normal =OpenAL= listener calls. + +#+begin_src C /* * Class: com_aurellem_send_AudioSend * Method: naddListener @@ -541,7 +574,19 @@ alListenerf(param, v1); alcMakeContextCurrent(current); } +#+end_src +*** Initilazation +=initDevice= is called from the Java side after LWJGL has created its +context, and before any calls to =addListener=. It establishes the +LWJGL context as the master context. + +=getAudioFormat= is a convienence function that uses JNI to build up a +=javax.sound.sampled.AudioFormat= object from data in the Device. This +way, there is no ambiguity about what the bits created by =step= and +returned by =getSamples= mean. + +#+begin_src C /* * Class: com_aurellem_send_AudioSend * Method: ninitDevice @@ -550,13 +595,10 @@ JNIEXPORT void JNICALL Java_com_aurellem_send_AudioSend_ninitDevice (JNIEnv *env, jclass clazz, jlong device){ UNUSED(env);UNUSED(clazz); - ALCdevice *Device = (ALCdevice*) ((intptr_t)device); init(Device); - } - /* * Class: com_aurellem_send_AudioSend * Method: ngetAudioFormat @@ -571,9 +613,6 @@ (*env)->GetMethodID(env, AudioFormatClass, "", "(FIIZZ)V"); ALCdevice *Device = (ALCdevice*) ((intptr_t)device); - - //float frequency - int isSigned; switch (Device->FmtType) { @@ -584,11 +623,6 @@ float frequency = Device->Frequency; int bitsPerFrame = (8 * BytesFromDevFmt(Device->FmtType)); int channels = Device->NumChan; - - - //printf("freq = %f, bpf = %d, channels = %d, signed? = %d\n", - // frequency, bitsPerFrame, channels, isSigned); - jobject format = (*env)-> NewObject( env,AudioFormatClass,AudioFormatConstructor, @@ -599,7 +633,13 @@ 0); return format; } +#+end_src +*** Boring Device management stuff +This code is more-or-less copied verbatim from the other =OpenAL= +backends. It's the basis for =OpenAL='s primitive object system. + +#+begin_src C //////////////////// Device Initilization / Management static const ALCchar sendDevice[] = "Multiple Audio Send"; @@ -685,57 +725,60 @@ } #+end_src +* The Java interface, =AudioSend= +The Java interface to the Send Device follows naturally from the JNI +definitions. It is included here for completeness. The only thing here +of note is the =deviceID=. This is available from LWJGL, but to only +way to get it is reflection. Unfornatuently, there is no other way to +control the Send device than to obtain a pointer to it. +#+include: "../java/src/com/aurellem/send/AudioSend.java" src java :exports code +* Finally, Ears in clojure! +Now that the infastructure is complete (modulo a few patches to +jMonkeyEngine3 to support accessing this modified version of =OpenAL= +that are not worth discussing), the clojure ear abstraction is rather +simple. Just as there were =SceneProcessors= for vision, there are +now =SoundProcessors= for hearing. - +#+include "../../jmeCapture/src/com/aurellem/capture/audio/SoundProcessor.java" src java #+srcname: ears #+begin_src clojure -(ns cortex.hearing) -(use 'cortex.world) -(use 'cortex.import) -(use 'clojure.contrib.def) -(cortex.import/mega-import-jme3) -(rlm.rlm-commands/help) -(import java.nio.ByteBuffer) -(import java.awt.image.BufferedImage) -(import java.awt.Color) -(import java.awt.Dimension) -(import java.awt.Graphics) -(import java.awt.Graphics2D) -(import java.awt.event.WindowAdapter) -(import java.awt.event.WindowEvent) -(import java.awt.image.BufferedImage) -(import java.nio.ByteBuffer) -(import javax.swing.JFrame) -(import javax.swing.JPanel) -(import javax.swing.SwingUtilities) -(import javax.swing.ImageIcon) -(import javax.swing.JOptionPane) -(import java.awt.image.ImageObserver) - -(import 'com.jme3.capture.SoundProcessor) - - +(ns cortex.hearing + "Simulate the sense of hearing in jMonkeyEngine3. Enables multiple + listeners at different positions in the same world. Passes vectors + of floats in the range [-1.0 -- 1.0] in PCM format to any arbitray + function." + {:author "Robert McIntyre"} + (:use (cortex world util)) + (:import java.nio.ByteBuffer) + (:import org.tritonus.share.sampled.FloatSampleTools) + (:import com.aurellem.capture.audio.SoundProcessor) + (:import javax.sound.sampled.AudioFormat)) + (defn sound-processor - "deals with converting ByteBuffers into Arrays of bytes so that the - continuation functions can be defined in terms of immutable stuff." + "Deals with converting ByteBuffers into Vectors of floats so that + the continuation functions can be defined in terms of immutable + stuff." [continuation] (proxy [SoundProcessor] [] (cleanup []) (process - [#^ByteBuffer audioSamples numSamples] - (no-exceptions - (let [byte-array (byte-array numSamples)] - (.get audioSamples byte-array 0 numSamples) - (continuation - (vec byte-array))))))) + [#^ByteBuffer audioSamples numSamples #^AudioFormat audioFormat] + (let [bytes (byte-array numSamples) + floats (float-array numSamples)] + (.get audioSamples bytes 0 numSamples) + (FloatSampleTools/byte2floatInterleaved + bytes 0 floats 0 + (/ numSamples (.getFrameSize audioFormat)) audioFormat) + (continuation + (vec floats)))))) (defn add-ear - "add an ear to the world. The continuation function will be called + "Add an ear to the world. The continuation function will be called on the FFT or the sounds which the ear hears in the given timeframe. Sound is 3D." [world listener continuation] @@ -744,45 +787,39 @@ (.registerSoundProcessor renderer listener (sound-processor continuation)) listener)) - #+end_src - +* Example #+srcname: test-hearing #+begin_src clojure :results silent -(ns test.hearing) -(use 'cortex.world) -(use 'cortex.import) -(use 'clojure.contrib.def) -(use 'body.ear) -(cortex.import/mega-import-jme3) -(rlm.rlm-commands/help) +(ns test.hearing + (:use (cortex world util hearing)) + (:import (com.jme3.audio AudioNode Listener)) + (:import com.jme3.scene.Node)) (defn setup-fn [world] (let [listener (Listener.)] - (add-ear world listener #(println (nth % 0))))) + (add-ear world listener #(println-repl (nth % 0))))) (defn play-sound [node world value] (if (not value) (do (.playSource (.getAudioRenderer world) node)))) -(defn test-world [] - (let [node1 (AudioNode. (asset-manager) "Sounds/pure.wav" false false)] - (world - (Node.) - {"key-space" (partial play-sound node1)} - setup-fn - no-op - ))) - - +(defn test-basic-hearing [] + (.start + (let [node1 (AudioNode. (asset-manager) "Sounds/pure.wav" false false)] + (world + (Node.) + {"key-space" (partial play-sound node1)} + setup-fn + no-op)))) #+end_src - - -* Example +This extremely basic program prints out the first sample it encounters +at every time stamp. You can see the rendered sound begin printed at +the REPL. * COMMENT Code Generation @@ -798,3 +835,5 @@ #+begin_src C :tangle ../Alc/backends/send.c <> #+end_src + +