audio-send: org/ear.org comparison

comparison org/ear.org @ 19:22ac5a0367cd

finally, a first pass at ear.org

author	Robert McIntyre <rlm@mit.edu>
date	Thu, 03 Nov 2011 14:54:45 -0700
parents	1e201037f666
children	e8ae40c9848c

comparison

equal deleted inserted replaced

-:1e201037f666
+:22ac5a0367cd
 To make many AI creatures in jMonkeyEngine that can each hear the
 world from their own perspective, it is necessary to go all the way
 back to =OpenAL= and implement support for simulated hearing there.
+* Extending =OpenAL=
 ** =OpenAL= Devices
 =OpenAL= goes to great lengths to support many different systems, all
 with different sound capabilities and interfaces.  It acomplishes this
 difficult task by providing code for many different sound backends in
 available.
 Therefore, in order to support multiple listeners, and get the sound
 data in a form that the AIs can use, it is necessary to create a new
 Device, which supports this features.
 ** The Send Device
 Adding a device to OpenAL is rather tricky -- there are five separate
 files in the =OpenAL= source tree that must be modified to do so. I've
 documented this process [[./add-new-device.org][here]] for anyone who is interested.
 Most of the hard work in Context Synchronization is done in
 =syncSources()=. The only thing that =syncContexts()= has to worry
 about is automoatically creating new sources whenever a slave context
 does not have the same number of sources as the master context.
-* Context Creation
+** Context Creation
 #+begin_src C
 static void addContext(ALCdevice *Device, ALCcontext *context){
 send_data *data = (send_data*)Device->ExtraData;
 // expand array if necessary
 if (data->numContexts >= data->maxContexts){
 Here, the slave context is created, and it's data is stored in the
 device-wide =ExtraData= structure.  The =renderBuffer= that is created
 here is where the rendered sound samples for this slave context will
 eventually go.
-* Context Switching
+** Context Switching
 #+begin_src C
 ////////////////////  Context Switching
 /* A device brings along with it two pieces of state
 * which have to be swapped in and out with each context.
 The main loop synchronizes the master LWJGL context with all the slave
 contexts, then walks each context, rendering just that context to it's
 audio-sample storage buffer.
-* JNI Methods
+** JNI Methods
+At this point, we have the ability to create multiple listeners by
+using the master/slave context trick, and the rendered audio data is
+waiting patiently in internal buffers, one for each listener.  We need
+a way to transport this information to Java, and also a way to drive
+this device from Java.  The following JNI interface code is inspired
+by the way LWJGL interfaces with =OpenAL=.
+*** step
 #+begin_src C
 ////////////////////   JNI Methods
 #include "com_aurellem_send_AudioSend.h"
 JNIEXPORT void JNICALL Java_com_aurellem_send_AudioSend_nstep
 (JNIEnv *env, jclass clazz, jlong device, jint samples){
 UNUSED(env);UNUSED(clazz);UNUSED(device);
 renderData((ALCdevice*)((intptr_t)device), samples);
 }
+#+end_src
+This device, unlike most of the other devices in =OpenAL=, does not
+render sound unless asked. This enables the system to slow down or
+speed up depending on the needs of the AIs who are using it to
+listen.  If the device tried to render samples in real-time, a
+complicated AI whose mind takes 100 seconds of computer time to
+simulate 1 second of AI-time would miss almost all of the sound in
+its environment.
+*** getSamples
+#+begin_src C
 /*
 * Class:     com_aurellem_send_AudioSend
 * Method:    ngetSamples
 * Signature: (JLjava/nio/ByteBuffer;III)V
 */
 send_data *data = (send_data*)recorder->ExtraData;
 if ((ALuint)n > data->numContexts){return;}
 memcpy(buffer_address, data->contexts[n]->renderBuffer,
 	 BytesFromDevFmt(recorder->FmtType) * recorder->NumChan * samples);
 }
+#+end_src
+This is the transport layer between C and Java that will eventually
+allow us to access rendered sound data from clojure.
+*** Listener Management
+=addListener=, =setNthListenerf=, and =setNthListener3f= are
+necessary to change the properties of any listener other than the
+master one, since only the listener of the current active context is
+affected by the normal =OpenAL= listener calls.
+#+begin_src C
 /*
 * Class:     com_aurellem_send_AudioSend
 * Method:    naddListener
 * Signature: (J)V
 */
 if ((ALuint)contextNum > data->numContexts){return;}
 alcMakeContextCurrent(data->contexts[contextNum]->ctx);
 alListenerf(param, v1);
 alcMakeContextCurrent(current);
 }
+#+end_src
+*** Initilazation
+=initDevice= is called from the Java side after LWJGL has created its
+context, and before any calls to =addListener=. It establishes the
+LWJGL context as the master context.
+=getAudioFormat= is a convienence function that uses JNI to build up a
+=javax.sound.sampled.AudioFormat= object from data in the Device. This
+way, there is no ambiguity about what the bits created by =step= and
+returned by =getSamples= mean.
+#+begin_src C
 /*
 * Class:     com_aurellem_send_AudioSend
 * Method:    ninitDevice
 * Signature: (J)V
 */
 JNIEXPORT void JNICALL Java_com_aurellem_send_AudioSend_ninitDevice
 (JNIEnv *env, jclass clazz, jlong device){
 UNUSED(env);UNUSED(clazz);
 ALCdevice *Device = (ALCdevice*) ((intptr_t)device);
 init(Device);
+}
-}
 /*
 * Class:     com_aurellem_send_AudioSend
 * Method:    ngetAudioFormat
 * Signature: (J)Ljavax/sound/sampled/AudioFormat;
 (*env)->FindClass(env, "javax/sound/sampled/AudioFormat");
 jmethodID AudioFormatConstructor =
 (*env)->GetMethodID(env, AudioFormatClass, "<init>", "(FIIZZ)V");
 ALCdevice *Device = (ALCdevice*) ((intptr_t)device);
-//float frequency
 int isSigned;
 switch (Device->FmtType)
 {
 case DevFmtUByte:
 case DevFmtUShort: isSigned = 0; break;
 default : isSigned = 1;
 }
 float frequency = Device->Frequency;
 int bitsPerFrame = (8 * BytesFromDevFmt(Device->FmtType));
 int channels = Device->NumChan;
-//printf("freq = %f, bpf = %d, channels = %d, signed? = %d\n",
-//	 frequency, bitsPerFrame, channels, isSigned);
 jobject format = (*env)->
 NewObject(
 	      env,AudioFormatClass,AudioFormatConstructor,
 	      frequency,
 	      bitsPerFrame,
 	      channels,
 	      isSigned,
 	      0);
 return format;
 }
+#+end_src
+*** Boring Device management stuff
+This code is more-or-less copied verbatim from the other =OpenAL=
+backends. It's the basis for =OpenAL='s primitive object system.
+#+begin_src C
 ////////////////////   Device Initilization / Management
 static const ALCchar sendDevice[] = "Multiple Audio Send";
 static ALCboolean send_open_playback(ALCdevice *device,
 break;
 }
 }
 #+end_src
+* The Java interface, =AudioSend=
+The Java interface to the Send Device follows naturally from the JNI
+definitions. It is included here for completeness. The only thing here
+of note is the =deviceID=. This is available from LWJGL, but to only
+way to get it is reflection. Unfornatuently, there is no other way to
+control the Send device than to obtain a pointer to it.
+#+include: "../java/src/com/aurellem/send/AudioSend.java" src java :exports code
+* Finally, Ears in clojure!
+Now that the infastructure is complete (modulo a few patches to
+jMonkeyEngine3 to support accessing this modified version of =OpenAL=
+that are not worth discussing), the clojure ear abstraction is rather
+simple.  Just as there were =SceneProcessors= for vision, there are
+now =SoundProcessors= for hearing.
+#+include "../../jmeCapture/src/com/aurellem/capture/audio/SoundProcessor.java" src java
 #+srcname: ears
 #+begin_src clojure
-(ns cortex.hearing)
+(ns cortex.hearing
-(use 'cortex.world)
+"Simulate the sense of hearing in jMonkeyEngine3. Enables multiple
-(use 'cortex.import)
+listeners at different positions in the same world. Passes vectors
-(use 'clojure.contrib.def)
+of floats in the range [-1.0 -- 1.0] in PCM format to any arbitray
-(cortex.import/mega-import-jme3)
+function."
-(rlm.rlm-commands/help)
+{:author "Robert McIntyre"}
-(import java.nio.ByteBuffer)
+(:use (cortex world util))
-(import java.awt.image.BufferedImage)
+(:import java.nio.ByteBuffer)
-(import java.awt.Color)
+(:import org.tritonus.share.sampled.FloatSampleTools)
-(import java.awt.Dimension)
+(:import com.aurellem.capture.audio.SoundProcessor)
-(import java.awt.Graphics)
+(:import javax.sound.sampled.AudioFormat))
-(import java.awt.Graphics2D)
-(import java.awt.event.WindowAdapter)
-(import java.awt.event.WindowEvent)
-(import java.awt.image.BufferedImage)
-(import java.nio.ByteBuffer)
-(import javax.swing.JFrame)
-(import javax.swing.JPanel)
-(import javax.swing.SwingUtilities)
-(import javax.swing.ImageIcon)
-(import javax.swing.JOptionPane)
-(import java.awt.image.ImageObserver)
-(import 'com.jme3.capture.SoundProcessor)
 (defn sound-processor
-"deals with converting ByteBuffers into Arrays of bytes so that the
+"Deals with converting ByteBuffers into Vectors of floats so that
-continuation functions can be defined in terms of immutable stuff."
+the continuation functions can be defined in terms of immutable
+stuff."
 [continuation]
 (proxy [SoundProcessor] []
 (cleanup [])
 (process
-[#^ByteBuffer audioSamples numSamples]
+[#^ByteBuffer audioSamples numSamples #^AudioFormat audioFormat]
-(no-exceptions
+(let [bytes  (byte-array numSamples)
-(let [byte-array (byte-array numSamples)]
+floats (float-array numSamples)]
-	 (.get audioSamples byte-array 0 numSamples)
+(.get audioSamples bytes 0 numSamples)
-	 (continuation
+(FloatSampleTools/byte2floatInterleaved
-	  (vec byte-array)))))))
+bytes 0 floats 0
+(/ numSamples (.getFrameSize audioFormat)) audioFormat)
+(continuation
+(vec floats))))))
 (defn add-ear
-"add an ear to the world.  The continuation function will be called
+"Add an ear to the world.  The continuation function will be called
 on the FFT or the sounds which the ear hears in the given
 timeframe. Sound is 3D."
 [world listener continuation]
 (let [renderer (.getAudioRenderer world)]
 (.addListener renderer listener)
 (.registerSoundProcessor renderer listener
 			     (sound-processor continuation))
 listener))
+#+end_src
-#+end_src
+* Example
 #+srcname: test-hearing
 #+begin_src clojure :results silent
-(ns test.hearing)
+(ns test.hearing
-(use 'cortex.world)
+(:use (cortex world util hearing))
-(use 'cortex.import)
+(:import (com.jme3.audio AudioNode Listener))
-(use 'clojure.contrib.def)
+(:import com.jme3.scene.Node))
-(use 'body.ear)
-(cortex.import/mega-import-jme3)
-(rlm.rlm-commands/help)
 (defn setup-fn [world]
 (let [listener (Listener.)]
-(add-ear world listener #(println (nth % 0)))))
+(add-ear world listener #(println-repl (nth % 0)))))
 (defn play-sound [node world value]
 (if (not value)
 (do
 (.playSource (.getAudioRenderer world) node))))
-(defn test-world []
+(defn test-basic-hearing []
-(let [node1 (AudioNode. (asset-manager) "Sounds/pure.wav" false false)]
+(.start
-(world
+(let [node1 (AudioNode. (asset-manager) "Sounds/pure.wav" false false)]
-(Node.)
+(world
-{"key-space" (partial play-sound node1)}
+(Node.)
-setup-fn
+{"key-space" (partial play-sound node1)}
-no-op
+setup-fn
-)))
+no-op))))
+#+end_src
-#+end_src
+This extremely basic program prints out the first sample it encounters
+at every time stamp. You can see the rendered sound begin printed at
+the REPL.
-* Example
 * COMMENT Code Generation
 #+begin_src clojure :tangle ../../cortex/src/cortex/hearing.clj
 <<ears>>
 #+begin_src C :tangle ../Alc/backends/send.c
 <<send>>
 #+end_src

Mercurial > audio-send

comparison org/ear.org @ 19:22ac5a0367cd