|
rlm@0
|
1 #+title: The EARS!
|
|
rlm@0
|
2 #+author: Robert McIntyre
|
|
rlm@0
|
3 #+email: rlm@mit.edu
|
|
rlm@0
|
4 #+MATHJAX: align:"left" mathml:t path:"../aurellem/src/MathJax/MathJax.js"
|
|
rlm@0
|
5 #+STYLE: <link rel="stylesheet" type="text/css" href="../aurellem/src/css/aurellem.css"/>
|
|
rlm@0
|
6 #+BABEL: :exports both :noweb yes :cache no :mkdirp yes
|
|
rlm@0
|
7 #+INCLUDE: ../aurellem/src/templates/level-0.org
|
|
rlm@0
|
8 #+description: Simulating multiple listeners and the sense of hearing in uMonkeyEngine3
|
|
rlm@0
|
9
|
|
rlm@0
|
10
|
|
rlm@0
|
11
|
|
rlm@0
|
12
|
|
rlm@0
|
13 * Ears!
|
|
rlm@0
|
14
|
|
rlm@0
|
15 I want to be able to place ears in a similiar manner to how I place
|
|
rlm@0
|
16 the eyes. I want to be able to place ears in a unique spatial
|
|
rlm@0
|
17 position, and recieve as output at every tick the FFT of whatever
|
|
rlm@0
|
18 signals are happening at that point.
|
|
rlm@0
|
19
|
|
rlm@0
|
20 #+srcname: ear-header
|
|
rlm@0
|
21 #+begin_src clojure
|
|
rlm@0
|
22 (ns body.ear)
|
|
rlm@0
|
23 (use 'cortex.world)
|
|
rlm@0
|
24 (use 'cortex.import)
|
|
rlm@0
|
25 (use 'clojure.contrib.def)
|
|
rlm@0
|
26 (cortex.import/mega-import-jme3)
|
|
rlm@0
|
27 (rlm.rlm-commands/help)
|
|
rlm@0
|
28 (import java.nio.ByteBuffer)
|
|
rlm@0
|
29 (import java.awt.image.BufferedImage)
|
|
rlm@0
|
30 (import java.awt.Color)
|
|
rlm@0
|
31 (import java.awt.Dimension)
|
|
rlm@0
|
32 (import java.awt.Graphics)
|
|
rlm@0
|
33 (import java.awt.Graphics2D)
|
|
rlm@0
|
34 (import java.awt.event.WindowAdapter)
|
|
rlm@0
|
35 (import java.awt.event.WindowEvent)
|
|
rlm@0
|
36 (import java.awt.image.BufferedImage)
|
|
rlm@0
|
37 (import java.nio.ByteBuffer)
|
|
rlm@0
|
38 (import javax.swing.JFrame)
|
|
rlm@0
|
39 (import javax.swing.JPanel)
|
|
rlm@0
|
40 (import javax.swing.SwingUtilities)
|
|
rlm@0
|
41 (import javax.swing.ImageIcon)
|
|
rlm@0
|
42 (import javax.swing.JOptionPane)
|
|
rlm@0
|
43 (import java.awt.image.ImageObserver)
|
|
rlm@0
|
44 #+end_src
|
|
rlm@0
|
45
|
|
rlm@0
|
46 JMonkeyEngine3's audio system works as follows:
|
|
rlm@0
|
47 first, an appropiate audio renderer is created during initialization
|
|
rlm@0
|
48 and depending on the context. On my computer, this is the
|
|
rlm@0
|
49 LwjglAudioRenderer.
|
|
rlm@0
|
50
|
|
rlm@0
|
51 The LwjglAudioRenderer sets a few internal state variables depending
|
|
rlm@0
|
52 on what capabilities the audio system has.
|
|
rlm@0
|
53
|
|
rlm@0
|
54 may very well need to make my own AudioRenderer
|
|
rlm@0
|
55
|
|
rlm@0
|
56 #+srcname: ear-body-1
|
|
rlm@0
|
57 #+begin_src clojure :results silent
|
|
rlm@0
|
58 (in-ns 'body.ear)
|
|
rlm@0
|
59 (import 'com.jme3.capture.SoundProcessor)
|
|
rlm@0
|
60
|
|
rlm@0
|
61
|
|
rlm@0
|
62 (defn sound-processor
|
|
rlm@0
|
63 "deals with converting ByteBuffers into Arrays of bytes so that the
|
|
rlm@0
|
64 continuation functions can be defined in terms of immutable stuff."
|
|
rlm@0
|
65 [continuation]
|
|
rlm@0
|
66 (proxy [SoundProcessor] []
|
|
rlm@0
|
67 (cleanup [])
|
|
rlm@0
|
68 (process
|
|
rlm@0
|
69 [#^ByteBuffer audioSamples numSamples]
|
|
rlm@0
|
70 (no-exceptions
|
|
rlm@0
|
71 (let [byte-array (byte-array numSamples)]
|
|
rlm@0
|
72 (.get audioSamples byte-array 0 numSamples)
|
|
rlm@0
|
73 (continuation
|
|
rlm@0
|
74 (vec byte-array)))))))
|
|
rlm@0
|
75
|
|
rlm@0
|
76
|
|
rlm@0
|
77 (defn add-ear
|
|
rlm@0
|
78 "add an ear to the world. The continuation function will be called
|
|
rlm@0
|
79 on the FFT or the sounds which the ear hears in the given
|
|
rlm@0
|
80 timeframe. Sound is 3D."
|
|
rlm@0
|
81 [world listener continuation]
|
|
rlm@0
|
82 (let [renderer (.getAudioRenderer world)]
|
|
rlm@0
|
83 (.addListener renderer listener)
|
|
rlm@0
|
84 (.registerSoundProcessor renderer listener
|
|
rlm@0
|
85 (sound-processor continuation))
|
|
rlm@0
|
86 listener))
|
|
rlm@0
|
87
|
|
rlm@0
|
88 #+end_src
|
|
rlm@0
|
89
|
|
rlm@0
|
90
|
|
rlm@0
|
91
|
|
rlm@0
|
92 #+srcname: test-hearing
|
|
rlm@0
|
93 #+begin_src clojure :results silent
|
|
rlm@0
|
94 (ns test.hearing)
|
|
rlm@0
|
95 (use 'cortex.world)
|
|
rlm@0
|
96 (use 'cortex.import)
|
|
rlm@0
|
97 (use 'clojure.contrib.def)
|
|
rlm@0
|
98 (use 'body.ear)
|
|
rlm@0
|
99 (cortex.import/mega-import-jme3)
|
|
rlm@0
|
100 (rlm.rlm-commands/help)
|
|
rlm@0
|
101
|
|
rlm@0
|
102 (defn setup-fn [world]
|
|
rlm@0
|
103 (let [listener (Listener.)]
|
|
rlm@0
|
104 (add-ear world listener #(println (nth % 0)))))
|
|
rlm@0
|
105
|
|
rlm@0
|
106 (defn play-sound [node world value]
|
|
rlm@0
|
107 (if (not value)
|
|
rlm@0
|
108 (do
|
|
rlm@0
|
109 (.playSource (.getAudioRenderer world) node))))
|
|
rlm@0
|
110
|
|
rlm@0
|
111 (defn test-world []
|
|
rlm@0
|
112 (let [node1 (AudioNode. (asset-manager) "Sounds/pure.wav" false false)]
|
|
rlm@0
|
113 (world
|
|
rlm@0
|
114 (Node.)
|
|
rlm@0
|
115 {"key-space" (partial play-sound node1)}
|
|
rlm@0
|
116 setup-fn
|
|
rlm@0
|
117 no-op
|
|
rlm@0
|
118 )))
|
|
rlm@0
|
119
|
|
rlm@0
|
120
|
|
rlm@0
|
121 #+end_src
|
|
rlm@0
|
122
|
|
rlm@0
|
123
|
|
rlm@0
|
124
|
|
rlm@0
|
125 * COMMENT Code Generation
|
|
rlm@0
|
126
|
|
rlm@0
|
127 #+begin_src clojure :tangle /home/r/cortex/src/body/ear.clj
|
|
rlm@0
|
128 <<ear-header>>
|
|
rlm@0
|
129 <<ear-body-1>>
|
|
rlm@0
|
130 #+end_src
|
|
rlm@0
|
131
|
|
rlm@0
|
132
|
|
rlm@0
|
133 #+begin_src clojure :tangle /home/r/cortex/src/test/hearing.clj
|
|
rlm@0
|
134 <<test-hearing>>
|
|
rlm@0
|
135 #+end_src
|
|
rlm@0
|
136
|
|
rlm@0
|
137
|
