1.1 --- a/org/test-creature.org	Sun Jan 22 04:53:58 2012 -0700
     1.2 +++ b/org/test-creature.org	Mon Jan 23 03:41:26 2012 -0700
     1.3 @@ -655,7 +655,6 @@
     1.4      [(apply max (map first dimensions))
     1.5       (apply max (map second dimensions))]))
     1.6  
     1.7 -
     1.8  (defn creature-eyes
     1.9    "The eye nodes which are children of the \"eyes\" node in the
    1.10    creature."
    1.11 @@ -664,45 +663,62 @@
    1.12      (seq (.getChildren eye-node))
    1.13      (do (println-repl "could not find eyes node") [])))
    1.14  
    1.15 +;; Here's how vision will work.
    1.16  
    1.17 -  ;; Here's how vision will work.
    1.18 +;; Make the continuation in scene-processor take FrameBuffer,
    1.19 +;; byte-buffer, BufferedImage already sized to the correct
    1.20 +;; dimensions. the continuation will decide wether to "mix" them
    1.21 +;; into the BufferedImage, lazily ignore them, or mix them halfway
    1.22 +;; and call c/graphics card routines.
    1.23  
    1.24 -  ;; Make the continuation in scene-processor take FrameBuffer,
    1.25 -  ;; byte-buffer, BufferedImage already sized to the correct
    1.26 -  ;; dimensions. the continuation will decide wether to "mix" them
    1.27 -  ;; into the BufferedImage, lazily ignore them, or mix them halfway
    1.28 -  ;; and call c/graphics card routines.
    1.29 -  
    1.30 -  ;; (vision creature) will take an optional :skip argument which will
    1.31 -  ;; inform the continuations in scene processor to skip the given
    1.32 -  ;; number of cycles; 0 means that no cycles will be skipped.
    1.33 +;; (vision creature) will take an optional :skip argument which will
    1.34 +;; inform the continuations in scene processor to skip the given
    1.35 +;; number of cycles; 0 means that no cycles will be skipped.
    1.36  
    1.37 -  ;; (vision creature) will return [init-functions sensor-functions].
    1.38 -  ;; The init-functions are each single-arg functions that take the
    1.39 -  ;; world and register the cameras and must each be called before the
    1.40 -  ;; corresponding sensor-functions.  Each init-function returns the
    1.41 -  ;; viewport for that eye which can be manipulated, saved, etc. Each
    1.42 -  ;; sensor-function is a thunk and will return data in the same
    1.43 -  ;; format as the tactile-sensor functions; the structure is
    1.44 -  ;; [topology, sensor-data]. Internally, these sensor-functions
    1.45 -  ;; maintain a reference to sensor-data which is periodically updated
    1.46 -  ;; by the continuation function established by its init-function.
    1.47 -  ;; They can be queried every cycle, but their information may not
    1.48 -  ;; necessairly be different every cycle.
    1.49 +;; (vision creature) will return [init-functions sensor-functions].
    1.50 +;; The init-functions are each single-arg functions that take the
    1.51 +;; world and register the cameras and must each be called before the
    1.52 +;; corresponding sensor-functions.  Each init-function returns the
    1.53 +;; viewport for that eye which can be manipulated, saved, etc. Each
    1.54 +;; sensor-function is a thunk and will return data in the same
    1.55 +;; format as the tactile-sensor functions; the structure is
    1.56 +;; [topology, sensor-data]. Internally, these sensor-functions
    1.57 +;; maintain a reference to sensor-data which is periodically updated
    1.58 +;; by the continuation function established by its init-function.
    1.59 +;; They can be queried every cycle, but their information may not
    1.60 +;; necessairly be different every cycle.
    1.61  
    1.62 -  ;; Each eye in the creature in blender will work the same way as
    1.63 -  ;; joints -- a one dimensional object with no geometry whose local
    1.64 -  ;; coordinate system determines the orientation of the resulting
    1.65 -  ;; eye. All eyes will have a parent named "eyes" just as all joints
    1.66 -  ;; have a parent named "joints". The resulting camera will be a
    1.67 -  ;; ChaseCamera or a CameraNode bound to the geo that is closest to
    1.68 -  ;; the eye marker. The eye marker will contain the metadata for the
    1.69 -  ;; eye, and will be moved by it's bound geometry. The dimensions of
    1.70 -  ;; the eye's camera are equal to the dimensions of the eye's "UV"
    1.71 -  ;; map. 
    1.72 +;; Each eye in the creature in blender will work the same way as
    1.73 +;; joints -- a zero dimensional object with no geometry whose local
    1.74 +;; coordinate system determines the orientation of the resulting
    1.75 +;; eye. All eyes will have a parent named "eyes" just as all joints
    1.76 +;; have a parent named "joints". The resulting camera will be a
    1.77 +;; ChaseCamera or a CameraNode bound to the geo that is closest to
    1.78 +;; the eye marker. The eye marker will contain the metadata for the
    1.79 +;; eye, and will be moved by it's bound geometry. The dimensions of
    1.80 +;; the eye's camera are equal to the dimensions of the eye's "UV"
    1.81 +;; map.
    1.82  
    1.83 -(defn eye-target
    1.84 -  "The closest object in creature to eye."
    1.85 +
    1.86 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
    1.87 +
    1.88 +;; Ears work the same way as vision.
    1.89 +
    1.90 +;; (hearing creature) will return [init-functions
    1.91 +;; sensor-functions]. The init functions each take the world and
    1.92 +;; register a SoundProcessor that does foureier transforms on the
    1.93 +;; incommong sound data, making it available to each sensor function.
    1.94 +
    1.95 +(defn creature-ears
    1.96 +  "The ear nodes which are children of the \"ears\" node in the
    1.97 +  creature."
    1.98 +  [#^Node creature]
    1.99 +  (if-let [ear-node (.getChild creature "ears")]
   1.100 +    (seq (.getChildren ear-node))
   1.101 +    (do (println-repl "could not find ears node") [])))
   1.102 +
   1.103 +(defn closest-node
   1.104 +  "The closest object in creature to the given node."
   1.105    [#^Node creature #^Node eye]
   1.106    (loop [radius (float 0.01)]
   1.107      (let [results (CollisionResults.)]
   1.108 @@ -715,14 +731,14 @@
   1.109          (.getGeometry target)
   1.110          (recur (float (* 2 radius)))))))
   1.111  
   1.112 -(defn bind-camera
   1.113 -  "Bind the camera to the Spatial such that it will maintain its
   1.114 +(defn bind-sense
   1.115 +  "Bind the sense to the Spatial such that it will maintain its
   1.116     current position relative to the Spatial no matter how the spatial
   1.117 -   moves."
   1.118 -  [#^Spatial obj #^Camera cam]
   1.119 -  (let [cam-offset (.subtract (.getLocation cam)
   1.120 -                              (.getWorldTranslation obj))
   1.121 -        initial-cam-rotation (Quaternion. (.getRotation cam))
   1.122 +   moves. 'sense can be either a Camera or Listener object."
   1.123 +  [#^Spatial obj sense]
   1.124 +  (let [sense-offset (.subtract (.getLocation sense)
   1.125 +                                (.getWorldTranslation obj))
   1.126 +        initial-sense-rotation (Quaternion. (.getRotation sense))
   1.127          base-anti-rotation (.inverse (.getWorldRotation obj))]
   1.128      (.addControl
   1.129       obj
   1.130 @@ -730,20 +746,82 @@
   1.131         (controlUpdate [tpf]
   1.132           (let [total-rotation
   1.133                 (.mult base-anti-rotation (.getWorldRotation obj))]
   1.134 -           (.setLocation cam
   1.135 +           (.setLocation sense
   1.136                           (.add
   1.137 -                          (.mult total-rotation cam-offset)
   1.138 +                          (.mult total-rotation sense-offset)
   1.139                            (.getWorldTranslation obj)))
   1.140 -           (.setRotation cam
   1.141 -                         (.mult total-rotation initial-cam-rotation))))
   1.142 +           (.setRotation sense
   1.143 +                         (.mult total-rotation initial-sense-rotation))))
   1.144         (controlRender [_ _])))))
   1.145  
   1.146  
   1.147 +(defn update-listener-velocity
   1.148 +  "Update the listener's velocity every update loop."
   1.149 +  [#^Spatial obj #^Listener lis]
   1.150 +  (let [old-position (atom (.getLocation lis))]
   1.151 +    (.addControl
   1.152 +     obj
   1.153 +     (proxy [AbstractControl] []
   1.154 +       (controlUpdate [tpf]
   1.155 +         (let [new-position (.getLocation lis)]
   1.156 +           (.setVelocity
   1.157 +            lis 
   1.158 +            (.mult (.subtract new-position @old-position)
   1.159 +                   (float (/ tpf))))
   1.160 +           (reset! old-position new-position)))
   1.161 +       (controlRender [_ _])))))
   1.162 +
   1.163 +(import com.aurellem.capture.audio.AudioSendRenderer)
   1.164 +
   1.165 +(defn attach-ear
   1.166 +  [#^Application world #^Node creature #^Spatial ear continuation]
   1.167 +  (let [target (closest-node creature ear)
   1.168 +        lis (Listener.)
   1.169 +        audio-renderer (.getAudioRenderer world)
   1.170 +        sp (sound-processor continuation)]
   1.171 +    (println-repl "audio-renderer is " audio-renderer)
   1.172 +    (.setLocation lis (.getWorldTranslation ear))
   1.173 +    (.setRotation lis (.getWorldRotation ear))
   1.174 +    (bind-sense target lis)
   1.175 +    (update-listener-velocity target lis)
   1.176 +    (.addListener audio-renderer lis)
   1.177 +    (.registerSoundProcessor audio-renderer lis sp)))
   1.178 +
   1.179 +(defn enable-hearing
   1.180 +  [#^Node creature #^Spatial ear]
   1.181 +  (let [hearing-data (atom [])]
   1.182 +    [(fn [world]
   1.183 +       (attach-ear world creature ear
   1.184 +                   (fn [data]
   1.185 +                     (reset! hearing-data (vec data)))))
   1.186 +     [(fn []
   1.187 +        (let [data @hearing-data
   1.188 +              topology              
   1.189 +              (vec (map #(vector % 0) (range 0 (count data))))
   1.190 +              scaled-data
   1.191 +              (vec
   1.192 +               (map
   1.193 +                #(rem (int (* 255 (/ (+ 1 %) 2)))  256)
   1.194 +                   data))]
   1.195 +          (println-repl (take 10 scaled-data))
   1.196 +          [topology scaled-data]))
   1.197 +        ]]))
   1.198 +
   1.199 +(defn hearing
   1.200 +  [#^Node creature]
   1.201 +  (reduce
   1.202 +   (fn [[init-a senses-a]
   1.203 +        [init-b senses-b]]
   1.204 +     [(conj init-a init-b)
   1.205 +      (into senses-a senses-b)])
   1.206 +   [[][]]      
   1.207 +   (for [ear (creature-ears creature)]
   1.208 +     (enable-hearing creature ear))))
   1.209 +
   1.210  (defn attach-eye
   1.211    "Attach a Camera to the appropiate area and return the Camera."
   1.212    [#^Node creature #^Spatial eye]
   1.213 -
   1.214 -  (let [target (eye-target creature eye)
   1.215 +  (let [target (closest-node creature eye)
   1.216          [cam-width cam-height] (eye-dimensions eye)
   1.217          cam (Camera. cam-width cam-height)]
   1.218      (.setLocation cam (.getWorldTranslation eye))
   1.219 @@ -751,52 +829,49 @@
   1.220      (.setFrustumPerspective
   1.221       cam 45 (/ (.getWidth cam) (.getHeight cam))
   1.222       1 1000)
   1.223 -                                            
   1.224 -    (bind-camera target cam)
   1.225 +    (bind-sense target cam)
   1.226      cam))
   1.227  
   1.228 -
   1.229  (def presets
   1.230    {:all    0xFFFFFF
   1.231     :red    0xFF0000
   1.232     :blue   0x0000FF
   1.233     :green  0x00FF00})
   1.234  
   1.235 -
   1.236 -
   1.237  (defn enable-vision
   1.238    "return [init-function sensor-functions] for a particular eye"
   1.239    [#^Node creature #^Spatial eye & {skip :skip :or {skip 0}}]
   1.240    (let [retinal-map (retina-sensor-image eye)
   1.241 -        vision-image (atom nil)
   1.242 -        camera (attach-eye creature eye)]
   1.243 -    [
   1.244 -    (fn [world]
   1.245 -      (add-eye
   1.246 -       world camera
   1.247 -       (let [counter  (atom 0)]
   1.248 -         (fn [r fb bb bi]
   1.249 -           (if (zero? (rem (swap! counter inc) (inc skip)))
   1.250 -             (reset! vision-image (BufferedImage! r fb bb bi)))))))
   1.251 -    (vec
   1.252 -     (map
   1.253 -      (fn [[key image]]
   1.254 -        (let [whites (white-coordinates image)
   1.255 -              topology (vec (collapse whites))
   1.256 -              mask (presets key)]
   1.257 -          (fn []
   1.258 -            (vector
   1.259 -             topology
   1.260 -             (vec 
   1.261 -              (for [[x y] whites]
   1.262 -                (bit-and
   1.263 -                 mask (.getRGB @vision-image x y))))))))
   1.264 -        retinal-map))
   1.265 -    ]))
   1.266 +        camera (attach-eye creature eye)
   1.267 +        vision-image
   1.268 +        (atom
   1.269 +         (BufferedImage. (.getWidth camera)
   1.270 +                         (.getHeight camera)
   1.271 +                         BufferedImage/TYPE_BYTE_BINARY))]
   1.272 +    [(fn [world]
   1.273 +       (add-eye
   1.274 +        world camera
   1.275 +        (let [counter  (atom 0)]
   1.276 +          (fn [r fb bb bi]
   1.277 +            (if (zero? (rem (swap! counter inc) (inc skip)))
   1.278 +              (reset! vision-image (BufferedImage! r fb bb bi)))))))
   1.279 +     (vec
   1.280 +      (map
   1.281 +       (fn [[key image]]
   1.282 +         (let [whites (white-coordinates image)
   1.283 +               topology (vec (collapse whites))
   1.284 +               mask (presets key)]
   1.285 +           (fn []
   1.286 +             (vector
   1.287 +              topology
   1.288 +              (vec 
   1.289 +               (for [[x y] whites]
   1.290 +                 (bit-and
   1.291 +                  mask (.getRGB @vision-image x y))))))))
   1.292 +       retinal-map))]))
   1.293  
   1.294  (defn vision
   1.295    [#^Node creature & {skip :skip :or {skip 0}}]
   1.296 -
   1.297    (reduce
   1.298     (fn [[init-a senses-a]
   1.299          [init-b senses-b]]
   1.300 @@ -827,8 +902,8 @@
   1.301          (dorun
   1.302           (for [i (range (count coords))]
   1.303             (.setRGB image ((coords i) 0) ((coords i) 1)
   1.304 -                    ({0 -16777216
   1.305 -                      1 -1} (sensor-data i)))))
   1.306 +                    ({0 0x000000
   1.307 +                      1 0xFFFFFF} (sensor-data i)))))
   1.308          (vi image)))))
   1.309  
   1.310  (defn debug-vision-window
   1.311 @@ -844,9 +919,33 @@
   1.312                     (sensor-data i))))
   1.313          (vi image)))))
   1.314  
   1.315 +(defn debug-hearing-window
   1.316 +  "view audio data"
   1.317 +  [height]
   1.318 +  (let [vi (view-image)]
   1.319 +    (fn [[coords sensor-data]]
   1.320 +      (let [image (BufferedImage. (count coords) height
   1.321 +                                  BufferedImage/TYPE_INT_RGB)]
   1.322 +        (dorun
   1.323 +         (for [x (range (count coords))]
   1.324 +           (dorun
   1.325 +            (for [y (range height)]
   1.326 +              (let [raw-sensor (sensor-data x)]
   1.327 +                (.setRGB image x y
   1.328 +                         (+ raw-sensor
   1.329 +                            (bit-shift-left raw-sensor 8)
   1.330 +                            (bit-shift-left raw-sensor 16))))))))
   1.331 +        (vi image)))))
   1.332 +        
   1.333 +         
   1.334 +
   1.335  ;;(defn test-touch [world creature]
   1.336  
   1.337  
   1.338 +
   1.339 +
   1.340 +
   1.341 +
   1.342  (defn test-creature [thing]
   1.343    (let [x-axis
   1.344          (box 1 0.01 0.01 :physical? false :color ColorRGBA/Red)
   1.345 @@ -860,6 +959,13 @@
   1.346          [init-vision-fns vision-data] (vision creature)
   1.347          vision-debug (map (fn [_] (debug-vision-window)) vision-data)
   1.348          me (sphere 0.5 :color ColorRGBA/Blue :physical? false)
   1.349 +        [init-hearing-fns hearing-senses] (hearing creature)
   1.350 +        hearing-windows (map (fn [_] (debug-hearing-window 50))
   1.351 +                             hearing-senses)
   1.352 +        bang (AudioNode. (asset-manager)
   1.353 +                         "Sounds/dream.wav" false)
   1.354 +        ;; dream
   1.355 +
   1.356          ]
   1.357    (world
   1.358     (nodify [creature
   1.359 @@ -868,11 +974,18 @@
   1.360              x-axis y-axis z-axis
   1.361              me
   1.362              ])
   1.363 -   standard-debug-controls
   1.364 +   (merge standard-debug-controls
   1.365 +          {"key-return"
   1.366 +           (fn [_ value]
   1.367 +             (if value
   1.368 +               (do
   1.369 +                 (println-repl "play-sound")
   1.370 +                 (.play bang))))})
   1.371     (fn [world]
   1.372       (light-up-everything world)
   1.373       (enable-debug world)
   1.374       (dorun (map #(% world) init-vision-fns))
   1.375 +     (dorun (map #(% world) init-hearing-fns))
   1.376       
   1.377       (add-eye world
   1.378                (attach-eye creature (test-eye))
   1.379 @@ -889,14 +1002,20 @@
   1.380     (fn [world tpf]
   1.381       ;;(dorun 
   1.382       ;; (map #(%1 %2) touch-nerves (repeat (.getRootNode world))))
   1.383 -    
   1.384 +     
   1.385 +
   1.386 +     
   1.387       (dorun
   1.388        (map #(%1 (%2 (.getRootNode world)))
   1.389             touch-debug-windows touch-nerves))
   1.390 +     
   1.391       (dorun
   1.392        (map #(%1 (%2))
   1.393             vision-debug vision-data))
   1.394 -
   1.395 +     (dorun
   1.396 +      (map #(%1 (%2)) hearing-windows hearing-senses))
   1.397 +     
   1.398 +     
   1.399       ;;(println-repl (vision-data))
   1.400       (.setLocalTranslation me (.getLocation (.getCamera world)))
   1.401       
   1.402 @@ -995,7 +1114,7 @@
   1.403                            (.setLocation camera-pos)
   1.404                            (.lookAt Vector3f/ZERO
   1.405                                     Vector3f/UNIT_X))]
   1.406 -                 (bind-camera rock cam)
   1.407 +                 (bind-sense rock cam)
   1.408                   
   1.409                   (.setTimer world (RatchetTimer. 60))
   1.410                   (add-eye world cam (comp (view-image) BufferedImage!))
   1.411 @@ -1004,6 +1123,7 @@
   1.412               (fn [_ _] (println-repl rot)))))
   1.413         
   1.414  
   1.415 +
   1.416  #+end_src
   1.417  
   1.418  #+results: body-1