# HG changeset patch # User Robert McIntyre # Date 1327315286 25200 # Node ID 91773e8ec50f48e82d87e0f53af2c16c791c435b # Parent b591da250afc6a5771b785a92adf1fa7cbcbaa66 got hearing diaplay working diff -r b591da250afc -r 91773e8ec50f assets/Models/creature1/try-again.blend Binary file assets/Models/creature1/try-again.blend has changed diff -r b591da250afc -r 91773e8ec50f org/test-creature.org --- a/org/test-creature.org Sun Jan 22 04:53:58 2012 -0700 +++ b/org/test-creature.org Mon Jan 23 03:41:26 2012 -0700 @@ -655,7 +655,6 @@ [(apply max (map first dimensions)) (apply max (map second dimensions))])) - (defn creature-eyes "The eye nodes which are children of the \"eyes\" node in the creature." @@ -664,45 +663,62 @@ (seq (.getChildren eye-node)) (do (println-repl "could not find eyes node") []))) +;; Here's how vision will work. - ;; Here's how vision will work. +;; Make the continuation in scene-processor take FrameBuffer, +;; byte-buffer, BufferedImage already sized to the correct +;; dimensions. the continuation will decide wether to "mix" them +;; into the BufferedImage, lazily ignore them, or mix them halfway +;; and call c/graphics card routines. - ;; Make the continuation in scene-processor take FrameBuffer, - ;; byte-buffer, BufferedImage already sized to the correct - ;; dimensions. the continuation will decide wether to "mix" them - ;; into the BufferedImage, lazily ignore them, or mix them halfway - ;; and call c/graphics card routines. - - ;; (vision creature) will take an optional :skip argument which will - ;; inform the continuations in scene processor to skip the given - ;; number of cycles; 0 means that no cycles will be skipped. +;; (vision creature) will take an optional :skip argument which will +;; inform the continuations in scene processor to skip the given +;; number of cycles; 0 means that no cycles will be skipped. - ;; (vision creature) will return [init-functions sensor-functions]. - ;; The init-functions are each single-arg functions that take the - ;; world and register the cameras and must each be called before the - ;; corresponding sensor-functions. Each init-function returns the - ;; viewport for that eye which can be manipulated, saved, etc. Each - ;; sensor-function is a thunk and will return data in the same - ;; format as the tactile-sensor functions; the structure is - ;; [topology, sensor-data]. Internally, these sensor-functions - ;; maintain a reference to sensor-data which is periodically updated - ;; by the continuation function established by its init-function. - ;; They can be queried every cycle, but their information may not - ;; necessairly be different every cycle. +;; (vision creature) will return [init-functions sensor-functions]. +;; The init-functions are each single-arg functions that take the +;; world and register the cameras and must each be called before the +;; corresponding sensor-functions. Each init-function returns the +;; viewport for that eye which can be manipulated, saved, etc. Each +;; sensor-function is a thunk and will return data in the same +;; format as the tactile-sensor functions; the structure is +;; [topology, sensor-data]. Internally, these sensor-functions +;; maintain a reference to sensor-data which is periodically updated +;; by the continuation function established by its init-function. +;; They can be queried every cycle, but their information may not +;; necessairly be different every cycle. - ;; Each eye in the creature in blender will work the same way as - ;; joints -- a one dimensional object with no geometry whose local - ;; coordinate system determines the orientation of the resulting - ;; eye. All eyes will have a parent named "eyes" just as all joints - ;; have a parent named "joints". The resulting camera will be a - ;; ChaseCamera or a CameraNode bound to the geo that is closest to - ;; the eye marker. The eye marker will contain the metadata for the - ;; eye, and will be moved by it's bound geometry. The dimensions of - ;; the eye's camera are equal to the dimensions of the eye's "UV" - ;; map. +;; Each eye in the creature in blender will work the same way as +;; joints -- a zero dimensional object with no geometry whose local +;; coordinate system determines the orientation of the resulting +;; eye. All eyes will have a parent named "eyes" just as all joints +;; have a parent named "joints". The resulting camera will be a +;; ChaseCamera or a CameraNode bound to the geo that is closest to +;; the eye marker. The eye marker will contain the metadata for the +;; eye, and will be moved by it's bound geometry. The dimensions of +;; the eye's camera are equal to the dimensions of the eye's "UV" +;; map. -(defn eye-target - "The closest object in creature to eye." + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Ears work the same way as vision. + +;; (hearing creature) will return [init-functions +;; sensor-functions]. The init functions each take the world and +;; register a SoundProcessor that does foureier transforms on the +;; incommong sound data, making it available to each sensor function. + +(defn creature-ears + "The ear nodes which are children of the \"ears\" node in the + creature." + [#^Node creature] + (if-let [ear-node (.getChild creature "ears")] + (seq (.getChildren ear-node)) + (do (println-repl "could not find ears node") []))) + +(defn closest-node + "The closest object in creature to the given node." [#^Node creature #^Node eye] (loop [radius (float 0.01)] (let [results (CollisionResults.)] @@ -715,14 +731,14 @@ (.getGeometry target) (recur (float (* 2 radius))))))) -(defn bind-camera - "Bind the camera to the Spatial such that it will maintain its +(defn bind-sense + "Bind the sense to the Spatial such that it will maintain its current position relative to the Spatial no matter how the spatial - moves." - [#^Spatial obj #^Camera cam] - (let [cam-offset (.subtract (.getLocation cam) - (.getWorldTranslation obj)) - initial-cam-rotation (Quaternion. (.getRotation cam)) + moves. 'sense can be either a Camera or Listener object." + [#^Spatial obj sense] + (let [sense-offset (.subtract (.getLocation sense) + (.getWorldTranslation obj)) + initial-sense-rotation (Quaternion. (.getRotation sense)) base-anti-rotation (.inverse (.getWorldRotation obj))] (.addControl obj @@ -730,20 +746,82 @@ (controlUpdate [tpf] (let [total-rotation (.mult base-anti-rotation (.getWorldRotation obj))] - (.setLocation cam + (.setLocation sense (.add - (.mult total-rotation cam-offset) + (.mult total-rotation sense-offset) (.getWorldTranslation obj))) - (.setRotation cam - (.mult total-rotation initial-cam-rotation)))) + (.setRotation sense + (.mult total-rotation initial-sense-rotation)))) (controlRender [_ _]))))) +(defn update-listener-velocity + "Update the listener's velocity every update loop." + [#^Spatial obj #^Listener lis] + (let [old-position (atom (.getLocation lis))] + (.addControl + obj + (proxy [AbstractControl] [] + (controlUpdate [tpf] + (let [new-position (.getLocation lis)] + (.setVelocity + lis + (.mult (.subtract new-position @old-position) + (float (/ tpf)))) + (reset! old-position new-position))) + (controlRender [_ _]))))) + +(import com.aurellem.capture.audio.AudioSendRenderer) + +(defn attach-ear + [#^Application world #^Node creature #^Spatial ear continuation] + (let [target (closest-node creature ear) + lis (Listener.) + audio-renderer (.getAudioRenderer world) + sp (sound-processor continuation)] + (println-repl "audio-renderer is " audio-renderer) + (.setLocation lis (.getWorldTranslation ear)) + (.setRotation lis (.getWorldRotation ear)) + (bind-sense target lis) + (update-listener-velocity target lis) + (.addListener audio-renderer lis) + (.registerSoundProcessor audio-renderer lis sp))) + +(defn enable-hearing + [#^Node creature #^Spatial ear] + (let [hearing-data (atom [])] + [(fn [world] + (attach-ear world creature ear + (fn [data] + (reset! hearing-data (vec data))))) + [(fn [] + (let [data @hearing-data + topology + (vec (map #(vector % 0) (range 0 (count data)))) + scaled-data + (vec + (map + #(rem (int (* 255 (/ (+ 1 %) 2))) 256) + data))] + (println-repl (take 10 scaled-data)) + [topology scaled-data])) + ]])) + +(defn hearing + [#^Node creature] + (reduce + (fn [[init-a senses-a] + [init-b senses-b]] + [(conj init-a init-b) + (into senses-a senses-b)]) + [[][]] + (for [ear (creature-ears creature)] + (enable-hearing creature ear)))) + (defn attach-eye "Attach a Camera to the appropiate area and return the Camera." [#^Node creature #^Spatial eye] - - (let [target (eye-target creature eye) + (let [target (closest-node creature eye) [cam-width cam-height] (eye-dimensions eye) cam (Camera. cam-width cam-height)] (.setLocation cam (.getWorldTranslation eye)) @@ -751,52 +829,49 @@ (.setFrustumPerspective cam 45 (/ (.getWidth cam) (.getHeight cam)) 1 1000) - - (bind-camera target cam) + (bind-sense target cam) cam)) - (def presets {:all 0xFFFFFF :red 0xFF0000 :blue 0x0000FF :green 0x00FF00}) - - (defn enable-vision "return [init-function sensor-functions] for a particular eye" [#^Node creature #^Spatial eye & {skip :skip :or {skip 0}}] (let [retinal-map (retina-sensor-image eye) - vision-image (atom nil) - camera (attach-eye creature eye)] - [ - (fn [world] - (add-eye - world camera - (let [counter (atom 0)] - (fn [r fb bb bi] - (if (zero? (rem (swap! counter inc) (inc skip))) - (reset! vision-image (BufferedImage! r fb bb bi))))))) - (vec - (map - (fn [[key image]] - (let [whites (white-coordinates image) - topology (vec (collapse whites)) - mask (presets key)] - (fn [] - (vector - topology - (vec - (for [[x y] whites] - (bit-and - mask (.getRGB @vision-image x y)))))))) - retinal-map)) - ])) + camera (attach-eye creature eye) + vision-image + (atom + (BufferedImage. (.getWidth camera) + (.getHeight camera) + BufferedImage/TYPE_BYTE_BINARY))] + [(fn [world] + (add-eye + world camera + (let [counter (atom 0)] + (fn [r fb bb bi] + (if (zero? (rem (swap! counter inc) (inc skip))) + (reset! vision-image (BufferedImage! r fb bb bi))))))) + (vec + (map + (fn [[key image]] + (let [whites (white-coordinates image) + topology (vec (collapse whites)) + mask (presets key)] + (fn [] + (vector + topology + (vec + (for [[x y] whites] + (bit-and + mask (.getRGB @vision-image x y)))))))) + retinal-map))])) (defn vision [#^Node creature & {skip :skip :or {skip 0}}] - (reduce (fn [[init-a senses-a] [init-b senses-b]] @@ -827,8 +902,8 @@ (dorun (for [i (range (count coords))] (.setRGB image ((coords i) 0) ((coords i) 1) - ({0 -16777216 - 1 -1} (sensor-data i))))) + ({0 0x000000 + 1 0xFFFFFF} (sensor-data i))))) (vi image))))) (defn debug-vision-window @@ -844,9 +919,33 @@ (sensor-data i)))) (vi image))))) +(defn debug-hearing-window + "view audio data" + [height] + (let [vi (view-image)] + (fn [[coords sensor-data]] + (let [image (BufferedImage. (count coords) height + BufferedImage/TYPE_INT_RGB)] + (dorun + (for [x (range (count coords))] + (dorun + (for [y (range height)] + (let [raw-sensor (sensor-data x)] + (.setRGB image x y + (+ raw-sensor + (bit-shift-left raw-sensor 8) + (bit-shift-left raw-sensor 16)))))))) + (vi image))))) + + + ;;(defn test-touch [world creature] + + + + (defn test-creature [thing] (let [x-axis (box 1 0.01 0.01 :physical? false :color ColorRGBA/Red) @@ -860,6 +959,13 @@ [init-vision-fns vision-data] (vision creature) vision-debug (map (fn [_] (debug-vision-window)) vision-data) me (sphere 0.5 :color ColorRGBA/Blue :physical? false) + [init-hearing-fns hearing-senses] (hearing creature) + hearing-windows (map (fn [_] (debug-hearing-window 50)) + hearing-senses) + bang (AudioNode. (asset-manager) + "Sounds/dream.wav" false) + ;; dream + ] (world (nodify [creature @@ -868,11 +974,18 @@ x-axis y-axis z-axis me ]) - standard-debug-controls + (merge standard-debug-controls + {"key-return" + (fn [_ value] + (if value + (do + (println-repl "play-sound") + (.play bang))))}) (fn [world] (light-up-everything world) (enable-debug world) (dorun (map #(% world) init-vision-fns)) + (dorun (map #(% world) init-hearing-fns)) (add-eye world (attach-eye creature (test-eye)) @@ -889,14 +1002,20 @@ (fn [world tpf] ;;(dorun ;; (map #(%1 %2) touch-nerves (repeat (.getRootNode world)))) - + + + (dorun (map #(%1 (%2 (.getRootNode world))) touch-debug-windows touch-nerves)) + (dorun (map #(%1 (%2)) vision-debug vision-data)) - + (dorun + (map #(%1 (%2)) hearing-windows hearing-senses)) + + ;;(println-repl (vision-data)) (.setLocalTranslation me (.getLocation (.getCamera world))) @@ -995,7 +1114,7 @@ (.setLocation camera-pos) (.lookAt Vector3f/ZERO Vector3f/UNIT_X))] - (bind-camera rock cam) + (bind-sense rock cam) (.setTimer world (RatchetTimer. 60)) (add-eye world cam (comp (view-image) BufferedImage!)) @@ -1004,6 +1123,7 @@ (fn [_ _] (println-repl rot))))) + #+end_src #+results: body-1