cortex: org/test-creature.org comparison

comparison org/test-creature.org @ 123:91773e8ec50f

got hearing diaplay working

author	Robert McIntyre <rlm@mit.edu>
date	Mon, 23 Jan 2012 03:41:26 -0700
parents	b591da250afc
children	90154bd674e9

comparison

equal deleted inserted replaced

-:b591da250afc
+:91773e8ec50f
 (map #(vector (.getWidth %) (.getHeight %))
 (vals (retina-sensor-image eye)))]
 [(apply max (map first dimensions))
 (apply max (map second dimensions))]))
 (defn creature-eyes
 "The eye nodes which are children of the \"eyes\" node in the
 creature."
 [#^Node creature]
 (if-let [eye-node (.getChild creature "eyes")]
 (seq (.getChildren eye-node))
 (do (println-repl "could not find eyes node") [])))
+;; Here's how vision will work.
-;; Here's how vision will work.
+;; Make the continuation in scene-processor take FrameBuffer,
-;; Make the continuation in scene-processor take FrameBuffer,
+;; byte-buffer, BufferedImage already sized to the correct
-;; byte-buffer, BufferedImage already sized to the correct
+;; dimensions. the continuation will decide wether to "mix" them
-;; dimensions. the continuation will decide wether to "mix" them
+;; into the BufferedImage, lazily ignore them, or mix them halfway
-;; into the BufferedImage, lazily ignore them, or mix them halfway
+;; and call c/graphics card routines.
-;; and call c/graphics card routines.
+;; (vision creature) will take an optional :skip argument which will
-;; (vision creature) will take an optional :skip argument which will
+;; inform the continuations in scene processor to skip the given
-;; inform the continuations in scene processor to skip the given
+;; number of cycles; 0 means that no cycles will be skipped.
-;; number of cycles; 0 means that no cycles will be skipped.
+;; (vision creature) will return [init-functions sensor-functions].
-;; (vision creature) will return [init-functions sensor-functions].
+;; The init-functions are each single-arg functions that take the
-;; The init-functions are each single-arg functions that take the
+;; world and register the cameras and must each be called before the
-;; world and register the cameras and must each be called before the
+;; corresponding sensor-functions.  Each init-function returns the
-;; corresponding sensor-functions.  Each init-function returns the
+;; viewport for that eye which can be manipulated, saved, etc. Each
-;; viewport for that eye which can be manipulated, saved, etc. Each
+;; sensor-function is a thunk and will return data in the same
-;; sensor-function is a thunk and will return data in the same
+;; format as the tactile-sensor functions; the structure is
-;; format as the tactile-sensor functions; the structure is
+;; [topology, sensor-data]. Internally, these sensor-functions
-;; [topology, sensor-data]. Internally, these sensor-functions
+;; maintain a reference to sensor-data which is periodically updated
-;; maintain a reference to sensor-data which is periodically updated
+;; by the continuation function established by its init-function.
-;; by the continuation function established by its init-function.
+;; They can be queried every cycle, but their information may not
-;; They can be queried every cycle, but their information may not
+;; necessairly be different every cycle.
-;; necessairly be different every cycle.
+;; Each eye in the creature in blender will work the same way as
-;; Each eye in the creature in blender will work the same way as
+;; joints -- a zero dimensional object with no geometry whose local
-;; joints -- a one dimensional object with no geometry whose local
+;; coordinate system determines the orientation of the resulting
-;; coordinate system determines the orientation of the resulting
+;; eye. All eyes will have a parent named "eyes" just as all joints
-;; eye. All eyes will have a parent named "eyes" just as all joints
+;; have a parent named "joints". The resulting camera will be a
-;; have a parent named "joints". The resulting camera will be a
+;; ChaseCamera or a CameraNode bound to the geo that is closest to
-;; ChaseCamera or a CameraNode bound to the geo that is closest to
+;; the eye marker. The eye marker will contain the metadata for the
-;; the eye marker. The eye marker will contain the metadata for the
+;; eye, and will be moved by it's bound geometry. The dimensions of
-;; eye, and will be moved by it's bound geometry. The dimensions of
+;; the eye's camera are equal to the dimensions of the eye's "UV"
-;; the eye's camera are equal to the dimensions of the eye's "UV"
+;; map.
-;; map.
-(defn eye-target
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-"The closest object in creature to eye."
+;; Ears work the same way as vision.
+;; (hearing creature) will return [init-functions
+;; sensor-functions]. The init functions each take the world and
+;; register a SoundProcessor that does foureier transforms on the
+;; incommong sound data, making it available to each sensor function.
+(defn creature-ears
+"The ear nodes which are children of the \"ears\" node in the
+creature."
+[#^Node creature]
+(if-let [ear-node (.getChild creature "ears")]
+(seq (.getChildren ear-node))
+(do (println-repl "could not find ears node") [])))
+(defn closest-node
+"The closest object in creature to the given node."
 [#^Node creature #^Node eye]
 (loop [radius (float 0.01)]
 (let [results (CollisionResults.)]
 (.collideWith
 creature
 results)
 (if-let [target (first results)]
 (.getGeometry target)
 (recur (float (* 2 radius)))))))
-(defn bind-camera
+(defn bind-sense
-"Bind the camera to the Spatial such that it will maintain its
+"Bind the sense to the Spatial such that it will maintain its
 current position relative to the Spatial no matter how the spatial
-moves."
+moves. 'sense can be either a Camera or Listener object."
-[#^Spatial obj #^Camera cam]
+[#^Spatial obj sense]
-(let [cam-offset (.subtract (.getLocation cam)
+(let [sense-offset (.subtract (.getLocation sense)
 (.getWorldTranslation obj))
-initial-cam-rotation (Quaternion. (.getRotation cam))
+initial-sense-rotation (Quaternion. (.getRotation sense))
 base-anti-rotation (.inverse (.getWorldRotation obj))]
 (.addControl
 obj
 (proxy [AbstractControl] []
 (controlUpdate [tpf]
 (let [total-rotation
 (.mult base-anti-rotation (.getWorldRotation obj))]
-(.setLocation cam
+(.setLocation sense
 (.add
-(.mult total-rotation cam-offset)
+(.mult total-rotation sense-offset)
 (.getWorldTranslation obj)))
-(.setRotation cam
+(.setRotation sense
-(.mult total-rotation initial-cam-rotation))))
+(.mult total-rotation initial-sense-rotation))))
 (controlRender [_ _])))))
+(defn update-listener-velocity
+"Update the listener's velocity every update loop."
+[#^Spatial obj #^Listener lis]
+(let [old-position (atom (.getLocation lis))]
+(.addControl
+obj
+(proxy [AbstractControl] []
+(controlUpdate [tpf]
+(let [new-position (.getLocation lis)]
+(.setVelocity
+lis
+(.mult (.subtract new-position @old-position)
+(float (/ tpf))))
+(reset! old-position new-position)))
+(controlRender [_ _])))))
+(import com.aurellem.capture.audio.AudioSendRenderer)
+(defn attach-ear
+[#^Application world #^Node creature #^Spatial ear continuation]
+(let [target (closest-node creature ear)
+lis (Listener.)
+audio-renderer (.getAudioRenderer world)
+sp (sound-processor continuation)]
+(println-repl "audio-renderer is " audio-renderer)
+(.setLocation lis (.getWorldTranslation ear))
+(.setRotation lis (.getWorldRotation ear))
+(bind-sense target lis)
+(update-listener-velocity target lis)
+(.addListener audio-renderer lis)
+(.registerSoundProcessor audio-renderer lis sp)))
+(defn enable-hearing
+[#^Node creature #^Spatial ear]
+(let [hearing-data (atom [])]
+[(fn [world]
+(attach-ear world creature ear
+(fn [data]
+(reset! hearing-data (vec data)))))
+[(fn []
+(let [data @hearing-data
+topology
+(vec (map #(vector % 0) (range 0 (count data))))
+scaled-data
+(vec
+(map
+#(rem (int (* 255 (/ (+ 1 %) 2)))  256)
+data))]
+(println-repl (take 10 scaled-data))
+[topology scaled-data]))
+]]))
+(defn hearing
+[#^Node creature]
+(reduce
+(fn [[init-a senses-a]
+[init-b senses-b]]
+[(conj init-a init-b)
+(into senses-a senses-b)])
+[[][]]
+(for [ear (creature-ears creature)]
+(enable-hearing creature ear))))
 (defn attach-eye
 "Attach a Camera to the appropiate area and return the Camera."
 [#^Node creature #^Spatial eye]
+(let [target (closest-node creature eye)
-(let [target (eye-target creature eye)
 [cam-width cam-height] (eye-dimensions eye)
 cam (Camera. cam-width cam-height)]
 (.setLocation cam (.getWorldTranslation eye))
 (.setRotation cam (.getWorldRotation eye))
 (.setFrustumPerspective
 cam 45 (/ (.getWidth cam) (.getHeight cam))
 1 1000)
+(bind-sense target cam)
-(bind-camera target cam)
 cam))
 (def presets
 {:all    0xFFFFFF
 :red    0xFF0000
 :blue   0x0000FF
 :green  0x00FF00})
 (defn enable-vision
 "return [init-function sensor-functions] for a particular eye"
 [#^Node creature #^Spatial eye & {skip :skip :or {skip 0}}]
 (let [retinal-map (retina-sensor-image eye)
-vision-image (atom nil)
+camera (attach-eye creature eye)
-camera (attach-eye creature eye)]
+vision-image
-[
+(atom
-(fn [world]
+(BufferedImage. (.getWidth camera)
-(add-eye
+(.getHeight camera)
-world camera
+BufferedImage/TYPE_BYTE_BINARY))]
-(let [counter  (atom 0)]
+[(fn [world]
-(fn [r fb bb bi]
+(add-eye
-(if (zero? (rem (swap! counter inc) (inc skip)))
+world camera
-(reset! vision-image (BufferedImage! r fb bb bi)))))))
+(let [counter  (atom 0)]
-(vec
+(fn [r fb bb bi]
-(map
+(if (zero? (rem (swap! counter inc) (inc skip)))
-(fn [[key image]]
+(reset! vision-image (BufferedImage! r fb bb bi)))))))
-(let [whites (white-coordinates image)
+(vec
-topology (vec (collapse whites))
+(map
-mask (presets key)]
+(fn [[key image]]
-(fn []
+(let [whites (white-coordinates image)
-(vector
+topology (vec (collapse whites))
-topology
+mask (presets key)]
-(vec
+(fn []
-(for [[x y] whites]
+(vector
-(bit-and
+topology
-mask (.getRGB @vision-image x y))))))))
+(vec
-retinal-map))
+(for [[x y] whites]
-]))
+(bit-and
+mask (.getRGB @vision-image x y))))))))
+retinal-map))]))
 (defn vision
 [#^Node creature & {skip :skip :or {skip 0}}]
 (reduce
 (fn [[init-a senses-a]
 [init-b senses-b]]
 [(conj init-a init-b)
 (into senses-a senses-b)])
 [[coords sensor-data]]
 (let [image (points->image coords)]
 (dorun
 (for [i (range (count coords))]
 (.setRGB image ((coords i) 0) ((coords i) 1)
-({0 -16777216
+({0 0x000000
-1 -1} (sensor-data i)))))
+1 0xFFFFFF} (sensor-data i)))))
 (vi image)))))
 (defn debug-vision-window
 "creates function that offers a debug view of sensor data"
 []
 (for [i (range (count coords))]
 (.setRGB image ((coords i) 0) ((coords i) 1)
 (sensor-data i))))
 (vi image)))))
+(defn debug-hearing-window
+"view audio data"
+[height]
+(let [vi (view-image)]
+(fn [[coords sensor-data]]
+(let [image (BufferedImage. (count coords) height
+BufferedImage/TYPE_INT_RGB)]
+(dorun
+(for [x (range (count coords))]
+(dorun
+(for [y (range height)]
+(let [raw-sensor (sensor-data x)]
+(.setRGB image x y
+(+ raw-sensor
+(bit-shift-left raw-sensor 8)
+(bit-shift-left raw-sensor 16))))))))
+(vi image)))))
 ;;(defn test-touch [world creature]
 (defn test-creature [thing]
 (let [x-axis
 (box 1 0.01 0.01 :physical? false :color ColorRGBA/Red)
 touch-nerves (touch creature)
 touch-debug-windows (map (fn [_] (debug-window)) touch-nerves)
 [init-vision-fns vision-data] (vision creature)
 vision-debug (map (fn [_] (debug-vision-window)) vision-data)
 me (sphere 0.5 :color ColorRGBA/Blue :physical? false)
+[init-hearing-fns hearing-senses] (hearing creature)
+hearing-windows (map (fn [_] (debug-hearing-window 50))
+hearing-senses)
+bang (AudioNode. (asset-manager)
+"Sounds/dream.wav" false)
+;; dream
 ]
 (world
 (nodify [creature
 (box 10 2 10 :position (Vector3f. 0 -9 0)
 :color ColorRGBA/Gray :mass 0)
 x-axis y-axis z-axis
 me
 ])
-standard-debug-controls
+(merge standard-debug-controls
+{"key-return"
+(fn [_ value]
+(if value
+(do
+(println-repl "play-sound")
+(.play bang))))})
 (fn [world]
 (light-up-everything world)
 (enable-debug world)
 (dorun (map #(% world) init-vision-fns))
+(dorun (map #(% world) init-hearing-fns))
 (add-eye world
 (attach-eye creature (test-eye))
 (comp (view-image) BufferedImage!))
 ;;(set-gravity world (Vector3f. 0 0 0))
 )
 (fn [world tpf]
 ;;(dorun
 ;; (map #(%1 %2) touch-nerves (repeat (.getRootNode world))))
 (dorun
 (map #(%1 (%2 (.getRootNode world)))
 touch-debug-windows touch-nerves))
 (dorun
 (map #(%1 (%2))
 vision-debug vision-data))
+(dorun
+(map #(%1 (%2)) hearing-windows hearing-senses))
 ;;(println-repl (vision-data))
 (.setLocalTranslation me (.getLocation (.getCamera world)))
 )
 (let
 [cam (doto (.clone (.getCamera world))
 (.setLocation camera-pos)
 (.lookAt Vector3f/ZERO
 Vector3f/UNIT_X))]
-(bind-camera rock cam)
+(bind-sense rock cam)
 (.setTimer world (RatchetTimer. 60))
 (add-eye world cam (comp (view-image) BufferedImage!))
 (add-eye world (.getCamera world) no-op))
 )
 (fn [_ _] (println-repl rot)))))
 #+end_src
 #+results: body-1
 : #'cortex.silly/test-creature

Mercurial > cortex

comparison org/test-creature.org @ 123:91773e8ec50f