cortex: org/eyes.org comparison

comparison org/eyes.org @ 151:aaacf087504c

refactored vision code

author	Robert McIntyre <rlm@mit.edu>
date	Fri, 03 Feb 2012 05:52:18 -0700
parents	9d0fe7f54e14
children	9e6a30b8c99a

comparison

equal deleted inserted replaced

-:e1232043656a
+:aaacf087504c
 * Vision
 I want to make creatures with eyes. Each eye can be independely moved
 and should see its own version of the world depending on where it is.
+Here's how vision will work.
+Make the continuation in scene-processor take FrameBuffer,
+byte-buffer, BufferedImage already sized to the correct
+dimensions. the continuation will decide wether to "mix" them
+into the BufferedImage, lazily ignore them, or mix them halfway
+and call c/graphics card routines.
+(vision creature) will take an optional :skip argument which will
+inform the continuations in scene processor to skip the given
+number of cycles 0 means that no cycles will be skipped.
+(vision creature) will return [init-functions sensor-functions].
+The init-functions are each single-arg functions that take the
+world and register the cameras and must each be called before the
+corresponding sensor-functions.  Each init-function returns the
+viewport for that eye which can be manipulated, saved, etc. Each
+sensor-function is a thunk and will return data in the same
+format as the tactile-sensor functions the structure is
+[topology, sensor-data]. Internally, these sensor-functions
+maintain a reference to sensor-data which is periodically updated
+by the continuation function established by its init-function.
+They can be queried every cycle, but their information may not
+necessairly be different every cycle.
+Each eye in the creature in blender will work the same way as
+joints -- a zero dimensional object with no geometry whose local
+coordinate system determines the orientation of the resulting
+eye. All eyes will have a parent named "eyes" just as all joints
+have a parent named "joints". The resulting camera will be a
+ChaseCamera or a CameraNode bound to the geo that is closest to
+the eye marker. The eye marker will contain the metadata for the
+eye, and will be moved by it's bound geometry. The dimensions of
+the eye's camera are equal to the dimensions of the eye's "UV"
+map.
 #+name: eyes
 #+begin_src clojure
 (ns cortex.vision
 "Simulate the sense of vision in jMonkeyEngine3. Enables multiple
 eyes from different positions to observe the same world, and pass
 the observed data to any arbitray function."
 {:author "Robert McIntyre"}
-(:use cortex.world)
+(:use (cortex world sense util))
 (:import com.jme3.post.SceneProcessor)
 (:import (com.jme3.util BufferUtils Screenshots))
 (:import java.nio.ByteBuffer)
 (:import java.awt.image.BufferedImage)
 (:import com.jme3.renderer.ViewPort)
 (:import com.jme3.math.ColorRGBA)
-(:import com.jme3.renderer.Renderer))
+(:import com.jme3.renderer.Renderer)
+(:import jme3tools.converters.ImageToAwt)
+(:import com.jme3.scene.Node))
+(cortex.import/mega-import-jme3)
 (defn vision-pipeline
 "Create a SceneProcessor object which wraps a vision processing
 continuation function. The continuation is a function that takes
 (doto viewport
 (.setClearFlags true true true)
 (.setBackgroundColor ColorRGBA/Black)
 (.addProcessor (vision-pipeline continuation))
 (.attachScene (.getRootNode world)))))
+(defn retina-sensor-image
+"Return a map of pixel selection functions to BufferedImages
+describing the distribution of light-sensitive components on this
+geometry's surface. Each function creates an integer from the rgb
+values found in the pixel. :red, :green, :blue, :gray are already
+defined as extracting the red green blue and average components
+respectively."
+[#^Spatial eye]
+(if-let [eye-map (meta-data eye "eye")]
+(map-vals
+#(ImageToAwt/convert
+(.getImage (.loadTexture (asset-manager) %))
+false false 0)
+(eval (read-string eye-map)))))
+(defn eye-dimensions
+"returns the width and height specified in the metadata of the eye"
+[#^Spatial eye]
+(let [dimensions
+(map #(vector (.getWidth %) (.getHeight %))
+(vals (retina-sensor-image eye)))]
+[(apply max (map first dimensions))
+(apply max (map second dimensions))]))
+(defn creature-eyes
+;;dylan
+"Return the children of the creature's \"eyes\" node."
+;;"The eye nodes which are children of the \"eyes\" node in the
+;;creature."
+[#^Node creature]
+(if-let [eye-node (.getChild creature "eyes")]
+(seq (.getChildren eye-node))
+(do (println-repl "could not find eyes node") [])))
+(defn attach-eye
+"Attach a Camera to the appropiate area and return the Camera."
+[#^Node creature #^Spatial eye]
+(let [target (closest-node creature eye)
+[cam-width cam-height] (eye-dimensions eye)
+cam (Camera. cam-width cam-height)]
+(.setLocation cam (.getWorldTranslation eye))
+(.setRotation cam (.getWorldRotation eye))
+(.setFrustumPerspective
+cam 45 (/ (.getWidth cam) (.getHeight cam))
+1 1000)
+(bind-sense target cam)
+cam))
+(def presets
+{:all    0xFFFFFF
+:red    0xFF0000
+:blue   0x0000FF
+:green  0x00FF00})
+(defn enable-vision
+"return [init-function sensor-functions] for a particular eye"
+[#^Node creature #^Spatial eye & {skip :skip :or {skip 0}}]
+(let [retinal-map (retina-sensor-image eye)
+camera (attach-eye creature eye)
+vision-image
+(atom
+(BufferedImage. (.getWidth camera)
+(.getHeight camera)
+BufferedImage/TYPE_BYTE_BINARY))]
+[(fn [world]
+(add-eye
+world camera
+(let [counter  (atom 0)]
+(fn [r fb bb bi]
+(if (zero? (rem (swap! counter inc) (inc skip)))
+(reset! vision-image (BufferedImage! r fb bb bi)))))))
+(vec
+(map
+(fn [[key image]]
+(let [whites (white-coordinates image)
+topology (vec (collapse whites))
+mask (presets key)]
+(fn []
+(vector
+topology
+(vec
+(for [[x y] whites]
+(bit-and
+mask (.getRGB @vision-image x y))))))))
+retinal-map))]))
+(defn vision
+[#^Node creature & {skip :skip :or {skip 0}}]
+(reduce
+(fn [[init-a senses-a]
+[init-b senses-b]]
+[(conj init-a init-b)
+(into senses-a senses-b)])
+[[][]]
+(for [eye (creature-eyes creature)]
+(enable-vision creature eye))))
 #+end_src
-#+results: eyes
-: #'cortex.vision/add-eye
 Note the use of continuation passing style for connecting the eye to a
 function to process the output. You can create any number of eyes, and
 each of them will see the world from their own =Camera=. Once every
 frame, the rendered image is copied to a =BufferedImage=, and that

Mercurial > cortex

comparison org/eyes.org @ 151:aaacf087504c