1.1 --- a/org/eyes.org	Sat Feb 04 04:07:25 2012 -0700
     1.2 +++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.3 @@ -1,309 +0,0 @@
     1.4 -#+title: Simulated Sense of Sight
     1.5 -#+author: Robert McIntyre
     1.6 -#+email: rlm@mit.edu
     1.7 -#+description: Simulated sight for AI research using JMonkeyEngine3 and clojure
     1.8 -#+keywords: computer vision, jMonkeyEngine3, clojure
     1.9 -#+SETUPFILE: ../../aurellem/org/setup.org
    1.10 -#+INCLUDE: ../../aurellem/org/level-0.org
    1.11 -#+babel: :mkdirp yes :noweb yes :exports both
    1.12 -
    1.13 -* Vision
    1.14 -
    1.15 -I want to make creatures with eyes. Each eye can be independely moved
    1.16 -and should see its own version of the world depending on where it is.
    1.17 -
    1.18 -Here's how vision will work.
    1.19 -
    1.20 -Make the continuation in scene-processor take FrameBuffer,
    1.21 -byte-buffer, BufferedImage already sized to the correct
    1.22 -dimensions. the continuation will decide wether to "mix" them
    1.23 -into the BufferedImage, lazily ignore them, or mix them halfway
    1.24 -and call c/graphics card routines.
    1.25 -
    1.26 -(vision creature) will take an optional :skip argument which will
    1.27 -inform the continuations in scene processor to skip the given
    1.28 -number of cycles 0 means that no cycles will be skipped.
    1.29 -
    1.30 -(vision creature) will return [init-functions sensor-functions].
    1.31 -The init-functions are each single-arg functions that take the
    1.32 -world and register the cameras and must each be called before the
    1.33 -corresponding sensor-functions.  Each init-function returns the
    1.34 -viewport for that eye which can be manipulated, saved, etc. Each
    1.35 -sensor-function is a thunk and will return data in the same
    1.36 -format as the tactile-sensor functions the structure is
    1.37 -[topology, sensor-data]. Internally, these sensor-functions
    1.38 -maintain a reference to sensor-data which is periodically updated
    1.39 -by the continuation function established by its init-function.
    1.40 -They can be queried every cycle, but their information may not
    1.41 -necessairly be different every cycle.
    1.42 -
    1.43 -Each eye in the creature in blender will work the same way as
    1.44 -joints -- a zero dimensional object with no geometry whose local
    1.45 -coordinate system determines the orientation of the resulting
    1.46 -eye. All eyes will have a parent named "eyes" just as all joints
    1.47 -have a parent named "joints". The resulting camera will be a
    1.48 -ChaseCamera or a CameraNode bound to the geo that is closest to
    1.49 -the eye marker. The eye marker will contain the metadata for the
    1.50 -eye, and will be moved by it's bound geometry. The dimensions of
    1.51 -the eye's camera are equal to the dimensions of the eye's "UV"
    1.52 -map.
    1.53 -
    1.54 -#+name: eyes
    1.55 -#+begin_src clojure 
    1.56 -(ns cortex.vision
    1.57 -  "Simulate the sense of vision in jMonkeyEngine3. Enables multiple
    1.58 -  eyes from different positions to observe the same world, and pass
    1.59 -  the observed data to any arbitray function."
    1.60 -  {:author "Robert McIntyre"}
    1.61 -  (:use (cortex world sense util))
    1.62 -  (:use clojure.contrib.def)
    1.63 -  (:import com.jme3.post.SceneProcessor)
    1.64 -  (:import (com.jme3.util BufferUtils Screenshots))
    1.65 -  (:import java.nio.ByteBuffer)
    1.66 -  (:import java.awt.image.BufferedImage)
    1.67 -  (:import com.jme3.renderer.ViewPort)
    1.68 -  (:import com.jme3.math.ColorRGBA)
    1.69 -  (:import com.jme3.renderer.Renderer)
    1.70 -  (:import com.jme3.scene.Node))
    1.71 -
    1.72 -(cortex.import/mega-import-jme3)
    1.73 -
    1.74 -
    1.75 -(defn vision-pipeline
    1.76 -  "Create a SceneProcessor object which wraps a vision processing
    1.77 -  continuation function. The continuation is a function that takes 
    1.78 -  [#^Renderer r #^FrameBuffer fb #^ByteBuffer b #^BufferedImage bi],
    1.79 -  each of which has already been appropiately sized."
    1.80 -  [continuation]
    1.81 -  (let [byte-buffer (atom nil)
    1.82 -	renderer (atom nil)
    1.83 -        image (atom nil)]
    1.84 -  (proxy [SceneProcessor] []
    1.85 -    (initialize
    1.86 -     [renderManager viewPort]
    1.87 -     (let [cam (.getCamera viewPort)
    1.88 -	   width (.getWidth cam)
    1.89 -	   height (.getHeight cam)]
    1.90 -       (reset! renderer (.getRenderer renderManager))
    1.91 -       (reset! byte-buffer
    1.92 -	     (BufferUtils/createByteBuffer
    1.93 -	      (* width height 4)))
    1.94 -        (reset! image (BufferedImage.
    1.95 -                      width height
    1.96 -                      BufferedImage/TYPE_4BYTE_ABGR))))
    1.97 -    (isInitialized [] (not (nil? @byte-buffer)))
    1.98 -    (reshape [_ _ _])
    1.99 -    (preFrame [_])
   1.100 -    (postQueue [_])
   1.101 -    (postFrame
   1.102 -     [#^FrameBuffer fb]
   1.103 -     (.clear @byte-buffer)
   1.104 -     (continuation @renderer fb @byte-buffer @image))
   1.105 -    (cleanup []))))
   1.106 -    
   1.107 -(defn frameBuffer->byteBuffer!
   1.108 -  "Transfer the data in the graphics card (Renderer, FrameBuffer) to
   1.109 -   the CPU (ByteBuffer)."  
   1.110 -  [#^Renderer r #^FrameBuffer fb #^ByteBuffer bb]
   1.111 -  (.readFrameBuffer r fb bb) bb)
   1.112 -
   1.113 -(defn byteBuffer->bufferedImage!
   1.114 -  "Convert the C-style BGRA image data in the ByteBuffer bb to the AWT
   1.115 -   style ABGR image data and place it in BufferedImage bi."
   1.116 -  [#^ByteBuffer bb #^BufferedImage bi]
   1.117 -  (Screenshots/convertScreenShot bb bi) bi)
   1.118 -
   1.119 -(defn BufferedImage!
   1.120 -  "Continuation which will grab the buffered image from the materials
   1.121 -   provided by (vision-pipeline)."
   1.122 -  [#^Renderer r #^FrameBuffer fb #^ByteBuffer bb #^BufferedImage bi]
   1.123 -  (byteBuffer->bufferedImage!
   1.124 -   (frameBuffer->byteBuffer! r fb bb) bi))
   1.125 -
   1.126 -(defn add-eye!
   1.127 -  "Add an eye to the world, calling continuation on every frame
   1.128 -  produced." 
   1.129 -  [#^Application world camera continuation]
   1.130 -  (let [width (.getWidth camera)
   1.131 -	height (.getHeight camera)
   1.132 -	render-manager (.getRenderManager world)
   1.133 -	viewport (.createMainView render-manager "eye-view" camera)]
   1.134 -    (doto viewport
   1.135 -      (.setClearFlags true true true)
   1.136 -      (.setBackgroundColor ColorRGBA/Black)
   1.137 -      (.addProcessor (vision-pipeline continuation))
   1.138 -      (.attachScene (.getRootNode world)))))
   1.139 -
   1.140 -(defn retina-sensor-image
   1.141 -  "Return a map of pixel selection functions to BufferedImages
   1.142 -   describing the distribution of light-sensitive components on this
   1.143 -   geometry's surface. Each function creates an integer from the rgb
   1.144 -   values found in the pixel. :red, :green, :blue, :gray are already
   1.145 -   defined as extracting the red green blue and average components
   1.146 -   respectively."
   1.147 -   [#^Spatial eye]
   1.148 -   (if-let [eye-map (meta-data eye "eye")]
   1.149 -     (map-vals
   1.150 -      load-image
   1.151 -      (eval (read-string eye-map)))))
   1.152 -
   1.153 -(defn eye-dimensions
   1.154 -  "returns the width and height specified in the metadata of the eye"
   1.155 -  [#^Spatial eye]
   1.156 -  (let [dimensions
   1.157 -          (map #(vector (.getWidth %) (.getHeight %))
   1.158 -               (vals (retina-sensor-image eye)))]
   1.159 -    [(apply max (map first dimensions))
   1.160 -     (apply max (map second dimensions))]))
   1.161 -
   1.162 -(defvar 
   1.163 -  ^{:arglists '([creature])}
   1.164 -  eyes
   1.165 -  (sense-nodes "eyes")
   1.166 -  "Return the children of the creature's \"eyes\" node.")
   1.167 -
   1.168 -(defn attach-eye
   1.169 -  "Attach a Camera to the appropiate area and return the Camera."
   1.170 -  [#^Node creature #^Spatial eye]
   1.171 -  (let [target (closest-node creature eye)
   1.172 -        [cam-width cam-height] (eye-dimensions eye)
   1.173 -        cam (Camera. cam-width cam-height)]
   1.174 -    (.setLocation cam (.getWorldTranslation eye))
   1.175 -    (.setRotation cam (.getWorldRotation eye))
   1.176 -    (.setFrustumPerspective
   1.177 -     cam 45 (/ (.getWidth cam) (.getHeight cam))
   1.178 -     1 1000)
   1.179 -    (bind-sense target cam)
   1.180 -    cam))
   1.181 -
   1.182 -(def presets
   1.183 -  {:all    0xFFFFFF
   1.184 -   :red    0xFF0000
   1.185 -   :blue   0x0000FF
   1.186 -   :green  0x00FF00})
   1.187 -
   1.188 -(defn enable-vision
   1.189 -  "return [init-function sensor-functions] for a particular eye"
   1.190 -  [#^Node creature #^Spatial eye & {skip :skip :or {skip 0}}]
   1.191 -  (let [retinal-map (retina-sensor-image eye)
   1.192 -        camera (attach-eye creature eye)
   1.193 -        vision-image
   1.194 -        (atom
   1.195 -         (BufferedImage. (.getWidth camera)
   1.196 -                         (.getHeight camera)
   1.197 -                         BufferedImage/TYPE_BYTE_BINARY))]
   1.198 -    [(fn [world]
   1.199 -       (add-eye!
   1.200 -        world camera
   1.201 -        (let [counter  (atom 0)]
   1.202 -          (fn [r fb bb bi]
   1.203 -            (if (zero? (rem (swap! counter inc) (inc skip)))
   1.204 -              (reset! vision-image (BufferedImage! r fb bb bi)))))))
   1.205 -     (vec
   1.206 -      (map
   1.207 -       (fn [[key image]]
   1.208 -         (let [whites (white-coordinates image)
   1.209 -               topology (vec (collapse whites))
   1.210 -               mask (presets key)]
   1.211 -           (fn []
   1.212 -             (vector
   1.213 -              topology
   1.214 -              (vec 
   1.215 -               (for [[x y] whites]
   1.216 -                 (bit-and
   1.217 -                  mask (.getRGB @vision-image x y))))))))
   1.218 -       retinal-map))]))
   1.219 -
   1.220 -(defn vision
   1.221 -  [#^Node creature & {skip :skip :or {skip 0}}]
   1.222 -  (reduce
   1.223 -   (fn [[init-a senses-a]
   1.224 -        [init-b senses-b]]
   1.225 -     [(conj init-a init-b)
   1.226 -      (into senses-a senses-b)])
   1.227 -   [[][]]      
   1.228 -   (for [eye (eyes creature)]
   1.229 -     (enable-vision creature eye))))
   1.230 -
   1.231 -
   1.232 -#+end_src
   1.233 -
   1.234 -
   1.235 -Note the use of continuation passing style for connecting the eye to a
   1.236 -function to process the output. You can create any number of eyes, and
   1.237 -each of them will see the world from their own =Camera=. Once every
   1.238 -frame, the rendered image is copied to a =BufferedImage=, and that
   1.239 -data is sent off to the continuation function. Moving the =Camera=
   1.240 -which was used to create the eye will change what the eye sees.
   1.241 -
   1.242 -* Example
   1.243 -
   1.244 -#+name: test-vision
   1.245 -#+begin_src clojure
   1.246 -(ns cortex.test.vision
   1.247 -  (:use (cortex world util vision))
   1.248 -  (:import java.awt.image.BufferedImage)
   1.249 -  (:import javax.swing.JPanel)
   1.250 -  (:import javax.swing.SwingUtilities)
   1.251 -  (:import java.awt.Dimension)
   1.252 -  (:import javax.swing.JFrame)
   1.253 -  (:import com.jme3.math.ColorRGBA)
   1.254 -  (:import com.jme3.scene.Node)
   1.255 -  (:import com.jme3.math.Vector3f))
   1.256 -
   1.257 -(defn test-two-eyes
   1.258 -  "Testing vision:
   1.259 -   Tests the vision system by creating two views of the same rotating
   1.260 -   object from different angles and displaying both of those views in
   1.261 -   JFrames.
   1.262 -
   1.263 -   You should see a rotating cube, and two windows,
   1.264 -   each displaying a different view of the cube."
   1.265 -  []
   1.266 -  (let [candy
   1.267 -        (box 1 1 1 :physical? false :color ColorRGBA/Blue)]
   1.268 -    (world
   1.269 -     (doto (Node.)
   1.270 -       (.attachChild candy))
   1.271 -     {}
   1.272 -     (fn [world]
   1.273 -       (let [cam (.clone (.getCamera world))
   1.274 -             width (.getWidth cam)
   1.275 -             height (.getHeight cam)]
   1.276 -         (add-eye! world cam 
   1.277 -                  ;;no-op
   1.278 -                  (comp (view-image) BufferedImage!)
   1.279 -                  )
   1.280 -         (add-eye! world
   1.281 -                  (doto (.clone cam)
   1.282 -                    (.setLocation (Vector3f. -10 0 0))
   1.283 -                    (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))
   1.284 -                  ;;no-op
   1.285 -                  (comp (view-image) BufferedImage!))
   1.286 -         ;; This is here to restore the main view
   1.287 -         ;; after the other views have completed processing
   1.288 -         (add-eye! world (.getCamera world) no-op)))
   1.289 -     (fn [world tpf]
   1.290 -       (.rotate candy (* tpf 0.2) 0 0)))))
   1.291 -#+end_src
   1.292 -
   1.293 -#+results: test-vision
   1.294 -: #'cortex.test.vision/test-two-eyes
   1.295 -
   1.296 -The example code will create two videos of the same rotating object
   1.297 -from different angles. It can be used both for stereoscopic vision
   1.298 -simulation or for simulating multiple creatures, each with their own
   1.299 -sense of vision.
   1.300 -
   1.301 -- As a neat bonus, this idea behind simulated vision also enables one
   1.302 -  to [[../../cortex/html/capture-video.html][capture live video feeds from jMonkeyEngine]].
   1.303 -
   1.304 -
   1.305 -* COMMENT code generation
   1.306 -#+begin_src clojure :tangle ../src/cortex/vision.clj
   1.307 -<<eyes>>
   1.308 -#+end_src
   1.309 -
   1.310 -#+begin_src clojure :tangle ../src/cortex/test/vision.clj
   1.311 -<<test-vision>>
   1.312 -#+end_src

     2.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     2.2 +++ b/org/vision.org	Sat Feb 04 04:08:08 2012 -0700
     2.3 @@ -0,0 +1,309 @@
     2.4 +#+title: Simulated Sense of Sight
     2.5 +#+author: Robert McIntyre
     2.6 +#+email: rlm@mit.edu
     2.7 +#+description: Simulated sight for AI research using JMonkeyEngine3 and clojure
     2.8 +#+keywords: computer vision, jMonkeyEngine3, clojure
     2.9 +#+SETUPFILE: ../../aurellem/org/setup.org
    2.10 +#+INCLUDE: ../../aurellem/org/level-0.org
    2.11 +#+babel: :mkdirp yes :noweb yes :exports both
    2.12 +
    2.13 +* Vision
    2.14 +
    2.15 +I want to make creatures with eyes. Each eye can be independely moved
    2.16 +and should see its own version of the world depending on where it is.
    2.17 +
    2.18 +Here's how vision will work.
    2.19 +
    2.20 +Make the continuation in scene-processor take FrameBuffer,
    2.21 +byte-buffer, BufferedImage already sized to the correct
    2.22 +dimensions. the continuation will decide wether to "mix" them
    2.23 +into the BufferedImage, lazily ignore them, or mix them halfway
    2.24 +and call c/graphics card routines.
    2.25 +
    2.26 +(vision creature) will take an optional :skip argument which will
    2.27 +inform the continuations in scene processor to skip the given
    2.28 +number of cycles 0 means that no cycles will be skipped.
    2.29 +
    2.30 +(vision creature) will return [init-functions sensor-functions].
    2.31 +The init-functions are each single-arg functions that take the
    2.32 +world and register the cameras and must each be called before the
    2.33 +corresponding sensor-functions.  Each init-function returns the
    2.34 +viewport for that eye which can be manipulated, saved, etc. Each
    2.35 +sensor-function is a thunk and will return data in the same
    2.36 +format as the tactile-sensor functions the structure is
    2.37 +[topology, sensor-data]. Internally, these sensor-functions
    2.38 +maintain a reference to sensor-data which is periodically updated
    2.39 +by the continuation function established by its init-function.
    2.40 +They can be queried every cycle, but their information may not
    2.41 +necessairly be different every cycle.
    2.42 +
    2.43 +Each eye in the creature in blender will work the same way as
    2.44 +joints -- a zero dimensional object with no geometry whose local
    2.45 +coordinate system determines the orientation of the resulting
    2.46 +eye. All eyes will have a parent named "eyes" just as all joints
    2.47 +have a parent named "joints". The resulting camera will be a
    2.48 +ChaseCamera or a CameraNode bound to the geo that is closest to
    2.49 +the eye marker. The eye marker will contain the metadata for the
    2.50 +eye, and will be moved by it's bound geometry. The dimensions of
    2.51 +the eye's camera are equal to the dimensions of the eye's "UV"
    2.52 +map.
    2.53 +
    2.54 +#+name: eyes
    2.55 +#+begin_src clojure 
    2.56 +(ns cortex.vision
    2.57 +  "Simulate the sense of vision in jMonkeyEngine3. Enables multiple
    2.58 +  eyes from different positions to observe the same world, and pass
    2.59 +  the observed data to any arbitray function."
    2.60 +  {:author "Robert McIntyre"}
    2.61 +  (:use (cortex world sense util))
    2.62 +  (:use clojure.contrib.def)
    2.63 +  (:import com.jme3.post.SceneProcessor)
    2.64 +  (:import (com.jme3.util BufferUtils Screenshots))
    2.65 +  (:import java.nio.ByteBuffer)
    2.66 +  (:import java.awt.image.BufferedImage)
    2.67 +  (:import com.jme3.renderer.ViewPort)
    2.68 +  (:import com.jme3.math.ColorRGBA)
    2.69 +  (:import com.jme3.renderer.Renderer)
    2.70 +  (:import com.jme3.scene.Node))
    2.71 +
    2.72 +(cortex.import/mega-import-jme3)
    2.73 +
    2.74 +
    2.75 +(defn vision-pipeline
    2.76 +  "Create a SceneProcessor object which wraps a vision processing
    2.77 +  continuation function. The continuation is a function that takes 
    2.78 +  [#^Renderer r #^FrameBuffer fb #^ByteBuffer b #^BufferedImage bi],
    2.79 +  each of which has already been appropiately sized."
    2.80 +  [continuation]
    2.81 +  (let [byte-buffer (atom nil)
    2.82 +	renderer (atom nil)
    2.83 +        image (atom nil)]
    2.84 +  (proxy [SceneProcessor] []
    2.85 +    (initialize
    2.86 +     [renderManager viewPort]
    2.87 +     (let [cam (.getCamera viewPort)
    2.88 +	   width (.getWidth cam)
    2.89 +	   height (.getHeight cam)]
    2.90 +       (reset! renderer (.getRenderer renderManager))
    2.91 +       (reset! byte-buffer
    2.92 +	     (BufferUtils/createByteBuffer
    2.93 +	      (* width height 4)))
    2.94 +        (reset! image (BufferedImage.
    2.95 +                      width height
    2.96 +                      BufferedImage/TYPE_4BYTE_ABGR))))
    2.97 +    (isInitialized [] (not (nil? @byte-buffer)))
    2.98 +    (reshape [_ _ _])
    2.99 +    (preFrame [_])
   2.100 +    (postQueue [_])
   2.101 +    (postFrame
   2.102 +     [#^FrameBuffer fb]
   2.103 +     (.clear @byte-buffer)
   2.104 +     (continuation @renderer fb @byte-buffer @image))
   2.105 +    (cleanup []))))
   2.106 +    
   2.107 +(defn frameBuffer->byteBuffer!
   2.108 +  "Transfer the data in the graphics card (Renderer, FrameBuffer) to
   2.109 +   the CPU (ByteBuffer)."  
   2.110 +  [#^Renderer r #^FrameBuffer fb #^ByteBuffer bb]
   2.111 +  (.readFrameBuffer r fb bb) bb)
   2.112 +
   2.113 +(defn byteBuffer->bufferedImage!
   2.114 +  "Convert the C-style BGRA image data in the ByteBuffer bb to the AWT
   2.115 +   style ABGR image data and place it in BufferedImage bi."
   2.116 +  [#^ByteBuffer bb #^BufferedImage bi]
   2.117 +  (Screenshots/convertScreenShot bb bi) bi)
   2.118 +
   2.119 +(defn BufferedImage!
   2.120 +  "Continuation which will grab the buffered image from the materials
   2.121 +   provided by (vision-pipeline)."
   2.122 +  [#^Renderer r #^FrameBuffer fb #^ByteBuffer bb #^BufferedImage bi]
   2.123 +  (byteBuffer->bufferedImage!
   2.124 +   (frameBuffer->byteBuffer! r fb bb) bi))
   2.125 +
   2.126 +(defn add-eye!
   2.127 +  "Add an eye to the world, calling continuation on every frame
   2.128 +  produced." 
   2.129 +  [#^Application world camera continuation]
   2.130 +  (let [width (.getWidth camera)
   2.131 +	height (.getHeight camera)
   2.132 +	render-manager (.getRenderManager world)
   2.133 +	viewport (.createMainView render-manager "eye-view" camera)]
   2.134 +    (doto viewport
   2.135 +      (.setClearFlags true true true)
   2.136 +      (.setBackgroundColor ColorRGBA/Black)
   2.137 +      (.addProcessor (vision-pipeline continuation))
   2.138 +      (.attachScene (.getRootNode world)))))
   2.139 +
   2.140 +(defn retina-sensor-image
   2.141 +  "Return a map of pixel selection functions to BufferedImages
   2.142 +   describing the distribution of light-sensitive components on this
   2.143 +   geometry's surface. Each function creates an integer from the rgb
   2.144 +   values found in the pixel. :red, :green, :blue, :gray are already
   2.145 +   defined as extracting the red green blue and average components
   2.146 +   respectively."
   2.147 +   [#^Spatial eye]
   2.148 +   (if-let [eye-map (meta-data eye "eye")]
   2.149 +     (map-vals
   2.150 +      load-image
   2.151 +      (eval (read-string eye-map)))))
   2.152 +
   2.153 +(defn eye-dimensions
   2.154 +  "returns the width and height specified in the metadata of the eye"
   2.155 +  [#^Spatial eye]
   2.156 +  (let [dimensions
   2.157 +          (map #(vector (.getWidth %) (.getHeight %))
   2.158 +               (vals (retina-sensor-image eye)))]
   2.159 +    [(apply max (map first dimensions))
   2.160 +     (apply max (map second dimensions))]))
   2.161 +
   2.162 +(defvar 
   2.163 +  ^{:arglists '([creature])}
   2.164 +  eyes
   2.165 +  (sense-nodes "eyes")
   2.166 +  "Return the children of the creature's \"eyes\" node.")
   2.167 +
   2.168 +(defn attach-eye
   2.169 +  "Attach a Camera to the appropiate area and return the Camera."
   2.170 +  [#^Node creature #^Spatial eye]
   2.171 +  (let [target (closest-node creature eye)
   2.172 +        [cam-width cam-height] (eye-dimensions eye)
   2.173 +        cam (Camera. cam-width cam-height)]
   2.174 +    (.setLocation cam (.getWorldTranslation eye))
   2.175 +    (.setRotation cam (.getWorldRotation eye))
   2.176 +    (.setFrustumPerspective
   2.177 +     cam 45 (/ (.getWidth cam) (.getHeight cam))
   2.178 +     1 1000)
   2.179 +    (bind-sense target cam)
   2.180 +    cam))
   2.181 +
   2.182 +(def presets
   2.183 +  {:all    0xFFFFFF
   2.184 +   :red    0xFF0000
   2.185 +   :blue   0x0000FF
   2.186 +   :green  0x00FF00})
   2.187 +
   2.188 +(defn enable-vision
   2.189 +  "return [init-function sensor-functions] for a particular eye"
   2.190 +  [#^Node creature #^Spatial eye & {skip :skip :or {skip 0}}]
   2.191 +  (let [retinal-map (retina-sensor-image eye)
   2.192 +        camera (attach-eye creature eye)
   2.193 +        vision-image
   2.194 +        (atom
   2.195 +         (BufferedImage. (.getWidth camera)
   2.196 +                         (.getHeight camera)
   2.197 +                         BufferedImage/TYPE_BYTE_BINARY))]
   2.198 +    [(fn [world]
   2.199 +       (add-eye!
   2.200 +        world camera
   2.201 +        (let [counter  (atom 0)]
   2.202 +          (fn [r fb bb bi]
   2.203 +            (if (zero? (rem (swap! counter inc) (inc skip)))
   2.204 +              (reset! vision-image (BufferedImage! r fb bb bi)))))))
   2.205 +     (vec
   2.206 +      (map
   2.207 +       (fn [[key image]]
   2.208 +         (let [whites (white-coordinates image)
   2.209 +               topology (vec (collapse whites))
   2.210 +               mask (presets key)]
   2.211 +           (fn []
   2.212 +             (vector
   2.213 +              topology
   2.214 +              (vec 
   2.215 +               (for [[x y] whites]
   2.216 +                 (bit-and
   2.217 +                  mask (.getRGB @vision-image x y))))))))
   2.218 +       retinal-map))]))
   2.219 +
   2.220 +(defn vision
   2.221 +  [#^Node creature & {skip :skip :or {skip 0}}]
   2.222 +  (reduce
   2.223 +   (fn [[init-a senses-a]
   2.224 +        [init-b senses-b]]
   2.225 +     [(conj init-a init-b)
   2.226 +      (into senses-a senses-b)])
   2.227 +   [[][]]      
   2.228 +   (for [eye (eyes creature)]
   2.229 +     (enable-vision creature eye))))
   2.230 +
   2.231 +
   2.232 +#+end_src
   2.233 +
   2.234 +
   2.235 +Note the use of continuation passing style for connecting the eye to a
   2.236 +function to process the output. You can create any number of eyes, and
   2.237 +each of them will see the world from their own =Camera=. Once every
   2.238 +frame, the rendered image is copied to a =BufferedImage=, and that
   2.239 +data is sent off to the continuation function. Moving the =Camera=
   2.240 +which was used to create the eye will change what the eye sees.
   2.241 +
   2.242 +* Example
   2.243 +
   2.244 +#+name: test-vision
   2.245 +#+begin_src clojure
   2.246 +(ns cortex.test.vision
   2.247 +  (:use (cortex world util vision))
   2.248 +  (:import java.awt.image.BufferedImage)
   2.249 +  (:import javax.swing.JPanel)
   2.250 +  (:import javax.swing.SwingUtilities)
   2.251 +  (:import java.awt.Dimension)
   2.252 +  (:import javax.swing.JFrame)
   2.253 +  (:import com.jme3.math.ColorRGBA)
   2.254 +  (:import com.jme3.scene.Node)
   2.255 +  (:import com.jme3.math.Vector3f))
   2.256 +
   2.257 +(defn test-two-eyes
   2.258 +  "Testing vision:
   2.259 +   Tests the vision system by creating two views of the same rotating
   2.260 +   object from different angles and displaying both of those views in
   2.261 +   JFrames.
   2.262 +
   2.263 +   You should see a rotating cube, and two windows,
   2.264 +   each displaying a different view of the cube."
   2.265 +  []
   2.266 +  (let [candy
   2.267 +        (box 1 1 1 :physical? false :color ColorRGBA/Blue)]
   2.268 +    (world
   2.269 +     (doto (Node.)
   2.270 +       (.attachChild candy))
   2.271 +     {}
   2.272 +     (fn [world]
   2.273 +       (let [cam (.clone (.getCamera world))
   2.274 +             width (.getWidth cam)
   2.275 +             height (.getHeight cam)]
   2.276 +         (add-eye! world cam 
   2.277 +                  ;;no-op
   2.278 +                  (comp (view-image) BufferedImage!)
   2.279 +                  )
   2.280 +         (add-eye! world
   2.281 +                  (doto (.clone cam)
   2.282 +                    (.setLocation (Vector3f. -10 0 0))
   2.283 +                    (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))
   2.284 +                  ;;no-op
   2.285 +                  (comp (view-image) BufferedImage!))
   2.286 +         ;; This is here to restore the main view
   2.287 +         ;; after the other views have completed processing
   2.288 +         (add-eye! world (.getCamera world) no-op)))
   2.289 +     (fn [world tpf]
   2.290 +       (.rotate candy (* tpf 0.2) 0 0)))))
   2.291 +#+end_src
   2.292 +
   2.293 +#+results: test-vision
   2.294 +: #'cortex.test.vision/test-two-eyes
   2.295 +
   2.296 +The example code will create two videos of the same rotating object
   2.297 +from different angles. It can be used both for stereoscopic vision
   2.298 +simulation or for simulating multiple creatures, each with their own
   2.299 +sense of vision.
   2.300 +
   2.301 +- As a neat bonus, this idea behind simulated vision also enables one
   2.302 +  to [[../../cortex/html/capture-video.html][capture live video feeds from jMonkeyEngine]].
   2.303 +
   2.304 +
   2.305 +* COMMENT code generation
   2.306 +#+begin_src clojure :tangle ../src/cortex/vision.clj
   2.307 +<<eyes>>
   2.308 +#+end_src
   2.309 +
   2.310 +#+begin_src clojure :tangle ../src/cortex/test/vision.clj
   2.311 +<<test-vision>>
   2.312 +#+end_src