1.1 --- a/org/eyes.org	Wed Nov 02 11:03:12 2011 -0700
     1.2 +++ b/org/eyes.org	Thu Nov 03 08:28:26 2011 -0700
     1.3 @@ -1,47 +1,37 @@
     1.4 -#+title: Eyes
     1.5 +#+title: Simulated Sense of Sight
     1.6  #+author: Robert McIntyre
     1.7  #+email: rlm@mit.edu
     1.8 -#+description: Simulating senses for AI research using JMonkeyEngine3
     1.9 +#+description: Simulated sight for AI research using JMonkeyEngine3
    1.10 +#+keywords: computer vision, jMonkeyEngine3, clojure
    1.11  #+SETUPFILE: ../../aurellem/org/setup.org
    1.12  #+INCLUDE: ../../aurellem/org/level-0.org
    1.13  #+babel: :mkdirp yes :noweb yes :exports both
    1.14  
    1.15 +* Vision
    1.16  
    1.17 -
    1.18 -** Eyes
    1.19 -
    1.20 -Ultimately I want to make creatures with eyes. Each eye can be
    1.21 -independely moved and should see its own version of the world
    1.22 -depending on where it is.
    1.23 +I want to make creatures with eyes. Each eye can be independely moved
    1.24 +and should see its own version of the world depending on where it is.
    1.25  #+srcname: eyes
    1.26  #+begin_src clojure 
    1.27 -(ns body.eye)
    1.28 -(use 'cortex.world)
    1.29 -(use 'cortex.import)
    1.30 -(use 'clojure.contrib.def)
    1.31 -(cortex.import/mega-import-jme3)
    1.32 -(rlm.rlm-commands/help)
    1.33 -(import java.nio.ByteBuffer)
    1.34 -(import java.awt.image.BufferedImage)
    1.35 -(import java.awt.Color)
    1.36 -(import java.awt.Dimension)
    1.37 -(import java.awt.Graphics)
    1.38 -(import java.awt.Graphics2D)
    1.39 -(import java.awt.event.WindowAdapter)
    1.40 -(import java.awt.event.WindowEvent)
    1.41 -(import java.awt.image.BufferedImage)
    1.42 -(import java.nio.ByteBuffer)
    1.43 -(import javax.swing.JFrame)
    1.44 -(import javax.swing.JPanel)
    1.45 -(import javax.swing.SwingUtilities)
    1.46 -(import javax.swing.ImageIcon)
    1.47 -(import javax.swing.JOptionPane)
    1.48 -(import java.awt.image.ImageObserver)
    1.49 +(ns cortex.vision
    1.50 +  "Simulate the sense of vision in jMonkeyEngine3. Enables multiple
    1.51 +  eyes from different positions to observe the same world, and pass
    1.52 +  the observed data to any arbitray function."
    1.53 +  {:author "Robert McIntyre"}
    1.54 +  (:use cortex.world)
    1.55 +  (:import com.jme3.post.SceneProcessor)
    1.56 +  (:import (com.jme3.util Screenshots BufferUtils))
    1.57 +  (:import java.nio.ByteBuffer)
    1.58 +  (:import java.awt.image.BufferedImage)
    1.59 +  (:import com.jme3.renderer.ViewPort)
    1.60 +  (:import com.jme3.math.ColorRGBA))
    1.61  
    1.62  (defn scene-processor
    1.63 -  "deals with converting FrameBuffers to BufferedImages so
    1.64 -   that the continuation function can be defined only in terms
    1.65 -   of what it does with BufferedImages"
    1.66 +  "Create a SceneProcessor object which wraps a vision processing
    1.67 +  continuation function. The SceneProcessor will take care of
    1.68 +  converting the rendered frame to a BufferedImage and passing that
    1.69 +  BufferedImage to the continuation.  The continuation should be a
    1.70 +  function that takes a BufferedImage."
    1.71    [continuation]
    1.72    (let [byte-buffer (atom nil)
    1.73  	renderer (atom nil)
    1.74 @@ -56,8 +46,9 @@
    1.75         (reset! byte-buffer
    1.76  	     (BufferUtils/createByteBuffer
    1.77  	      (* width height 4)))
    1.78 -       (reset! image (BufferedImage. width height
    1.79 -				     BufferedImage/TYPE_4BYTE_ABGR))))
    1.80 +       (reset! image (BufferedImage.
    1.81 +                      width height
    1.82 +                      BufferedImage/TYPE_4BYTE_ABGR))))
    1.83      (isInitialized [] (not (nil? @byte-buffer)))
    1.84      (reshape [_ _ _])
    1.85      (preFrame [_])
    1.86 @@ -71,179 +62,108 @@
    1.87      (cleanup []))))
    1.88      
    1.89  (defn add-eye
    1.90 -  "Add an eye to the world, and call continuation on
    1.91 -   every frame produced"
    1.92 +  "Add an eye to the world, calling continuation on every frame
    1.93 +  produced." 
    1.94    [world camera continuation]
    1.95    (let [width (.getWidth camera)
    1.96  	height (.getHeight camera)
    1.97  	render-manager (.getRenderManager world)
    1.98  	viewport (.createMainView render-manager "eye-view" camera)]
    1.99      (doto viewport
   1.100 -      (.setBackgroundColor ColorRGBA/Black)
   1.101        (.setClearFlags true true true)
   1.102 +      (.setBackgroundColor ColorRGBA/Gray)
   1.103        (.addProcessor (scene-processor continuation))
   1.104        (.attachScene (.getRootNode world)))))
   1.105  
   1.106 -(defn make-display-frame [display width height]
   1.107 -  (SwingUtilities/invokeLater
   1.108 -   (fn []
   1.109 -     (.setPreferredSize display (Dimension. width height))
   1.110 -     (doto (JFrame. "Eye Camera!")
   1.111 -       (-> (.getContentPane) (.add display))
   1.112 -       (.setDefaultCloseOperation JFrame/DISPOSE_ON_CLOSE)
   1.113 -       (.pack)
   1.114 -       (.setLocationRelativeTo nil)
   1.115 -       (.setResizable false)
   1.116 -       (.setVisible true)))))
   1.117 -	   
   1.118 -(defn image-monitor [#^BufferedImage image]
   1.119 -  (proxy [JPanel] []
   1.120 -    (paintComponent 
   1.121 -     [g]
   1.122 -     (proxy-super paintComponent g)
   1.123 -     (locking image
   1.124 -       (.drawImage g image 0 0
   1.125 -		   (proxy [ImageObserver]
   1.126 -		       []
   1.127 -		     (imageUpdate
   1.128 -		      []
   1.129 -		      (proxy-super imageUpdate))))))))
   1.130 +#+end_src
   1.131  
   1.132 -(defn movie-image []
   1.133 -  (let [setup
   1.134 -	(runonce
   1.135 -	 (fn [#^BufferedImage image]
   1.136 -	   (let [width (.getWidth image)
   1.137 -		 height (.getHeight image)
   1.138 -		 display (image-monitor image)
   1.139 -		 frame (make-display-frame display width height)]
   1.140 -	     display)))]
   1.141 -    (fn [#^BufferedImage image]
   1.142 -      (.repaint (setup image)))))
   1.143 -	     
   1.144 +Note the use of continuation passing style for connecting the eye to a
   1.145 +function to process the output. You can create any number of eyes, and
   1.146 +each of them will see the world from their own =Camera=. Once every
   1.147 +frame, the rendered image is copied to a =BufferedImage=, and that
   1.148 +data is sent off to the continuation function. Moving the =Camera=
   1.149 +which was used to create the eye will change what the eye sees.
   1.150  
   1.151 -(defn observer
   1.152 -  "place thy eye!"
   1.153 -  [world camera]
   1.154 -  (let [eye camera
   1.155 -	width (.getWidth eye)
   1.156 -	height (.getHeight eye)]
   1.157 -    (no-exceptions
   1.158 -     (add-eye
   1.159 -      world
   1.160 -      eye
   1.161 -      (movie-image)))))
   1.162 -#+end_src
   1.163 +* Example
   1.164  
   1.165  #+srcname: test-vision
   1.166  #+begin_src clojure
   1.167 +(ns test.vision
   1.168 +  (:use (cortex world util vision))
   1.169 +  (:import java.awt.image.BufferedImage)
   1.170 +  (:import javax.swing.JPanel)
   1.171 +  (:import javax.swing.SwingUtilities)
   1.172 +  (:import java.awt.Dimension)
   1.173 +  (:import javax.swing.JFrame)
   1.174 +  (:import com.jme3.math.ColorRGBA)
   1.175 +  (:import com.jme3.scene.Node))
   1.176  
   1.177 -(ns test.vision)
   1.178 -(use 'cortex.world)
   1.179 -(use 'cortex.import)
   1.180 -(use 'clojure.contrib.def)
   1.181 -(use 'body.eye)
   1.182 -(cortex.import/mega-import-jme3)
   1.183 -(rlm.rlm-commands/help)
   1.184 -(import java.nio.ByteBuffer)
   1.185 -(import java.awt.image.BufferedImage)
   1.186 -(import java.awt.Color)
   1.187 -(import java.awt.Dimension)
   1.188 -(import java.awt.Graphics)
   1.189 -(import java.awt.Graphics2D)
   1.190 -(import java.awt.event.WindowAdapter)
   1.191 -(import java.awt.event.WindowEvent)
   1.192 -(import java.awt.image.BufferedImage)
   1.193 -(import java.nio.ByteBuffer)
   1.194 -(import javax.swing.JFrame)
   1.195 -(import javax.swing.JPanel)
   1.196 -(import javax.swing.SwingUtilities)
   1.197 -(import javax.swing.ImageIcon)
   1.198 -(import javax.swing.JOptionPane)
   1.199 -(import java.awt.image.ImageObserver)
   1.200 +(defn view-image
   1.201 +  "Initailizes a JPanel on which you may draw a BufferedImage of the
   1.202 +  given width and height.  Returns a function that accepts a
   1.203 +  BufferedImage and draws it to the JPanel."
   1.204 +  [width height]
   1.205 +  (let [image
   1.206 +        (atom
   1.207 +         (BufferedImage. width height BufferedImage/TYPE_4BYTE_ABGR))
   1.208 +        panel 
   1.209 +        (proxy [JPanel] []
   1.210 +          (paint
   1.211 +            [graphics]
   1.212 +            (proxy-super paintComponent graphics)
   1.213 +            (.drawImage graphics @image 0 0 nil)
   1.214 +            ))]
   1.215 +    
   1.216 +    (SwingUtilities/invokeLater
   1.217 +     (fn []
   1.218 +       (.setPreferredSize panel (Dimension. width height))
   1.219 +       (doto (JFrame. "Eye Camera!")
   1.220 +         (-> (.getContentPane) (.add panel))
   1.221 +         (.pack)
   1.222 +         (.setLocationRelativeTo nil)
   1.223 +         (.setResizable false)
   1.224 +         (.setVisible true))))
   1.225 +    (fn [#^BufferedImage i]
   1.226 +      (reset! image i)
   1.227 +      (.repaint panel))))
   1.228  
   1.229 -
   1.230 -(def width 200)
   1.231 -(def height 200)
   1.232 -
   1.233 -(defn camera []
   1.234 -  (doto (Camera. width height)
   1.235 -    (.setFrustumPerspective 45 1 1 1000)
   1.236 -    (.setLocation (Vector3f. -3 0 -5))
   1.237 -    (.lookAt Vector3f/ZERO Vector3f/UNIT_Y)))
   1.238 -
   1.239 -(defn camera2 []
   1.240 -  (doto (Camera. width height)
   1.241 -    (.setFrustumPerspective 45 1 1 1000)
   1.242 -    (.setLocation (Vector3f. 3 0 -5))
   1.243 -    (.lookAt Vector3f/ZERO Vector3f/UNIT_Y)))
   1.244 -
   1.245 -(defn setup-fn [world]
   1.246 -  (let [eye (camera)
   1.247 -	width (.getWidth eye)
   1.248 -	height (.getHeight eye)]
   1.249 -    (no-exceptions
   1.250 -     (add-eye
   1.251 -      world
   1.252 -      eye
   1.253 -      (runonce visual))
   1.254 -     (add-eye
   1.255 -      world
   1.256 -      (camera2)
   1.257 -      (runonce visual)))))
   1.258 -
   1.259 -(defn spider-eye [position]
   1.260 -  (doto (Camera. 200 200 )
   1.261 -    (.setFrustumPerspective 45 1 1 1000)
   1.262 -    (.setLocation position)
   1.263 -    (.lookAt Vector3f/ZERO Vector3f/UNIT_Y)))
   1.264 -  
   1.265 -(defn setup-fn* [world]
   1.266 -  (let [eye (camera)
   1.267 -	width (.getWidth eye)
   1.268 -	height (.getHeight eye)]
   1.269 -    ;;(.setClearFlags (.getViewPort world) true true true)
   1.270 -    (observer world (.getCamera world))
   1.271 -    (observer world (spider-eye (Vector3f.  3  0 -5)))
   1.272 -    ;;(observer world (spider-eye (Vector3f.  0  0 -5)))
   1.273 -    ;; (observer world (spider-eye (Vector3f. -3  0 -5)))
   1.274 -    ;; (observer world (spider-eye (Vector3f.  0  3 -5)))
   1.275 -    ;; (observer world (spider-eye (Vector3f.  0 -3 -5)))
   1.276 -    ;; (observer world (spider-eye (Vector3f.  3  3 -5)))
   1.277 -    ;; (observer world (spider-eye (Vector3f. -3  3 -5)))
   1.278 -    ;; (observer world (spider-eye (Vector3f.  3 -3 -5)))
   1.279 -    ;; (observer world (spider-eye (Vector3f. -3 -3 -5)))
   1.280 -
   1.281 -    )
   1.282 -  world)
   1.283 -
   1.284 -(defn test-world []
   1.285 -  (let [thing (box 1 1 1 :physical? false)]
   1.286 -    (world
   1.287 -     (doto (Node.)
   1.288 -       (.attachChild thing))
   1.289 -     {}
   1.290 -     setup-fn
   1.291 -     (fn [world tpf]
   1.292 -       (.rotate thing (* tpf 0.2) 0 0)
   1.293 -       ))))
   1.294 -
   1.295 -
   1.296 +(defn test-vision 
   1.297 +  "Tests the vision system by creating two views of the same rotating
   1.298 +  object from different angles and displaying both of those views in
   1.299 +  JFrames."  []
   1.300 +  (.start
   1.301 +   (let [candy
   1.302 +         (box 1 1 1 :physical? false :color ColorRGBA/Blue)]
   1.303 +     (world (doto (Node.)
   1.304 +              (.attachChild candy))
   1.305 +            {}
   1.306 +            (fn [world]
   1.307 +              (let [cam (.clone (.getCamera world))
   1.308 +                    width (.getWidth cam)
   1.309 +                    height (.getHeight cam)]
   1.310 +                (add-eye world
   1.311 +                         (doto (.clone cam)
   1.312 +                           (.setLocation (Vector3f. -10 0 0))
   1.313 +                           (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))
   1.314 +                         (view-image width height))
   1.315 +                (add-eye world cam
   1.316 +                         (view-image width height))
   1.317 +                ;; this is here to restore the main view
   1.318 +                ;; after the other views have completed processing
   1.319 +                (add-eye world (.getCamera world) no-op)
   1.320 +                ))
   1.321 +            (fn [world tpf]
   1.322 +              (.rotate candy (* tpf 0.2) 0 0))))))
   1.323  #+end_src
   1.324  
   1.325 -
   1.326 -#+results: eyes
   1.327 -: #'body.eye/test-world
   1.328 -
   1.329 -Note the use of continuation passing style for connecting the eye to a
   1.330 -function to process the output. The example code will create two
   1.331 -videos of the same rotating cube from different angles, sutiable for
   1.332 -stereoscopic vision.
   1.333 -
   1.334 +The example code will create two videos of the same rotating object
   1.335 +from different angles. It can be used both for stereoscopic vision
   1.336 +simulation or for simulating multiple creatures, each with their own
   1.337 +sense of vision.
   1.338  
   1.339  
   1.340  * COMMENT code generation
   1.341 -#+begin_src clojure :tangle ../src/body/eye.clj
   1.342 +#+begin_src clojure :tangle ../src/cortex/vision.clj
   1.343  <<eyes>>
   1.344  #+end_src
   1.345