1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/org/vision.org	Sat Feb 04 04:08:08 2012 -0700
     1.3 @@ -0,0 +1,309 @@
     1.4 +#+title: Simulated Sense of Sight
     1.5 +#+author: Robert McIntyre
     1.6 +#+email: rlm@mit.edu
     1.7 +#+description: Simulated sight for AI research using JMonkeyEngine3 and clojure
     1.8 +#+keywords: computer vision, jMonkeyEngine3, clojure
     1.9 +#+SETUPFILE: ../../aurellem/org/setup.org
    1.10 +#+INCLUDE: ../../aurellem/org/level-0.org
    1.11 +#+babel: :mkdirp yes :noweb yes :exports both
    1.12 +
    1.13 +* Vision
    1.14 +
    1.15 +I want to make creatures with eyes. Each eye can be independely moved
    1.16 +and should see its own version of the world depending on where it is.
    1.17 +
    1.18 +Here's how vision will work.
    1.19 +
    1.20 +Make the continuation in scene-processor take FrameBuffer,
    1.21 +byte-buffer, BufferedImage already sized to the correct
    1.22 +dimensions. the continuation will decide wether to "mix" them
    1.23 +into the BufferedImage, lazily ignore them, or mix them halfway
    1.24 +and call c/graphics card routines.
    1.25 +
    1.26 +(vision creature) will take an optional :skip argument which will
    1.27 +inform the continuations in scene processor to skip the given
    1.28 +number of cycles 0 means that no cycles will be skipped.
    1.29 +
    1.30 +(vision creature) will return [init-functions sensor-functions].
    1.31 +The init-functions are each single-arg functions that take the
    1.32 +world and register the cameras and must each be called before the
    1.33 +corresponding sensor-functions.  Each init-function returns the
    1.34 +viewport for that eye which can be manipulated, saved, etc. Each
    1.35 +sensor-function is a thunk and will return data in the same
    1.36 +format as the tactile-sensor functions the structure is
    1.37 +[topology, sensor-data]. Internally, these sensor-functions
    1.38 +maintain a reference to sensor-data which is periodically updated
    1.39 +by the continuation function established by its init-function.
    1.40 +They can be queried every cycle, but their information may not
    1.41 +necessairly be different every cycle.
    1.42 +
    1.43 +Each eye in the creature in blender will work the same way as
    1.44 +joints -- a zero dimensional object with no geometry whose local
    1.45 +coordinate system determines the orientation of the resulting
    1.46 +eye. All eyes will have a parent named "eyes" just as all joints
    1.47 +have a parent named "joints". The resulting camera will be a
    1.48 +ChaseCamera or a CameraNode bound to the geo that is closest to
    1.49 +the eye marker. The eye marker will contain the metadata for the
    1.50 +eye, and will be moved by it's bound geometry. The dimensions of
    1.51 +the eye's camera are equal to the dimensions of the eye's "UV"
    1.52 +map.
    1.53 +
    1.54 +#+name: eyes
    1.55 +#+begin_src clojure 
    1.56 +(ns cortex.vision
    1.57 +  "Simulate the sense of vision in jMonkeyEngine3. Enables multiple
    1.58 +  eyes from different positions to observe the same world, and pass
    1.59 +  the observed data to any arbitray function."
    1.60 +  {:author "Robert McIntyre"}
    1.61 +  (:use (cortex world sense util))
    1.62 +  (:use clojure.contrib.def)
    1.63 +  (:import com.jme3.post.SceneProcessor)
    1.64 +  (:import (com.jme3.util BufferUtils Screenshots))
    1.65 +  (:import java.nio.ByteBuffer)
    1.66 +  (:import java.awt.image.BufferedImage)
    1.67 +  (:import com.jme3.renderer.ViewPort)
    1.68 +  (:import com.jme3.math.ColorRGBA)
    1.69 +  (:import com.jme3.renderer.Renderer)
    1.70 +  (:import com.jme3.scene.Node))
    1.71 +
    1.72 +(cortex.import/mega-import-jme3)
    1.73 +
    1.74 +
    1.75 +(defn vision-pipeline
    1.76 +  "Create a SceneProcessor object which wraps a vision processing
    1.77 +  continuation function. The continuation is a function that takes 
    1.78 +  [#^Renderer r #^FrameBuffer fb #^ByteBuffer b #^BufferedImage bi],
    1.79 +  each of which has already been appropiately sized."
    1.80 +  [continuation]
    1.81 +  (let [byte-buffer (atom nil)
    1.82 +	renderer (atom nil)
    1.83 +        image (atom nil)]
    1.84 +  (proxy [SceneProcessor] []
    1.85 +    (initialize
    1.86 +     [renderManager viewPort]
    1.87 +     (let [cam (.getCamera viewPort)
    1.88 +	   width (.getWidth cam)
    1.89 +	   height (.getHeight cam)]
    1.90 +       (reset! renderer (.getRenderer renderManager))
    1.91 +       (reset! byte-buffer
    1.92 +	     (BufferUtils/createByteBuffer
    1.93 +	      (* width height 4)))
    1.94 +        (reset! image (BufferedImage.
    1.95 +                      width height
    1.96 +                      BufferedImage/TYPE_4BYTE_ABGR))))
    1.97 +    (isInitialized [] (not (nil? @byte-buffer)))
    1.98 +    (reshape [_ _ _])
    1.99 +    (preFrame [_])
   1.100 +    (postQueue [_])
   1.101 +    (postFrame
   1.102 +     [#^FrameBuffer fb]
   1.103 +     (.clear @byte-buffer)
   1.104 +     (continuation @renderer fb @byte-buffer @image))
   1.105 +    (cleanup []))))
   1.106 +    
   1.107 +(defn frameBuffer->byteBuffer!
   1.108 +  "Transfer the data in the graphics card (Renderer, FrameBuffer) to
   1.109 +   the CPU (ByteBuffer)."  
   1.110 +  [#^Renderer r #^FrameBuffer fb #^ByteBuffer bb]
   1.111 +  (.readFrameBuffer r fb bb) bb)
   1.112 +
   1.113 +(defn byteBuffer->bufferedImage!
   1.114 +  "Convert the C-style BGRA image data in the ByteBuffer bb to the AWT
   1.115 +   style ABGR image data and place it in BufferedImage bi."
   1.116 +  [#^ByteBuffer bb #^BufferedImage bi]
   1.117 +  (Screenshots/convertScreenShot bb bi) bi)
   1.118 +
   1.119 +(defn BufferedImage!
   1.120 +  "Continuation which will grab the buffered image from the materials
   1.121 +   provided by (vision-pipeline)."
   1.122 +  [#^Renderer r #^FrameBuffer fb #^ByteBuffer bb #^BufferedImage bi]
   1.123 +  (byteBuffer->bufferedImage!
   1.124 +   (frameBuffer->byteBuffer! r fb bb) bi))
   1.125 +
   1.126 +(defn add-eye!
   1.127 +  "Add an eye to the world, calling continuation on every frame
   1.128 +  produced." 
   1.129 +  [#^Application world camera continuation]
   1.130 +  (let [width (.getWidth camera)
   1.131 +	height (.getHeight camera)
   1.132 +	render-manager (.getRenderManager world)
   1.133 +	viewport (.createMainView render-manager "eye-view" camera)]
   1.134 +    (doto viewport
   1.135 +      (.setClearFlags true true true)
   1.136 +      (.setBackgroundColor ColorRGBA/Black)
   1.137 +      (.addProcessor (vision-pipeline continuation))
   1.138 +      (.attachScene (.getRootNode world)))))
   1.139 +
   1.140 +(defn retina-sensor-image
   1.141 +  "Return a map of pixel selection functions to BufferedImages
   1.142 +   describing the distribution of light-sensitive components on this
   1.143 +   geometry's surface. Each function creates an integer from the rgb
   1.144 +   values found in the pixel. :red, :green, :blue, :gray are already
   1.145 +   defined as extracting the red green blue and average components
   1.146 +   respectively."
   1.147 +   [#^Spatial eye]
   1.148 +   (if-let [eye-map (meta-data eye "eye")]
   1.149 +     (map-vals
   1.150 +      load-image
   1.151 +      (eval (read-string eye-map)))))
   1.152 +
   1.153 +(defn eye-dimensions
   1.154 +  "returns the width and height specified in the metadata of the eye"
   1.155 +  [#^Spatial eye]
   1.156 +  (let [dimensions
   1.157 +          (map #(vector (.getWidth %) (.getHeight %))
   1.158 +               (vals (retina-sensor-image eye)))]
   1.159 +    [(apply max (map first dimensions))
   1.160 +     (apply max (map second dimensions))]))
   1.161 +
   1.162 +(defvar 
   1.163 +  ^{:arglists '([creature])}
   1.164 +  eyes
   1.165 +  (sense-nodes "eyes")
   1.166 +  "Return the children of the creature's \"eyes\" node.")
   1.167 +
   1.168 +(defn attach-eye
   1.169 +  "Attach a Camera to the appropiate area and return the Camera."
   1.170 +  [#^Node creature #^Spatial eye]
   1.171 +  (let [target (closest-node creature eye)
   1.172 +        [cam-width cam-height] (eye-dimensions eye)
   1.173 +        cam (Camera. cam-width cam-height)]
   1.174 +    (.setLocation cam (.getWorldTranslation eye))
   1.175 +    (.setRotation cam (.getWorldRotation eye))
   1.176 +    (.setFrustumPerspective
   1.177 +     cam 45 (/ (.getWidth cam) (.getHeight cam))
   1.178 +     1 1000)
   1.179 +    (bind-sense target cam)
   1.180 +    cam))
   1.181 +
   1.182 +(def presets
   1.183 +  {:all    0xFFFFFF
   1.184 +   :red    0xFF0000
   1.185 +   :blue   0x0000FF
   1.186 +   :green  0x00FF00})
   1.187 +
   1.188 +(defn enable-vision
   1.189 +  "return [init-function sensor-functions] for a particular eye"
   1.190 +  [#^Node creature #^Spatial eye & {skip :skip :or {skip 0}}]
   1.191 +  (let [retinal-map (retina-sensor-image eye)
   1.192 +        camera (attach-eye creature eye)
   1.193 +        vision-image
   1.194 +        (atom
   1.195 +         (BufferedImage. (.getWidth camera)
   1.196 +                         (.getHeight camera)
   1.197 +                         BufferedImage/TYPE_BYTE_BINARY))]
   1.198 +    [(fn [world]
   1.199 +       (add-eye!
   1.200 +        world camera
   1.201 +        (let [counter  (atom 0)]
   1.202 +          (fn [r fb bb bi]
   1.203 +            (if (zero? (rem (swap! counter inc) (inc skip)))
   1.204 +              (reset! vision-image (BufferedImage! r fb bb bi)))))))
   1.205 +     (vec
   1.206 +      (map
   1.207 +       (fn [[key image]]
   1.208 +         (let [whites (white-coordinates image)
   1.209 +               topology (vec (collapse whites))
   1.210 +               mask (presets key)]
   1.211 +           (fn []
   1.212 +             (vector
   1.213 +              topology
   1.214 +              (vec 
   1.215 +               (for [[x y] whites]
   1.216 +                 (bit-and
   1.217 +                  mask (.getRGB @vision-image x y))))))))
   1.218 +       retinal-map))]))
   1.219 +
   1.220 +(defn vision
   1.221 +  [#^Node creature & {skip :skip :or {skip 0}}]
   1.222 +  (reduce
   1.223 +   (fn [[init-a senses-a]
   1.224 +        [init-b senses-b]]
   1.225 +     [(conj init-a init-b)
   1.226 +      (into senses-a senses-b)])
   1.227 +   [[][]]      
   1.228 +   (for [eye (eyes creature)]
   1.229 +     (enable-vision creature eye))))
   1.230 +
   1.231 +
   1.232 +#+end_src
   1.233 +
   1.234 +
   1.235 +Note the use of continuation passing style for connecting the eye to a
   1.236 +function to process the output. You can create any number of eyes, and
   1.237 +each of them will see the world from their own =Camera=. Once every
   1.238 +frame, the rendered image is copied to a =BufferedImage=, and that
   1.239 +data is sent off to the continuation function. Moving the =Camera=
   1.240 +which was used to create the eye will change what the eye sees.
   1.241 +
   1.242 +* Example
   1.243 +
   1.244 +#+name: test-vision
   1.245 +#+begin_src clojure
   1.246 +(ns cortex.test.vision
   1.247 +  (:use (cortex world util vision))
   1.248 +  (:import java.awt.image.BufferedImage)
   1.249 +  (:import javax.swing.JPanel)
   1.250 +  (:import javax.swing.SwingUtilities)
   1.251 +  (:import java.awt.Dimension)
   1.252 +  (:import javax.swing.JFrame)
   1.253 +  (:import com.jme3.math.ColorRGBA)
   1.254 +  (:import com.jme3.scene.Node)
   1.255 +  (:import com.jme3.math.Vector3f))
   1.256 +
   1.257 +(defn test-two-eyes
   1.258 +  "Testing vision:
   1.259 +   Tests the vision system by creating two views of the same rotating
   1.260 +   object from different angles and displaying both of those views in
   1.261 +   JFrames.
   1.262 +
   1.263 +   You should see a rotating cube, and two windows,
   1.264 +   each displaying a different view of the cube."
   1.265 +  []
   1.266 +  (let [candy
   1.267 +        (box 1 1 1 :physical? false :color ColorRGBA/Blue)]
   1.268 +    (world
   1.269 +     (doto (Node.)
   1.270 +       (.attachChild candy))
   1.271 +     {}
   1.272 +     (fn [world]
   1.273 +       (let [cam (.clone (.getCamera world))
   1.274 +             width (.getWidth cam)
   1.275 +             height (.getHeight cam)]
   1.276 +         (add-eye! world cam 
   1.277 +                  ;;no-op
   1.278 +                  (comp (view-image) BufferedImage!)
   1.279 +                  )
   1.280 +         (add-eye! world
   1.281 +                  (doto (.clone cam)
   1.282 +                    (.setLocation (Vector3f. -10 0 0))
   1.283 +                    (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))
   1.284 +                  ;;no-op
   1.285 +                  (comp (view-image) BufferedImage!))
   1.286 +         ;; This is here to restore the main view
   1.287 +         ;; after the other views have completed processing
   1.288 +         (add-eye! world (.getCamera world) no-op)))
   1.289 +     (fn [world tpf]
   1.290 +       (.rotate candy (* tpf 0.2) 0 0)))))
   1.291 +#+end_src
   1.292 +
   1.293 +#+results: test-vision
   1.294 +: #'cortex.test.vision/test-two-eyes
   1.295 +
   1.296 +The example code will create two videos of the same rotating object
   1.297 +from different angles. It can be used both for stereoscopic vision
   1.298 +simulation or for simulating multiple creatures, each with their own
   1.299 +sense of vision.
   1.300 +
   1.301 +- As a neat bonus, this idea behind simulated vision also enables one
   1.302 +  to [[../../cortex/html/capture-video.html][capture live video feeds from jMonkeyEngine]].
   1.303 +
   1.304 +
   1.305 +* COMMENT code generation
   1.306 +#+begin_src clojure :tangle ../src/cortex/vision.clj
   1.307 +<<eyes>>
   1.308 +#+end_src
   1.309 +
   1.310 +#+begin_src clojure :tangle ../src/cortex/test/vision.clj
   1.311 +<<test-vision>>
   1.312 +#+end_src