1.1 --- a/org/vision.org	Thu Feb 09 09:04:17 2012 -0700
     1.2 +++ b/org/vision.org	Fri Feb 10 02:19:24 2012 -0700
     1.3 @@ -160,21 +160,50 @@
     1.4  
     1.5  #+name: add-eye
     1.6  #+begin_src clojure
     1.7 +(in-ns 'cortex.vision)
     1.8 +
     1.9 +(import com.jme3.math.Vector3f)
    1.10 +
    1.11 +(def blender-rotation-correction
    1.12 +  (doto (Quaternion.)
    1.13 +    (.fromRotationMatrix
    1.14 +     (doto (Matrix3f.)
    1.15 +       (.setColumn 0
    1.16 +                   (Vector3f. 1 0 0))
    1.17 +       (.setColumn 1
    1.18 +                   (Vector3f. 0 -1 0))
    1.19 +       (.setColumn 2
    1.20 +                   (Vector3f. 0 0 -1)))
    1.21 +
    1.22 +     (doto (Matrix3f.)
    1.23 +       (.setColumn 0
    1.24 +                   (Vector3f. 
    1.25 +
    1.26 +
    1.27  (defn add-eye!
    1.28    "Create a Camera centered on the current position of 'eye which
    1.29     follows the closest physical node in 'creature and sends visual
    1.30 -   data to 'continuation."
    1.31 +   data to 'continuation. The camera will point in the X direction and
    1.32 +   use the Z vector as up as determined by the rotation of these
    1.33 +   vectors in blender coordinate space. Use XZY rotation for the node
    1.34 +   in blender."
    1.35    [#^Node creature #^Spatial eye]
    1.36    (let [target (closest-node creature eye)
    1.37          [cam-width cam-height] (eye-dimensions eye)
    1.38 -        cam (Camera. cam-width cam-height)]
    1.39 +        cam (Camera. cam-width cam-height)
    1.40 +        rot (.getWorldRotation eye)]
    1.41      (.setLocation cam (.getWorldTranslation eye))
    1.42 -    (.setRotation cam (.getWorldRotation eye))
    1.43 +    (.lookAtDirection cam (.mult rot Vector3f/UNIT_X)
    1.44 +                      ;; this part is consistent with using Z in
    1.45 +                      ;; blender as the UP vector.
    1.46 +                      (.mult rot Vector3f/UNIT_Y))
    1.47 +
    1.48 +    (println-repl "eye unit-z ->"  (.mult rot Vector3f/UNIT_Z))
    1.49 +    (println-repl "eye unit-y ->"  (.mult rot Vector3f/UNIT_Y))
    1.50 +    (println-repl "eye unit-x ->"  (.mult rot Vector3f/UNIT_X))
    1.51      (.setFrustumPerspective
    1.52 -     cam 45 (/ (.getWidth cam) (.getHeight cam))
    1.53 -     1 1000)
    1.54 -    (bind-sense target cam)
    1.55 -    cam))
    1.56 +     cam 45 (/ (.getWidth cam) (.getHeight cam)) 1 1000)
    1.57 +    (bind-sense target cam) cam))
    1.58  #+end_src
    1.59  
    1.60  Here, the camera is created based on metadata on the eye-node and
    1.61 @@ -286,7 +315,8 @@
    1.62    "Return the children of the creature's \"eyes\" node.")
    1.63  #+end_src
    1.64  
    1.65 -Then, 
    1.66 +Then, add the camera created by =(add-eye!)= to the simulation by
    1.67 +creating a new viewport.
    1.68  
    1.69  #+begin_src clojure
    1.70  (defn add-camera!
    1.71 @@ -302,12 +332,26 @@
    1.72        (.setBackgroundColor ColorRGBA/Black)
    1.73        (.addProcessor (vision-pipeline continuation))
    1.74        (.attachScene (.getRootNode world)))))
    1.75 +#+end_src
    1.76  
    1.77  
    1.78 +The continuation function registers the viewport with the simulation
    1.79 +the first time it is called, and uses the CPU to extract the
    1.80 +appropriate pixels from the rendered image and weight them by each
    1.81 +sensors sensitivity. I have the option to do this filtering in native
    1.82 +code for a slight gain in speed. I could also do it in the GPU for a
    1.83 +massive gain in speed. =(vision-kernel)= generates a list of such
    1.84 +continuation functions, one for each channel of the eye.
    1.85  
    1.86 +#+begin_src clojure 
    1.87 +(in-ns 'cortex.vision)
    1.88  
    1.89 +(defrecord attached-viewport [vision-fn viewport-fn]
    1.90 +  clojure.lang.IFn
    1.91 +  (invoke [this world] (vision-fn world))
    1.92 +  (applyTo [this args] (apply vision-fn args)))
    1.93  
    1.94 -(defn vision-fn
    1.95 +(defn vision-kernel
    1.96    "Returns a list of functions, each of which will return a color
    1.97     channel's worth of visual information when called inside a running
    1.98     simulation."
    1.99 @@ -335,20 +379,47 @@
   1.100           (let [whites (white-coordinates image)
   1.101                 topology (vec (collapse whites))
   1.102                 mask (color-channel-presets key key)]
   1.103 -           (fn [world]
   1.104 -             (register-eye! world)
   1.105 -             (vector
   1.106 -              topology
   1.107 -              (vec 
   1.108 -               (for [[x y] whites]
   1.109 -                 (bit-and
   1.110 -                  mask (.getRGB @vision-image x y))))))))
   1.111 -       retinal-map))))
   1.112 +           (attached-viewport.
   1.113 +            (fn [world]
   1.114 +              (register-eye! world)
   1.115 +              (vector
   1.116 +               topology
   1.117 +               (vec 
   1.118 +                (for [[x y] whites]
   1.119 +                  (bit-and
   1.120 +                   mask (.getRGB @vision-image x y))))))
   1.121 +            register-eye!)))
   1.122 +         retinal-map))))
   1.123  
   1.124 +(defn gen-fix-display
   1.125 +  "Create a function to call to restore a simulation's display when it
   1.126 +   is disrupted by a Viewport."
   1.127 +  []
   1.128 +  (runonce
   1.129 +   (fn [world]
   1.130 +     (add-camera! world (.getCamera world) no-op))))
   1.131  
   1.132 -;; TODO maybe should add a viewport-manipulation function to
   1.133 -;; automatically change viewport settings, attach shadow filters, etc.
   1.134 +#+end_src
   1.135  
   1.136 +Note that since each of the functions generated by =(vision-kernel)=
   1.137 +shares the same =(register-eye!)= function, the eye will be registered
   1.138 +only once the first time any of the functions from the list returned
   1.139 +by =(vision-kernel)= is called.  Each of the functions returned by
   1.140 +=(vision-kernel)= also allows access to the =Viewport= through which
   1.141 +it recieves images.
   1.142 +
   1.143 +The in-game display can be disrupted by all the viewports that the
   1.144 +functions greated by =(vision-kernel)= add. This doesn't affect the
   1.145 +simulation or the simulated senses, but can be annoying.
   1.146 +=(gen-fix-display)= restores the in-simulation display.
   1.147 +
   1.148 +** Vision!
   1.149 +
   1.150 +All the hard work has been done, all that remains is to apply
   1.151 +=(vision-kernel)= to each eye in the creature and gather the results
   1.152 +into one list of functions.
   1.153 +
   1.154 +#+begin_src clojure
   1.155  (defn vision!
   1.156    "Returns a function which returns visual sensory data when called
   1.157     inside a running simulation"
   1.158 @@ -356,8 +427,16 @@
   1.159    (reduce
   1.160     concat 
   1.161     (for [eye (eyes creature)]
   1.162 -     (vision-fn creature eye))))
   1.163 +     (vision-kernel creature eye))))
   1.164 +#+end_src
   1.165  
   1.166 +** Visualization of Vision
   1.167 +
   1.168 +It's vital to have a visual representation for each sense. Here I use
   1.169 +=(view-sense)= to construct a function that will create a display for
   1.170 +visual data.
   1.171 +
   1.172 +#+begin_src clojure 
   1.173  (defn view-vision
   1.174    "Creates a function which accepts a list of visual sensor-data and
   1.175    displays each element of the list to the screen." 
   1.176 @@ -371,31 +450,20 @@
   1.177            (.setRGB image ((coords i) 0) ((coords i) 1)
   1.178                     (sensor-data i))))
   1.179         image))))
   1.180 -
   1.181  #+end_src
   1.182  
   1.183 +* Tests
   1.184  
   1.185 -Note the use of continuation passing style for connecting the eye to a
   1.186 -function to process the output. You can create any number of eyes, and
   1.187 -each of them will see the world from their own =Camera=. Once every
   1.188 -frame, the rendered image is copied to a =BufferedImage=, and that
   1.189 -data is sent off to the continuation function. Moving the =Camera=
   1.190 -which was used to create the eye will change what the eye sees.
   1.191 +** Basic Test
   1.192  
   1.193 -* Example
   1.194 +This is a basic test for the vision system.  It only tests the
   1.195 +vision-pipeline and does not deal with loadig eyes from a blender
   1.196 +file. The code creates two videos of the same rotating cube from
   1.197 +different angles. 
   1.198  
   1.199 -#+name: test-vision
   1.200 +#+name: test-1
   1.201  #+begin_src clojure
   1.202 -(ns cortex.test.vision
   1.203 -  (:use (cortex world util vision))
   1.204 -  (:import java.awt.image.BufferedImage)
   1.205 -  (:import javax.swing.JPanel)
   1.206 -  (:import javax.swing.SwingUtilities)
   1.207 -  (:import java.awt.Dimension)
   1.208 -  (:import javax.swing.JFrame)
   1.209 -  (:import com.jme3.math.ColorRGBA)
   1.210 -  (:import com.jme3.scene.Node)
   1.211 -  (:import com.jme3.math.Vector3f))
   1.212 +(in-ns 'cortex.test.vision)
   1.213  
   1.214  (defn test-two-eyes
   1.215    "Testing vision:
   1.216 @@ -417,15 +485,18 @@
   1.217               width (.getWidth cam)
   1.218               height (.getHeight cam)]
   1.219           (add-camera! world cam 
   1.220 -                  ;;no-op
   1.221 -                  (comp (view-image) BufferedImage!)
   1.222 -                  )
   1.223 +                      (comp
   1.224 +                       (view-image
   1.225 +                        (File. "/home/r/proj/cortex/render/vision/1"))
   1.226 +                        BufferedImage!))
   1.227           (add-camera! world
   1.228                    (doto (.clone cam)
   1.229                      (.setLocation (Vector3f. -10 0 0))
   1.230                      (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))
   1.231 -                  ;;no-op
   1.232 -                  (comp (view-image) BufferedImage!))
   1.233 +                  (comp
   1.234 +                   (view-image
   1.235 +                    (File. "/home/r/proj/cortex/render/vision/2"))
   1.236 +                        BufferedImage!))
   1.237           ;; This is here to restore the main view
   1.238           ;; after the other views have completed processing
   1.239           (add-camera! world (.getCamera world) no-op)))
   1.240 @@ -433,6 +504,98 @@
   1.241         (.rotate candy (* tpf 0.2) 0 0)))))
   1.242  #+end_src
   1.243  
   1.244 +#+begin_html
   1.245 +<div class="figure">
   1.246 +<video controls="controls" width="755">
   1.247 +  <source src="../video/spinning-cube.ogg" type="video/ogg"
   1.248 +	  preload="none" poster="../images/aurellem-1280x480.png" />
   1.249 +</video>
   1.250 +<p>A rotating cube viewed from two different perspectives.</p>
   1.251 +</div>
   1.252 +#+end_html
   1.253 +
   1.254 +Creating multiple eyes like this can be used for stereoscopic vision
   1.255 +simulation in a single creature or for simulating multiple creatures,
   1.256 +each with their own sense of vision.
   1.257 +
   1.258 +** Adding Vision to the Worm
   1.259 +
   1.260 +To the worm from the last post, we add a new node that describes its
   1.261 +eyes.
   1.262 +
   1.263 +#+attr_html: width=755
   1.264 +#+caption: The worm with newly added empty nodes describing a single eye.
   1.265 +[[../images/worm-with-eye.png]]
   1.266 +
   1.267 +The node highlighted in yellow is the root level "eyes" node.  It has
   1.268 +a single node, highlighted in orange, which describes a single
   1.269 +eye. This is the "eye" node. The two nodes which are not highlighted describe the single joint
   1.270 +of the worm.
   1.271 +
   1.272 +The metadata of the eye-node is:
   1.273 +
   1.274 +#+begin_src clojure :results verbatim :exports both
   1.275 +(cortex.sense/meta-data
   1.276 + (.getChild
   1.277 +  (.getChild (cortex.test.body/worm)
   1.278 +            "eyes") "eye") "eye")
   1.279 +#+end_src
   1.280 +
   1.281 +#+results:
   1.282 +: "(let [retina \"Models/test-creature/retina-small.png\"]
   1.283 +:     {:all retina :red retina :green retina :blue retina})"
   1.284 +
   1.285 +This is the approximation to the human eye described earlier.
   1.286 +
   1.287 +#+begin_src clojure
   1.288 +(in-ns 'cortex.test.vision)
   1.289 +
   1.290 +(import com.aurellem.capture.Capture)
   1.291 +
   1.292 +(defn test-worm-vision [] 
   1.293 +  (let [the-worm (doto (worm)(body!))
   1.294 +        vision (vision! the-worm)
   1.295 +        vision-display (view-vision)
   1.296 +        fix-display (gen-fix-display)
   1.297 +        me (sphere 0.5 :color ColorRGBA/Blue :physical? false)
   1.298 +        x-axis
   1.299 +        (box 1 0.01 0.01 :physical? false :color ColorRGBA/Red
   1.300 +             :position (Vector3f. 0 -5 0))
   1.301 +        y-axis
   1.302 +        (box 0.01 1 0.01 :physical? false :color ColorRGBA/Green
   1.303 +             :position (Vector3f. 0 -5 0))
   1.304 +        z-axis
   1.305 +        (box 0.01 0.01 1 :physical? false :color ColorRGBA/Blue
   1.306 +             :position (Vector3f. 0 -5 0))]
   1.307 +
   1.308 +    (world (nodify [(floor) the-worm x-axis y-axis z-axis me])
   1.309 +           standard-debug-controls
   1.310 +           (fn [world]
   1.311 +             (light-up-everything world)
   1.312 +             ;; add a view from the worm's perspective
   1.313 +             (add-camera!
   1.314 +              world
   1.315 +              (add-eye! the-worm
   1.316 +                        (.getChild 
   1.317 +                         (.getChild the-worm "eyes") "eye"))
   1.318 +              (comp
   1.319 +               (view-image
   1.320 +                (File. "/home/r/proj/cortex/render/worm-vision/worm-view"))
   1.321 +               BufferedImage!))
   1.322 +             (set-gravity world Vector3f/ZERO)
   1.323 +             (Capture/captureVideo
   1.324 +              world
   1.325 +              (File. "/home/r/proj/cortex/render/worm-vision/main-view")))
   1.326 +           (fn [world _ ]
   1.327 +             (.setLocalTranslation me (.getLocation (.getCamera world)))
   1.328 +             (vision-display
   1.329 +              (map #(% world) vision)
   1.330 +              (File. "/home/r/proj/cortex/render/worm-vision"))
   1.331 +             (fix-display world)))))
   1.332 +#+end_src
   1.333 +
   1.334 +* Headers
   1.335 +
   1.336  #+name: vision-header
   1.337  #+begin_src clojure 
   1.338  (ns cortex.vision
   1.339 @@ -456,10 +619,23 @@
   1.340    (:import (com.jme3.scene Node Spatial)))
   1.341  #+end_src
   1.342  
   1.343 -The example code will create two videos of the same rotating object
   1.344 -from different angles. It can be used both for stereoscopic vision
   1.345 -simulation or for simulating multiple creatures, each with their own
   1.346 -sense of vision.
   1.347 +#+name: test-header
   1.348 +#+begin_src clojure
   1.349 +(ns cortex.test.vision
   1.350 +  (:use (cortex world sense util body vision))
   1.351 +  (:use cortex.test.body)
   1.352 +  (:import java.awt.image.BufferedImage)
   1.353 +  (:import javax.swing.JPanel)
   1.354 +  (:import javax.swing.SwingUtilities)
   1.355 +  (:import java.awt.Dimension)
   1.356 +  (:import javax.swing.JFrame)
   1.357 +  (:import com.jme3.math.ColorRGBA)
   1.358 +  (:import com.jme3.scene.Node)
   1.359 +  (:import com.jme3.math.Vector3f)
   1.360 +  (:import java.io.File))
   1.361 +#+end_src
   1.362 +
   1.363 +
   1.364  
   1.365  - As a neat bonus, this idea behind simulated vision also enables one
   1.366    to [[../../cortex/html/capture-video.html][capture live video feeds from jMonkeyEngine]].
   1.367 @@ -471,5 +647,6 @@
   1.368  #+end_src
   1.369  
   1.370  #+begin_src clojure :tangle ../src/cortex/test/vision.clj
   1.371 -<<test-vision>>
   1.372 +<<test-header>>
   1.373 +<<test-1>>
   1.374  #+end_src