1.1 --- a/.hgignore	Thu Feb 09 09:04:17 2012 -0700
     1.2 +++ b/.hgignore	Fri Feb 10 02:19:24 2012 -0700
     1.3 @@ -7,6 +7,10 @@
     1.4  render*
     1.5  sources*
     1.6  *.hdr
     1.7 +libbulletjme64.so
     1.8 +liblwjgl64.so
     1.9 +libopenal64.so
    1.10 +
    1.11  
    1.12  syntax: regexp
    1.13  ^.*blend\d$
    1.14 \ No newline at end of file

     2.1 Binary file assets/Models/test-creature/worm.blend has changed

     3.1 Binary file images/worm-with-eye.png has changed

     4.1 --- a/org/body.org	Thu Feb 09 09:04:17 2012 -0700
     4.2 +++ b/org/body.org	Fri Feb 10 02:19:24 2012 -0700
     4.3 @@ -491,13 +491,15 @@
     4.4  #+begin_src clojure 
     4.5  (in-ns 'cortex.test.body)
     4.6  
     4.7 +(defn worm []
     4.8 + (load-blender-model
     4.9 +            "Models/test-creature/worm.blend"))
    4.10 +
    4.11  (defn worm-1 []
    4.12    (let [timer (RatchetTimer. 60)]
    4.13      (world
    4.14       (nodify
    4.15 -      [(doto
    4.16 -           (load-blender-model
    4.17 -            "Models/test-creature/worm.blend")
    4.18 +      [(doto (worm)
    4.19           (body!))
    4.20         (floor)])
    4.21       (merge standard-debug-controls debug-control)

     5.1 --- a/org/sense.org	Thu Feb 09 09:04:17 2012 -0700
     5.2 +++ b/org/sense.org	Fri Feb 10 02:19:24 2012 -0700
     5.3 @@ -271,15 +271,27 @@
     5.4     its own JFrame."
     5.5    [sense-display-kernel]
     5.6    (let [windows (atom [])]
     5.7 -    (fn [data]
     5.8 -      (if (> (count data) (count @windows))
     5.9 -        (reset! 
    5.10 -         windows (map (fn [_] (view-image)) (range (count data)))))
    5.11 -      (dorun
    5.12 -       (map
    5.13 -        (fn [display datum]
    5.14 -          (display (sense-display-kernel datum)))
    5.15 -        @windows data)))))
    5.16 +    (fn this
    5.17 +      ([data]
    5.18 +       (this data nil))
    5.19 +      ([data save-to]
    5.20 +         (if (> (count data) (count @windows))
    5.21 +           (reset! 
    5.22 +            windows
    5.23 +            (doall 
    5.24 +             (map
    5.25 +             (fn [idx]
    5.26 +               (if save-to
    5.27 +                 (let [dir (File. save-to (str idx))]
    5.28 +                   (.mkdir dir)
    5.29 +                   (view-image dir))
    5.30 +                 (view-image))) (range (count data))))))
    5.31 +         (dorun
    5.32 +          (map
    5.33 +           (fn [display datum]
    5.34 +             (display (sense-display-kernel datum)))
    5.35 +           @windows data))))))
    5.36 +         
    5.37  
    5.38  (defn points->image
    5.39    "Take a collection of points and visuliaze it as a BufferedImage."
    5.40 @@ -464,33 +476,35 @@
    5.41  
    5.42  *** Combine Frames with ImageMagick
    5.43  #+begin_src clojure :results silent
    5.44 -(in-ns 'user)
    5.45 -(import java.io.File)
    5.46 -(use 'clojure.contrib.shell-out)
    5.47 -(let
    5.48 -    [idx (atom -1)
    5.49 -     left (rest 
    5.50 -           (sort
    5.51 -           (file-seq (File. "/home/r/proj/cortex/render/bind-sense0/"))))
    5.52 -     right (rest
    5.53 +(ns cortex.video.magick
    5.54 +  (:import java.io.File)
    5.55 +  (:use clojure.contrib.shell-out))
    5.56 +
    5.57 +(defn combine-images []
    5.58 +  (let    
    5.59 +      [idx (atom -1)
    5.60 +       left (rest 
    5.61 +             (sort
    5.62 +              (file-seq (File. "/home/r/proj/cortex/render/bind-sense0/"))))
    5.63 +       right (rest
    5.64 +              (sort
    5.65 +               (file-seq
    5.66 +                (File. "/home/r/proj/cortex/render/bind-sense1/"))))
    5.67 +       sub (rest
    5.68              (sort
    5.69               (file-seq
    5.70 -              (File. "/home/r/proj/cortex/render/bind-sense1/"))))
    5.71 -     sub (rest
    5.72 -          (sort
    5.73 -           (file-seq
    5.74 -            (File. "/home/r/proj/cortex/render/bind-senseB/"))))
    5.75 -     sub* (concat sub (repeat 1000 (last sub)))]
    5.76 -  (dorun
    5.77 -   (map  
    5.78 -    (fn [im-1 im-2 sub]
    5.79 -      (sh "convert" (.getCanonicalPath im-1)
    5.80 -          (.getCanonicalPath im-2) "+append"
    5.81 -          (.getCanonicalPath sub) "-append"
    5.82 -          (.getCanonicalPath
    5.83 -           (File. "/home/r/proj/cortex/render/bind-sense/"
    5.84 -                  (format "%07d.png" (swap! idx inc))))))
    5.85 -    left right sub*)))
    5.86 +              (File. "/home/r/proj/cortex/render/bind-senseB/"))))
    5.87 +       sub* (concat sub (repeat 1000 (last sub)))]
    5.88 +    (dorun
    5.89 +     (map  
    5.90 +      (fn [im-1 im-2 sub]
    5.91 +        (sh "convert" (.getCanonicalPath im-1)
    5.92 +            (.getCanonicalPath im-2) "+append"
    5.93 +            (.getCanonicalPath sub) "-append"
    5.94 +            (.getCanonicalPath
    5.95 +             (File. "/home/r/proj/cortex/render/bind-sense/"
    5.96 +                    (format "%07d.png" (swap! idx inc))))))
    5.97 +      left right sub*))))
    5.98  #+end_src
    5.99  
   5.100  *** Encode Frames with ffmpeg
   5.101 @@ -559,3 +573,7 @@
   5.102  <<test-header>>
   5.103  <<test>>
   5.104  #+end_src
   5.105 +
   5.106 +#+begin_src clojure :tangle ../src/cortex/video/magick.clj
   5.107 +<<magick>>
   5.108 +#+end_src

     6.1 --- a/org/vision.org	Thu Feb 09 09:04:17 2012 -0700
     6.2 +++ b/org/vision.org	Fri Feb 10 02:19:24 2012 -0700
     6.3 @@ -160,21 +160,50 @@
     6.4  
     6.5  #+name: add-eye
     6.6  #+begin_src clojure
     6.7 +(in-ns 'cortex.vision)
     6.8 +
     6.9 +(import com.jme3.math.Vector3f)
    6.10 +
    6.11 +(def blender-rotation-correction
    6.12 +  (doto (Quaternion.)
    6.13 +    (.fromRotationMatrix
    6.14 +     (doto (Matrix3f.)
    6.15 +       (.setColumn 0
    6.16 +                   (Vector3f. 1 0 0))
    6.17 +       (.setColumn 1
    6.18 +                   (Vector3f. 0 -1 0))
    6.19 +       (.setColumn 2
    6.20 +                   (Vector3f. 0 0 -1)))
    6.21 +
    6.22 +     (doto (Matrix3f.)
    6.23 +       (.setColumn 0
    6.24 +                   (Vector3f. 
    6.25 +
    6.26 +
    6.27  (defn add-eye!
    6.28    "Create a Camera centered on the current position of 'eye which
    6.29     follows the closest physical node in 'creature and sends visual
    6.30 -   data to 'continuation."
    6.31 +   data to 'continuation. The camera will point in the X direction and
    6.32 +   use the Z vector as up as determined by the rotation of these
    6.33 +   vectors in blender coordinate space. Use XZY rotation for the node
    6.34 +   in blender."
    6.35    [#^Node creature #^Spatial eye]
    6.36    (let [target (closest-node creature eye)
    6.37          [cam-width cam-height] (eye-dimensions eye)
    6.38 -        cam (Camera. cam-width cam-height)]
    6.39 +        cam (Camera. cam-width cam-height)
    6.40 +        rot (.getWorldRotation eye)]
    6.41      (.setLocation cam (.getWorldTranslation eye))
    6.42 -    (.setRotation cam (.getWorldRotation eye))
    6.43 +    (.lookAtDirection cam (.mult rot Vector3f/UNIT_X)
    6.44 +                      ;; this part is consistent with using Z in
    6.45 +                      ;; blender as the UP vector.
    6.46 +                      (.mult rot Vector3f/UNIT_Y))
    6.47 +
    6.48 +    (println-repl "eye unit-z ->"  (.mult rot Vector3f/UNIT_Z))
    6.49 +    (println-repl "eye unit-y ->"  (.mult rot Vector3f/UNIT_Y))
    6.50 +    (println-repl "eye unit-x ->"  (.mult rot Vector3f/UNIT_X))
    6.51      (.setFrustumPerspective
    6.52 -     cam 45 (/ (.getWidth cam) (.getHeight cam))
    6.53 -     1 1000)
    6.54 -    (bind-sense target cam)
    6.55 -    cam))
    6.56 +     cam 45 (/ (.getWidth cam) (.getHeight cam)) 1 1000)
    6.57 +    (bind-sense target cam) cam))
    6.58  #+end_src
    6.59  
    6.60  Here, the camera is created based on metadata on the eye-node and
    6.61 @@ -286,7 +315,8 @@
    6.62    "Return the children of the creature's \"eyes\" node.")
    6.63  #+end_src
    6.64  
    6.65 -Then, 
    6.66 +Then, add the camera created by =(add-eye!)= to the simulation by
    6.67 +creating a new viewport.
    6.68  
    6.69  #+begin_src clojure
    6.70  (defn add-camera!
    6.71 @@ -302,12 +332,26 @@
    6.72        (.setBackgroundColor ColorRGBA/Black)
    6.73        (.addProcessor (vision-pipeline continuation))
    6.74        (.attachScene (.getRootNode world)))))
    6.75 +#+end_src
    6.76  
    6.77  
    6.78 +The continuation function registers the viewport with the simulation
    6.79 +the first time it is called, and uses the CPU to extract the
    6.80 +appropriate pixels from the rendered image and weight them by each
    6.81 +sensors sensitivity. I have the option to do this filtering in native
    6.82 +code for a slight gain in speed. I could also do it in the GPU for a
    6.83 +massive gain in speed. =(vision-kernel)= generates a list of such
    6.84 +continuation functions, one for each channel of the eye.
    6.85  
    6.86 +#+begin_src clojure 
    6.87 +(in-ns 'cortex.vision)
    6.88  
    6.89 +(defrecord attached-viewport [vision-fn viewport-fn]
    6.90 +  clojure.lang.IFn
    6.91 +  (invoke [this world] (vision-fn world))
    6.92 +  (applyTo [this args] (apply vision-fn args)))
    6.93  
    6.94 -(defn vision-fn
    6.95 +(defn vision-kernel
    6.96    "Returns a list of functions, each of which will return a color
    6.97     channel's worth of visual information when called inside a running
    6.98     simulation."
    6.99 @@ -335,20 +379,47 @@
   6.100           (let [whites (white-coordinates image)
   6.101                 topology (vec (collapse whites))
   6.102                 mask (color-channel-presets key key)]
   6.103 -           (fn [world]
   6.104 -             (register-eye! world)
   6.105 -             (vector
   6.106 -              topology
   6.107 -              (vec 
   6.108 -               (for [[x y] whites]
   6.109 -                 (bit-and
   6.110 -                  mask (.getRGB @vision-image x y))))))))
   6.111 -       retinal-map))))
   6.112 +           (attached-viewport.
   6.113 +            (fn [world]
   6.114 +              (register-eye! world)
   6.115 +              (vector
   6.116 +               topology
   6.117 +               (vec 
   6.118 +                (for [[x y] whites]
   6.119 +                  (bit-and
   6.120 +                   mask (.getRGB @vision-image x y))))))
   6.121 +            register-eye!)))
   6.122 +         retinal-map))))
   6.123  
   6.124 +(defn gen-fix-display
   6.125 +  "Create a function to call to restore a simulation's display when it
   6.126 +   is disrupted by a Viewport."
   6.127 +  []
   6.128 +  (runonce
   6.129 +   (fn [world]
   6.130 +     (add-camera! world (.getCamera world) no-op))))
   6.131  
   6.132 -;; TODO maybe should add a viewport-manipulation function to
   6.133 -;; automatically change viewport settings, attach shadow filters, etc.
   6.134 +#+end_src
   6.135  
   6.136 +Note that since each of the functions generated by =(vision-kernel)=
   6.137 +shares the same =(register-eye!)= function, the eye will be registered
   6.138 +only once the first time any of the functions from the list returned
   6.139 +by =(vision-kernel)= is called.  Each of the functions returned by
   6.140 +=(vision-kernel)= also allows access to the =Viewport= through which
   6.141 +it recieves images.
   6.142 +
   6.143 +The in-game display can be disrupted by all the viewports that the
   6.144 +functions greated by =(vision-kernel)= add. This doesn't affect the
   6.145 +simulation or the simulated senses, but can be annoying.
   6.146 +=(gen-fix-display)= restores the in-simulation display.
   6.147 +
   6.148 +** Vision!
   6.149 +
   6.150 +All the hard work has been done, all that remains is to apply
   6.151 +=(vision-kernel)= to each eye in the creature and gather the results
   6.152 +into one list of functions.
   6.153 +
   6.154 +#+begin_src clojure
   6.155  (defn vision!
   6.156    "Returns a function which returns visual sensory data when called
   6.157     inside a running simulation"
   6.158 @@ -356,8 +427,16 @@
   6.159    (reduce
   6.160     concat 
   6.161     (for [eye (eyes creature)]
   6.162 -     (vision-fn creature eye))))
   6.163 +     (vision-kernel creature eye))))
   6.164 +#+end_src
   6.165  
   6.166 +** Visualization of Vision
   6.167 +
   6.168 +It's vital to have a visual representation for each sense. Here I use
   6.169 +=(view-sense)= to construct a function that will create a display for
   6.170 +visual data.
   6.171 +
   6.172 +#+begin_src clojure 
   6.173  (defn view-vision
   6.174    "Creates a function which accepts a list of visual sensor-data and
   6.175    displays each element of the list to the screen." 
   6.176 @@ -371,31 +450,20 @@
   6.177            (.setRGB image ((coords i) 0) ((coords i) 1)
   6.178                     (sensor-data i))))
   6.179         image))))
   6.180 -
   6.181  #+end_src
   6.182  
   6.183 +* Tests
   6.184  
   6.185 -Note the use of continuation passing style for connecting the eye to a
   6.186 -function to process the output. You can create any number of eyes, and
   6.187 -each of them will see the world from their own =Camera=. Once every
   6.188 -frame, the rendered image is copied to a =BufferedImage=, and that
   6.189 -data is sent off to the continuation function. Moving the =Camera=
   6.190 -which was used to create the eye will change what the eye sees.
   6.191 +** Basic Test
   6.192  
   6.193 -* Example
   6.194 +This is a basic test for the vision system.  It only tests the
   6.195 +vision-pipeline and does not deal with loadig eyes from a blender
   6.196 +file. The code creates two videos of the same rotating cube from
   6.197 +different angles. 
   6.198  
   6.199 -#+name: test-vision
   6.200 +#+name: test-1
   6.201  #+begin_src clojure
   6.202 -(ns cortex.test.vision
   6.203 -  (:use (cortex world util vision))
   6.204 -  (:import java.awt.image.BufferedImage)
   6.205 -  (:import javax.swing.JPanel)
   6.206 -  (:import javax.swing.SwingUtilities)
   6.207 -  (:import java.awt.Dimension)
   6.208 -  (:import javax.swing.JFrame)
   6.209 -  (:import com.jme3.math.ColorRGBA)
   6.210 -  (:import com.jme3.scene.Node)
   6.211 -  (:import com.jme3.math.Vector3f))
   6.212 +(in-ns 'cortex.test.vision)
   6.213  
   6.214  (defn test-two-eyes
   6.215    "Testing vision:
   6.216 @@ -417,15 +485,18 @@
   6.217               width (.getWidth cam)
   6.218               height (.getHeight cam)]
   6.219           (add-camera! world cam 
   6.220 -                  ;;no-op
   6.221 -                  (comp (view-image) BufferedImage!)
   6.222 -                  )
   6.223 +                      (comp
   6.224 +                       (view-image
   6.225 +                        (File. "/home/r/proj/cortex/render/vision/1"))
   6.226 +                        BufferedImage!))
   6.227           (add-camera! world
   6.228                    (doto (.clone cam)
   6.229                      (.setLocation (Vector3f. -10 0 0))
   6.230                      (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))
   6.231 -                  ;;no-op
   6.232 -                  (comp (view-image) BufferedImage!))
   6.233 +                  (comp
   6.234 +                   (view-image
   6.235 +                    (File. "/home/r/proj/cortex/render/vision/2"))
   6.236 +                        BufferedImage!))
   6.237           ;; This is here to restore the main view
   6.238           ;; after the other views have completed processing
   6.239           (add-camera! world (.getCamera world) no-op)))
   6.240 @@ -433,6 +504,98 @@
   6.241         (.rotate candy (* tpf 0.2) 0 0)))))
   6.242  #+end_src
   6.243  
   6.244 +#+begin_html
   6.245 +<div class="figure">
   6.246 +<video controls="controls" width="755">
   6.247 +  <source src="../video/spinning-cube.ogg" type="video/ogg"
   6.248 +	  preload="none" poster="../images/aurellem-1280x480.png" />
   6.249 +</video>
   6.250 +<p>A rotating cube viewed from two different perspectives.</p>
   6.251 +</div>
   6.252 +#+end_html
   6.253 +
   6.254 +Creating multiple eyes like this can be used for stereoscopic vision
   6.255 +simulation in a single creature or for simulating multiple creatures,
   6.256 +each with their own sense of vision.
   6.257 +
   6.258 +** Adding Vision to the Worm
   6.259 +
   6.260 +To the worm from the last post, we add a new node that describes its
   6.261 +eyes.
   6.262 +
   6.263 +#+attr_html: width=755
   6.264 +#+caption: The worm with newly added empty nodes describing a single eye.
   6.265 +[[../images/worm-with-eye.png]]
   6.266 +
   6.267 +The node highlighted in yellow is the root level "eyes" node.  It has
   6.268 +a single node, highlighted in orange, which describes a single
   6.269 +eye. This is the "eye" node. The two nodes which are not highlighted describe the single joint
   6.270 +of the worm.
   6.271 +
   6.272 +The metadata of the eye-node is:
   6.273 +
   6.274 +#+begin_src clojure :results verbatim :exports both
   6.275 +(cortex.sense/meta-data
   6.276 + (.getChild
   6.277 +  (.getChild (cortex.test.body/worm)
   6.278 +            "eyes") "eye") "eye")
   6.279 +#+end_src
   6.280 +
   6.281 +#+results:
   6.282 +: "(let [retina \"Models/test-creature/retina-small.png\"]
   6.283 +:     {:all retina :red retina :green retina :blue retina})"
   6.284 +
   6.285 +This is the approximation to the human eye described earlier.
   6.286 +
   6.287 +#+begin_src clojure
   6.288 +(in-ns 'cortex.test.vision)
   6.289 +
   6.290 +(import com.aurellem.capture.Capture)
   6.291 +
   6.292 +(defn test-worm-vision [] 
   6.293 +  (let [the-worm (doto (worm)(body!))
   6.294 +        vision (vision! the-worm)
   6.295 +        vision-display (view-vision)
   6.296 +        fix-display (gen-fix-display)
   6.297 +        me (sphere 0.5 :color ColorRGBA/Blue :physical? false)
   6.298 +        x-axis
   6.299 +        (box 1 0.01 0.01 :physical? false :color ColorRGBA/Red
   6.300 +             :position (Vector3f. 0 -5 0))
   6.301 +        y-axis
   6.302 +        (box 0.01 1 0.01 :physical? false :color ColorRGBA/Green
   6.303 +             :position (Vector3f. 0 -5 0))
   6.304 +        z-axis
   6.305 +        (box 0.01 0.01 1 :physical? false :color ColorRGBA/Blue
   6.306 +             :position (Vector3f. 0 -5 0))]
   6.307 +
   6.308 +    (world (nodify [(floor) the-worm x-axis y-axis z-axis me])
   6.309 +           standard-debug-controls
   6.310 +           (fn [world]
   6.311 +             (light-up-everything world)
   6.312 +             ;; add a view from the worm's perspective
   6.313 +             (add-camera!
   6.314 +              world
   6.315 +              (add-eye! the-worm
   6.316 +                        (.getChild 
   6.317 +                         (.getChild the-worm "eyes") "eye"))
   6.318 +              (comp
   6.319 +               (view-image
   6.320 +                (File. "/home/r/proj/cortex/render/worm-vision/worm-view"))
   6.321 +               BufferedImage!))
   6.322 +             (set-gravity world Vector3f/ZERO)
   6.323 +             (Capture/captureVideo
   6.324 +              world
   6.325 +              (File. "/home/r/proj/cortex/render/worm-vision/main-view")))
   6.326 +           (fn [world _ ]
   6.327 +             (.setLocalTranslation me (.getLocation (.getCamera world)))
   6.328 +             (vision-display
   6.329 +              (map #(% world) vision)
   6.330 +              (File. "/home/r/proj/cortex/render/worm-vision"))
   6.331 +             (fix-display world)))))
   6.332 +#+end_src
   6.333 +
   6.334 +* Headers
   6.335 +
   6.336  #+name: vision-header
   6.337  #+begin_src clojure 
   6.338  (ns cortex.vision
   6.339 @@ -456,10 +619,23 @@
   6.340    (:import (com.jme3.scene Node Spatial)))
   6.341  #+end_src
   6.342  
   6.343 -The example code will create two videos of the same rotating object
   6.344 -from different angles. It can be used both for stereoscopic vision
   6.345 -simulation or for simulating multiple creatures, each with their own
   6.346 -sense of vision.
   6.347 +#+name: test-header
   6.348 +#+begin_src clojure
   6.349 +(ns cortex.test.vision
   6.350 +  (:use (cortex world sense util body vision))
   6.351 +  (:use cortex.test.body)
   6.352 +  (:import java.awt.image.BufferedImage)
   6.353 +  (:import javax.swing.JPanel)
   6.354 +  (:import javax.swing.SwingUtilities)
   6.355 +  (:import java.awt.Dimension)
   6.356 +  (:import javax.swing.JFrame)
   6.357 +  (:import com.jme3.math.ColorRGBA)
   6.358 +  (:import com.jme3.scene.Node)
   6.359 +  (:import com.jme3.math.Vector3f)
   6.360 +  (:import java.io.File))
   6.361 +#+end_src
   6.362 +
   6.363 +
   6.364  
   6.365  - As a neat bonus, this idea behind simulated vision also enables one
   6.366    to [[../../cortex/html/capture-video.html][capture live video feeds from jMonkeyEngine]].
   6.367 @@ -471,5 +647,6 @@
   6.368  #+end_src
   6.369  
   6.370  #+begin_src clojure :tangle ../src/cortex/test/vision.clj
   6.371 -<<test-vision>>
   6.372 +<<test-header>>
   6.373 +<<test-1>>
   6.374  #+end_src