1.1 --- a/org/vision.org	Fri Feb 10 02:19:24 2012 -0700
     1.2 +++ b/org/vision.org	Fri Feb 10 11:34:07 2012 -0700
     1.3 @@ -8,8 +8,7 @@
     1.4  #+babel: :mkdirp yes :noweb yes :exports both
     1.5  
     1.6  * Vision
     1.7 -
     1.8 -
     1.9 +  
    1.10  Vision is one of the most important senses for humans, so I need to
    1.11  build a simulated sense of vision for my AI. I will do this with
    1.12  simulated eyes. Each eye can be independely moved and should see its
    1.13 @@ -143,8 +142,6 @@
    1.14  They can be queried every cycle, but their information may not
    1.15  necessairly be different every cycle.
    1.16  
    1.17 -
    1.18 -
    1.19  * Physical Eyes
    1.20  
    1.21  The vision pipeline described above handles the flow of rendered
    1.22 @@ -162,24 +159,6 @@
    1.23  #+begin_src clojure
    1.24  (in-ns 'cortex.vision)
    1.25  
    1.26 -(import com.jme3.math.Vector3f)
    1.27 -
    1.28 -(def blender-rotation-correction
    1.29 -  (doto (Quaternion.)
    1.30 -    (.fromRotationMatrix
    1.31 -     (doto (Matrix3f.)
    1.32 -       (.setColumn 0
    1.33 -                   (Vector3f. 1 0 0))
    1.34 -       (.setColumn 1
    1.35 -                   (Vector3f. 0 -1 0))
    1.36 -       (.setColumn 2
    1.37 -                   (Vector3f. 0 0 -1)))
    1.38 -
    1.39 -     (doto (Matrix3f.)
    1.40 -       (.setColumn 0
    1.41 -                   (Vector3f. 
    1.42 -
    1.43 -
    1.44  (defn add-eye!
    1.45    "Create a Camera centered on the current position of 'eye which
    1.46     follows the closest physical node in 'creature and sends visual
    1.47 @@ -197,10 +176,6 @@
    1.48                        ;; this part is consistent with using Z in
    1.49                        ;; blender as the UP vector.
    1.50                        (.mult rot Vector3f/UNIT_Y))
    1.51 -
    1.52 -    (println-repl "eye unit-z ->"  (.mult rot Vector3f/UNIT_Z))
    1.53 -    (println-repl "eye unit-y ->"  (.mult rot Vector3f/UNIT_Y))
    1.54 -    (println-repl "eye unit-x ->"  (.mult rot Vector3f/UNIT_X))
    1.55      (.setFrustumPerspective
    1.56       cam 45 (/ (.getWidth cam) (.getHeight cam)) 1 1000)
    1.57      (bind-sense target cam) cam))
    1.58 @@ -279,6 +254,7 @@
    1.59  dimansions of the smallest image required to contain all the retinal
    1.60  sensor maps.
    1.61  
    1.62 +#+name: retina
    1.63  #+begin_src clojure
    1.64  (defn retina-sensor-profile
    1.65    "Return a map of pixel sensitivity numbers to BufferedImages
    1.66 @@ -306,7 +282,7 @@
    1.67  
    1.68  First off, get the children of the "eyes" empty node to find all the
    1.69  eyes the creature has.
    1.70 -
    1.71 +#+name: eye-node
    1.72  #+begin_src clojure
    1.73  (defvar 
    1.74    ^{:arglists '([creature])}
    1.75 @@ -318,6 +294,7 @@
    1.76  Then, add the camera created by =(add-eye!)= to the simulation by
    1.77  creating a new viewport.
    1.78  
    1.79 +#+name: add-camera
    1.80  #+begin_src clojure
    1.81  (defn add-camera!
    1.82    "Add a camera to the world, calling continuation on every frame
    1.83 @@ -343,6 +320,7 @@
    1.84  massive gain in speed. =(vision-kernel)= generates a list of such
    1.85  continuation functions, one for each channel of the eye.
    1.86  
    1.87 +#+name: kernel
    1.88  #+begin_src clojure 
    1.89  (in-ns 'cortex.vision)
    1.90  
    1.91 @@ -351,6 +329,21 @@
    1.92    (invoke [this world] (vision-fn world))
    1.93    (applyTo [this args] (apply vision-fn args)))
    1.94  
    1.95 +(defn pixel-sense [sensitivity pixel]
    1.96 +  (let [s-r (bit-shift-right (bit-and 0xFF0000 sensitivity) 16)
    1.97 +        s-g (bit-shift-right (bit-and 0x00FF00 sensitivity)  8)
    1.98 +        s-b (bit-and 0x0000FF sensitivity)
    1.99 +
   1.100 +        p-r (bit-shift-right (bit-and 0xFF0000 pixel)       16)
   1.101 +        p-g (bit-shift-right (bit-and 0x00FF00 pixel)        8)
   1.102 +        p-b (bit-and 0x0000FF pixel)
   1.103 +
   1.104 +        total-sensitivity (* 255 (+ s-r s-g s-b))]
   1.105 +        (float (/ (+ (* s-r p-r)
   1.106 +                     (* s-g p-g)
   1.107 +                     (* s-b p-b))
   1.108 +                  total-sensitivity))))
   1.109 +
   1.110  (defn vision-kernel
   1.111    "Returns a list of functions, each of which will return a color
   1.112     channel's worth of visual information when called inside a running
   1.113 @@ -378,7 +371,7 @@
   1.114         (fn [[key image]]
   1.115           (let [whites (white-coordinates image)
   1.116                 topology (vec (collapse whites))
   1.117 -               mask (color-channel-presets key key)]
   1.118 +               sensitivity (sensitivity-presets key key)]
   1.119             (attached-viewport.
   1.120              (fn [world]
   1.121                (register-eye! world)
   1.122 @@ -386,8 +379,9 @@
   1.123                 topology
   1.124                 (vec 
   1.125                  (for [[x y] whites]
   1.126 -                  (bit-and
   1.127 -                   mask (.getRGB @vision-image x y))))))
   1.128 +                  (pixel-sense 
   1.129 +                   sensitivity
   1.130 +                   (.getRGB @vision-image x y))))))
   1.131              register-eye!)))
   1.132           retinal-map))))
   1.133  
   1.134 @@ -398,7 +392,6 @@
   1.135    (runonce
   1.136     (fn [world]
   1.137       (add-camera! world (.getCamera world) no-op))))
   1.138 -
   1.139  #+end_src
   1.140  
   1.141  Note that since each of the functions generated by =(vision-kernel)=
   1.142 @@ -419,6 +412,7 @@
   1.143  =(vision-kernel)= to each eye in the creature and gather the results
   1.144  into one list of functions.
   1.145  
   1.146 +#+name: main
   1.147  #+begin_src clojure
   1.148  (defn vision!
   1.149    "Returns a function which returns visual sensory data when called
   1.150 @@ -436,7 +430,10 @@
   1.151  =(view-sense)= to construct a function that will create a display for
   1.152  visual data.
   1.153  
   1.154 +#+name: display
   1.155  #+begin_src clojure 
   1.156 +(in-ns 'cortex.vision)
   1.157 +
   1.158  (defn view-vision
   1.159    "Creates a function which accepts a list of visual sensor-data and
   1.160    displays each element of the list to the screen." 
   1.161 @@ -448,7 +445,7 @@
   1.162         (dorun
   1.163          (for [i (range (count coords))]
   1.164            (.setRGB image ((coords i) 0) ((coords i) 1)
   1.165 -                   (sensor-data i))))
   1.166 +                   (gray (int (* 255 (sensor-data i)))))))
   1.167         image))))
   1.168  #+end_src
   1.169  
   1.170 @@ -547,10 +544,18 @@
   1.171  
   1.172  This is the approximation to the human eye described earlier.
   1.173  
   1.174 +#+name: test-2
   1.175  #+begin_src clojure
   1.176  (in-ns 'cortex.test.vision)
   1.177  
   1.178 -(import com.aurellem.capture.Capture)
   1.179 +(defn change-color [obj color]
   1.180 +  (println-repl obj)
   1.181 +  (if obj
   1.182 +    (.setColor  (.getMaterial obj) "Color" color)))
   1.183 +
   1.184 +(defn colored-cannon-ball [color]
   1.185 +  (comp #(change-color % color)
   1.186 +         (fire-cannon-ball)))
   1.187  
   1.188  (defn test-worm-vision [] 
   1.189    (let [the-worm (doto (worm)(body!))
   1.190 @@ -566,12 +571,19 @@
   1.191               :position (Vector3f. 0 -5 0))
   1.192          z-axis
   1.193          (box 0.01 0.01 1 :physical? false :color ColorRGBA/Blue
   1.194 -             :position (Vector3f. 0 -5 0))]
   1.195 +             :position (Vector3f. 0 -5 0))
   1.196 +        timer (RatchetTimer. 60)]
   1.197  
   1.198      (world (nodify [(floor) the-worm x-axis y-axis z-axis me])
   1.199 -           standard-debug-controls
   1.200 +           (assoc standard-debug-controls
   1.201 +             "key-r" (colored-cannon-ball ColorRGBA/Red)
   1.202 +             "key-b" (colored-cannon-ball ColorRGBA/Blue)
   1.203 +             "key-g" (colored-cannon-ball ColorRGBA/Green))
   1.204             (fn [world]
   1.205               (light-up-everything world)
   1.206 +             (speed-up world)
   1.207 +             (.setTimer world timer)
   1.208 +             (display-dialated-time world timer)
   1.209               ;; add a view from the worm's perspective
   1.210               (add-camera!
   1.211                world
   1.212 @@ -583,9 +595,11 @@
   1.213                  (File. "/home/r/proj/cortex/render/worm-vision/worm-view"))
   1.214                 BufferedImage!))
   1.215               (set-gravity world Vector3f/ZERO)
   1.216 -             (Capture/captureVideo
   1.217 -              world
   1.218 -              (File. "/home/r/proj/cortex/render/worm-vision/main-view")))
   1.219 +             (try 
   1.220 +               (Capture/captureVideo
   1.221 +                world
   1.222 +                (File. "/home/r/proj/cortex/render/worm-vision/main-view"))))
   1.223 +               
   1.224             (fn [world _ ]
   1.225               (.setLocalTranslation me (.getLocation (.getCamera world)))
   1.226               (vision-display
   1.227 @@ -594,6 +608,69 @@
   1.228               (fix-display world)))))
   1.229  #+end_src
   1.230  
   1.231 +** Methods to Generate the Worm Video
   1.232 +#+name: magick2
   1.233 +#+begin_src clojure
   1.234 +(ns cortex.video.magick2
   1.235 +  (:import java.io.File)
   1.236 +  (:use clojure.contrib.shell-out))
   1.237 +
   1.238 +(defn images [path]
   1.239 +  (sort (rest (file-seq (File. path)))))
   1.240 +
   1.241 +(def base "/home/r/proj/cortex/render/worm-vision/")
   1.242 +
   1.243 +(defn pics [file]
   1.244 +  (images (str base file)))
   1.245 +
   1.246 +(defn combine-images []
   1.247 +  (let [main-view (pics "main-view")
   1.248 +        worm-view (pics "worm-view")
   1.249 +        blue   (pics "0")
   1.250 +        green  (pics "1")
   1.251 +        red    (pics "2")
   1.252 +        gray   (pics "3")
   1.253 +        blender (let [b-pics (pics "blender")]
   1.254 +                  (concat b-pics (repeat 9001 (last b-pics))))
   1.255 +        background (repeat 9001 (File. (str base "background.png")))
   1.256 +        targets (map
   1.257 +                 #(File. (str base "out/" (format "%07d.png" %)))
   1.258 +                 (range 0 (count main-view)))]
   1.259 +    (dorun
   1.260 +     (pmap
   1.261 +      (comp
   1.262 +       (fn [[background main-view worm-view red green blue gray blender target]]
   1.263 +         (println target)
   1.264 +         (sh "convert"
   1.265 +             background
   1.266 +             main-view "-geometry" "+18+17"    "-composite"
   1.267 +             worm-view "-geometry" "+677+17"   "-composite"
   1.268 +             green     "-geometry" "+685+430"  "-composite"
   1.269 +             red       "-geometry" "+788+430"  "-composite"
   1.270 +             blue      "-geometry" "+894+430"  "-composite"
   1.271 +             gray      "-geometry" "+1000+430" "-composite"
   1.272 +             blender   "-geometry" "+0+0"      "-composite"
   1.273 +             target))
   1.274 +       (fn [& args] (map #(.getCanonicalPath %) args)))
   1.275 +      background main-view worm-view red green blue gray blender targets))))
   1.276 +#+end_src
   1.277 +
   1.278 +#+begin_src sh :results silent
   1.279 +cd /home/r/proj/cortex/render/worm-vision
   1.280 +ffmpeg -r 25 -b 9001k -i out/%07d.png -vcodec libtheora worm-vision.ogg 
   1.281 +#+end_src
   1.282 +
   1.283 +* Demonstration of Vision
   1.284 +#+begin_html
   1.285 +<div class="figure">
   1.286 +<video controls="controls" width="755">
   1.287 +  <source src="../video/worm-vision.ogg" type="video/ogg"
   1.288 +	  preload="none" poster="../images/aurellem-1280x480.png" />
   1.289 +</video>
   1.290 +<p>Simulated Vision in a Virtual Environment</p>
   1.291 +</div>
   1.292 +#+end_html
   1.293 +
   1.294  * Headers
   1.295  
   1.296  #+name: vision-header
   1.297 @@ -602,7 +679,7 @@
   1.298    "Simulate the sense of vision in jMonkeyEngine3. Enables multiple
   1.299    eyes from different positions to observe the same world, and pass
   1.300    the observed data to any arbitray function. Automatically reads
   1.301 -  eye-nodes from specially prepared blender files and instanttiates
   1.302 +  eye-nodes from specially prepared blender files and instantiates
   1.303    them in the world as actual eyes."
   1.304    {:author "Robert McIntyre"}
   1.305    (:use (cortex world sense util))
   1.306 @@ -612,7 +689,7 @@
   1.307    (:import java.nio.ByteBuffer)
   1.308    (:import java.awt.image.BufferedImage)
   1.309    (:import (com.jme3.renderer ViewPort Camera))
   1.310 -  (:import com.jme3.math.ColorRGBA)
   1.311 +  (:import (com.jme3.math ColorRGBA Vector3f Matrix3f))
   1.312    (:import com.jme3.renderer.Renderer)
   1.313    (:import com.jme3.app.Application)
   1.314    (:import com.jme3.texture.FrameBuffer)
   1.315 @@ -632,21 +709,46 @@
   1.316    (:import com.jme3.math.ColorRGBA)
   1.317    (:import com.jme3.scene.Node)
   1.318    (:import com.jme3.math.Vector3f)
   1.319 -  (:import java.io.File))
   1.320 +  (:import java.io.File)
   1.321 +  (:import (com.aurellem.capture Capture RatchetTimer)))
   1.322  #+end_src
   1.323  
   1.324 +* Onward!
   1.325 +  - As a neat bonus, this idea behind simulated vision also enables one
   1.326 +    to [[../../cortex/html/capture-video.html][capture live video feeds from jMonkeyEngine]].
   1.327 +  - Now that we have vision, it's time to tackle [[./hearing.org][hearing]].
   1.328  
   1.329 -
   1.330 -- As a neat bonus, this idea behind simulated vision also enables one
   1.331 -  to [[../../cortex/html/capture-video.html][capture live video feeds from jMonkeyEngine]].
   1.332 +* Source Listing
   1.333 +  - [[../src/cortex/vision.clj][cortex.vision]]
   1.334 +  - [[../src/cortex/test/vision.clj][cortex.test.vision]]
   1.335 +  - [[../src/cortex/video/magick2.clj][cortex.video.magick2]]
   1.336 +  - [[../assets/Models/subtitles/worm-vision-subtitles.blend][worm-vision-subtitles.blend]]
   1.337 +#+html: <ul> <li> <a href="../org/sense.org">This org file</a> </li> </ul>
   1.338 +  - [[http://hg.bortreb.com ][source-repository]]
   1.339 + 
   1.340  
   1.341  
   1.342  * COMMENT Generate Source
   1.343  #+begin_src clojure :tangle ../src/cortex/vision.clj
   1.344 -<<eyes>>
   1.345 +<<vision-header>>
   1.346 +<<pipeline-1>>
   1.347 +<<pipeline-2>>
   1.348 +<<retina>>
   1.349 +<<add-eye>>
   1.350 +<<sensitivity>>
   1.351 +<<eye-node>>
   1.352 +<<add-camera>>
   1.353 +<<kernel>>
   1.354 +<<main>>
   1.355 +<<display>>
   1.356  #+end_src
   1.357  
   1.358  #+begin_src clojure :tangle ../src/cortex/test/vision.clj
   1.359  <<test-header>>
   1.360  <<test-1>>
   1.361 +<<test-2>>
   1.362  #+end_src
   1.363 +
   1.364 +#+begin_src clojure :tangle ../src/cortex/video/magick2.clj
   1.365 +<<magick2>>
   1.366 +#+end_src