#+title: Simulated Sense of Sight

 #+author: Robert McIntyre

 #+email: rlm@mit.edu

 #+description: Simulated sight for AI research using JMonkeyEngine3 and clojure

 #+keywords: computer vision, jMonkeyEngine3, clojure

 #+SETUPFILE: ../../aurellem/org/setup.org

 #+INCLUDE: ../../aurellem/org/level-0.org

 #+babel: :mkdirp yes :noweb yes :exports both

 

 * Vision

 

 I want to make creatures with eyes. Each eye can be independely moved

 and should see its own version of the world depending on where it is.

 

 Here's how vision will work.

 

 Make the continuation in scene-processor take FrameBuffer,

 byte-buffer, BufferedImage already sized to the correct

 dimensions. the continuation will decide wether to "mix" them

 into the BufferedImage, lazily ignore them, or mix them halfway

 and call c/graphics card routines.

 

 (vision creature) will take an optional :skip argument which will

 inform the continuations in scene processor to skip the given

 number of cycles 0 means that no cycles will be skipped.

 

 (vision creature) will return [init-functions sensor-functions].

 The init-functions are each single-arg functions that take the

 world and register the cameras and must each be called before the

 corresponding sensor-functions.  Each init-function returns the

 viewport for that eye which can be manipulated, saved, etc. Each

 sensor-function is a thunk and will return data in the same

 format as the tactile-sensor functions the structure is

 [topology, sensor-data]. Internally, these sensor-functions

 maintain a reference to sensor-data which is periodically updated

 by the continuation function established by its init-function.

 They can be queried every cycle, but their information may not

 necessairly be different every cycle.

 

 Each eye in the creature in blender will work the same way as

 joints -- a zero dimensional object with no geometry whose local

 coordinate system determines the orientation of the resulting

 eye. All eyes will have a parent named "eyes" just as all joints

 have a parent named "joints". The resulting camera will be a

 ChaseCamera or a CameraNode bound to the geo that is closest to

 the eye marker. The eye marker will contain the metadata for the

 eye, and will be moved by it's bound geometry. The dimensions of

 the eye's camera are equal to the dimensions of the eye's "UV"

 map.

 

 #+name: eyes

 #+begin_src clojure 

 (ns cortex.vision

   "Simulate the sense of vision in jMonkeyEngine3. Enables multiple

   eyes from different positions to observe the same world, and pass

   the observed data to any arbitray function. Automatically reads

   eye-nodes from specially prepared blender files and instanttiates

   them in the world as actual eyes."

   {:author "Robert McIntyre"}

   (:use (cortex world sense util))

   (:use clojure.contrib.def)

   (:import com.jme3.post.SceneProcessor)

   (:import (com.jme3.util BufferUtils Screenshots))

   (:import java.nio.ByteBuffer)

   (:import java.awt.image.BufferedImage)

   (:import (com.jme3.renderer ViewPort Camera))

   (:import com.jme3.math.ColorRGBA)

   (:import com.jme3.renderer.Renderer)

   (:import com.jme3.app.Application)

   (:import com.jme3.texture.FrameBuffer)

   (:import (com.jme3.scene Node Spatial)))

 

 (defn vision-pipeline

   "Create a SceneProcessor object which wraps a vision processing

   continuation function. The continuation is a function that takes 

   [#^Renderer r #^FrameBuffer fb #^ByteBuffer b #^BufferedImage bi],

   each of which has already been appropiately sized."

   [continuation]

   (let [byte-buffer (atom nil)

 	renderer (atom nil)

         image (atom nil)]

   (proxy [SceneProcessor] []

     (initialize

      [renderManager viewPort]

      (let [cam (.getCamera viewPort)

 	   width (.getWidth cam)

 	   height (.getHeight cam)]

        (reset! renderer (.getRenderer renderManager))

        (reset! byte-buffer

 	     (BufferUtils/createByteBuffer

 	      (* width height 4)))

         (reset! image (BufferedImage.

                       width height

                       BufferedImage/TYPE_4BYTE_ABGR))))

     (isInitialized [] (not (nil? @byte-buffer)))

     (reshape [_ _ _])

     (preFrame [_])

     (postQueue [_])

     (postFrame

      [#^FrameBuffer fb]

      (.clear @byte-buffer)

      (continuation @renderer fb @byte-buffer @image))

     (cleanup []))))

     

 (defn frameBuffer->byteBuffer!

   "Transfer the data in the graphics card (Renderer, FrameBuffer) to

    the CPU (ByteBuffer)."  

   [#^Renderer r #^FrameBuffer fb #^ByteBuffer bb]

   (.readFrameBuffer r fb bb) bb)

 

 (defn byteBuffer->bufferedImage!

   "Convert the C-style BGRA image data in the ByteBuffer bb to the AWT

    style ABGR image data and place it in BufferedImage bi."

   [#^ByteBuffer bb #^BufferedImage bi]

   (Screenshots/convertScreenShot bb bi) bi)

 

 (defn BufferedImage!

   "Continuation which will grab the buffered image from the materials

    provided by (vision-pipeline)."

   [#^Renderer r #^FrameBuffer fb #^ByteBuffer bb #^BufferedImage bi]

   (byteBuffer->bufferedImage!

    (frameBuffer->byteBuffer! r fb bb) bi))

 

 (defn add-camera!

   "Add a camera to the world, calling continuation on every frame

   produced." 

   [#^Application world camera continuation]

   (let [width (.getWidth camera)

 	height (.getHeight camera)

 	render-manager (.getRenderManager world)

 	viewport (.createMainView render-manager "eye-view" camera)]

     (doto viewport

       (.setClearFlags true true true)

       (.setBackgroundColor ColorRGBA/Black)

       (.addProcessor (vision-pipeline continuation))

       (.attachScene (.getRootNode world)))))

 

 (defn retina-sensor-profile

   "Return a map of pixel selection functions to BufferedImages

    describing the distribution of light-sensitive components of this

    eye. Each function creates an integer from the rgb values found in

    the pixel. :red, :green, :blue, :gray are already defined as

    extracting the red, green, blue, and average components

    respectively."

    [#^Spatial eye]

    (if-let [eye-map (meta-data eye "eye")]

      (map-vals

       load-image

       (eval (read-string eye-map)))))

 

 (defn eye-dimensions

   "Returns [width, height] specified in the metadata of the eye"

   [#^Spatial eye]

   (let [dimensions

           (map #(vector (.getWidth %) (.getHeight %))

                (vals (retina-sensor-profile eye)))]

     [(apply max (map first dimensions))

      (apply max (map second dimensions))]))

 

 (defvar 

   ^{:arglists '([creature])}

   eyes

   (sense-nodes "eyes")

   "Return the children of the creature's \"eyes\" node.")

 

 (defn add-eye!

   "Create a Camera centered on the current position of 'eye which

    follows the closest physical node in 'creature and sends visual

    data to 'continuation."

   [#^Node creature #^Spatial eye]

   (let [target (closest-node creature eye)

         [cam-width cam-height] (eye-dimensions eye)

         cam (Camera. cam-width cam-height)]

     (.setLocation cam (.getWorldTranslation eye))

     (.setRotation cam (.getWorldRotation eye))

     (.setFrustumPerspective

      cam 45 (/ (.getWidth cam) (.getHeight cam))

      1 1000)

     (bind-sense target cam)

     cam))

 

 (defvar color-channel-presets

   {:all    0xFFFFFF

    :red    0xFF0000

    :blue   0x0000FF

    :green  0x00FF00}

   "Bitmasks for common RGB color channels")

 

 (defn vision-fn

   "Returns a list of functions, each of which will return a color

    channel's worth of visual information when called inside a running

    simulation."

   [#^Node creature #^Spatial eye & {skip :skip :or {skip 0}}]

   (let [retinal-map (retina-sensor-profile eye)

         camera (add-eye! creature eye)

         vision-image

         (atom

          (BufferedImage. (.getWidth camera)

                          (.getHeight camera)

                          BufferedImage/TYPE_BYTE_BINARY))

         register-eye!

         (runonce

          (fn [world]

            (add-camera!

             world camera

             (let [counter  (atom 0)]

               (fn [r fb bb bi]

                 (if (zero? (rem (swap! counter inc) (inc skip)))

                   (reset! vision-image

                           (BufferedImage! r fb bb bi))))))))]

      (vec

       (map

        (fn [[key image]]

          (let [whites (white-coordinates image)

                topology (vec (collapse whites))

                mask (color-channel-presets key)]

            (fn [world]

              (register-eye! world)

              (vector

               topology

               (vec 

                (for [[x y] whites]

                  (bit-and

                   mask (.getRGB @vision-image x y))))))))

        retinal-map))))

 

 

 ;; TODO maybe should add a viewport-manipulation function to

 ;; automatically change viewport settings, attach shadow filters, etc.

 

 (defn vision!

   "Returns a function which returns visual sensory data when called

    inside a running simulation"

   [#^Node creature & {skip :skip :or {skip 0}}]

   (reduce

    concat 

    (for [eye (eyes creature)]

      (vision-fn creature eye))))

 

 (defn view-vision

   "Creates a function which accepts a list of visual sensor-data and

   displays each element of the list to the screen." 

   []

   (view-sense

    (fn 

      [[coords sensor-data]]

      (let [image (points->image coords)]

        (dorun

         (for [i (range (count coords))]

           (.setRGB image ((coords i) 0) ((coords i) 1)

                    (sensor-data i))))

        image))))

 

 #+end_src

 

 

 Note the use of continuation passing style for connecting the eye to a

 function to process the output. You can create any number of eyes, and

 each of them will see the world from their own =Camera=. Once every

 frame, the rendered image is copied to a =BufferedImage=, and that

 data is sent off to the continuation function. Moving the =Camera=

 which was used to create the eye will change what the eye sees.

 

 * Example

 

 #+name: test-vision

 #+begin_src clojure

 (ns cortex.test.vision

   (:use (cortex world util vision))

   (:import java.awt.image.BufferedImage)

   (:import javax.swing.JPanel)

   (:import javax.swing.SwingUtilities)

   (:import java.awt.Dimension)

   (:import javax.swing.JFrame)

   (:import com.jme3.math.ColorRGBA)

   (:import com.jme3.scene.Node)

   (:import com.jme3.math.Vector3f))

 

 (defn test-two-eyes

   "Testing vision:

    Tests the vision system by creating two views of the same rotating

    object from different angles and displaying both of those views in

    JFrames.

 

    You should see a rotating cube, and two windows,

    each displaying a different view of the cube."

   []

   (let [candy

         (box 1 1 1 :physical? false :color ColorRGBA/Blue)]

     (world

      (doto (Node.)

        (.attachChild candy))

      {}

      (fn [world]

        (let [cam (.clone (.getCamera world))

              width (.getWidth cam)

              height (.getHeight cam)]

          (add-camera! world cam 

                   ;;no-op

                   (comp (view-image) BufferedImage!)

                   )

          (add-camera! world

                   (doto (.clone cam)

                     (.setLocation (Vector3f. -10 0 0))

                     (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))

                   ;;no-op

                   (comp (view-image) BufferedImage!))

          ;; This is here to restore the main view

          ;; after the other views have completed processing

          (add-camera! world (.getCamera world) no-op)))

      (fn [world tpf]

        (.rotate candy (* tpf 0.2) 0 0)))))

 #+end_src

 

 #+results: test-vision

 : #'cortex.test.vision/test-two-eyes

 

 The example code will create two videos of the same rotating object

 from different angles. It can be used both for stereoscopic vision

 simulation or for simulating multiple creatures, each with their own

 sense of vision.

 

 - As a neat bonus, this idea behind simulated vision also enables one

   to [[../../cortex/html/capture-video.html][capture live video feeds from jMonkeyEngine]].

 

 

 * COMMENT code generation

 #+begin_src clojure :tangle ../src/cortex/vision.clj

 <<eyes>>

 #+end_src

 

 #+begin_src clojure :tangle ../src/cortex/test/vision.clj

 <<test-vision>>

 #+end_src