#+title: Simulated Sense of Sight

 #+author: Robert McIntyre

 #+email: rlm@mit.edu

 #+description: Simulated sight for AI research using JMonkeyEngine3 and clojure

 #+keywords: computer vision, jMonkeyEngine3, clojure

 #+SETUPFILE: ../../aurellem/org/setup.org

 #+INCLUDE: ../../aurellem/org/level-0.org

 #+babel: :mkdirp yes :noweb yes :exports both

 

 * Vision

 

 I want to make creatures with eyes. Each eye can be independely moved

 and should see its own version of the world depending on where it is.

 #+name: eyes

 #+begin_src clojure 

 (ns cortex.vision

   "Simulate the sense of vision in jMonkeyEngine3. Enables multiple

   eyes from different positions to observe the same world, and pass

   the observed data to any arbitray function."

   {:author "Robert McIntyre"}

   (:use cortex.world)

   (:import com.jme3.post.SceneProcessor)

   (:import (com.jme3.util BufferUtils))

   (:import java.nio.ByteBuffer)

   (:import java.awt.image.BufferedImage)

   (:import com.jme3.renderer.ViewPort)

   (:import com.jme3.math.ColorRGBA))

 

 (defn scene-processor

   "Create a SceneProcessor object which wraps a vision processing

   continuation function. The SceneProcessor will take care of

   converting the rendered frame to a BufferedImage and passing that

   BufferedImage to the continuation.  The continuation should be a

   function that takes a ByteBuffer which represents the image."

   [continuation]

   (let [byte-buffer (atom nil)

 	renderer (atom nil)]

   (proxy [SceneProcessor] []

     (initialize

      [renderManager viewPort]

      (let [cam (.getCamera viewPort)

 	   width (.getWidth cam)

 	   height (.getHeight cam)]

        (reset! renderer (.getRenderer renderManager))

        (reset! byte-buffer

 	     (BufferUtils/createByteBuffer

 	      (* width height 4)))))

     (isInitialized [] (not (nil? @byte-buffer)))

     (reshape [_ _ _])

     (preFrame [_])

     (postQueue [_])

     (postFrame

      [#^FrameBuffer fb]

      (.clear @byte-buffer)

      ;;(.readFrameBuffer @renderer fb @byte-buffer)

      (continuation @byte-buffer))

     (cleanup []))))

     

 (defn buffer->image! [width height]

   (let [image (BufferedImage. width height

                               BufferedImage/TYPE_4BYTE_ABGR)]

     (fn [byte-buffer]

       (Screenshots/convertScreenShot byte-buffer image)

       image)))

 

 (defn add-eye

   "Add an eye to the world, calling continuation on every frame

   produced." 

   [world camera continuation]

   (let [width (.getWidth camera)

 	height (.getHeight camera)

 	render-manager (.getRenderManager world)

 	viewport (.createMainView render-manager "eye-view" camera)]

     (doto viewport

       (.setClearFlags true true true)

       (.setBackgroundColor ColorRGBA/Black)

       (.addProcessor (scene-processor continuation))

       (.attachScene (.getRootNode world)))))

 #+end_src

 

 #+results: eyes

 : #'cortex.vision/add-eye

 

 Note the use of continuation passing style for connecting the eye to a

 function to process the output. You can create any number of eyes, and

 each of them will see the world from their own =Camera=. Once every

 frame, the rendered image is copied to a =BufferedImage=, and that

 data is sent off to the continuation function. Moving the =Camera=

 which was used to create the eye will change what the eye sees.

 

 * Example

 

 #+name: test-vision

 #+begin_src clojure

 (ns cortex.test.vision

   (:use (cortex world util vision))

   (:import java.awt.image.BufferedImage)

   (:import javax.swing.JPanel)

   (:import javax.swing.SwingUtilities)

   (:import java.awt.Dimension)

   (:import javax.swing.JFrame)

   (:import com.jme3.math.ColorRGBA)

   (:import com.jme3.scene.Node)

   (:import com.jme3.math.Vector3f)

   (:import (com.jme3.util Screenshots)))

 

 (defn view-image

   "Initailizes a JPanel on which you may draw a BufferedImage.

    Returns a function that accepts a BufferedImage and draws it to the

    JPanel."

   []

   (let [image

         (atom

          (BufferedImage. 1 1 BufferedImage/TYPE_4BYTE_ABGR))

         panel 

         (proxy [JPanel] []

           (paint

             [graphics]

             (proxy-super paintComponent graphics)

             (.drawImage graphics @image 0 0 nil)))

         frame (JFrame. "Display Image")]

     (SwingUtilities/invokeLater

      (fn []

        (doto frame

          (-> (.getContentPane) (.add panel))

          (.pack)

          (.setLocationRelativeTo nil)

          (.setResizable true)

          (.setVisible true))))

     (fn [#^BufferedImage i]

       (reset! image i)

       (.setSize frame (+ 8 (.getWidth i)) (+ 28 (.getHeight i)))

       (.repaint panel 0 0 (.getWidth i) (.getHeight i)))))

 

 (defn test-two-eyes

   "Testing vision:

    Tests the vision system by creating two views of the same rotating

    object from different angles and displaying both of those views in

    JFrames.

 

    You should see a rotating cube, and two windows,

    each displaying a different view of the cube."

   []

   (let [candy

         (box 1 1 1 :physical? false :color ColorRGBA/Blue)]

     (world

      (doto (Node.)

        (.attachChild candy))

      {}

      (fn [world]

        (let [cam (.clone (.getCamera world))

              width (.getWidth cam)

              height (.getHeight cam)]

          (add-eye world cam 

                   no-op

                   ;;(comp (view-image) (buffer->image! width height))

                   )

          (add-eye world

                   (doto (.clone cam)

                     (.setLocation (Vector3f. -10 0 0))

                     (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))

                   no-op

                   ;;(comp (view-image) (buffer->image! width height))

                   )

          

          

          ;; This is here to restore the main view

          ;; after the other views have completed processing

          (add-eye world (.getCamera world) no-op)))

      (fn [world tpf]

        (.rotate candy (* tpf 0.2) 0 0)))))

 #+end_src

 

 #+results: test-vision

 : #'cortex.test.vision/test-two-eyes

 

 The example code will create two videos of the same rotating object

 from different angles. It can be used both for stereoscopic vision

 simulation or for simulating multiple creatures, each with their own

 sense of vision.

 

 - As a neat bonus, this idea behind simulated vision also enables one

   to [[../../cortex/html/capture-video.html][capture live video feeds from jMonkeyEngine]].

 

 

 * COMMENT code generation

 #+begin_src clojure :tangle ../src/cortex/vision.clj

 <<eyes>>

 #+end_src

 

 #+begin_src clojure :tangle ../src/cortex/test/vision.clj

 <<test-vision>>

 #+end_src