rlm@34: #+title: Simulated Sense of Sight
rlm@23: #+author: Robert McIntyre
rlm@23: #+email: rlm@mit.edu
rlm@38: #+description: Simulated sight for AI research using JMonkeyEngine3 and clojure
rlm@34: #+keywords: computer vision, jMonkeyEngine3, clojure
rlm@23: #+SETUPFILE: ../../aurellem/org/setup.org
rlm@23: #+INCLUDE: ../../aurellem/org/level-0.org
rlm@23: #+babel: :mkdirp yes :noweb yes :exports both
rlm@23: 
rlm@34: * Vision
rlm@23: 
rlm@34: I want to make creatures with eyes. Each eye can be independely moved
rlm@34: and should see its own version of the world depending on where it is.
rlm@23: #+srcname: eyes
rlm@23: #+begin_src clojure 
rlm@34: (ns cortex.vision
rlm@34:   "Simulate the sense of vision in jMonkeyEngine3. Enables multiple
rlm@34:   eyes from different positions to observe the same world, and pass
rlm@34:   the observed data to any arbitray function."
rlm@34:   {:author "Robert McIntyre"}
rlm@34:   (:use cortex.world)
rlm@34:   (:import com.jme3.post.SceneProcessor)
rlm@34:   (:import (com.jme3.util Screenshots BufferUtils))
rlm@34:   (:import java.nio.ByteBuffer)
rlm@34:   (:import java.awt.image.BufferedImage)
rlm@34:   (:import com.jme3.renderer.ViewPort)
rlm@34:   (:import com.jme3.math.ColorRGBA))
rlm@23: 
rlm@23: (defn scene-processor
rlm@34:   "Create a SceneProcessor object which wraps a vision processing
rlm@34:   continuation function. The SceneProcessor will take care of
rlm@34:   converting the rendered frame to a BufferedImage and passing that
rlm@34:   BufferedImage to the continuation.  The continuation should be a
rlm@34:   function that takes a BufferedImage."
rlm@23:   [continuation]
rlm@23:   (let [byte-buffer (atom nil)
rlm@23: 	renderer (atom nil)
rlm@23: 	image (atom nil)]
rlm@23:   (proxy [SceneProcessor] []
rlm@23:     (initialize
rlm@23:      [renderManager viewPort]
rlm@23:      (let [cam (.getCamera viewPort)
rlm@23: 	   width (.getWidth cam)
rlm@23: 	   height (.getHeight cam)]
rlm@23:        (reset! renderer (.getRenderer renderManager))
rlm@23:        (reset! byte-buffer
rlm@23: 	     (BufferUtils/createByteBuffer
rlm@23: 	      (* width height 4)))
rlm@34:        (reset! image (BufferedImage.
rlm@34:                       width height
rlm@34:                       BufferedImage/TYPE_4BYTE_ABGR))))
rlm@23:     (isInitialized [] (not (nil? @byte-buffer)))
rlm@23:     (reshape [_ _ _])
rlm@23:     (preFrame [_])
rlm@23:     (postQueue [_])
rlm@23:     (postFrame
rlm@23:      [#^FrameBuffer fb]
rlm@23:      (.clear @byte-buffer)
rlm@23:      (.readFrameBuffer @renderer fb @byte-buffer)
rlm@23:      (Screenshots/convertScreenShot @byte-buffer @image)
rlm@23:      (continuation @image))
rlm@23:     (cleanup []))))
rlm@23:     
rlm@23: (defn add-eye
rlm@34:   "Add an eye to the world, calling continuation on every frame
rlm@34:   produced." 
rlm@23:   [world camera continuation]
rlm@23:   (let [width (.getWidth camera)
rlm@23: 	height (.getHeight camera)
rlm@23: 	render-manager (.getRenderManager world)
rlm@23: 	viewport (.createMainView render-manager "eye-view" camera)]
rlm@23:     (doto viewport
rlm@23:       (.setClearFlags true true true)
rlm@34:       (.setBackgroundColor ColorRGBA/Gray)
rlm@23:       (.addProcessor (scene-processor continuation))
rlm@23:       (.attachScene (.getRootNode world)))))
rlm@34: #+end_src
rlm@23: 
rlm@34: Note the use of continuation passing style for connecting the eye to a
rlm@34: function to process the output. You can create any number of eyes, and
rlm@34: each of them will see the world from their own =Camera=. Once every
rlm@34: frame, the rendered image is copied to a =BufferedImage=, and that
rlm@34: data is sent off to the continuation function. Moving the =Camera=
rlm@34: which was used to create the eye will change what the eye sees.
rlm@23: 
rlm@34: * Example
rlm@23: 
rlm@23: #+srcname: test-vision
rlm@23: #+begin_src clojure
rlm@34: (ns test.vision
rlm@34:   (:use (cortex world util vision))
rlm@34:   (:import java.awt.image.BufferedImage)
rlm@34:   (:import javax.swing.JPanel)
rlm@34:   (:import javax.swing.SwingUtilities)
rlm@34:   (:import java.awt.Dimension)
rlm@34:   (:import javax.swing.JFrame)
rlm@34:   (:import com.jme3.math.ColorRGBA)
rlm@34:   (:import com.jme3.scene.Node))
rlm@23: 
rlm@34: (defn view-image
rlm@34:   "Initailizes a JPanel on which you may draw a BufferedImage of the
rlm@34:   given width and height.  Returns a function that accepts a
rlm@34:   BufferedImage and draws it to the JPanel."
rlm@34:   [width height]
rlm@34:   (let [image
rlm@34:         (atom
rlm@34:          (BufferedImage. width height BufferedImage/TYPE_4BYTE_ABGR))
rlm@34:         panel 
rlm@34:         (proxy [JPanel] []
rlm@34:           (paint
rlm@34:             [graphics]
rlm@34:             (proxy-super paintComponent graphics)
rlm@35:             (.drawImage graphics @image 0 0 nil)))]
rlm@34:     (SwingUtilities/invokeLater
rlm@34:      (fn []
rlm@34:        (.setPreferredSize panel (Dimension. width height))
rlm@34:        (doto (JFrame. "Eye Camera!")
rlm@34:          (-> (.getContentPane) (.add panel))
rlm@34:          (.pack)
rlm@34:          (.setLocationRelativeTo nil)
rlm@34:          (.setResizable false)
rlm@34:          (.setVisible true))))
rlm@34:     (fn [#^BufferedImage i]
rlm@34:       (reset! image i)
rlm@34:       (.repaint panel))))
rlm@23: 
rlm@36: (defn test-two-eyes
rlm@34:   "Tests the vision system by creating two views of the same rotating
rlm@34:   object from different angles and displaying both of those views in
rlm@36:   JFrames."  
rlm@36:   []
rlm@34:   (.start
rlm@34:    (let [candy
rlm@34:          (box 1 1 1 :physical? false :color ColorRGBA/Blue)]
rlm@34:      (world (doto (Node.)
rlm@34:               (.attachChild candy))
rlm@34:             {}
rlm@34:             (fn [world]
rlm@34:               (let [cam (.clone (.getCamera world))
rlm@34:                     width (.getWidth cam)
rlm@34:                     height (.getHeight cam)]
rlm@35:                 (add-eye world cam (view-image width height))
rlm@34:                 (add-eye world
rlm@34:                          (doto (.clone cam)
rlm@34:                            (.setLocation (Vector3f. -10 0 0))
rlm@34:                            (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))
rlm@34:                          (view-image width height))
rlm@35:                 ;; This is here to restore the main view
rlm@34:                 ;; after the other views have completed processing
rlm@35:                 (add-eye world (.getCamera world) no-op)))
rlm@34:             (fn [world tpf]
rlm@34:               (.rotate candy (* tpf 0.2) 0 0))))))
rlm@23: #+end_src
rlm@23: 
rlm@34: The example code will create two videos of the same rotating object
rlm@34: from different angles. It can be used both for stereoscopic vision
rlm@34: simulation or for simulating multiple creatures, each with their own
rlm@34: sense of vision.
rlm@24: 
rlm@35: - As a neat bonus, this idea behind simulated vision also enables one
rlm@35:   to [[../../cortex/html/capture-video.html][capture live video feeds from jMonkeyEngine]].
rlm@35: 
rlm@24: 
rlm@24: * COMMENT code generation
rlm@34: #+begin_src clojure :tangle ../src/cortex/vision.clj
rlm@24: <<eyes>>
rlm@24: #+end_src
rlm@24: 
rlm@24: #+begin_src clojure :tangle ../src/test/vision.clj
rlm@24: <<test-vision>>
rlm@24: #+end_src