rlm@23: #+title: Eyes
rlm@23: #+author: Robert McIntyre
rlm@23: #+email: rlm@mit.edu
rlm@23: #+description: Simulating senses for AI research using JMonkeyEngine3
rlm@23: #+SETUPFILE: ../../aurellem/org/setup.org
rlm@23: #+INCLUDE: ../../aurellem/org/level-0.org
rlm@23: #+babel: :mkdirp yes :noweb yes :exports both
rlm@23: 
rlm@23: 
rlm@23: 
rlm@23: ** Eyes
rlm@23: 
rlm@23: Ultimately I want to make creatures with eyes. Each eye can be
rlm@23: independely moved and should see its own version of the world
rlm@23: depending on where it is.
rlm@23: #+srcname: eyes
rlm@23: #+begin_src clojure 
rlm@23: (ns body.eye)
rlm@23: (use 'cortex.world)
rlm@23: (use 'cortex.import)
rlm@23: (use 'clojure.contrib.def)
rlm@23: (cortex.import/mega-import-jme3)
rlm@23: (rlm.rlm-commands/help)
rlm@23: (import java.nio.ByteBuffer)
rlm@23: (import java.awt.image.BufferedImage)
rlm@23: (import java.awt.Color)
rlm@23: (import java.awt.Dimension)
rlm@23: (import java.awt.Graphics)
rlm@23: (import java.awt.Graphics2D)
rlm@23: (import java.awt.event.WindowAdapter)
rlm@23: (import java.awt.event.WindowEvent)
rlm@23: (import java.awt.image.BufferedImage)
rlm@23: (import java.nio.ByteBuffer)
rlm@23: (import javax.swing.JFrame)
rlm@23: (import javax.swing.JPanel)
rlm@23: (import javax.swing.SwingUtilities)
rlm@23: (import javax.swing.ImageIcon)
rlm@23: (import javax.swing.JOptionPane)
rlm@23: (import java.awt.image.ImageObserver)
rlm@23: 
rlm@23: 
rlm@23: 
rlm@23: (defn scene-processor
rlm@23:   "deals with converting FrameBuffers to BufferedImages so
rlm@23:    that the continuation function can be defined only in terms
rlm@23:    of what it does with BufferedImages"
rlm@23:   [continuation]
rlm@23:   (let [byte-buffer (atom nil)
rlm@23: 	renderer (atom nil)
rlm@23: 	image (atom nil)]
rlm@23:   (proxy [SceneProcessor] []
rlm@23:     (initialize
rlm@23:      [renderManager viewPort]
rlm@23:      (let [cam (.getCamera viewPort)
rlm@23: 	   width (.getWidth cam)
rlm@23: 	   height (.getHeight cam)]
rlm@23:        (reset! renderer (.getRenderer renderManager))
rlm@23:        (reset! byte-buffer
rlm@23: 	     (BufferUtils/createByteBuffer
rlm@23: 	      (* width height 4)))
rlm@23:        (reset! image (BufferedImage. width height
rlm@23: 				     BufferedImage/TYPE_4BYTE_ABGR))))
rlm@23:     (isInitialized [] (not (nil? @byte-buffer)))
rlm@23:     (reshape [_ _ _])
rlm@23:     (preFrame [_])
rlm@23:     (postQueue [_])
rlm@23:     (postFrame
rlm@23:      [#^FrameBuffer fb]
rlm@23:      (.clear @byte-buffer)
rlm@23:      (.readFrameBuffer @renderer fb @byte-buffer)
rlm@23:      (Screenshots/convertScreenShot @byte-buffer @image)
rlm@23:      (continuation @image))
rlm@23:     (cleanup []))))
rlm@23:     
rlm@23: (defn add-eye
rlm@23:   "Add an eye to the world, and call continuation on
rlm@23:    every frame produced"
rlm@23:   [world camera continuation]
rlm@23:   (let [width (.getWidth camera)
rlm@23: 	height (.getHeight camera)
rlm@23: 	render-manager (.getRenderManager world)
rlm@23: 	viewport (.createMainView render-manager "eye-view" camera)]
rlm@23:     (doto viewport
rlm@23:       (.setBackgroundColor ColorRGBA/Black)
rlm@23:       (.setClearFlags true true true)
rlm@23:       (.addProcessor (scene-processor continuation))
rlm@23:       (.attachScene (.getRootNode world)))))
rlm@23: 
rlm@23: (defn make-display-frame [display width height]
rlm@23:   (SwingUtilities/invokeLater
rlm@23:    (fn []
rlm@23:      (.setPreferredSize display (Dimension. width height))
rlm@23:      (doto (JFrame. "Eye Camera!")
rlm@23:        (-> (.getContentPane) (.add display))
rlm@23:        (.setDefaultCloseOperation JFrame/DISPOSE_ON_CLOSE)
rlm@23:        (.pack)
rlm@23:        (.setLocationRelativeTo nil)
rlm@23:        (.setResizable false)
rlm@23:        (.setVisible true)))))
rlm@23: 	   
rlm@23: (defn image-monitor [#^BufferedImage image]
rlm@23:   (proxy [JPanel] []
rlm@23:     (paintComponent 
rlm@23:      [g]
rlm@23:      (proxy-super paintComponent g)
rlm@23:      (locking image
rlm@23:        (.drawImage g image 0 0
rlm@23: 		   (proxy [ImageObserver]
rlm@23: 		       []
rlm@23: 		     (imageUpdate
rlm@23: 		      []
rlm@23: 		      (proxy-super imageUpdate))))))))
rlm@23: 
rlm@23: (defn movie-image []
rlm@23:   (let [setup
rlm@23: 	(runonce
rlm@23: 	 (fn [#^BufferedImage image]
rlm@23: 	   (let [width (.getWidth image)
rlm@23: 		 height (.getHeight image)
rlm@23: 		 display (image-monitor image)
rlm@23: 		 frame (make-display-frame display width height)]
rlm@23: 	     display)))]
rlm@23:     (fn [#^BufferedImage image]
rlm@23:       (.repaint (setup image)))))
rlm@23: 	     
rlm@23: 
rlm@23: (defn observer
rlm@23:   "place thy eye!"
rlm@23:   [world camera]
rlm@23:   (let [eye camera
rlm@23: 	width (.getWidth eye)
rlm@23: 	height (.getHeight eye)]
rlm@23:     (no-exceptions
rlm@23:      (add-eye
rlm@23:       world
rlm@23:       eye
rlm@23:       (movie-image)))))
rlm@23: #+end_src
rlm@23: 
rlm@23: #+srcname: test-vision
rlm@23: #+begin_src clojure
rlm@23: 
rlm@23: (ns test.vision)
rlm@23: (use 'cortex.world)
rlm@23: (use 'cortex.import)
rlm@23: (use 'clojure.contrib.def)
rlm@23: (use 'body.eye)
rlm@23: (cortex.import/mega-import-jme3)
rlm@23: (rlm.rlm-commands/help)
rlm@23: (import java.nio.ByteBuffer)
rlm@23: (import java.awt.image.BufferedImage)
rlm@23: (import java.awt.Color)
rlm@23: (import java.awt.Dimension)
rlm@23: (import java.awt.Graphics)
rlm@23: (import java.awt.Graphics2D)
rlm@23: (import java.awt.event.WindowAdapter)
rlm@23: (import java.awt.event.WindowEvent)
rlm@23: (import java.awt.image.BufferedImage)
rlm@23: (import java.nio.ByteBuffer)
rlm@23: (import javax.swing.JFrame)
rlm@23: (import javax.swing.JPanel)
rlm@23: (import javax.swing.SwingUtilities)
rlm@23: (import javax.swing.ImageIcon)
rlm@23: (import javax.swing.JOptionPane)
rlm@23: (import java.awt.image.ImageObserver)
rlm@23: 
rlm@23: 
rlm@23: (def width 200)
rlm@23: (def height 200)
rlm@23: 
rlm@23: (defn camera []
rlm@23:   (doto (Camera. width height)
rlm@23:     (.setFrustumPerspective 45 1 1 1000)
rlm@23:     (.setLocation (Vector3f. -3 0 -5))
rlm@23:     (.lookAt Vector3f/ZERO Vector3f/UNIT_Y)))
rlm@23: 
rlm@23: (defn camera2 []
rlm@23:   (doto (Camera. width height)
rlm@23:     (.setFrustumPerspective 45 1 1 1000)
rlm@23:     (.setLocation (Vector3f. 3 0 -5))
rlm@23:     (.lookAt Vector3f/ZERO Vector3f/UNIT_Y)))
rlm@23: 
rlm@23: (defn setup-fn [world]
rlm@23:   (let [eye (camera)
rlm@23: 	width (.getWidth eye)
rlm@23: 	height (.getHeight eye)]
rlm@23:     (no-exceptions
rlm@23:      (add-eye
rlm@23:       world
rlm@23:       eye
rlm@23:       (runonce visual))
rlm@23:      (add-eye
rlm@23:       world
rlm@23:       (camera2)
rlm@23:       (runonce visual)))))
rlm@23: 
rlm@23: (defn spider-eye [position]
rlm@23:   (doto (Camera. 200 200 )
rlm@23:     (.setFrustumPerspective 45 1 1 1000)
rlm@23:     (.setLocation position)
rlm@23:     (.lookAt Vector3f/ZERO Vector3f/UNIT_Y)))
rlm@23:   
rlm@23: (defn setup-fn* [world]
rlm@23:   (let [eye (camera)
rlm@23: 	width (.getWidth eye)
rlm@23: 	height (.getHeight eye)]
rlm@23:     ;;(.setClearFlags (.getViewPort world) true true true)
rlm@23:     (observer world (.getCamera world))
rlm@23:     (observer world (spider-eye (Vector3f.  3  0 -5)))
rlm@23:     ;;(observer world (spider-eye (Vector3f.  0  0 -5)))
rlm@23:     ;; (observer world (spider-eye (Vector3f. -3  0 -5)))
rlm@23:     ;; (observer world (spider-eye (Vector3f.  0  3 -5)))
rlm@23:     ;; (observer world (spider-eye (Vector3f.  0 -3 -5)))
rlm@23:     ;; (observer world (spider-eye (Vector3f.  3  3 -5)))
rlm@23:     ;; (observer world (spider-eye (Vector3f. -3  3 -5)))
rlm@23:     ;; (observer world (spider-eye (Vector3f.  3 -3 -5)))
rlm@23:     ;; (observer world (spider-eye (Vector3f. -3 -3 -5)))
rlm@23: 
rlm@23:     )
rlm@23:   world)
rlm@23: 
rlm@23: (defn test-world []
rlm@23:   (let [thing (box 1 1 1 :physical? false)]
rlm@23:     (world
rlm@23:      (doto (Node.)
rlm@23:        (.attachChild thing))
rlm@23:      {}
rlm@23:      setup-fn
rlm@23:      (fn [world tpf]
rlm@23:        (.rotate thing (* tpf 0.2) 0 0)
rlm@23:        ))))
rlm@23: 
rlm@23: 
rlm@23: #+end_src
rlm@23: 
rlm@23: 
rlm@23: #+results: eyes
rlm@23: : #'body.eye/test-world
rlm@23: 
rlm@23: Note the use of continuation passing style for connecting the eye to a
rlm@23: function to process the output. The example code will create two
rlm@23: videos of the same rotating cube from different angles, sutiable for
rlm@23: stereoscopic vision.
rlm@24: 
rlm@24: 
rlm@24: 
rlm@24: * COMMENT code generation
rlm@24: #+begin_src clojure :tangle ../src/body/eye.clj
rlm@24: <<eyes>>
rlm@24: #+end_src
rlm@24: 
rlm@24: #+begin_src clojure :tangle ../src/test/vision.clj
rlm@24: <<test-vision>>
rlm@24: #+end_src