view org/eyes.org @ 108:92b857b6145d

slow implementation of UV-mapped touch is complete
author Robert McIntyre <rlm@mit.edu>
date Sun, 15 Jan 2012 04:08:31 -0700
parents b7a3ba5e879b
children 128fa71ee188
line wrap: on
line source
1 #+title: Simulated Sense of Sight
2 #+author: Robert McIntyre
3 #+email: rlm@mit.edu
4 #+description: Simulated sight for AI research using JMonkeyEngine3 and clojure
5 #+keywords: computer vision, jMonkeyEngine3, clojure
6 #+SETUPFILE: ../../aurellem/org/setup.org
7 #+INCLUDE: ../../aurellem/org/level-0.org
8 #+babel: :mkdirp yes :noweb yes :exports both
10 * Vision
12 I want to make creatures with eyes. Each eye can be independely moved
13 and should see its own version of the world depending on where it is.
14 #+name: eyes
15 #+begin_src clojure
16 (ns cortex.vision
17 "Simulate the sense of vision in jMonkeyEngine3. Enables multiple
18 eyes from different positions to observe the same world, and pass
19 the observed data to any arbitray function."
20 {:author "Robert McIntyre"}
21 (:use cortex.world)
22 (:import com.jme3.post.SceneProcessor)
23 (:import (com.jme3.util Screenshots BufferUtils))
24 (:import java.nio.ByteBuffer)
25 (:import java.awt.image.BufferedImage)
26 (:import com.jme3.renderer.ViewPort)
27 (:import com.jme3.math.ColorRGBA))
29 (defn scene-processor
30 "Create a SceneProcessor object which wraps a vision processing
31 continuation function. The SceneProcessor will take care of
32 converting the rendered frame to a BufferedImage and passing that
33 BufferedImage to the continuation. The continuation should be a
34 function that takes a BufferedImage."
35 [continuation]
36 (let [byte-buffer (atom nil)
37 renderer (atom nil)
38 image (atom nil)]
39 (proxy [SceneProcessor] []
40 (initialize
41 [renderManager viewPort]
42 (let [cam (.getCamera viewPort)
43 width (.getWidth cam)
44 height (.getHeight cam)]
45 (reset! renderer (.getRenderer renderManager))
46 (reset! byte-buffer
47 (BufferUtils/createByteBuffer
48 (* width height 4)))
49 (reset! image (BufferedImage.
50 width height
51 BufferedImage/TYPE_4BYTE_ABGR))))
52 (isInitialized [] (not (nil? @byte-buffer)))
53 (reshape [_ _ _])
54 (preFrame [_])
55 (postQueue [_])
56 (postFrame
57 [#^FrameBuffer fb]
58 (.clear @byte-buffer)
59 (.readFrameBuffer @renderer fb @byte-buffer)
60 (Screenshots/convertScreenShot @byte-buffer @image)
61 (continuation @image))
62 (cleanup []))))
64 (defn add-eye
65 "Add an eye to the world, calling continuation on every frame
66 produced."
67 [world camera continuation]
68 (let [width (.getWidth camera)
69 height (.getHeight camera)
70 render-manager (.getRenderManager world)
71 viewport (.createMainView render-manager "eye-view" camera)]
72 (doto viewport
73 (.setClearFlags true true true)
74 (.setBackgroundColor ColorRGBA/Gray)
75 (.addProcessor (scene-processor continuation))
76 (.attachScene (.getRootNode world)))))
77 #+end_src
79 Note the use of continuation passing style for connecting the eye to a
80 function to process the output. You can create any number of eyes, and
81 each of them will see the world from their own =Camera=. Once every
82 frame, the rendered image is copied to a =BufferedImage=, and that
83 data is sent off to the continuation function. Moving the =Camera=
84 which was used to create the eye will change what the eye sees.
86 * Example
88 #+name: test-vision
89 #+begin_src clojure
90 (ns cortex.test.vision
91 (:use (cortex world util vision))
92 (:import java.awt.image.BufferedImage)
93 (:import javax.swing.JPanel)
94 (:import javax.swing.SwingUtilities)
95 (:import java.awt.Dimension)
96 (:import javax.swing.JFrame)
97 (:import com.jme3.math.ColorRGBA)
98 (:import com.jme3.scene.Node)
99 (:import com.jme3.math.Vector3f))
101 (defn view-image
102 "Initailizes a JPanel on which you may draw a BufferedImage.
103 Returns a function that accepts a BufferedImage and draws it to the
104 JPanel."
105 []
106 (let [image
107 (atom
108 (BufferedImage. 1 1 BufferedImage/TYPE_4BYTE_ABGR))
109 panel
110 (proxy [JPanel] []
111 (paint
112 [graphics]
113 (proxy-super paintComponent graphics)
114 (.drawImage graphics @image 0 0 nil)))
115 frame (JFrame. "Display Image")]
116 (SwingUtilities/invokeLater
117 (fn []
118 (doto frame
119 (-> (.getContentPane) (.add panel))
120 (.pack)
121 (.setLocationRelativeTo nil)
122 (.setResizable true)
123 (.setVisible true))))
124 (fn [#^BufferedImage i]
125 (reset! image i)
126 (.setSize frame (+ 8 (.getWidth i)) (+ 28 (.getHeight i)))
127 (.repaint panel 0 0 (.getWidth i) (.getHeight i)))))
129 (defn test-two-eyes
130 "Testing vision:
131 Tests the vision system by creating two views of the same rotating
132 object from different angles and displaying both of those views in
133 JFrames.
135 You should see a rotating cube, and two windows,
136 each displaying a different view of the cube."
137 []
138 (let [candy
139 (box 1 1 1 :physical? false :color ColorRGBA/Blue)]
140 (world (doto (Node.)
141 (.attachChild candy))
142 {}
143 (fn [world]
144 (let [cam (.clone (.getCamera world))
145 width (.getWidth cam)
146 height (.getHeight cam)]
147 (add-eye world cam (view-image))
148 (add-eye world
149 (doto (.clone cam)
150 (.setLocation (Vector3f. -10 0 0))
151 (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))
152 (view-image))
153 ;; This is here to restore the main view
154 ;; after the other views have completed processing
155 (add-eye world (.getCamera world) no-op)))
156 (fn [world tpf]
157 (.rotate candy (* tpf 0.2) 0 0)))))
158 #+end_src
160 The example code will create two videos of the same rotating object
161 from different angles. It can be used both for stereoscopic vision
162 simulation or for simulating multiple creatures, each with their own
163 sense of vision.
165 - As a neat bonus, this idea behind simulated vision also enables one
166 to [[../../cortex/html/capture-video.html][capture live video feeds from jMonkeyEngine]].
169 * COMMENT code generation
170 #+begin_src clojure :tangle ../src/cortex/vision.clj
171 <<eyes>>
172 #+end_src
174 #+begin_src clojure :tangle ../src/cortex/test/vision.clj
175 <<test-vision>>
176 #+end_src