Mercurial > cortex

comparison org/eyes.org @ 34:183744c179e6

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
MASSIVE cleanup, especially in the vision code
author Robert McIntyre <rlm@mit.edu>
date Thu, 03 Nov 2011 08:28:26 -0700
parents 775d97247dd0
children 5eb5c6f0590b
comparison
equal deleted inserted replaced
33:c377222528e6 34:183744c179e6
1 #+title: Eyes 1 #+title: Simulated Sense of Sight
2 #+author: Robert McIntyre 2 #+author: Robert McIntyre
3 #+email: rlm@mit.edu 3 #+email: rlm@mit.edu
4 #+description: Simulating senses for AI research using JMonkeyEngine3 4 #+description: Simulated sight for AI research using JMonkeyEngine3
5 #+keywords: computer vision, jMonkeyEngine3, clojure
5 #+SETUPFILE: ../../aurellem/org/setup.org 6 #+SETUPFILE: ../../aurellem/org/setup.org
6 #+INCLUDE: ../../aurellem/org/level-0.org 7 #+INCLUDE: ../../aurellem/org/level-0.org
7 #+babel: :mkdirp yes :noweb yes :exports both 8 #+babel: :mkdirp yes :noweb yes :exports both
8 9
10 * Vision
9 11
10 12 I want to make creatures with eyes. Each eye can be independely moved
11 ** Eyes 13 and should see its own version of the world depending on where it is.
12
13 Ultimately I want to make creatures with eyes. Each eye can be
14 independely moved and should see its own version of the world
15 depending on where it is.
16 #+srcname: eyes 14 #+srcname: eyes
17 #+begin_src clojure 15 #+begin_src clojure
18 (ns body.eye) 16 (ns cortex.vision
19 (use 'cortex.world) 17 "Simulate the sense of vision in jMonkeyEngine3. Enables multiple
20 (use 'cortex.import) 18 eyes from different positions to observe the same world, and pass
21 (use 'clojure.contrib.def) 19 the observed data to any arbitray function."
22 (cortex.import/mega-import-jme3) 20 {:author "Robert McIntyre"}
23 (rlm.rlm-commands/help) 21 (:use cortex.world)
24 (import java.nio.ByteBuffer) 22 (:import com.jme3.post.SceneProcessor)
25 (import java.awt.image.BufferedImage) 23 (:import (com.jme3.util Screenshots BufferUtils))
26 (import java.awt.Color) 24 (:import java.nio.ByteBuffer)
27 (import java.awt.Dimension) 25 (:import java.awt.image.BufferedImage)
28 (import java.awt.Graphics) 26 (:import com.jme3.renderer.ViewPort)
29 (import java.awt.Graphics2D) 27 (:import com.jme3.math.ColorRGBA))
30 (import java.awt.event.WindowAdapter)
31 (import java.awt.event.WindowEvent)
32 (import java.awt.image.BufferedImage)
33 (import java.nio.ByteBuffer)
34 (import javax.swing.JFrame)
35 (import javax.swing.JPanel)
36 (import javax.swing.SwingUtilities)
37 (import javax.swing.ImageIcon)
38 (import javax.swing.JOptionPane)
39 (import java.awt.image.ImageObserver)
40 28
41 (defn scene-processor 29 (defn scene-processor
42 "deals with converting FrameBuffers to BufferedImages so 30 "Create a SceneProcessor object which wraps a vision processing
43 that the continuation function can be defined only in terms 31 continuation function. The SceneProcessor will take care of
44 of what it does with BufferedImages" 32 converting the rendered frame to a BufferedImage and passing that
33 BufferedImage to the continuation. The continuation should be a
34 function that takes a BufferedImage."
45 [continuation] 35 [continuation]
46 (let [byte-buffer (atom nil) 36 (let [byte-buffer (atom nil)
47 renderer (atom nil) 37 renderer (atom nil)
48 image (atom nil)] 38 image (atom nil)]
49 (proxy [SceneProcessor] [] 39 (proxy [SceneProcessor] []
54 height (.getHeight cam)] 44 height (.getHeight cam)]
55 (reset! renderer (.getRenderer renderManager)) 45 (reset! renderer (.getRenderer renderManager))
56 (reset! byte-buffer 46 (reset! byte-buffer
57 (BufferUtils/createByteBuffer 47 (BufferUtils/createByteBuffer
58 (* width height 4))) 48 (* width height 4)))
59 (reset! image (BufferedImage. width height 49 (reset! image (BufferedImage.
60 BufferedImage/TYPE_4BYTE_ABGR)))) 50 width height
51 BufferedImage/TYPE_4BYTE_ABGR))))
61 (isInitialized [] (not (nil? @byte-buffer))) 52 (isInitialized [] (not (nil? @byte-buffer)))
62 (reshape [_ _ _]) 53 (reshape [_ _ _])
63 (preFrame [_]) 54 (preFrame [_])
64 (postQueue [_]) 55 (postQueue [_])
65 (postFrame 56 (postFrame
69 (Screenshots/convertScreenShot @byte-buffer @image) 60 (Screenshots/convertScreenShot @byte-buffer @image)
70 (continuation @image)) 61 (continuation @image))
71 (cleanup [])))) 62 (cleanup []))))
72 63
73 (defn add-eye 64 (defn add-eye
74 "Add an eye to the world, and call continuation on 65 "Add an eye to the world, calling continuation on every frame
75 every frame produced" 66 produced."
76 [world camera continuation] 67 [world camera continuation]
77 (let [width (.getWidth camera) 68 (let [width (.getWidth camera)
78 height (.getHeight camera) 69 height (.getHeight camera)
79 render-manager (.getRenderManager world) 70 render-manager (.getRenderManager world)
80 viewport (.createMainView render-manager "eye-view" camera)] 71 viewport (.createMainView render-manager "eye-view" camera)]
81 (doto viewport 72 (doto viewport
82 (.setBackgroundColor ColorRGBA/Black)
83 (.setClearFlags true true true) 73 (.setClearFlags true true true)
74 (.setBackgroundColor ColorRGBA/Gray)
84 (.addProcessor (scene-processor continuation)) 75 (.addProcessor (scene-processor continuation))
85 (.attachScene (.getRootNode world))))) 76 (.attachScene (.getRootNode world)))))
86 77
87 (defn make-display-frame [display width height] 78 #+end_src
88 (SwingUtilities/invokeLater
89 (fn []
90 (.setPreferredSize display (Dimension. width height))
91 (doto (JFrame. "Eye Camera!")
92 (-> (.getContentPane) (.add display))
93 (.setDefaultCloseOperation JFrame/DISPOSE_ON_CLOSE)
94 (.pack)
95 (.setLocationRelativeTo nil)
96 (.setResizable false)
97 (.setVisible true)))))
98
99 (defn image-monitor [#^BufferedImage image]
100 (proxy [JPanel] []
101 (paintComponent
102 [g]
103 (proxy-super paintComponent g)
104 (locking image
105 (.drawImage g image 0 0
106 (proxy [ImageObserver]
107 []
108 (imageUpdate
109 []
110 (proxy-super imageUpdate))))))))
111 79
112 (defn movie-image [] 80 Note the use of continuation passing style for connecting the eye to a
113 (let [setup 81 function to process the output. You can create any number of eyes, and
114 (runonce 82 each of them will see the world from their own =Camera=. Once every
115 (fn [#^BufferedImage image] 83 frame, the rendered image is copied to a =BufferedImage=, and that
116 (let [width (.getWidth image) 84 data is sent off to the continuation function. Moving the =Camera=
117 height (.getHeight image) 85 which was used to create the eye will change what the eye sees.
118 display (image-monitor image)
119 frame (make-display-frame display width height)]
120 display)))]
121 (fn [#^BufferedImage image]
122 (.repaint (setup image)))))
123
124 86
125 (defn observer 87 * Example
126 "place thy eye!"
127 [world camera]
128 (let [eye camera
129 width (.getWidth eye)
130 height (.getHeight eye)]
131 (no-exceptions
132 (add-eye
133 world
134 eye
135 (movie-image)))))
136 #+end_src
137 88
138 #+srcname: test-vision 89 #+srcname: test-vision
139 #+begin_src clojure 90 #+begin_src clojure
91 (ns test.vision
92 (:use (cortex world util vision))
93 (:import java.awt.image.BufferedImage)
94 (:import javax.swing.JPanel)
95 (:import javax.swing.SwingUtilities)
96 (:import java.awt.Dimension)
97 (:import javax.swing.JFrame)
98 (:import com.jme3.math.ColorRGBA)
99 (:import com.jme3.scene.Node))
140 100
141 (ns test.vision) 101 (defn view-image
142 (use 'cortex.world) 102 "Initailizes a JPanel on which you may draw a BufferedImage of the
143 (use 'cortex.import) 103 given width and height. Returns a function that accepts a
144 (use 'clojure.contrib.def) 104 BufferedImage and draws it to the JPanel."
145 (use 'body.eye) 105 [width height]
146 (cortex.import/mega-import-jme3) 106 (let [image
147 (rlm.rlm-commands/help) 107 (atom
148 (import java.nio.ByteBuffer) 108 (BufferedImage. width height BufferedImage/TYPE_4BYTE_ABGR))
149 (import java.awt.image.BufferedImage) 109 panel
150 (import java.awt.Color) 110 (proxy [JPanel] []
151 (import java.awt.Dimension) 111 (paint
152 (import java.awt.Graphics) 112 [graphics]
153 (import java.awt.Graphics2D) 113 (proxy-super paintComponent graphics)
154 (import java.awt.event.WindowAdapter) 114 (.drawImage graphics @image 0 0 nil)
155 (import java.awt.event.WindowEvent) 115 ))]
156 (import java.awt.image.BufferedImage) 116
157 (import java.nio.ByteBuffer) 117 (SwingUtilities/invokeLater
158 (import javax.swing.JFrame) 118 (fn []
159 (import javax.swing.JPanel) 119 (.setPreferredSize panel (Dimension. width height))
160 (import javax.swing.SwingUtilities) 120 (doto (JFrame. "Eye Camera!")
161 (import javax.swing.ImageIcon) 121 (-> (.getContentPane) (.add panel))
162 (import javax.swing.JOptionPane) 122 (.pack)
163 (import java.awt.image.ImageObserver) 123 (.setLocationRelativeTo nil)
124 (.setResizable false)
125 (.setVisible true))))
126 (fn [#^BufferedImage i]
127 (reset! image i)
128 (.repaint panel))))
164 129
165 130 (defn test-vision
166 (def width 200) 131 "Tests the vision system by creating two views of the same rotating
167 (def height 200) 132 object from different angles and displaying both of those views in
168 133 JFrames." []
169 (defn camera [] 134 (.start
170 (doto (Camera. width height) 135 (let [candy
171 (.setFrustumPerspective 45 1 1 1000) 136 (box 1 1 1 :physical? false :color ColorRGBA/Blue)]
172 (.setLocation (Vector3f. -3 0 -5)) 137 (world (doto (Node.)
173 (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))) 138 (.attachChild candy))
174 139 {}
175 (defn camera2 [] 140 (fn [world]
176 (doto (Camera. width height) 141 (let [cam (.clone (.getCamera world))
177 (.setFrustumPerspective 45 1 1 1000) 142 width (.getWidth cam)
178 (.setLocation (Vector3f. 3 0 -5)) 143 height (.getHeight cam)]
179 (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))) 144 (add-eye world
180 145 (doto (.clone cam)
181 (defn setup-fn [world] 146 (.setLocation (Vector3f. -10 0 0))
182 (let [eye (camera) 147 (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))
183 width (.getWidth eye) 148 (view-image width height))
184 height (.getHeight eye)] 149 (add-eye world cam
185 (no-exceptions 150 (view-image width height))
186 (add-eye 151 ;; this is here to restore the main view
187 world 152 ;; after the other views have completed processing
188 eye 153 (add-eye world (.getCamera world) no-op)
189 (runonce visual)) 154 ))
190 (add-eye 155 (fn [world tpf]
191 world 156 (.rotate candy (* tpf 0.2) 0 0))))))
192 (camera2)
193 (runonce visual)))))
194
195 (defn spider-eye [position]
196 (doto (Camera. 200 200 )
197 (.setFrustumPerspective 45 1 1 1000)
198 (.setLocation position)
199 (.lookAt Vector3f/ZERO Vector3f/UNIT_Y)))
200
201 (defn setup-fn* [world]
202 (let [eye (camera)
203 width (.getWidth eye)
204 height (.getHeight eye)]
205 ;;(.setClearFlags (.getViewPort world) true true true)
206 (observer world (.getCamera world))
207 (observer world (spider-eye (Vector3f. 3 0 -5)))
208 ;;(observer world (spider-eye (Vector3f. 0 0 -5)))
209 ;; (observer world (spider-eye (Vector3f. -3 0 -5)))
210 ;; (observer world (spider-eye (Vector3f. 0 3 -5)))
211 ;; (observer world (spider-eye (Vector3f. 0 -3 -5)))
212 ;; (observer world (spider-eye (Vector3f. 3 3 -5)))
213 ;; (observer world (spider-eye (Vector3f. -3 3 -5)))
214 ;; (observer world (spider-eye (Vector3f. 3 -3 -5)))
215 ;; (observer world (spider-eye (Vector3f. -3 -3 -5)))
216
217 )
218 world)
219
220 (defn test-world []
221 (let [thing (box 1 1 1 :physical? false)]
222 (world
223 (doto (Node.)
224 (.attachChild thing))
225 {}
226 setup-fn
227 (fn [world tpf]
228 (.rotate thing (* tpf 0.2) 0 0)
229 ))))
230
231
232 #+end_src 157 #+end_src
233 158
234 159 The example code will create two videos of the same rotating object
235 #+results: eyes 160 from different angles. It can be used both for stereoscopic vision
236 : #'body.eye/test-world 161 simulation or for simulating multiple creatures, each with their own
237 162 sense of vision.
238 Note the use of continuation passing style for connecting the eye to a
239 function to process the output. The example code will create two
240 videos of the same rotating cube from different angles, sutiable for
241 stereoscopic vision.
242
243 163
244 164
245 * COMMENT code generation 165 * COMMENT code generation
246 #+begin_src clojure :tangle ../src/body/eye.clj 166 #+begin_src clojure :tangle ../src/cortex/vision.clj
247 <<eyes>> 167 <<eyes>>
248 #+end_src 168 #+end_src
249 169
250 #+begin_src clojure :tangle ../src/test/vision.clj 170 #+begin_src clojure :tangle ../src/test/vision.clj
251 <<test-vision>> 171 <<test-vision>>