|
rlm@34
|
1 #+title: Simulated Sense of Sight
|
|
rlm@23
|
2 #+author: Robert McIntyre
|
|
rlm@23
|
3 #+email: rlm@mit.edu
|
|
rlm@38
|
4 #+description: Simulated sight for AI research using JMonkeyEngine3 and clojure
|
|
rlm@34
|
5 #+keywords: computer vision, jMonkeyEngine3, clojure
|
|
rlm@23
|
6 #+SETUPFILE: ../../aurellem/org/setup.org
|
|
rlm@23
|
7 #+INCLUDE: ../../aurellem/org/level-0.org
|
|
rlm@23
|
8 #+babel: :mkdirp yes :noweb yes :exports both
|
|
rlm@23
|
9
|
|
rlm@34
|
10 * Vision
|
|
rlm@23
|
11
|
|
rlm@34
|
12 I want to make creatures with eyes. Each eye can be independely moved
|
|
rlm@34
|
13 and should see its own version of the world depending on where it is.
|
|
rlm@23
|
14 #+srcname: eyes
|
|
rlm@23
|
15 #+begin_src clojure
|
|
rlm@34
|
16 (ns cortex.vision
|
|
rlm@34
|
17 "Simulate the sense of vision in jMonkeyEngine3. Enables multiple
|
|
rlm@34
|
18 eyes from different positions to observe the same world, and pass
|
|
rlm@34
|
19 the observed data to any arbitray function."
|
|
rlm@34
|
20 {:author "Robert McIntyre"}
|
|
rlm@34
|
21 (:use cortex.world)
|
|
rlm@34
|
22 (:import com.jme3.post.SceneProcessor)
|
|
rlm@34
|
23 (:import (com.jme3.util Screenshots BufferUtils))
|
|
rlm@34
|
24 (:import java.nio.ByteBuffer)
|
|
rlm@34
|
25 (:import java.awt.image.BufferedImage)
|
|
rlm@34
|
26 (:import com.jme3.renderer.ViewPort)
|
|
rlm@34
|
27 (:import com.jme3.math.ColorRGBA))
|
|
rlm@23
|
28
|
|
rlm@23
|
29 (defn scene-processor
|
|
rlm@34
|
30 "Create a SceneProcessor object which wraps a vision processing
|
|
rlm@34
|
31 continuation function. The SceneProcessor will take care of
|
|
rlm@34
|
32 converting the rendered frame to a BufferedImage and passing that
|
|
rlm@34
|
33 BufferedImage to the continuation. The continuation should be a
|
|
rlm@34
|
34 function that takes a BufferedImage."
|
|
rlm@23
|
35 [continuation]
|
|
rlm@23
|
36 (let [byte-buffer (atom nil)
|
|
rlm@23
|
37 renderer (atom nil)
|
|
rlm@23
|
38 image (atom nil)]
|
|
rlm@23
|
39 (proxy [SceneProcessor] []
|
|
rlm@23
|
40 (initialize
|
|
rlm@23
|
41 [renderManager viewPort]
|
|
rlm@23
|
42 (let [cam (.getCamera viewPort)
|
|
rlm@23
|
43 width (.getWidth cam)
|
|
rlm@23
|
44 height (.getHeight cam)]
|
|
rlm@23
|
45 (reset! renderer (.getRenderer renderManager))
|
|
rlm@23
|
46 (reset! byte-buffer
|
|
rlm@23
|
47 (BufferUtils/createByteBuffer
|
|
rlm@23
|
48 (* width height 4)))
|
|
rlm@34
|
49 (reset! image (BufferedImage.
|
|
rlm@34
|
50 width height
|
|
rlm@34
|
51 BufferedImage/TYPE_4BYTE_ABGR))))
|
|
rlm@23
|
52 (isInitialized [] (not (nil? @byte-buffer)))
|
|
rlm@23
|
53 (reshape [_ _ _])
|
|
rlm@23
|
54 (preFrame [_])
|
|
rlm@23
|
55 (postQueue [_])
|
|
rlm@23
|
56 (postFrame
|
|
rlm@23
|
57 [#^FrameBuffer fb]
|
|
rlm@23
|
58 (.clear @byte-buffer)
|
|
rlm@23
|
59 (.readFrameBuffer @renderer fb @byte-buffer)
|
|
rlm@23
|
60 (Screenshots/convertScreenShot @byte-buffer @image)
|
|
rlm@23
|
61 (continuation @image))
|
|
rlm@23
|
62 (cleanup []))))
|
|
rlm@23
|
63
|
|
rlm@23
|
64 (defn add-eye
|
|
rlm@34
|
65 "Add an eye to the world, calling continuation on every frame
|
|
rlm@34
|
66 produced."
|
|
rlm@23
|
67 [world camera continuation]
|
|
rlm@23
|
68 (let [width (.getWidth camera)
|
|
rlm@23
|
69 height (.getHeight camera)
|
|
rlm@23
|
70 render-manager (.getRenderManager world)
|
|
rlm@23
|
71 viewport (.createMainView render-manager "eye-view" camera)]
|
|
rlm@23
|
72 (doto viewport
|
|
rlm@23
|
73 (.setClearFlags true true true)
|
|
rlm@34
|
74 (.setBackgroundColor ColorRGBA/Gray)
|
|
rlm@23
|
75 (.addProcessor (scene-processor continuation))
|
|
rlm@23
|
76 (.attachScene (.getRootNode world)))))
|
|
rlm@34
|
77 #+end_src
|
|
rlm@23
|
78
|
|
rlm@34
|
79 Note the use of continuation passing style for connecting the eye to a
|
|
rlm@34
|
80 function to process the output. You can create any number of eyes, and
|
|
rlm@34
|
81 each of them will see the world from their own =Camera=. Once every
|
|
rlm@34
|
82 frame, the rendered image is copied to a =BufferedImage=, and that
|
|
rlm@34
|
83 data is sent off to the continuation function. Moving the =Camera=
|
|
rlm@34
|
84 which was used to create the eye will change what the eye sees.
|
|
rlm@23
|
85
|
|
rlm@34
|
86 * Example
|
|
rlm@23
|
87
|
|
rlm@23
|
88 #+srcname: test-vision
|
|
rlm@23
|
89 #+begin_src clojure
|
|
rlm@34
|
90 (ns test.vision
|
|
rlm@34
|
91 (:use (cortex world util vision))
|
|
rlm@34
|
92 (:import java.awt.image.BufferedImage)
|
|
rlm@34
|
93 (:import javax.swing.JPanel)
|
|
rlm@34
|
94 (:import javax.swing.SwingUtilities)
|
|
rlm@34
|
95 (:import java.awt.Dimension)
|
|
rlm@34
|
96 (:import javax.swing.JFrame)
|
|
rlm@34
|
97 (:import com.jme3.math.ColorRGBA)
|
|
rlm@45
|
98 (:import com.jme3.scene.Node)
|
|
rlm@45
|
99 (:import com.jme3.math.Vector3f))
|
|
rlm@23
|
100
|
|
rlm@34
|
101 (defn view-image
|
|
rlm@34
|
102 "Initailizes a JPanel on which you may draw a BufferedImage of the
|
|
rlm@34
|
103 given width and height. Returns a function that accepts a
|
|
rlm@34
|
104 BufferedImage and draws it to the JPanel."
|
|
rlm@34
|
105 [width height]
|
|
rlm@34
|
106 (let [image
|
|
rlm@34
|
107 (atom
|
|
rlm@34
|
108 (BufferedImage. width height BufferedImage/TYPE_4BYTE_ABGR))
|
|
rlm@34
|
109 panel
|
|
rlm@34
|
110 (proxy [JPanel] []
|
|
rlm@34
|
111 (paint
|
|
rlm@34
|
112 [graphics]
|
|
rlm@34
|
113 (proxy-super paintComponent graphics)
|
|
rlm@35
|
114 (.drawImage graphics @image 0 0 nil)))]
|
|
rlm@34
|
115 (SwingUtilities/invokeLater
|
|
rlm@34
|
116 (fn []
|
|
rlm@34
|
117 (.setPreferredSize panel (Dimension. width height))
|
|
rlm@34
|
118 (doto (JFrame. "Eye Camera!")
|
|
rlm@34
|
119 (-> (.getContentPane) (.add panel))
|
|
rlm@34
|
120 (.pack)
|
|
rlm@34
|
121 (.setLocationRelativeTo nil)
|
|
rlm@34
|
122 (.setResizable false)
|
|
rlm@34
|
123 (.setVisible true))))
|
|
rlm@34
|
124 (fn [#^BufferedImage i]
|
|
rlm@34
|
125 (reset! image i)
|
|
rlm@34
|
126 (.repaint panel))))
|
|
rlm@23
|
127
|
|
rlm@36
|
128 (defn test-two-eyes
|
|
rlm@34
|
129 "Tests the vision system by creating two views of the same rotating
|
|
rlm@34
|
130 object from different angles and displaying both of those views in
|
|
rlm@36
|
131 JFrames."
|
|
rlm@36
|
132 []
|
|
rlm@34
|
133 (.start
|
|
rlm@34
|
134 (let [candy
|
|
rlm@34
|
135 (box 1 1 1 :physical? false :color ColorRGBA/Blue)]
|
|
rlm@34
|
136 (world (doto (Node.)
|
|
rlm@34
|
137 (.attachChild candy))
|
|
rlm@34
|
138 {}
|
|
rlm@34
|
139 (fn [world]
|
|
rlm@34
|
140 (let [cam (.clone (.getCamera world))
|
|
rlm@34
|
141 width (.getWidth cam)
|
|
rlm@34
|
142 height (.getHeight cam)]
|
|
rlm@35
|
143 (add-eye world cam (view-image width height))
|
|
rlm@34
|
144 (add-eye world
|
|
rlm@34
|
145 (doto (.clone cam)
|
|
rlm@34
|
146 (.setLocation (Vector3f. -10 0 0))
|
|
rlm@34
|
147 (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))
|
|
rlm@34
|
148 (view-image width height))
|
|
rlm@35
|
149 ;; This is here to restore the main view
|
|
rlm@34
|
150 ;; after the other views have completed processing
|
|
rlm@35
|
151 (add-eye world (.getCamera world) no-op)))
|
|
rlm@34
|
152 (fn [world tpf]
|
|
rlm@34
|
153 (.rotate candy (* tpf 0.2) 0 0))))))
|
|
rlm@23
|
154 #+end_src
|
|
rlm@23
|
155
|
|
rlm@34
|
156 The example code will create two videos of the same rotating object
|
|
rlm@34
|
157 from different angles. It can be used both for stereoscopic vision
|
|
rlm@34
|
158 simulation or for simulating multiple creatures, each with their own
|
|
rlm@34
|
159 sense of vision.
|
|
rlm@24
|
160
|
|
rlm@35
|
161 - As a neat bonus, this idea behind simulated vision also enables one
|
|
rlm@35
|
162 to [[../../cortex/html/capture-video.html][capture live video feeds from jMonkeyEngine]].
|
|
rlm@35
|
163
|
|
rlm@24
|
164
|
|
rlm@24
|
165 * COMMENT code generation
|
|
rlm@34
|
166 #+begin_src clojure :tangle ../src/cortex/vision.clj
|
|
rlm@24
|
167 <<eyes>>
|
|
rlm@24
|
168 #+end_src
|
|
rlm@24
|
169
|
|
rlm@24
|
170 #+begin_src clojure :tangle ../src/test/vision.clj
|
|
rlm@24
|
171 <<test-vision>>
|
|
rlm@24
|
172 #+end_src
|
