|
rlm@34
|
1 #+title: Simulated Sense of Sight
|
|
rlm@23
|
2 #+author: Robert McIntyre
|
|
rlm@23
|
3 #+email: rlm@mit.edu
|
|
rlm@38
|
4 #+description: Simulated sight for AI research using JMonkeyEngine3 and clojure
|
|
rlm@34
|
5 #+keywords: computer vision, jMonkeyEngine3, clojure
|
|
rlm@23
|
6 #+SETUPFILE: ../../aurellem/org/setup.org
|
|
rlm@23
|
7 #+INCLUDE: ../../aurellem/org/level-0.org
|
|
rlm@23
|
8 #+babel: :mkdirp yes :noweb yes :exports both
|
|
rlm@23
|
9
|
|
rlm@34
|
10 * Vision
|
|
rlm@23
|
11
|
|
rlm@34
|
12 I want to make creatures with eyes. Each eye can be independely moved
|
|
rlm@34
|
13 and should see its own version of the world depending on where it is.
|
|
rlm@66
|
14 #+name: eyes
|
|
rlm@23
|
15 #+begin_src clojure
|
|
rlm@34
|
16 (ns cortex.vision
|
|
rlm@34
|
17 "Simulate the sense of vision in jMonkeyEngine3. Enables multiple
|
|
rlm@34
|
18 eyes from different positions to observe the same world, and pass
|
|
rlm@34
|
19 the observed data to any arbitray function."
|
|
rlm@34
|
20 {:author "Robert McIntyre"}
|
|
rlm@34
|
21 (:use cortex.world)
|
|
rlm@34
|
22 (:import com.jme3.post.SceneProcessor)
|
|
rlm@113
|
23 (:import (com.jme3.util BufferUtils Screenshots))
|
|
rlm@34
|
24 (:import java.nio.ByteBuffer)
|
|
rlm@34
|
25 (:import java.awt.image.BufferedImage)
|
|
rlm@34
|
26 (:import com.jme3.renderer.ViewPort)
|
|
rlm@113
|
27 (:import com.jme3.math.ColorRGBA)
|
|
rlm@113
|
28 (:import com.jme3.renderer.Renderer))
|
|
rlm@23
|
29
|
|
rlm@113
|
30
|
|
rlm@113
|
31 (defn vision-pipeline
|
|
rlm@34
|
32 "Create a SceneProcessor object which wraps a vision processing
|
|
rlm@113
|
33 continuation function. The continuation is a function that takes
|
|
rlm@113
|
34 [#^Renderer r #^FrameBuffer fb #^ByteBuffer b #^BufferedImage bi],
|
|
rlm@113
|
35 each of which has already been appropiately sized."
|
|
rlm@23
|
36 [continuation]
|
|
rlm@23
|
37 (let [byte-buffer (atom nil)
|
|
rlm@113
|
38 renderer (atom nil)
|
|
rlm@113
|
39 image (atom nil)]
|
|
rlm@23
|
40 (proxy [SceneProcessor] []
|
|
rlm@23
|
41 (initialize
|
|
rlm@23
|
42 [renderManager viewPort]
|
|
rlm@23
|
43 (let [cam (.getCamera viewPort)
|
|
rlm@23
|
44 width (.getWidth cam)
|
|
rlm@23
|
45 height (.getHeight cam)]
|
|
rlm@23
|
46 (reset! renderer (.getRenderer renderManager))
|
|
rlm@23
|
47 (reset! byte-buffer
|
|
rlm@23
|
48 (BufferUtils/createByteBuffer
|
|
rlm@113
|
49 (* width height 4)))
|
|
rlm@113
|
50 (reset! image (BufferedImage.
|
|
rlm@113
|
51 width height
|
|
rlm@113
|
52 BufferedImage/TYPE_4BYTE_ABGR))))
|
|
rlm@23
|
53 (isInitialized [] (not (nil? @byte-buffer)))
|
|
rlm@23
|
54 (reshape [_ _ _])
|
|
rlm@23
|
55 (preFrame [_])
|
|
rlm@23
|
56 (postQueue [_])
|
|
rlm@23
|
57 (postFrame
|
|
rlm@23
|
58 [#^FrameBuffer fb]
|
|
rlm@23
|
59 (.clear @byte-buffer)
|
|
rlm@113
|
60 (continuation @renderer fb @byte-buffer @image))
|
|
rlm@23
|
61 (cleanup []))))
|
|
rlm@23
|
62
|
|
rlm@113
|
63 (defn frameBuffer->byteBuffer!
|
|
rlm@113
|
64 "Transfer the data in the graphics card (Renderer, FrameBuffer) to
|
|
rlm@113
|
65 the CPU (ByteBuffer)."
|
|
rlm@113
|
66 [#^Renderer r #^FrameBuffer fb #^ByteBuffer bb]
|
|
rlm@113
|
67 (.readFrameBuffer r fb bb) bb)
|
|
rlm@113
|
68
|
|
rlm@113
|
69 (defn byteBuffer->bufferedImage!
|
|
rlm@113
|
70 "Convert the C-style BGRA image data in the ByteBuffer bb to the AWT
|
|
rlm@113
|
71 style ABGR image data and place it in BufferedImage bi."
|
|
rlm@113
|
72 [#^ByteBuffer bb #^BufferedImage bi]
|
|
rlm@113
|
73 (Screenshots/convertScreenShot bb bi) bi)
|
|
rlm@113
|
74
|
|
rlm@113
|
75 (defn BufferedImage!
|
|
rlm@113
|
76 "Continuation which will grab the buffered image from the materials
|
|
rlm@113
|
77 provided by (vision-pipeline)."
|
|
rlm@113
|
78 [#^Renderer r #^FrameBuffer fb #^ByteBuffer bb #^BufferedImage bi]
|
|
rlm@113
|
79 (byteBuffer->bufferedImage!
|
|
rlm@113
|
80 (frameBuffer->byteBuffer! r fb bb) bi))
|
|
rlm@112
|
81
|
|
rlm@23
|
82 (defn add-eye
|
|
rlm@34
|
83 "Add an eye to the world, calling continuation on every frame
|
|
rlm@34
|
84 produced."
|
|
rlm@23
|
85 [world camera continuation]
|
|
rlm@23
|
86 (let [width (.getWidth camera)
|
|
rlm@23
|
87 height (.getHeight camera)
|
|
rlm@23
|
88 render-manager (.getRenderManager world)
|
|
rlm@23
|
89 viewport (.createMainView render-manager "eye-view" camera)]
|
|
rlm@23
|
90 (doto viewport
|
|
rlm@23
|
91 (.setClearFlags true true true)
|
|
rlm@112
|
92 (.setBackgroundColor ColorRGBA/Black)
|
|
rlm@113
|
93 (.addProcessor (vision-pipeline continuation))
|
|
rlm@23
|
94 (.attachScene (.getRootNode world)))))
|
|
rlm@34
|
95 #+end_src
|
|
rlm@23
|
96
|
|
rlm@112
|
97 #+results: eyes
|
|
rlm@112
|
98 : #'cortex.vision/add-eye
|
|
rlm@112
|
99
|
|
rlm@34
|
100 Note the use of continuation passing style for connecting the eye to a
|
|
rlm@34
|
101 function to process the output. You can create any number of eyes, and
|
|
rlm@34
|
102 each of them will see the world from their own =Camera=. Once every
|
|
rlm@34
|
103 frame, the rendered image is copied to a =BufferedImage=, and that
|
|
rlm@34
|
104 data is sent off to the continuation function. Moving the =Camera=
|
|
rlm@34
|
105 which was used to create the eye will change what the eye sees.
|
|
rlm@23
|
106
|
|
rlm@34
|
107 * Example
|
|
rlm@23
|
108
|
|
rlm@66
|
109 #+name: test-vision
|
|
rlm@23
|
110 #+begin_src clojure
|
|
rlm@68
|
111 (ns cortex.test.vision
|
|
rlm@34
|
112 (:use (cortex world util vision))
|
|
rlm@34
|
113 (:import java.awt.image.BufferedImage)
|
|
rlm@34
|
114 (:import javax.swing.JPanel)
|
|
rlm@34
|
115 (:import javax.swing.SwingUtilities)
|
|
rlm@34
|
116 (:import java.awt.Dimension)
|
|
rlm@34
|
117 (:import javax.swing.JFrame)
|
|
rlm@34
|
118 (:import com.jme3.math.ColorRGBA)
|
|
rlm@45
|
119 (:import com.jme3.scene.Node)
|
|
rlm@113
|
120 (:import com.jme3.math.Vector3f))
|
|
rlm@23
|
121
|
|
rlm@36
|
122 (defn test-two-eyes
|
|
rlm@69
|
123 "Testing vision:
|
|
rlm@69
|
124 Tests the vision system by creating two views of the same rotating
|
|
rlm@69
|
125 object from different angles and displaying both of those views in
|
|
rlm@69
|
126 JFrames.
|
|
rlm@69
|
127
|
|
rlm@69
|
128 You should see a rotating cube, and two windows,
|
|
rlm@69
|
129 each displaying a different view of the cube."
|
|
rlm@36
|
130 []
|
|
rlm@58
|
131 (let [candy
|
|
rlm@58
|
132 (box 1 1 1 :physical? false :color ColorRGBA/Blue)]
|
|
rlm@112
|
133 (world
|
|
rlm@112
|
134 (doto (Node.)
|
|
rlm@112
|
135 (.attachChild candy))
|
|
rlm@112
|
136 {}
|
|
rlm@112
|
137 (fn [world]
|
|
rlm@112
|
138 (let [cam (.clone (.getCamera world))
|
|
rlm@112
|
139 width (.getWidth cam)
|
|
rlm@112
|
140 height (.getHeight cam)]
|
|
rlm@112
|
141 (add-eye world cam
|
|
rlm@113
|
142 ;;no-op
|
|
rlm@113
|
143 (comp (view-image) BufferedImage!)
|
|
rlm@112
|
144 )
|
|
rlm@112
|
145 (add-eye world
|
|
rlm@112
|
146 (doto (.clone cam)
|
|
rlm@112
|
147 (.setLocation (Vector3f. -10 0 0))
|
|
rlm@112
|
148 (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))
|
|
rlm@113
|
149 ;;no-op
|
|
rlm@113
|
150 (comp (view-image) BufferedImage!))
|
|
rlm@112
|
151 ;; This is here to restore the main view
|
|
rlm@112
|
152 ;; after the other views have completed processing
|
|
rlm@112
|
153 (add-eye world (.getCamera world) no-op)))
|
|
rlm@112
|
154 (fn [world tpf]
|
|
rlm@112
|
155 (.rotate candy (* tpf 0.2) 0 0)))))
|
|
rlm@23
|
156 #+end_src
|
|
rlm@23
|
157
|
|
rlm@112
|
158 #+results: test-vision
|
|
rlm@112
|
159 : #'cortex.test.vision/test-two-eyes
|
|
rlm@112
|
160
|
|
rlm@34
|
161 The example code will create two videos of the same rotating object
|
|
rlm@34
|
162 from different angles. It can be used both for stereoscopic vision
|
|
rlm@34
|
163 simulation or for simulating multiple creatures, each with their own
|
|
rlm@34
|
164 sense of vision.
|
|
rlm@24
|
165
|
|
rlm@35
|
166 - As a neat bonus, this idea behind simulated vision also enables one
|
|
rlm@35
|
167 to [[../../cortex/html/capture-video.html][capture live video feeds from jMonkeyEngine]].
|
|
rlm@35
|
168
|
|
rlm@24
|
169
|
|
rlm@24
|
170 * COMMENT code generation
|
|
rlm@34
|
171 #+begin_src clojure :tangle ../src/cortex/vision.clj
|
|
rlm@24
|
172 <<eyes>>
|
|
rlm@24
|
173 #+end_src
|
|
rlm@24
|
174
|
|
rlm@68
|
175 #+begin_src clojure :tangle ../src/cortex/test/vision.clj
|
|
rlm@24
|
176 <<test-vision>>
|
|
rlm@24
|
177 #+end_src
|
