view org/eyes.org @ 145:7a49b81ca1bf

finally got proprioception working to my satisfaction
author Robert McIntyre <rlm@mit.edu>
date Thu, 02 Feb 2012 12:49:11 -0700
parents 9d0fe7f54e14
children aaacf087504c
line wrap: on
line source
1 #+title: Simulated Sense of Sight
2 #+author: Robert McIntyre
3 #+email: rlm@mit.edu
4 #+description: Simulated sight for AI research using JMonkeyEngine3 and clojure
5 #+keywords: computer vision, jMonkeyEngine3, clojure
6 #+SETUPFILE: ../../aurellem/org/setup.org
7 #+INCLUDE: ../../aurellem/org/level-0.org
8 #+babel: :mkdirp yes :noweb yes :exports both
10 * Vision
12 I want to make creatures with eyes. Each eye can be independely moved
13 and should see its own version of the world depending on where it is.
14 #+name: eyes
15 #+begin_src clojure
16 (ns cortex.vision
17 "Simulate the sense of vision in jMonkeyEngine3. Enables multiple
18 eyes from different positions to observe the same world, and pass
19 the observed data to any arbitray function."
20 {:author "Robert McIntyre"}
21 (:use cortex.world)
22 (:import com.jme3.post.SceneProcessor)
23 (:import (com.jme3.util BufferUtils Screenshots))
24 (:import java.nio.ByteBuffer)
25 (:import java.awt.image.BufferedImage)
26 (:import com.jme3.renderer.ViewPort)
27 (:import com.jme3.math.ColorRGBA)
28 (:import com.jme3.renderer.Renderer))
31 (defn vision-pipeline
32 "Create a SceneProcessor object which wraps a vision processing
33 continuation function. The continuation is a function that takes
34 [#^Renderer r #^FrameBuffer fb #^ByteBuffer b #^BufferedImage bi],
35 each of which has already been appropiately sized."
36 [continuation]
37 (let [byte-buffer (atom nil)
38 renderer (atom nil)
39 image (atom nil)]
40 (proxy [SceneProcessor] []
41 (initialize
42 [renderManager viewPort]
43 (let [cam (.getCamera viewPort)
44 width (.getWidth cam)
45 height (.getHeight cam)]
46 (reset! renderer (.getRenderer renderManager))
47 (reset! byte-buffer
48 (BufferUtils/createByteBuffer
49 (* width height 4)))
50 (reset! image (BufferedImage.
51 width height
52 BufferedImage/TYPE_4BYTE_ABGR))))
53 (isInitialized [] (not (nil? @byte-buffer)))
54 (reshape [_ _ _])
55 (preFrame [_])
56 (postQueue [_])
57 (postFrame
58 [#^FrameBuffer fb]
59 (.clear @byte-buffer)
60 (continuation @renderer fb @byte-buffer @image))
61 (cleanup []))))
63 (defn frameBuffer->byteBuffer!
64 "Transfer the data in the graphics card (Renderer, FrameBuffer) to
65 the CPU (ByteBuffer)."
66 [#^Renderer r #^FrameBuffer fb #^ByteBuffer bb]
67 (.readFrameBuffer r fb bb) bb)
69 (defn byteBuffer->bufferedImage!
70 "Convert the C-style BGRA image data in the ByteBuffer bb to the AWT
71 style ABGR image data and place it in BufferedImage bi."
72 [#^ByteBuffer bb #^BufferedImage bi]
73 (Screenshots/convertScreenShot bb bi) bi)
75 (defn BufferedImage!
76 "Continuation which will grab the buffered image from the materials
77 provided by (vision-pipeline)."
78 [#^Renderer r #^FrameBuffer fb #^ByteBuffer bb #^BufferedImage bi]
79 (byteBuffer->bufferedImage!
80 (frameBuffer->byteBuffer! r fb bb) bi))
82 (defn add-eye
83 "Add an eye to the world, calling continuation on every frame
84 produced."
85 [world camera continuation]
86 (let [width (.getWidth camera)
87 height (.getHeight camera)
88 render-manager (.getRenderManager world)
89 viewport (.createMainView render-manager "eye-view" camera)]
90 (doto viewport
91 (.setClearFlags true true true)
92 (.setBackgroundColor ColorRGBA/Black)
93 (.addProcessor (vision-pipeline continuation))
94 (.attachScene (.getRootNode world)))))
95 #+end_src
97 #+results: eyes
98 : #'cortex.vision/add-eye
100 Note the use of continuation passing style for connecting the eye to a
101 function to process the output. You can create any number of eyes, and
102 each of them will see the world from their own =Camera=. Once every
103 frame, the rendered image is copied to a =BufferedImage=, and that
104 data is sent off to the continuation function. Moving the =Camera=
105 which was used to create the eye will change what the eye sees.
107 * Example
109 #+name: test-vision
110 #+begin_src clojure
111 (ns cortex.test.vision
112 (:use (cortex world util vision))
113 (:import java.awt.image.BufferedImage)
114 (:import javax.swing.JPanel)
115 (:import javax.swing.SwingUtilities)
116 (:import java.awt.Dimension)
117 (:import javax.swing.JFrame)
118 (:import com.jme3.math.ColorRGBA)
119 (:import com.jme3.scene.Node)
120 (:import com.jme3.math.Vector3f))
122 (defn test-two-eyes
123 "Testing vision:
124 Tests the vision system by creating two views of the same rotating
125 object from different angles and displaying both of those views in
126 JFrames.
128 You should see a rotating cube, and two windows,
129 each displaying a different view of the cube."
130 []
131 (let [candy
132 (box 1 1 1 :physical? false :color ColorRGBA/Blue)]
133 (world
134 (doto (Node.)
135 (.attachChild candy))
136 {}
137 (fn [world]
138 (let [cam (.clone (.getCamera world))
139 width (.getWidth cam)
140 height (.getHeight cam)]
141 (add-eye world cam
142 ;;no-op
143 (comp (view-image) BufferedImage!)
144 )
145 (add-eye world
146 (doto (.clone cam)
147 (.setLocation (Vector3f. -10 0 0))
148 (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))
149 ;;no-op
150 (comp (view-image) BufferedImage!))
151 ;; This is here to restore the main view
152 ;; after the other views have completed processing
153 (add-eye world (.getCamera world) no-op)))
154 (fn [world tpf]
155 (.rotate candy (* tpf 0.2) 0 0)))))
156 #+end_src
158 #+results: test-vision
159 : #'cortex.test.vision/test-two-eyes
161 The example code will create two videos of the same rotating object
162 from different angles. It can be used both for stereoscopic vision
163 simulation or for simulating multiple creatures, each with their own
164 sense of vision.
166 - As a neat bonus, this idea behind simulated vision also enables one
167 to [[../../cortex/html/capture-video.html][capture live video feeds from jMonkeyEngine]].
170 * COMMENT code generation
171 #+begin_src clojure :tangle ../src/cortex/vision.clj
172 <<eyes>>
173 #+end_src
175 #+begin_src clojure :tangle ../src/cortex/test/vision.clj
176 <<test-vision>>
177 #+end_src