cortex: org/vision.org comparison

comparison org/vision.org @ 168:1c8e9d389ea4

renamed eyes.org to vision.org

author	Robert McIntyre <rlm@mit.edu>
date	Sat, 04 Feb 2012 04:08:08 -0700
parents	org/eyes.org@9e6a30b8c99a
children	94b79c191fc7

comparison

equal deleted inserted replaced

-:9e6a30b8c99a
+:1c8e9d389ea4
+#+title: Simulated Sense of Sight
+#+author: Robert McIntyre
+#+email: rlm@mit.edu
+#+description: Simulated sight for AI research using JMonkeyEngine3 and clojure
+#+keywords: computer vision, jMonkeyEngine3, clojure
+#+SETUPFILE: ../../aurellem/org/setup.org
+#+INCLUDE: ../../aurellem/org/level-0.org
+#+babel: :mkdirp yes :noweb yes :exports both
+* Vision
+I want to make creatures with eyes. Each eye can be independely moved
+and should see its own version of the world depending on where it is.
+Here's how vision will work.
+Make the continuation in scene-processor take FrameBuffer,
+byte-buffer, BufferedImage already sized to the correct
+dimensions. the continuation will decide wether to "mix" them
+into the BufferedImage, lazily ignore them, or mix them halfway
+and call c/graphics card routines.
+(vision creature) will take an optional :skip argument which will
+inform the continuations in scene processor to skip the given
+number of cycles 0 means that no cycles will be skipped.
+(vision creature) will return [init-functions sensor-functions].
+The init-functions are each single-arg functions that take the
+world and register the cameras and must each be called before the
+corresponding sensor-functions.  Each init-function returns the
+viewport for that eye which can be manipulated, saved, etc. Each
+sensor-function is a thunk and will return data in the same
+format as the tactile-sensor functions the structure is
+[topology, sensor-data]. Internally, these sensor-functions
+maintain a reference to sensor-data which is periodically updated
+by the continuation function established by its init-function.
+They can be queried every cycle, but their information may not
+necessairly be different every cycle.
+Each eye in the creature in blender will work the same way as
+joints -- a zero dimensional object with no geometry whose local
+coordinate system determines the orientation of the resulting
+eye. All eyes will have a parent named "eyes" just as all joints
+have a parent named "joints". The resulting camera will be a
+ChaseCamera or a CameraNode bound to the geo that is closest to
+the eye marker. The eye marker will contain the metadata for the
+eye, and will be moved by it's bound geometry. The dimensions of
+the eye's camera are equal to the dimensions of the eye's "UV"
+map.
+#+name: eyes
+#+begin_src clojure
+(ns cortex.vision
+"Simulate the sense of vision in jMonkeyEngine3. Enables multiple
+eyes from different positions to observe the same world, and pass
+the observed data to any arbitray function."
+{:author "Robert McIntyre"}
+(:use (cortex world sense util))
+(:use clojure.contrib.def)
+(:import com.jme3.post.SceneProcessor)
+(:import (com.jme3.util BufferUtils Screenshots))
+(:import java.nio.ByteBuffer)
+(:import java.awt.image.BufferedImage)
+(:import com.jme3.renderer.ViewPort)
+(:import com.jme3.math.ColorRGBA)
+(:import com.jme3.renderer.Renderer)
+(:import com.jme3.scene.Node))
+(cortex.import/mega-import-jme3)
+(defn vision-pipeline
+"Create a SceneProcessor object which wraps a vision processing
+continuation function. The continuation is a function that takes
+[#^Renderer r #^FrameBuffer fb #^ByteBuffer b #^BufferedImage bi],
+each of which has already been appropiately sized."
+[continuation]
+(let [byte-buffer (atom nil)
+	renderer (atom nil)
+image (atom nil)]
+(proxy [SceneProcessor] []
+(initialize
+[renderManager viewPort]
+(let [cam (.getCamera viewPort)
+	   width (.getWidth cam)
+	   height (.getHeight cam)]
+(reset! renderer (.getRenderer renderManager))
+(reset! byte-buffer
+	     (BufferUtils/createByteBuffer
+	      (* width height 4)))
+(reset! image (BufferedImage.
+width height
+BufferedImage/TYPE_4BYTE_ABGR))))
+(isInitialized [] (not (nil? @byte-buffer)))
+(reshape [_ _ _])
+(preFrame [_])
+(postQueue [_])
+(postFrame
+[#^FrameBuffer fb]
+(.clear @byte-buffer)
+(continuation @renderer fb @byte-buffer @image))
+(cleanup []))))
+(defn frameBuffer->byteBuffer!
+"Transfer the data in the graphics card (Renderer, FrameBuffer) to
+the CPU (ByteBuffer)."
+[#^Renderer r #^FrameBuffer fb #^ByteBuffer bb]
+(.readFrameBuffer r fb bb) bb)
+(defn byteBuffer->bufferedImage!
+"Convert the C-style BGRA image data in the ByteBuffer bb to the AWT
+style ABGR image data and place it in BufferedImage bi."
+[#^ByteBuffer bb #^BufferedImage bi]
+(Screenshots/convertScreenShot bb bi) bi)
+(defn BufferedImage!
+"Continuation which will grab the buffered image from the materials
+provided by (vision-pipeline)."
+[#^Renderer r #^FrameBuffer fb #^ByteBuffer bb #^BufferedImage bi]
+(byteBuffer->bufferedImage!
+(frameBuffer->byteBuffer! r fb bb) bi))
+(defn add-eye!
+"Add an eye to the world, calling continuation on every frame
+produced."
+[#^Application world camera continuation]
+(let [width (.getWidth camera)
+	height (.getHeight camera)
+	render-manager (.getRenderManager world)
+	viewport (.createMainView render-manager "eye-view" camera)]
+(doto viewport
+(.setClearFlags true true true)
+(.setBackgroundColor ColorRGBA/Black)
+(.addProcessor (vision-pipeline continuation))
+(.attachScene (.getRootNode world)))))
+(defn retina-sensor-image
+"Return a map of pixel selection functions to BufferedImages
+describing the distribution of light-sensitive components on this
+geometry's surface. Each function creates an integer from the rgb
+values found in the pixel. :red, :green, :blue, :gray are already
+defined as extracting the red green blue and average components
+respectively."
+[#^Spatial eye]
+(if-let [eye-map (meta-data eye "eye")]
+(map-vals
+load-image
+(eval (read-string eye-map)))))
+(defn eye-dimensions
+"returns the width and height specified in the metadata of the eye"
+[#^Spatial eye]
+(let [dimensions
+(map #(vector (.getWidth %) (.getHeight %))
+(vals (retina-sensor-image eye)))]
+[(apply max (map first dimensions))
+(apply max (map second dimensions))]))
+(defvar
+^{:arglists '([creature])}
+eyes
+(sense-nodes "eyes")
+"Return the children of the creature's \"eyes\" node.")
+(defn attach-eye
+"Attach a Camera to the appropiate area and return the Camera."
+[#^Node creature #^Spatial eye]
+(let [target (closest-node creature eye)
+[cam-width cam-height] (eye-dimensions eye)
+cam (Camera. cam-width cam-height)]
+(.setLocation cam (.getWorldTranslation eye))
+(.setRotation cam (.getWorldRotation eye))
+(.setFrustumPerspective
+cam 45 (/ (.getWidth cam) (.getHeight cam))
+1 1000)
+(bind-sense target cam)
+cam))
+(def presets
+{:all    0xFFFFFF
+:red    0xFF0000
+:blue   0x0000FF
+:green  0x00FF00})
+(defn enable-vision
+"return [init-function sensor-functions] for a particular eye"
+[#^Node creature #^Spatial eye & {skip :skip :or {skip 0}}]
+(let [retinal-map (retina-sensor-image eye)
+camera (attach-eye creature eye)
+vision-image
+(atom
+(BufferedImage. (.getWidth camera)
+(.getHeight camera)
+BufferedImage/TYPE_BYTE_BINARY))]
+[(fn [world]
+(add-eye!
+world camera
+(let [counter  (atom 0)]
+(fn [r fb bb bi]
+(if (zero? (rem (swap! counter inc) (inc skip)))
+(reset! vision-image (BufferedImage! r fb bb bi)))))))
+(vec
+(map
+(fn [[key image]]
+(let [whites (white-coordinates image)
+topology (vec (collapse whites))
+mask (presets key)]
+(fn []
+(vector
+topology
+(vec
+(for [[x y] whites]
+(bit-and
+mask (.getRGB @vision-image x y))))))))
+retinal-map))]))
+(defn vision
+[#^Node creature & {skip :skip :or {skip 0}}]
+(reduce
+(fn [[init-a senses-a]
+[init-b senses-b]]
+[(conj init-a init-b)
+(into senses-a senses-b)])
+[[][]]
+(for [eye (eyes creature)]
+(enable-vision creature eye))))
+#+end_src
+Note the use of continuation passing style for connecting the eye to a
+function to process the output. You can create any number of eyes, and
+each of them will see the world from their own =Camera=. Once every
+frame, the rendered image is copied to a =BufferedImage=, and that
+data is sent off to the continuation function. Moving the =Camera=
+which was used to create the eye will change what the eye sees.
+* Example
+#+name: test-vision
+#+begin_src clojure
+(ns cortex.test.vision
+(:use (cortex world util vision))
+(:import java.awt.image.BufferedImage)
+(:import javax.swing.JPanel)
+(:import javax.swing.SwingUtilities)
+(:import java.awt.Dimension)
+(:import javax.swing.JFrame)
+(:import com.jme3.math.ColorRGBA)
+(:import com.jme3.scene.Node)
+(:import com.jme3.math.Vector3f))
+(defn test-two-eyes
+"Testing vision:
+Tests the vision system by creating two views of the same rotating
+object from different angles and displaying both of those views in
+JFrames.
+You should see a rotating cube, and two windows,
+each displaying a different view of the cube."
+[]
+(let [candy
+(box 1 1 1 :physical? false :color ColorRGBA/Blue)]
+(world
+(doto (Node.)
+(.attachChild candy))
+{}
+(fn [world]
+(let [cam (.clone (.getCamera world))
+width (.getWidth cam)
+height (.getHeight cam)]
+(add-eye! world cam
+;;no-op
+(comp (view-image) BufferedImage!)
+)
+(add-eye! world
+(doto (.clone cam)
+(.setLocation (Vector3f. -10 0 0))
+(.lookAt Vector3f/ZERO Vector3f/UNIT_Y))
+;;no-op
+(comp (view-image) BufferedImage!))
+;; This is here to restore the main view
+;; after the other views have completed processing
+(add-eye! world (.getCamera world) no-op)))
+(fn [world tpf]
+(.rotate candy (* tpf 0.2) 0 0)))))
+#+end_src
+#+results: test-vision
+: #'cortex.test.vision/test-two-eyes
+The example code will create two videos of the same rotating object
+from different angles. It can be used both for stereoscopic vision
+simulation or for simulating multiple creatures, each with their own
+sense of vision.
+- As a neat bonus, this idea behind simulated vision also enables one
+to [[../../cortex/html/capture-video.html][capture live video feeds from jMonkeyEngine]].
+* COMMENT code generation
+#+begin_src clojure :tangle ../src/cortex/vision.clj
+<<eyes>>
+#+end_src
+#+begin_src clojure :tangle ../src/cortex/test/vision.clj
+<<test-vision>>
+#+end_src

Mercurial > cortex

comparison org/vision.org @ 168:1c8e9d389ea4