Mercurial > cortex

annotate org/touch.org @ 228:0589c35f04f2

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
wrote intro for touch.org
author Robert McIntyre <rlm@mit.edu>
date Sat, 11 Feb 2012 18:42:27 -0700
parents 2a7f57e7efdb
children 6f1be9525e40
rev   line source
rlm@37 1 #+title: Simulated Sense of Touch
rlm@0 2 #+author: Robert McIntyre
rlm@0 3 #+email: rlm@mit.edu
rlm@37 4 #+description: Simulated touch for AI research using JMonkeyEngine and clojure.
rlm@37 5 #+keywords: simulation, tactile sense, jMonkeyEngine3, clojure
rlm@4 6 #+SETUPFILE: ../../aurellem/org/setup.org
rlm@4 7 #+INCLUDE: ../../aurellem/org/level-0.org
rlm@0 8
rlm@37 9 * Touch
rlm@0 10
rlm@226 11 Touch is critical to navigation and spatial reasoning and as such I
rlm@226 12 need a simulated version of it to give to my AI creatures.
rlm@0 13
rlm@228 14 However, touch in my virtual can not exactly correspond to human touch
rlm@228 15 because my creatures are made out of completely rigid segments that
rlm@228 16 don't deform like human skin.
rlm@228 17
rlm@228 18 Human skin has a wide array of touch sensors, each of which speciliaze
rlm@228 19 in detecting different vibrational modes and pressures. These sensors
rlm@228 20 can integrate a vast expanse of skin (i.e. your entire palm), or a
rlm@228 21 tiny patch of skin at the tip of your finger. The hairs of the skin
rlm@228 22 help detect objects before they even come into contact with the skin
rlm@228 23 proper.
rlm@228 24
rlm@228 25 Instead of measuring deformation or vibration, I surround each rigid
rlm@228 26 part with a plenitude of hair-like objects which do not interact with
rlm@228 27 the physical world. Physical objects can pass through them with no
rlm@228 28 effect. The hairs are able to measure contact with other objects, and
rlm@228 29 constantly report how much of their extent is covered. So, even though
rlm@228 30 the creature's body parts do not deform, the hairs create a margin
rlm@228 31 around those body parts which achieves a sense of touch which is a
rlm@228 32 hybrid between a human's sense of deformation and sense from hairs.
rlm@228 33
rlm@228 34 Implementing touch in jMonkeyEngine follows a different techinal route
rlm@228 35 than vision and hearing. Those two senses piggybacked off
rlm@228 36 jMonkeyEngine's 3D audio and video rendering subsystems. To simulate
rlm@228 37 Touch, I use jMonkeyEngine's physics system to execute many small
rlm@228 38 collision detections, one for each "hair".
rlm@228 39
rlm@228 40 * Sensor Related Functions
rlm@0 41 #+begin_src clojure
rlm@156 42 (defn sensors-in-triangle
rlm@178 43 "Locate the touch sensors in the triangle, returning a map of their
rlm@178 44 UV and geometry-relative coordinates."
rlm@156 45 [image mesh tri-index]
rlm@156 46 (let [width (.getWidth image)
rlm@156 47 height (.getHeight image)
rlm@156 48 UV-vertex-coords (triangle-UV-coord mesh width height tri-index)
rlm@156 49 bounds (convex-bounds UV-vertex-coords)
rlm@156 50
rlm@156 51 cutout-triangle (points->triangle UV-vertex-coords)
rlm@156 52 UV-sensor-coords
rlm@156 53 (filter (comp (partial inside-triangle? cutout-triangle)
rlm@156 54 (fn [[u v]] (Vector3f. u v 0)))
rlm@156 55 (white-coordinates image bounds))
rlm@156 56 UV->geometry (triangle-transformation
rlm@156 57 cutout-triangle
rlm@227 58 (mesh-triangle mesh tri-index))
rlm@156 59 geometry-sensor-coords
rlm@156 60 (map (fn [[u v]] (.mult UV->geometry (Vector3f. u v 0)))
rlm@156 61 UV-sensor-coords)]
rlm@156 62 {:UV UV-sensor-coords :geometry geometry-sensor-coords}))
rlm@156 63
rlm@156 64 (defn-memo locate-feelers
rlm@178 65 "Search the geometry's tactile UV profile for touch sensors,
rlm@178 66 returning their positions in geometry-relative coordinates."
rlm@156 67 [#^Geometry geo]
rlm@156 68 (let [mesh (.getMesh geo)
rlm@156 69 num-triangles (.getTriangleCount mesh)]
rlm@178 70 (if-let [image (tactile-sensor-profile geo)]
rlm@156 71 (map
rlm@156 72 (partial sensors-in-triangle image mesh)
rlm@156 73 (range num-triangles))
rlm@156 74 (repeat (.getTriangleCount mesh) {:UV nil :geometry nil}))))
rlm@156 75
rlm@178 76 (defn-memo touch-topology
rlm@178 77 "Return a sequence of vectors of the form [x y] describing the
rlm@178 78 \"topology\" of the tactile sensors. Points that are close together
rlm@178 79 in the touch-topology are generally close together in the simulation."
rlm@178 80 [#^Gemoetry geo]
rlm@156 81 (vec (collapse (reduce concat (map :UV (locate-feelers geo))))))
rlm@156 82
rlm@178 83 (defn-memo feeler-coordinates
rlm@178 84 "The location of the touch sensors in world-space coordinates."
rlm@178 85 [#^Geometry geo]
rlm@156 86 (vec (map :geometry (locate-feelers geo))))
rlm@228 87 #+end_src
rlm@156 88
rlm@228 89 * Triangle Manipulation Functions
rlm@228 90
rlm@228 91 #+begin_src clojure
rlm@228 92 (defn triangles
rlm@228 93 "Return a sequence of all the Triangles which compose a given
rlm@228 94 Geometry."
rlm@228 95 [#^Geometry geom]
rlm@228 96 (let
rlm@228 97 [mesh (.getMesh geom)
rlm@228 98 triangles (transient [])]
rlm@228 99 (dorun
rlm@228 100 (for [n (range (.getTriangleCount mesh))]
rlm@228 101 (let [tri (Triangle.)]
rlm@228 102 (.getTriangle mesh n tri)
rlm@228 103 ;; (.calculateNormal tri)
rlm@228 104 ;; (.calculateCenter tri)
rlm@228 105 (conj! triangles tri))))
rlm@228 106 (persistent! triangles)))
rlm@228 107
rlm@228 108 (defn mesh-triangle
rlm@228 109 "Get the triangle specified by triangle-index from the mesh within
rlm@228 110 bounds."
rlm@228 111 [#^Mesh mesh triangle-index]
rlm@228 112 (let [scratch (Triangle.)]
rlm@228 113 (.getTriangle mesh triangle-index scratch)
rlm@228 114 scratch))
rlm@228 115
rlm@228 116 (defn triangle-vertex-indices
rlm@228 117 "Get the triangle vertex indices of a given triangle from a given
rlm@228 118 mesh."
rlm@228 119 [#^Mesh mesh triangle-index]
rlm@228 120 (let [indices (int-array 3)]
rlm@228 121 (.getTriangle mesh triangle-index indices)
rlm@228 122 (vec indices)))
rlm@228 123
rlm@228 124 (defn vertex-UV-coord
rlm@228 125 "Get the UV-coordinates of the vertex named by vertex-index"
rlm@228 126 [#^Mesh mesh vertex-index]
rlm@228 127 (let [UV-buffer
rlm@228 128 (.getData
rlm@228 129 (.getBuffer
rlm@228 130 mesh
rlm@228 131 VertexBuffer$Type/TexCoord))]
rlm@228 132 [(.get UV-buffer (* vertex-index 2))
rlm@228 133 (.get UV-buffer (+ 1 (* vertex-index 2)))]))
rlm@228 134
rlm@228 135 (defn triangle-UV-coord
rlm@228 136 "Get the UV-cooridnates of the triangle's verticies."
rlm@228 137 [#^Mesh mesh width height triangle-index]
rlm@228 138 (map (fn [[u v]] (vector (* width u) (* height v)))
rlm@228 139 (map (partial vertex-UV-coord mesh)
rlm@228 140 (triangle-vertex-indices mesh triangle-index))))
rlm@228 141 #+end_src
rlm@228 142
rlm@228 143 * Schrapnel Conversion Functions
rlm@228 144 #+begin_src clojure
rlm@228 145 (defn triangle-seq [#^Triangle tri]
rlm@228 146 [(.get1 tri) (.get2 tri) (.get3 tri)])
rlm@228 147
rlm@228 148 (defn vector3f-seq [#^Vector3f v]
rlm@228 149 [(.getX v) (.getY v) (.getZ v)])
rlm@228 150
rlm@228 151 (defn point->vector2f [[u v]]
rlm@228 152 (Vector2f. u v))
rlm@228 153
rlm@228 154 (defn vector2f->vector3f [v]
rlm@228 155 (Vector3f. (.getX v) (.getY v) 0))
rlm@228 156
rlm@228 157 (defn map-triangle [f #^Triangle tri]
rlm@228 158 (Triangle.
rlm@228 159 (f 0 (.get1 tri))
rlm@228 160 (f 1 (.get2 tri))
rlm@228 161 (f 2 (.get3 tri))))
rlm@228 162
rlm@228 163 (defn points->triangle
rlm@228 164 "Convert a list of points into a triangle."
rlm@228 165 [points]
rlm@228 166 (apply #(Triangle. %1 %2 %3)
rlm@228 167 (map (fn [point]
rlm@228 168 (let [point (vec point)]
rlm@228 169 (Vector3f. (get point 0 0)
rlm@228 170 (get point 1 0)
rlm@228 171 (get point 2 0))))
rlm@228 172 (take 3 points))))
rlm@228 173 #+end_src
rlm@228 174
rlm@228 175 * Triangle Affine Transforms
rlm@228 176
rlm@228 177 #+begin_src clojure
rlm@228 178 (defn triangle->matrix4f
rlm@228 179 "Converts the triangle into a 4x4 matrix: The first three columns
rlm@228 180 contain the vertices of the triangle; the last contains the unit
rlm@228 181 normal of the triangle. The bottom row is filled with 1s."
rlm@228 182 [#^Triangle t]
rlm@228 183 (let [mat (Matrix4f.)
rlm@228 184 [vert-1 vert-2 vert-3]
rlm@228 185 ((comp vec map) #(.get t %) (range 3))
rlm@228 186 unit-normal (do (.calculateNormal t)(.getNormal t))
rlm@228 187 vertices [vert-1 vert-2 vert-3 unit-normal]]
rlm@228 188 (dorun
rlm@228 189 (for [row (range 4) col (range 3)]
rlm@228 190 (do
rlm@228 191 (.set mat col row (.get (vertices row)col))
rlm@228 192 (.set mat 3 row 1))))
rlm@228 193 mat))
rlm@228 194
rlm@228 195 (defn triangle-transformation
rlm@228 196 "Returns the affine transformation that converts each vertex in the
rlm@228 197 first triangle into the corresponding vertex in the second
rlm@228 198 triangle."
rlm@228 199 [#^Triangle tri-1 #^Triangle tri-2]
rlm@228 200 (.mult
rlm@228 201 (triangle->matrix4f tri-2)
rlm@228 202 (.invert (triangle->matrix4f tri-1))))
rlm@228 203 #+end_src
rlm@228 204
rlm@228 205 * Blender Meta-Data
rlm@228 206
rlm@228 207 #+begin_src clojure
rlm@228 208 (defn tactile-sensor-profile
rlm@228 209 "Return the touch-sensor distribution image in BufferedImage format,
rlm@228 210 or nil if it does not exist."
rlm@228 211 [#^Geometry obj]
rlm@228 212 (if-let [image-path (meta-data obj "touch")]
rlm@228 213 (load-image image-path)))
rlm@228 214 #+end_src
rlm@228 215
rlm@228 216 * Physics Collision Objects
rlm@228 217 #+begin_src clojure
rlm@228 218 (defn get-ray-origin
rlm@228 219 "Return the origin which a Ray would have to have to be in the exact
rlm@228 220 center of a particular Triangle in the Geometry in World
rlm@228 221 Coordinates."
rlm@228 222 [geom tri]
rlm@228 223 (let [new (Vector3f.)]
rlm@228 224 (.calculateCenter tri)
rlm@228 225 (.localToWorld geom (.getCenter tri) new) new))
rlm@228 226
rlm@228 227 (defn get-ray-direction
rlm@228 228 "Return the direction which a Ray would have to have to be to point
rlm@228 229 normal to the Triangle, in coordinates relative to the center of the
rlm@228 230 Triangle."
rlm@228 231 [geom tri]
rlm@228 232 (let [n+c (Vector3f.)]
rlm@228 233 (.calculateNormal tri)
rlm@228 234 (.calculateCenter tri)
rlm@228 235 (.localToWorld
rlm@228 236 geom
rlm@228 237 (.add (.getCenter tri) (.getNormal tri)) n+c)
rlm@228 238 (.subtract n+c (get-ray-origin geom tri))))
rlm@228 239 #+end_src
rlm@228 240
rlm@228 241 * Triangle Boundaries
rlm@228 242 #+begin_src clojure
rlm@228 243 (defn same-side?
rlm@228 244 "Given the points p1 and p2 and the reference point ref, is point p
rlm@228 245 on the same side of the line that goes through p1 and p2 as ref is?"
rlm@228 246 [p1 p2 ref p]
rlm@228 247 (<=
rlm@228 248 0
rlm@228 249 (.dot
rlm@228 250 (.cross (.subtract p2 p1) (.subtract p p1))
rlm@228 251 (.cross (.subtract p2 p1) (.subtract ref p1)))))
rlm@228 252
rlm@228 253 (defn inside-triangle?
rlm@228 254 "Is the point inside the triangle?"
rlm@228 255 {:author "Dylan Holmes"}
rlm@228 256 [#^Triangle tri #^Vector3f p]
rlm@228 257 (let [[vert-1 vert-2 vert-3] (triangle-seq tri)]
rlm@228 258 (and
rlm@228 259 (same-side? vert-1 vert-2 vert-3 p)
rlm@228 260 (same-side? vert-2 vert-3 vert-1 p)
rlm@228 261 (same-side? vert-3 vert-1 vert-2 p))))
rlm@228 262
rlm@228 263 (defn convex-bounds
rlm@228 264 "Returns the smallest square containing the given vertices, as a
rlm@228 265 vector of integers [left top width height]."
rlm@228 266 [uv-verts]
rlm@228 267 (let [xs (map first uv-verts)
rlm@228 268 ys (map second uv-verts)
rlm@228 269 x0 (Math/floor (apply min xs))
rlm@228 270 y0 (Math/floor (apply min ys))
rlm@228 271 x1 (Math/ceil (apply max xs))
rlm@228 272 y1 (Math/ceil (apply max ys))]
rlm@228 273 [x0 y0 (- x1 x0) (- y1 y0)]))
rlm@228 274 #+end_src
rlm@228 275
rlm@228 276 * Skin Creation
rlm@228 277
rlm@228 278 #+begin_src clojure
rlm@178 279 (defn touch-fn
rlm@178 280 "Returns a function which returns tactile sensory data when called
rlm@178 281 inside a running simulation."
rlm@178 282 [#^Geometry geo]
rlm@156 283 (let [feeler-coords (feeler-coordinates geo)
rlm@156 284 tris (triangles geo)
rlm@156 285 limit 0.1
rlm@156 286 ;;results (CollisionResults.)
rlm@156 287 ]
rlm@156 288 (if (empty? (touch-topology geo))
rlm@156 289 nil
rlm@156 290 (fn [node]
rlm@156 291 (let [sensor-origins
rlm@156 292 (map
rlm@156 293 #(map (partial local-to-world geo) %)
rlm@156 294 feeler-coords)
rlm@156 295 triangle-normals
rlm@156 296 (map (partial get-ray-direction geo)
rlm@156 297 tris)
rlm@156 298 rays
rlm@156 299 (flatten
rlm@156 300 (map (fn [origins norm]
rlm@156 301 (map #(doto (Ray. % norm)
rlm@156 302 (.setLimit limit)) origins))
rlm@156 303 sensor-origins triangle-normals))]
rlm@156 304 (vector
rlm@156 305 (touch-topology geo)
rlm@156 306 (vec
rlm@156 307 (for [ray rays]
rlm@156 308 (do
rlm@156 309 (let [results (CollisionResults.)]
rlm@156 310 (.collideWith node ray results)
rlm@156 311 (let [touch-objects
rlm@156 312 (filter #(not (= geo (.getGeometry %)))
rlm@156 313 results)]
rlm@156 314 (- 255
rlm@156 315 (if (empty? touch-objects) 255
rlm@156 316 (rem
rlm@156 317 (int
rlm@156 318 (* 255 (/ (.getDistance
rlm@156 319 (first touch-objects)) limit)))
rlm@156 320 256))))))))))))))
rlm@156 321
rlm@178 322 (defn touch!
rlm@178 323 "Endow the creature with the sense of touch. Returns a sequence of
rlm@178 324 functions, one for each body part with a tactile-sensor-proile,
rlm@178 325 each of which when called returns sensory data for that body part."
rlm@178 326 [#^Node creature]
rlm@178 327 (filter
rlm@178 328 (comp not nil?)
rlm@178 329 (map touch-fn
rlm@178 330 (filter #(isa? (class %) Geometry)
rlm@178 331 (node-seq creature)))))
rlm@228 332 #+end_src
rlm@156 333
rlm@228 334 * Visualizing Touch
rlm@228 335
rlm@228 336 #+begin_src clojure
rlm@188 337 (defn view-touch
rlm@189 338 "Creates a function which accepts a list of touch sensor-data and
rlm@189 339 displays each element to the screen."
rlm@188 340 []
rlm@187 341 (view-sense
rlm@187 342 (fn
rlm@187 343 [[coords sensor-data]]
rlm@187 344 (let [image (points->image coords)]
rlm@187 345 (dorun
rlm@187 346 (for [i (range (count coords))]
rlm@187 347 (.setRGB image ((coords i) 0) ((coords i) 1)
rlm@187 348 (gray (sensor-data i)))))
rlm@188 349 image))))
rlm@37 350 #+end_src
rlm@37 351
rlm@226 352 * Headers
rlm@226 353 #+begin_src clojure
rlm@226 354 (ns cortex.touch
rlm@226 355 "Simulate the sense of touch in jMonkeyEngine3. Enables any Geometry
rlm@226 356 to be outfitted with touch sensors with density determined by a UV
rlm@226 357 image. In this way a Geometry can know what parts of itself are
rlm@226 358 touching nearby objects. Reads specially prepared blender files to
rlm@226 359 construct this sense automatically."
rlm@226 360 {:author "Robert McIntyre"}
rlm@226 361 (:use (cortex world util sense))
rlm@226 362 (:use clojure.contrib.def)
rlm@226 363 (:import (com.jme3.scene Geometry Node Mesh))
rlm@226 364 (:import com.jme3.collision.CollisionResults)
rlm@226 365 (:import com.jme3.scene.VertexBuffer$Type)
rlm@226 366 (:import (com.jme3.math Triangle Vector3f Vector2f Ray Matrix4f)))
rlm@226 367 #+end_src
rlm@37 368
rlm@228 369 ;; Every Mesh has many triangles, each with its own index.
rlm@228 370 ;; Every vertex has its own index as well.
rlm@228 371
rlm@228 372 * Source Listing
rlm@228 373 * Next
rlm@228 374
rlm@228 375
rlm@226 376 * COMMENT Code Generation
rlm@39 377 #+begin_src clojure :tangle ../src/cortex/touch.clj
rlm@0 378 <<skin-main>>
rlm@0 379 #+end_src
rlm@0 380
rlm@68 381 #+begin_src clojure :tangle ../src/cortex/test/touch.clj
rlm@39 382 #+end_src
rlm@39 383
rlm@0 384
rlm@0 385
rlm@0 386
rlm@32 387
rlm@32 388
rlm@226 389