Mercurial > cortex
view org/test-creature.org @ 156:e8df6e76c3e5
refactored touch
author | Robert McIntyre <rlm@mit.edu> |
---|---|
date | Fri, 03 Feb 2012 06:15:34 -0700 |
parents | 95bf55614211 |
children | 84c67be00abe |
line wrap: on
line source
1 #+title: First attempt at a creature!2 #+author: Robert McIntyre3 #+email: rlm@mit.edu4 #+description:5 #+keywords: simulation, jMonkeyEngine3, clojure6 #+SETUPFILE: ../../aurellem/org/setup.org7 #+INCLUDE: ../../aurellem/org/level-0.org12 * Intro13 So far, I've made the following senses --14 - Vision15 - Hearing16 - Touch17 - Proprioception19 And one effector:20 - Movement22 However, the code so far has only enabled these senses, but has not23 actually implemented them. For example, there is still a lot of work24 to be done for vision. I need to be able to create an /eyeball/ in25 simulation that can be moved around and see the world from different26 angles. I also need to determine weather to use log-polar or cartesian27 for the visual input, and I need to determine how/wether to28 disceritise the visual input.30 I also want to be able to visualize both the sensors and the31 effectors in pretty pictures. This semi-retarted creature will be my32 first attempt at bringing everything together.34 * The creature's body36 Still going to do an eve-like body in blender, but due to problems37 importing the joints, etc into jMonkeyEngine3, I'm going to do all38 the connecting here in clojure code, using the names of the individual39 components and trial and error. Later, I'll maybe make some sort of40 creature-building modifications to blender that support whatever41 discreitized senses I'm going to make.43 #+name: body-144 #+begin_src clojure45 (ns cortex.silly46 "let's play!"47 {:author "Robert McIntyre"})49 ;; TODO remove this!50 (require 'cortex.import)51 (cortex.import/mega-import-jme3)52 (use '(cortex world util body hearing touch vision sense))54 (rlm.rlm-commands/help)55 (import java.awt.image.BufferedImage)56 (import javax.swing.JPanel)57 (import javax.swing.SwingUtilities)58 (import java.awt.Dimension)59 (import javax.swing.JFrame)60 (import java.awt.Dimension)61 (import com.aurellem.capture.RatchetTimer)62 (declare joint-create)63 (use 'clojure.contrib.def)65 (defn load-blender-model66 "Load a .blend file using an asset folder relative path."67 [^String model]68 (.loadModel69 (doto (asset-manager)70 (.registerLoader BlenderModelLoader (into-array String ["blend"])))71 model))73 (defn blender-to-jme74 "Convert from Blender coordinates to JME coordinates"75 [#^Vector3f in]76 (Vector3f. (.getX in)77 (.getZ in)78 (- (.getY in))))81 (defmulti joint-dispatch82 "Translate blender pseudo-joints into real JME joints."83 (fn [constraints & _]84 (:type constraints)))86 (defmethod joint-dispatch :point87 [constraints control-a control-b pivot-a pivot-b rotation]88 (println-repl "creating POINT2POINT joint")89 ;; bullet's point2point joints are BROKEN, so we must use the90 ;; generic 6DOF joint instead of an actual Point2Point joint!92 ;; should be able to do this:93 (comment94 (Point2PointJoint.95 control-a96 control-b97 pivot-a98 pivot-b))100 ;; but instead we must do this:101 (println-repl "substuting 6DOF joint for POINT2POINT joint!")102 (doto103 (SixDofJoint.104 control-a105 control-b106 pivot-a107 pivot-b108 false)109 (.setLinearLowerLimit Vector3f/ZERO)110 (.setLinearUpperLimit Vector3f/ZERO)111 ;;(.setAngularLowerLimit (Vector3f. 1 1 1))112 ;;(.setAngularUpperLimit (Vector3f. 0 0 0))114 ))117 (defmethod joint-dispatch :hinge118 [constraints control-a control-b pivot-a pivot-b rotation]119 (println-repl "creating HINGE joint")120 (let [axis121 (if-let122 [axis (:axis constraints)]123 axis124 Vector3f/UNIT_X)125 [limit-1 limit-2] (:limit constraints)126 hinge-axis127 (.mult128 rotation129 (blender-to-jme axis))]130 (doto131 (HingeJoint.132 control-a133 control-b134 pivot-a135 pivot-b136 hinge-axis137 hinge-axis)138 (.setLimit limit-1 limit-2))))140 (defmethod joint-dispatch :cone141 [constraints control-a control-b pivot-a pivot-b rotation]142 (let [limit-xz (:limit-xz constraints)143 limit-xy (:limit-xy constraints)144 twist (:twist constraints)]146 (println-repl "creating CONE joint")147 (println-repl rotation)148 (println-repl149 "UNIT_X --> " (.mult rotation (Vector3f. 1 0 0)))150 (println-repl151 "UNIT_Y --> " (.mult rotation (Vector3f. 0 1 0)))152 (println-repl153 "UNIT_Z --> " (.mult rotation (Vector3f. 0 0 1)))154 (doto155 (ConeJoint.156 control-a157 control-b158 pivot-a159 pivot-b160 rotation161 rotation)162 (.setLimit (float limit-xz)163 (float limit-xy)164 (float twist)))))166 (defn connect167 "here are some examples:168 {:type :point}169 {:type :hinge :limit [0 (/ Math/PI 2)] :axis (Vector3f. 0 1 0)}170 (:axis defaults to (Vector3f. 1 0 0) if not provided for hinge joints)172 {:type :cone :limit-xz 0]173 :limit-xy 0]174 :twist 0]} (use XZY rotation mode in blender!)"175 [#^Node obj-a #^Node obj-b #^Node joint]176 (let [control-a (.getControl obj-a RigidBodyControl)177 control-b (.getControl obj-b RigidBodyControl)178 joint-center (.getWorldTranslation joint)179 joint-rotation (.toRotationMatrix (.getWorldRotation joint))180 pivot-a (world-to-local obj-a joint-center)181 pivot-b (world-to-local obj-b joint-center)]183 (if-let [constraints184 (map-vals185 eval186 (read-string187 (meta-data joint "joint")))]188 ;; A side-effect of creating a joint registers189 ;; it with both physics objects which in turn190 ;; will register the joint with the physics system191 ;; when the simulation is started.192 (do193 (println-repl "creating joint between"194 (.getName obj-a) "and" (.getName obj-b))195 (joint-dispatch constraints196 control-a control-b197 pivot-a pivot-b198 joint-rotation))199 (println-repl "could not find joint meta-data!"))))204 (defn assemble-creature [#^Node pieces joints]205 (dorun206 (map207 (fn [geom]208 (let [physics-control209 (RigidBodyControl.210 (HullCollisionShape.211 (.getMesh geom))212 (if-let [mass (meta-data geom "mass")]213 (do214 (println-repl215 "setting" (.getName geom) "mass to" (float mass))216 (float mass))217 (float 1)))]219 (.addControl geom physics-control)))220 (filter #(isa? (class %) Geometry )221 (node-seq pieces))))222 (dorun223 (map224 (fn [joint]225 (let [[obj-a obj-b] (joint-targets pieces joint)]226 (connect obj-a obj-b joint)))227 joints))228 pieces)230 (declare blender-creature)232 (def hand "Models/creature1/one.blend")234 (def worm "Models/creature1/try-again.blend")236 (defn worm-model [] (load-blender-model worm))238 (defn x-ray [#^ColorRGBA color]239 (doto (Material. (asset-manager)240 "Common/MatDefs/Misc/Unshaded.j3md")241 (.setColor "Color" color)242 (-> (.getAdditionalRenderState)243 (.setDepthTest false))))245 (defn colorful []246 (.getChild (worm-model) "worm-21"))248 (import jme3tools.converters.ImageToAwt)250 (import ij.ImagePlus)254 (defn test-eye []255 (.getChild256 (.getChild (worm-model) "eyes")257 "eye"))261 ;; Ears work the same way as vision.263 ;; (hearing creature) will return [init-functions264 ;; sensor-functions]. The init functions each take the world and265 ;; register a SoundProcessor that does foureier transforms on the266 ;; incommong sound data, making it available to each sensor function.268 (defn creature-ears269 "Return the children of the creature's \"ears\" node."270 ;;dylan271 ;;"The ear nodes which are children of the \"ears\" node in the272 ;;creature."273 [#^Node creature]274 (if-let [ear-node (.getChild creature "ears")]275 (seq (.getChildren ear-node))276 (do (println-repl "could not find ears node") [])))279 ;;dylan (defn follow-sense, adjoin-sense, attach-stimuli,280 ;;anchor-qualia, augment-organ, with-organ283 (defn update-listener-velocity284 "Update the listener's velocity every update loop."285 [#^Spatial obj #^Listener lis]286 (let [old-position (atom (.getLocation lis))]287 (.addControl288 obj289 (proxy [AbstractControl] []290 (controlUpdate [tpf]291 (let [new-position (.getLocation lis)]292 (.setVelocity293 lis294 (.mult (.subtract new-position @old-position)295 (float (/ tpf))))296 (reset! old-position new-position)))297 (controlRender [_ _])))))299 (import com.aurellem.capture.audio.AudioSendRenderer)301 (defn attach-ear302 [#^Application world #^Node creature #^Spatial ear continuation]303 (let [target (closest-node creature ear)304 lis (Listener.)305 audio-renderer (.getAudioRenderer world)306 sp (sound-processor continuation)]307 (.setLocation lis (.getWorldTranslation ear))308 (.setRotation lis (.getWorldRotation ear))309 (bind-sense target lis)310 (update-listener-velocity target lis)311 (.addListener audio-renderer lis)312 (.registerSoundProcessor audio-renderer lis sp)))314 (defn enable-hearing315 [#^Node creature #^Spatial ear]316 (let [hearing-data (atom [])]317 [(fn [world]318 (attach-ear world creature ear319 (fn [data]320 (reset! hearing-data (vec data)))))321 [(fn []322 (let [data @hearing-data323 topology324 (vec (map #(vector % 0) (range 0 (count data))))325 scaled-data326 (vec327 (map328 #(rem (int (* 255 (/ (+ 1 %) 2))) 256)329 data))]330 [topology scaled-data]))331 ]]))333 (defn hearing334 [#^Node creature]335 (reduce336 (fn [[init-a senses-a]337 [init-b senses-b]]338 [(conj init-a init-b)339 (into senses-a senses-b)])340 [[][]]341 (for [ear (creature-ears creature)]342 (enable-hearing creature ear))))349 ;; lower level --- nodes350 ;; closest-node "parse/compile-x" -> makes organ, which is spatial, fn pair352 ;; higher level -- organs353 ;;355 ;; higher level --- sense/effector356 ;; these are the functions that provide world i/o, chinese-room style361 (defn blender-creature362 "Return a creature with all joints in place."363 [blender-path]364 (let [model (load-blender-model blender-path)365 joints (creature-joints model)]366 (assemble-creature model joints)))368 (defn gray-scale [num]369 (+ num370 (bit-shift-left num 8)371 (bit-shift-left num 16)))373 (defn debug-touch-window374 "creates function that offers a debug view of sensor data"375 []376 (let [vi (view-image)]377 (fn378 [[coords sensor-data]]379 (let [image (points->image coords)]380 (dorun381 (for [i (range (count coords))]382 (.setRGB image ((coords i) 0) ((coords i) 1)383 (gray-scale (sensor-data i)))))386 (vi image)))))388 (defn debug-vision-window389 "creates function that offers a debug view of sensor data"390 []391 (let [vi (view-image)]392 (fn393 [[coords sensor-data]]394 (let [image (points->image coords)]395 (dorun396 (for [i (range (count coords))]397 (.setRGB image ((coords i) 0) ((coords i) 1)398 (sensor-data i))))399 (vi image)))))401 (defn debug-hearing-window402 "view audio data"403 [height]404 (let [vi (view-image)]405 (fn [[coords sensor-data]]406 (let [image (BufferedImage. (count coords) height407 BufferedImage/TYPE_INT_RGB)]408 (dorun409 (for [x (range (count coords))]410 (dorun411 (for [y (range height)]412 (let [raw-sensor (sensor-data x)]413 (.setRGB image x y (gray-scale raw-sensor)))))))415 (vi image)))))419 ;;(defn test-touch [world creature]424 ;; here's how motor-control/ proprioception will work: Each muscle is425 ;; defined by a 1-D array of numbers (the "motor pool") each of which426 ;; represent muscle fibers. A muscle also has a scalar :strength427 ;; factor which determines how strong the muscle as a whole is.428 ;; The effector function for a muscle takes a number < (count429 ;; motor-pool) and that number is said to "activate" all the muscle430 ;; fibers whose index is lower than the number. Each fiber will apply431 ;; force in proportion to its value in the array. Lower values cause432 ;; less force. The lower values can be put at the "beginning" of the433 ;; 1-D array to simulate the layout of actual human muscles, which are434 ;; capable of more percise movements when exerting less force.436 ;; I don't know how to encode proprioception, so for now, just return437 ;; a function for each joint that returns a triplet of floats which438 ;; represent relative roll, pitch, and yaw. Write display code for439 ;; this though.441 (defn muscle-fiber-values442 "get motor pool strengths"443 [#^BufferedImage image]444 (vec445 (let [width (.getWidth image)]446 (for [x (range width)]447 (- 255448 (bit-and449 0x0000FF450 (.getRGB image x 0)))))))453 (defn creature-muscles454 "Return the children of the creature's \"muscles\" node."455 [#^Node creature]456 (if-let [muscle-node (.getChild creature "muscles")]457 (seq (.getChildren muscle-node))458 (do (println-repl "could not find muscles node") [])))460 (defn single-muscle [#^Node parts #^Node muscle]461 (let [target (closest-node parts muscle)462 axis463 (.mult (.getWorldRotation muscle) Vector3f/UNIT_Y)464 strength (meta-data muscle "strength")465 image-name (read-string (meta-data muscle "muscle"))466 image467 (ImageToAwt/convert468 (.getImage (.loadTexture (asset-manager) image-name))469 false false 0)470 fibers (muscle-fiber-values image)471 fiber-integral (reductions + fibers)472 force-index (vec473 (map474 #(float (* strength (/ % (last475 fiber-integral))))476 fiber-integral))477 control (.getControl target RigidBodyControl)]478 (fn [n]479 (let [pool-index (min n (count fibers))]480 (.applyTorque control (.mult axis (force-index n)))))))483 (defn enable-muscles484 "Must be called on a creature after RigidBodyControls have been485 created."486 [#^Node creature]487 (let [muscles (creature-muscles creature)]488 (for [muscle muscles]489 (single-muscle creature muscle))))491 (defn test-creature [thing]492 (let [x-axis493 (box 1 0.01 0.01 :physical? false :color ColorRGBA/Red)494 y-axis495 (box 0.01 1 0.01 :physical? false :color ColorRGBA/Green)496 z-axis497 (box 0.01 0.01 1 :physical? false :color ColorRGBA/Blue)498 creature (blender-creature thing)499 touch-nerves (touch creature)500 touch-debug-windows (map (fn [_] (debug-touch-window)) touch-nerves)501 [init-vision-fns vision-data] (vision creature)502 vision-debug (map (fn [_] (debug-vision-window)) vision-data)503 me (sphere 0.5 :color ColorRGBA/Blue :physical? false)504 [init-hearing-fns hearing-senses] (hearing creature)505 hearing-windows (map (fn [_] (debug-hearing-window 50))506 hearing-senses)507 bell (AudioNode. (asset-manager)508 "Sounds/pure.wav" false)509 prop (proprioception creature)510 prop-debug (proprioception-debug-window)512 muscle-fns (enable-muscles creature)513 ;; dream515 ]518 (apply519 world520 (with-movement521 (.getChild creature "worm-21")522 ["key-r" "key-t"523 "key-f" "key-g"524 "key-v" "key-b"]525 [10 10 10 10 1 1]526 [(nodify [creature527 (box 10 2 10 :position (Vector3f. 0 -9 0)528 :color ColorRGBA/Gray :mass 0)529 x-axis y-axis z-axis530 me531 ])532 (merge standard-debug-controls533 {"key-return"534 (fn [_ value]535 (if value536 (do537 (println-repl "play-sound")538 (.play bell))))539 "key-h"540 (fn [_ value]541 (if value542 (do543 (println-repl "muscle activating!")544 ((first muscle-fns) 199))))546 })547 (fn [world]548 (light-up-everything world)549 (enable-debug world)550 (dorun (map #(% world) init-vision-fns))551 (dorun (map #(% world) init-hearing-fns))553 (add-eye world554 (attach-eye creature (test-eye))555 (comp (view-image) BufferedImage!))557 (add-eye world (.getCamera world) no-op)558 ;;(set-gravity world (Vector3f. 0 0 0))559 ;;(com.aurellem.capture.Capture/captureVideo560 ;; world (file-str "/home/r/proj/ai-videos/hand"))561 ;;(.setTimer world (RatchetTimer. 60))562 (speed-up world)563 (set-gravity world (Vector3f. 0 0 0))564 )565 (fn [world tpf]566 ;;(dorun567 ;; (map #(%1 %2) touch-nerves (repeat (.getRootNode world))))569 (prop-debug (prop))571 (dorun572 (map #(%1 (%2 (.getRootNode world)))573 touch-debug-windows touch-nerves))575 (dorun576 (map #(%1 (%2))577 vision-debug vision-data))578 (dorun579 (map #(%1 (%2)) hearing-windows hearing-senses))582 ;;(println-repl (vision-data))583 (.setLocalTranslation me (.getLocation (.getCamera world)))586 )]587 ;;(let [timer (atom 0)]588 ;; (fn [_ _]589 ;; (swap! timer inc)590 ;; (if (= (rem @timer 60) 0)591 ;; (println-repl (float (/ @timer 60))))))592 ))))596 ;; the camera will stay in its initial position/rotation with relation597 ;; to the spatial.600 (defn follow-test601 "show a camera that stays in the same relative position to a blue cube."602 []603 (let [camera-pos (Vector3f. 0 30 0)604 rock (box 1 1 1 :color ColorRGBA/Blue605 :position (Vector3f. 0 10 0)606 :mass 30607 )608 rot (.getWorldRotation rock)610 table (box 3 1 10 :color ColorRGBA/Gray :mass 0611 :position (Vector3f. 0 -3 0))]613 (world614 (nodify [rock table])615 standard-debug-controls616 (fn [world]617 (let618 [cam (doto (.clone (.getCamera world))619 (.setLocation camera-pos)620 (.lookAt Vector3f/ZERO621 Vector3f/UNIT_X))]622 (bind-sense rock cam)624 (.setTimer world (RatchetTimer. 60))625 (add-eye world cam (comp (view-image) BufferedImage!))626 (add-eye world (.getCamera world) no-op))627 )628 (fn [_ _] (println-repl rot)))))632 #+end_src634 #+results: body-1635 : #'cortex.silly/follow-test638 * COMMENT purgatory639 #+begin_src clojure641 (defn bullet-trans* []642 (let [obj-a (box 1.5 0.5 0.5 :color ColorRGBA/Red643 :position (Vector3f. 5 0 0)644 :mass 90)645 obj-b (sphere 0.5 :color ColorRGBA/Blue646 :position (Vector3f. -5 0 0)647 :mass 0)648 control-a (.getControl obj-a RigidBodyControl)649 control-b (.getControl obj-b RigidBodyControl)650 move-up? (atom nil)651 move-down? (atom nil)652 move-left? (atom nil)653 move-right? (atom nil)654 roll-left? (atom nil)655 roll-right? (atom nil)656 force 100657 swivel658 (.toRotationMatrix659 (doto (Quaternion.)660 (.fromAngleAxis (/ Math/PI 2)661 Vector3f/UNIT_X)))662 x-move663 (doto (Matrix3f.)664 (.fromStartEndVectors Vector3f/UNIT_X665 (.normalize (Vector3f. 1 1 0))))667 timer (atom 0)]668 (doto669 (ConeJoint.670 control-a control-b671 (Vector3f. -8 0 0)672 (Vector3f. 2 0 0)673 ;;swivel swivel674 ;;Matrix3f/IDENTITY Matrix3f/IDENTITY675 x-move Matrix3f/IDENTITY676 )677 (.setCollisionBetweenLinkedBodys false)678 (.setLimit (* 1 (/ Math/PI 4)) ;; twist679 (* 1 (/ Math/PI 4)) ;; swing span in X-Y plane680 (* 0 (/ Math/PI 4)))) ;; swing span in Y-Z plane681 (world (nodify682 [obj-a obj-b])683 (merge standard-debug-controls684 {"key-r" (fn [_ pressed?] (reset! move-up? pressed?))685 "key-t" (fn [_ pressed?] (reset! move-down? pressed?))686 "key-f" (fn [_ pressed?] (reset! move-left? pressed?))687 "key-g" (fn [_ pressed?] (reset! move-right? pressed?))688 "key-v" (fn [_ pressed?] (reset! roll-left? pressed?))689 "key-b" (fn [_ pressed?] (reset! roll-right? pressed?))})691 (fn [world]692 (enable-debug world)693 (set-gravity world Vector3f/ZERO)694 )696 (fn [world _]698 (if @move-up?699 (.applyForce control-a700 (Vector3f. force 0 0)701 (Vector3f. 0 0 0)))702 (if @move-down?703 (.applyForce control-a704 (Vector3f. (- force) 0 0)705 (Vector3f. 0 0 0)))706 (if @move-left?707 (.applyForce control-a708 (Vector3f. 0 force 0)709 (Vector3f. 0 0 0)))710 (if @move-right?711 (.applyForce control-a712 (Vector3f. 0 (- force) 0)713 (Vector3f. 0 0 0)))715 (if @roll-left?716 (.applyForce control-a717 (Vector3f. 0 0 force)718 (Vector3f. 0 0 0)))719 (if @roll-right?720 (.applyForce control-a721 (Vector3f. 0 0 (- force))722 (Vector3f. 0 0 0)))724 (if (zero? (rem (swap! timer inc) 100))725 (.attachChild726 (.getRootNode world)727 (sphere 0.05 :color ColorRGBA/Yellow728 :physical? false :position729 (.getWorldTranslation obj-a)))))730 )731 ))733 (defn test-joint [joint]734 (let [[origin top bottom floor] (world-setup joint)735 control (.getControl top RigidBodyControl)736 move-up? (atom false)737 move-down? (atom false)738 move-left? (atom false)739 move-right? (atom false)740 roll-left? (atom false)741 roll-right? (atom false)742 timer (atom 0)]744 (world745 (nodify [top bottom floor origin])746 (merge standard-debug-controls747 {"key-r" (fn [_ pressed?] (reset! move-up? pressed?))748 "key-t" (fn [_ pressed?] (reset! move-down? pressed?))749 "key-f" (fn [_ pressed?] (reset! move-left? pressed?))750 "key-g" (fn [_ pressed?] (reset! move-right? pressed?))751 "key-v" (fn [_ pressed?] (reset! roll-left? pressed?))752 "key-b" (fn [_ pressed?] (reset! roll-right? pressed?))})754 (fn [world]755 (light-up-everything world)756 (enable-debug world)757 (set-gravity world (Vector3f. 0 0 0))758 )760 (fn [world _]761 (if (zero? (rem (swap! timer inc) 100))762 (do763 ;; (println-repl @timer)764 (.attachChild (.getRootNode world)765 (sphere 0.05 :color ColorRGBA/Yellow766 :position (.getWorldTranslation top)767 :physical? false))768 (.attachChild (.getRootNode world)769 (sphere 0.05 :color ColorRGBA/LightGray770 :position (.getWorldTranslation bottom)771 :physical? false))))773 (if @move-up?774 (.applyTorque control775 (.mult (.getPhysicsRotation control)776 (Vector3f. 0 0 10))))777 (if @move-down?778 (.applyTorque control779 (.mult (.getPhysicsRotation control)780 (Vector3f. 0 0 -10))))781 (if @move-left?782 (.applyTorque control783 (.mult (.getPhysicsRotation control)784 (Vector3f. 0 10 0))))785 (if @move-right?786 (.applyTorque control787 (.mult (.getPhysicsRotation control)788 (Vector3f. 0 -10 0))))789 (if @roll-left?790 (.applyTorque control791 (.mult (.getPhysicsRotation control)792 (Vector3f. -1 0 0))))793 (if @roll-right?794 (.applyTorque control795 (.mult (.getPhysicsRotation control)796 (Vector3f. 1 0 0))))))))797 #+end_src800 * COMMENT generate source801 #+begin_src clojure :tangle ../src/cortex/silly.clj802 <<body-1>>803 #+end_src