#+title: First attempt at a creature!

 #+author: Robert McIntyre

 #+email: rlm@mit.edu

 #+description: 

 #+keywords: simulation, jMonkeyEngine3, clojure

 #+SETUPFILE: ../../aurellem/org/setup.org

 #+INCLUDE: ../../aurellem/org/level-0.org

 

 

 

 

 * Intro 

 So far, I've made the following senses -- 

  - Vision

  - Hearing

  - Touch

  - Proprioception

 

 And one effector: 

  - Movement

 

 However, the code so far has only enabled these senses, but has not

 actually implemented them.  For example, there is still a lot of work

 to be done for vision. I need to be able to create an /eyeball/ in

 simulation that can be moved around and see the world from different

 angles. I also need to determine weather to use log-polar or cartesian

 for the visual input, and I need to determine how/wether to

 disceritise the visual input.

 

 I also want to be able to visualize both the sensors and the

 effectors in pretty pictures. This semi-retarted creature will be my

 first attempt at bringing everything together.

 

 * The creature's body

 

 Still going to do an eve-like body in blender, but due to problems

 importing the joints, etc into jMonkeyEngine3, I'm going to do all

 the connecting here in clojure code, using the names of the individual

 components and trial and error. Later, I'll maybe make some sort of

 creature-building modifications to blender that support whatever

 discritized senses I'm going to make.

 

 #+name: body-1

 #+begin_src clojure 

 (ns cortex.silly

   "let's play!"

   {:author "Robert McIntyre"})

 

 ;; TODO remove this!

 (require 'cortex.import)

 (cortex.import/mega-import-jme3)

 (use '(cortex world util body hearing touch vision sense

 proprioception movement))

 

 (rlm.rlm-commands/help)

 (import java.awt.image.BufferedImage)

 (import javax.swing.JPanel)

 (import javax.swing.SwingUtilities)

 (import java.awt.Dimension)

 (import javax.swing.JFrame)

 (import java.awt.Dimension)

 (import com.aurellem.capture.RatchetTimer)

 (declare joint-create)

 (use 'clojure.contrib.def)

 

 (defn load-blender-model

   "Load a .blend file using an asset folder relative path."

   [^String model]

   (.loadModel

    (doto (asset-manager)

      (.registerLoader BlenderModelLoader (into-array String ["blend"])))

    model))

 

 (defn blender-to-jme

   "Convert from Blender coordinates to JME coordinates"

   [#^Vector3f in]

   (Vector3f. (.getX in)

              (.getZ in)

              (- (.getY in))))

 

 

 (defmulti joint-dispatch

   "Translate blender pseudo-joints into real JME joints."

   (fn [constraints & _] 

     (:type constraints)))

 

 (defmethod joint-dispatch :point

   [constraints control-a control-b pivot-a pivot-b rotation]

   (println-repl "creating POINT2POINT joint")

   ;; bullet's point2point joints are BROKEN, so we must use the

   ;; generic 6DOF joint instead of an actual Point2Point joint!

 

   ;; should be able to do this:

   (comment 

     (Point2PointJoint.

      control-a

      control-b

      pivot-a

      pivot-b))

 

   ;; but instead we must do this:

   (println-repl "substuting 6DOF joint for POINT2POINT joint!")

   (doto

       (SixDofJoint.

        control-a

        control-b

        pivot-a

        pivot-b

        false)

     (.setLinearLowerLimit Vector3f/ZERO)

     (.setLinearUpperLimit Vector3f/ZERO)

     ;;(.setAngularLowerLimit (Vector3f. 1 1 1))

     ;;(.setAngularUpperLimit (Vector3f. 0 0 0))

 

 ))

 

 

 (defmethod joint-dispatch :hinge

   [constraints control-a control-b pivot-a pivot-b rotation]

   (println-repl "creating HINGE joint")

   (let [axis

         (if-let

             [axis (:axis constraints)]

           axis

           Vector3f/UNIT_X)

         [limit-1 limit-2] (:limit constraints)

         hinge-axis

         (.mult

          rotation

          (blender-to-jme axis))]

     (doto

         (HingeJoint.

          control-a

          control-b

          pivot-a

          pivot-b

          hinge-axis

          hinge-axis)

       (.setLimit limit-1 limit-2))))

 

 (defmethod joint-dispatch :cone

   [constraints control-a control-b pivot-a pivot-b rotation]

   (let [limit-xz (:limit-xz constraints)

         limit-xy (:limit-xy constraints)

         twist    (:twist constraints)]

     

     (println-repl "creating CONE joint")

     (println-repl rotation)

     (println-repl

      "UNIT_X --> " (.mult rotation (Vector3f. 1 0 0)))

     (println-repl

      "UNIT_Y --> " (.mult rotation (Vector3f. 0 1 0)))

     (println-repl

      "UNIT_Z --> " (.mult rotation (Vector3f. 0 0 1)))

     (doto

         (ConeJoint.

          control-a

          control-b

          pivot-a

          pivot-b

          rotation

          rotation)

       (.setLimit (float limit-xz)

                  (float limit-xy)

                  (float twist)))))

 

 (defn connect

   "here are some examples:

    {:type :point}

    {:type :hinge  :limit [0 (/ Math/PI 2)] :axis (Vector3f. 0 1 0)}

    (:axis defaults to (Vector3f. 1 0 0) if not provided for hinge joints)

 

    {:type :cone :limit-xz 0]

                 :limit-xy 0]

                 :twist 0]}   (use XZY rotation mode in blender!)"

   [#^Node obj-a #^Node obj-b #^Node joint]

   (let [control-a (.getControl obj-a RigidBodyControl)

         control-b (.getControl obj-b RigidBodyControl)

         joint-center (.getWorldTranslation joint)

         joint-rotation (.toRotationMatrix (.getWorldRotation joint))

         pivot-a (world-to-local obj-a joint-center)

         pivot-b (world-to-local obj-b joint-center)]

    

     (if-let [constraints

              (map-vals

               eval

               (read-string

                (meta-data joint "joint")))]

       ;; A side-effect of creating a joint registers

       ;; it with both physics objects which in turn

       ;; will register the joint with the physics system

       ;; when the simulation is started.

       (do

         (println-repl "creating joint between"

                       (.getName obj-a) "and" (.getName obj-b))

         (joint-dispatch constraints

                         control-a control-b

                         pivot-a pivot-b

                         joint-rotation))

       (println-repl "could not find joint meta-data!"))))

 

 

 

 

 (defn assemble-creature [#^Node pieces joints]

   (dorun

    (map

     (fn [geom]

       (let [physics-control

             (RigidBodyControl.

              (HullCollisionShape.

               (.getMesh geom))

              (if-let [mass (meta-data geom "mass")]

                (do

                  (println-repl

                   "setting" (.getName geom) "mass to" (float mass))

                  (float mass))

                (float 1)))]

                

         (.addControl geom physics-control)))

     (filter #(isa? (class %) Geometry )

             (node-seq pieces))))

   (dorun

    (map

     (fn [joint]

       (let [[obj-a obj-b] (joint-targets pieces joint)]

         (connect obj-a obj-b joint)))

     joints))

   pieces)

 

 (declare blender-creature)

 

 (def hand "Models/creature1/one.blend")

 

 (def worm "Models/creature1/try-again.blend")

 

 (defn worm-model [] (load-blender-model worm))

 

 (defn x-ray [#^ColorRGBA color]

   (doto (Material. (asset-manager)

                    "Common/MatDefs/Misc/Unshaded.j3md")

     (.setColor "Color" color)

     (-> (.getAdditionalRenderState)

         (.setDepthTest false))))

 

 (defn colorful []

   (.getChild (worm-model) "worm-21"))

 

 (import jme3tools.converters.ImageToAwt)

 

 (import ij.ImagePlus)

 

 

 

 (defn test-eye  []

   (.getChild 

    (.getChild (worm-model) "eyes")

    "eye"))

 

 

 

 ;; Ears work the same way as vision.

 

 ;; (hearing creature) will return [init-functions

 ;; sensor-functions]. The init functions each take the world and

 ;; register a SoundProcessor that does foureier transforms on the

 ;; incommong sound data, making it available to each sensor function.

 

 (defn creature-ears

   "Return the children of the creature's \"ears\" node."

   ;;dylan

   ;;"The ear nodes which are children of the \"ears\" node in the

   ;;creature."

   [#^Node creature]

   (if-let [ear-node (.getChild creature "ears")]

     (seq (.getChildren ear-node))

     (do (println-repl "could not find ears node") [])))

 

 

 ;;dylan (defn follow-sense, adjoin-sense, attach-stimuli,

 ;;anchor-qualia, augment-organ, with-organ

 

 

 (defn update-listener-velocity

   "Update the listener's velocity every update loop."

   [#^Spatial obj #^Listener lis]

   (let [old-position (atom (.getLocation lis))]

     (.addControl

      obj

      (proxy [AbstractControl] []

        (controlUpdate [tpf]

          (let [new-position (.getLocation lis)]

            (.setVelocity

             lis 

             (.mult (.subtract new-position @old-position)

                    (float (/ tpf))))

            (reset! old-position new-position)))

        (controlRender [_ _])))))

 

 (import com.aurellem.capture.audio.AudioSendRenderer)

 

 (defn attach-ear

   [#^Application world #^Node creature #^Spatial ear continuation]

   (let [target (closest-node creature ear)

         lis (Listener.)

         audio-renderer (.getAudioRenderer world)

         sp (sound-processor continuation)]

     (.setLocation lis (.getWorldTranslation ear))

     (.setRotation lis (.getWorldRotation ear))

     (bind-sense target lis)

     (update-listener-velocity target lis)

     (.addListener audio-renderer lis)

     (.registerSoundProcessor audio-renderer lis sp)))

 

 (defn enable-hearing

   [#^Node creature #^Spatial ear]

   (let [hearing-data (atom [])]

     [(fn [world]

        (attach-ear world creature ear

                    (fn [data]

                      (reset! hearing-data (vec data)))))

      [(fn []

         (let [data @hearing-data

               topology              

               (vec (map #(vector % 0) (range 0 (count data))))

               scaled-data

               (vec

                (map

                 #(rem (int (* 255 (/ (+ 1 %) 2)))  256)

                    data))]

           [topology scaled-data]))

         ]]))

 

 (defn hearing

   [#^Node creature]

   (reduce

    (fn [[init-a senses-a]

         [init-b senses-b]]

      [(conj init-a init-b)

       (into senses-a senses-b)])

    [[][]]      

    (for [ear (creature-ears creature)]

      (enable-hearing creature ear))))

 

 

 

 

 

 

 ;; lower level --- nodes

 ;; closest-node "parse/compile-x" -> makes organ, which is spatial, fn pair

 

 ;; higher level -- organs

 ;; 

 

 ;; higher level --- sense/effector

 ;; these are the functions that provide world i/o, chinese-room style

 

 

 

 

 (defn blender-creature

   "Return a creature with all joints in place."

   [blender-path]

   (let [model (load-blender-model blender-path)

         joints (creature-joints model)]

     (assemble-creature model joints)))

 

 (defn gray-scale [num]

   (+ num

      (bit-shift-left num 8)

      (bit-shift-left num 16)))

 

 (defn debug-touch-window

   "creates function that offers a debug view of sensor data"

   []

   (let [vi (view-image)]

     (fn 

       [[coords sensor-data]]

       (let [image (points->image coords)]

         (dorun

          (for [i (range (count coords))]

            (.setRGB image ((coords i) 0) ((coords i) 1)

                     (gray-scale (sensor-data i)))))

 

                    

         (vi image)))))

 

 (defn debug-vision-window

   "creates function that offers a debug view of sensor data"

   []

   (let [vi (view-image)]

     (fn 

       [[coords sensor-data]]

       (let [image (points->image coords)]

         (dorun

          (for [i (range (count coords))]

            (.setRGB image ((coords i) 0) ((coords i) 1)

                    (sensor-data i))))

         (vi image)))))

 

 (defn debug-hearing-window

   "view audio data"

   [height]

   (let [vi (view-image)]

     (fn [[coords sensor-data]]

       (let [image (BufferedImage. (count coords) height

                                   BufferedImage/TYPE_INT_RGB)]

         (dorun

          (for [x (range (count coords))]

            (dorun

             (for [y (range height)]

               (let [raw-sensor (sensor-data x)]

                 (.setRGB image x y (gray-scale raw-sensor)))))))

 

         (vi image)))))

         

          

 

 ;;(defn test-touch [world creature]

 

 

 

 

 

 (defn test-creature [thing]

   (let [x-axis

         (box 1 0.01 0.01 :physical? false :color ColorRGBA/Red)

         y-axis

         (box 0.01 1 0.01 :physical? false :color ColorRGBA/Green)

         z-axis

         (box 0.01 0.01 1 :physical? false :color ColorRGBA/Blue)

         creature (blender-creature thing)

         touch-nerves (touch creature)

         touch-debug-windows (map (fn [_] (debug-touch-window)) touch-nerves)

         [init-vision-fns vision-data] (vision creature)

         vision-debug (map (fn [_] (debug-vision-window)) vision-data)

         me (sphere 0.5 :color ColorRGBA/Blue :physical? false)

         [init-hearing-fns hearing-senses] (hearing creature)

         hearing-windows (map (fn [_] (debug-hearing-window 50))

                              hearing-senses)

         bell (AudioNode. (asset-manager)

                          "Sounds/pure.wav" false)

         prop (proprioception creature)

         prop-debug (proprioception-debug-window)

 

         muscle-fns (enable-muscles creature)

         ;; dream

 

         ]

     

     

     (apply

      world

      (with-movement

        (.getChild creature "worm-21")

        ["key-r" "key-t"

         "key-f" "key-g"

         "key-v" "key-b"]

        [10 10 10 10 1 1]

        [(nodify [creature

                  (box 10 2 10 :position (Vector3f. 0 -9 0)

                       :color ColorRGBA/Gray :mass 0)

                  x-axis y-axis z-axis

                  me

                  ])

         (merge standard-debug-controls

                {"key-return"

                 (fn [_ value]

                   (if value

                     (do

                       (println-repl "play-sound")

                       (.play bell))))

                 "key-h"

                 (fn [_ value]

                   (if value

                     (do

                       (println-repl "muscle activating!")

                       ((first muscle-fns)  199))))

 

                 })

         (fn [world]

           (light-up-everything world)

           (enable-debug world)

           (dorun (map #(% world) init-vision-fns))

           (dorun (map #(% world) init-hearing-fns))

           

           (add-eye world

                    (attach-eye creature (test-eye))

                    (comp (view-image) BufferedImage!))

           

           (add-eye world (.getCamera world) no-op)

           ;;(set-gravity world (Vector3f. 0 0 0))

           ;;(com.aurellem.capture.Capture/captureVideo

           ;; world (file-str "/home/r/proj/ai-videos/hand"))

           ;;(.setTimer world (RatchetTimer. 60))

           (speed-up world)

           (set-gravity world (Vector3f. 0 0 0))

           )

         (fn [world tpf]

           ;;(dorun 

           ;; (map #(%1 %2) touch-nerves (repeat (.getRootNode world))))

           

           (prop-debug (prop))

           

           (dorun

            (map #(%1 (%2 (.getRootNode world)))

                 touch-debug-windows touch-nerves))

           

           (dorun

            (map #(%1 (%2))

                 vision-debug vision-data))

           (dorun

            (map #(%1 (%2)) hearing-windows hearing-senses))

           

           

           ;;(println-repl (vision-data))

           (.setLocalTranslation me (.getLocation (.getCamera world)))

           

           

           )]

    ;;(let [timer (atom 0)]

    ;;  (fn [_ _]

    ;;    (swap! timer inc)

    ;;    (if (= (rem @timer 60) 0)

    ;;      (println-repl (float (/ @timer 60))))))

    ))))

 

    

 

 ;; the camera will stay in its initial position/rotation with relation

 ;; to the spatial.

 

 

 (defn follow-test

   "show a camera that stays in the same relative position to a blue cube."

   []

   (let [camera-pos (Vector3f. 0 30 0)

         rock (box 1 1 1 :color ColorRGBA/Blue

                      :position (Vector3f. 0 10 0)

                      :mass 30

                      )

         rot (.getWorldRotation rock)

         

         table (box 3 1 10 :color ColorRGBA/Gray :mass 0

                    :position (Vector3f. 0 -3 0))]

 

     (world

      (nodify [rock table])

              standard-debug-controls

              (fn [world]

                (let 

                    [cam (doto (.clone (.getCamera world))

                           (.setLocation camera-pos)

                           (.lookAt Vector3f/ZERO

                                    Vector3f/UNIT_X))]

                  (bind-sense rock cam)

                  

                  (.setTimer world (RatchetTimer. 60))

                  (add-eye world cam (comp (view-image) BufferedImage!))

                  (add-eye world (.getCamera world) no-op))

                )

              (fn [_ _] (println-repl rot)))))

        

 

 

 #+end_src

 

 #+results: body-1

 : #'cortex.silly/follow-test

 

 

 * COMMENT purgatory

 #+begin_src clojure

 

 (defn bullet-trans* []

   (let [obj-a (box 1.5 0.5 0.5 :color ColorRGBA/Red

                    :position (Vector3f. 5 0 0)

                    :mass 90)

         obj-b (sphere 0.5 :color ColorRGBA/Blue

                       :position (Vector3f. -5 0 0)

                       :mass 0)

         control-a (.getControl obj-a RigidBodyControl)

         control-b (.getControl obj-b RigidBodyControl)

         move-up? (atom nil)

         move-down? (atom nil)

         move-left? (atom nil)

         move-right? (atom nil)

         roll-left? (atom nil)

         roll-right? (atom nil)

         force 100

         swivel

         (.toRotationMatrix

          (doto (Quaternion.)

            (.fromAngleAxis (/ Math/PI 2)

                            Vector3f/UNIT_X)))

         x-move

         (doto (Matrix3f.)

           (.fromStartEndVectors Vector3f/UNIT_X

                                 (.normalize (Vector3f. 1 1 0))))

         

         timer (atom 0)]

     (doto 

         (ConeJoint.

          control-a control-b

          (Vector3f. -8 0 0)

          (Vector3f. 2 0 0)

          ;;swivel swivel

          ;;Matrix3f/IDENTITY Matrix3f/IDENTITY

          x-move Matrix3f/IDENTITY 

          )

       (.setCollisionBetweenLinkedBodys false)

       (.setLimit (* 1 (/ Math/PI 4))    ;; twist 

                  (* 1 (/ Math/PI 4))  ;; swing span in X-Y plane

                  (* 0 (/ Math/PI 4))))  ;; swing span in Y-Z plane

     (world (nodify

             [obj-a obj-b])

            (merge standard-debug-controls

                   {"key-r" (fn [_ pressed?] (reset! move-up? pressed?))

                    "key-t" (fn [_ pressed?] (reset! move-down? pressed?))

                    "key-f" (fn [_ pressed?] (reset! move-left? pressed?))

                    "key-g" (fn [_ pressed?] (reset! move-right? pressed?))

                    "key-v" (fn [_ pressed?] (reset! roll-left? pressed?))

                    "key-b" (fn [_ pressed?] (reset! roll-right? pressed?))})     

            

            (fn [world]

              (enable-debug world)

              (set-gravity world Vector3f/ZERO)

              )

                            

            (fn [world _]

 

              (if @move-up?

                (.applyForce control-a

                               (Vector3f. force 0 0)

                               (Vector3f. 0 0 0)))

              (if @move-down?

                (.applyForce control-a

                              (Vector3f. (- force) 0 0)

                              (Vector3f. 0 0 0)))

              (if @move-left?

                (.applyForce control-a

                              (Vector3f. 0 force 0)

                              (Vector3f. 0 0 0)))

              (if @move-right?

                (.applyForce control-a

                              (Vector3f. 0 (- force) 0)

                              (Vector3f. 0 0 0)))

 

              (if @roll-left?

                (.applyForce control-a

                              (Vector3f. 0 0 force)

                              (Vector3f. 0 0 0)))

              (if @roll-right?

                (.applyForce control-a

                              (Vector3f. 0 0 (- force))

                              (Vector3f. 0 0 0)))

 

              (if (zero? (rem (swap! timer inc) 100))

                (.attachChild

                 (.getRootNode world)

                 (sphere 0.05 :color ColorRGBA/Yellow

                         :physical? false :position

                         (.getWorldTranslation obj-a)))))

            )

     ))

 

 (defn test-joint [joint]

   (let [[origin top bottom floor] (world-setup joint)

         control (.getControl top RigidBodyControl)

         move-up? (atom false)

         move-down? (atom false)

         move-left? (atom false)

         move-right? (atom false)

         roll-left? (atom false)

         roll-right? (atom false)

         timer (atom 0)]

 

     (world

      (nodify [top bottom floor origin])

      (merge standard-debug-controls

             {"key-r" (fn [_ pressed?] (reset! move-up? pressed?))

              "key-t" (fn [_ pressed?] (reset! move-down? pressed?))

              "key-f" (fn [_ pressed?] (reset! move-left? pressed?))

              "key-g" (fn [_ pressed?] (reset! move-right? pressed?))

              "key-v" (fn [_ pressed?] (reset! roll-left? pressed?))

              "key-b" (fn [_ pressed?] (reset! roll-right? pressed?))})     

      

      (fn [world]

        (light-up-everything world)

        (enable-debug world)

        (set-gravity world (Vector3f. 0 0 0))

        )

      

      (fn [world _]

        (if (zero? (rem (swap! timer inc) 100))

          (do

          ;;  (println-repl @timer)

            (.attachChild (.getRootNode world)

                        (sphere 0.05 :color ColorRGBA/Yellow

                                :position (.getWorldTranslation top)

                                :physical? false))

            (.attachChild (.getRootNode world)

                          (sphere 0.05 :color ColorRGBA/LightGray

                                  :position (.getWorldTranslation bottom)

                                  :physical? false))))

 

        (if @move-up?

          (.applyTorque control

                        (.mult (.getPhysicsRotation control)

                               (Vector3f. 0 0 10))))

        (if @move-down?

          (.applyTorque control

                        (.mult (.getPhysicsRotation control)

                               (Vector3f. 0 0 -10))))

        (if @move-left?

          (.applyTorque control

                        (.mult (.getPhysicsRotation control)

                               (Vector3f. 0 10 0))))

        (if @move-right?

          (.applyTorque control

                        (.mult (.getPhysicsRotation control)

                               (Vector3f. 0 -10 0))))

        (if @roll-left?

          (.applyTorque control

                        (.mult (.getPhysicsRotation control)

                               (Vector3f. -1 0 0))))

        (if @roll-right?

          (.applyTorque control

                        (.mult (.getPhysicsRotation control)

                               (Vector3f. 1 0 0))))))))

 #+end_src

 

 

 * COMMENT generate source

 #+begin_src clojure :tangle ../src/cortex/silly.clj

 <<body-1>>

 #+end_src