1.1 --- a/org/self_organizing_touch.clj	Mon Apr 28 15:10:59 2014 -0400
     1.2 +++ b/org/self_organizing_touch.clj	Thu May 01 23:41:41 2014 -0400
     1.3 @@ -99,11 +99,6 @@
     1.4              (recur result xs)
     1.5              (recur (cons x result) xs))))))
     1.6  
     1.7 -(def all-touch-coordinates
     1.8 -  (concat
     1.9 -   (rect-region [0  15] [7  22])
    1.10 -   (rect-region [8   0] [14 29])
    1.11 -   (rect-region [15 15] [22 22])))
    1.12  
    1.13  (defn view-touch-region
    1.14    ([coords out]

     2.1 --- a/org/worm_learn.clj	Mon Apr 28 15:10:59 2014 -0400
     2.2 +++ b/org/worm_learn.clj	Thu May 01 23:41:41 2014 -0400
     2.3 @@ -20,8 +20,18 @@
     2.4  (dorun (cortex.import/mega-import-jme3))
     2.5  (rlm.rlm-commands/help)
     2.6  
     2.7 +
     2.8  (load-bullet)
     2.9  
    2.10 +(defn bin [digits]
    2.11 +  (fn [angles]
    2.12 +    (->> angles
    2.13 +         (flatten)
    2.14 +         (map (juxt #(Math/sin %) #(Math/cos %)))
    2.15 +         (flatten)
    2.16 +         (mapv #(Math/round (* % (Math/pow 10 (dec digits))))))))
    2.17 +
    2.18 +
    2.19  (def hand "Models/test-creature/hand.blend")
    2.20  
    2.21  (defn worm-model []
    2.22 @@ -162,6 +172,12 @@
    2.23           y (range y0 (inc y1))]
    2.24       [x y])))
    2.25  
    2.26 +(def all-touch-coordinates
    2.27 +  (concat
    2.28 +   (rect-region [0  15] [7  22])
    2.29 +   (rect-region [8   0] [14 29])
    2.30 +   (rect-region [15 15] [22 22])))
    2.31 +
    2.32  (def worm-segment-bottom (rect-region [8 15] [14 22]))
    2.33  
    2.34  (defn contact
    2.35 @@ -221,8 +237,6 @@
    2.36          (or (accept? (take 64 base-activity))
    2.37              (accept? (take 64 (drop 20 base-activity))))))))
    2.38  
    2.39 -
    2.40 -
    2.41  (def worm-segment-bottom-tip (rect-region [15 15] [22 22]))
    2.42  
    2.43  (def worm-segment-top-tip (rect-region [0 15] [7 22]))
    2.44 @@ -238,6 +252,36 @@
    2.45                (< 0.1 (contact worm-segment-top-tip    head-touch))))))
    2.46  
    2.47  
    2.48 +(defn draped?
    2.49 +  "Is the worm:
    2.50 +    -- not flat (the floor is not a 'chair')
    2.51 +    -- supported (not using its muscles to hold its position)
    2.52 +    -- stable (not changing its position)
    2.53 +    -- touching something (must register contact)"
    2.54 +  [experiences]
    2.55 +  (let [b2-hash (bin 2)
    2.56 +        touch (:touch (peek experiences))
    2.57 +        total-contact
    2.58 +        (reduce
    2.59 +         +
    2.60 +         (map #(contact all-touch-coordinates %)
    2.61 +              (rest touch)))]
    2.62 +    (println total-contact)
    2.63 +    (and (not (resting? experiences))
    2.64 +         (every?
    2.65 +          zero?
    2.66 +          (-> experiences
    2.67 +              (vector:last-n 25)
    2.68 +              (#(map :muscle %))
    2.69 +              (flatten)))
    2.70 +         (-> experiences
    2.71 +             (vector:last-n 20)
    2.72 +             (#(map (comp b2-hash flatten :proprioception) %))
    2.73 +             (set)
    2.74 +             (count) (= 1))
    2.75 +         (< 0.03 total-contact))))
    2.76 +
    2.77 +
    2.78  (declare phi-space phi-scan debug-experience) 
    2.79  
    2.80  
    2.81 @@ -254,6 +298,13 @@
    2.82    [(Vector3f. -0.0708936, -8.570261, 2.6487997)
    2.83     (Quaternion. -2.318909E-4, 0.9985348, 0.053941682, 0.004291452)])
    2.84  
    2.85 +(defn summon-chair
    2.86 +  "Create a chair in the world for the worm"
    2.87 +  [world]
    2.88 +  (let [chair (box 0.5 0.5 0.5 :position (Vector3f. 0 -5 -2)
    2.89 +                   :mass 350. :color ColorRGBA/Pink)]
    2.90 +    (add-element world chair (.getRootNode world))))
    2.91 +
    2.92  (defn worm-world-defaults []
    2.93    (let [direct-control (worm-direct-control worm-muscle-labels 40)]
    2.94      (merge direct-control     
    2.95 @@ -262,7 +313,11 @@
    2.96              :experiences (atom [])
    2.97              :experience-watch debug-experience
    2.98              :worm worm
    2.99 -            :end-frame nil})))
   2.100 +            :end-frame nil
   2.101 +            :keybindings
   2.102 +            (merge (:keybindings direct-control)
   2.103 +                   {"key-b" (fn [world pressed?]
   2.104 +                              (if pressed? (summon-chair world)))})})))
   2.105  
   2.106  (defn dir! [file]
   2.107    (if-not (.exists file)
   2.108 @@ -291,6 +346,7 @@
   2.109  (defn debug-experience
   2.110    [experiences text]
   2.111    (cond
   2.112 +   (draped? experiences)       (.setText text "Draped")
   2.113     (grand-circle? experiences) (.setText text "Grand Circle")
   2.114     (curled? experiences)       (.setText text "Curled")
   2.115     (wiggling? experiences)     (.setText text "Wiggling")
   2.116 @@ -390,14 +446,6 @@
   2.117          :experiences experiences})))
   2.118      @experiences))
   2.119  
   2.120 -(defn bin [digits]
   2.121 -  (fn [angles]
   2.122 -    (->> angles
   2.123 -         (flatten)
   2.124 -         (map (juxt #(Math/sin %) #(Math/cos %)))
   2.125 -         (flatten)
   2.126 -         (mapv #(Math/round (* % (Math/pow 10 (dec digits))))))))
   2.127 -
   2.128  ;; k-nearest neighbors with spatial binning. Only returns a result if
   2.129  ;; the propriceptive data is within 10% of a previously recorded
   2.130  ;; result in all dimensions.
   2.131 @@ -467,7 +515,7 @@
   2.132  ;;(infer-nils [1 nil 1 1]) [1 1 1 1]
   2.133  ;;(infer-nils [1 1 1 nil]) [1 1 1 0]
   2.134  ;;(infer-nils [nil 2 1 1]) [2 2 1 1]       
   2.135 -  
   2.136 +
   2.137  
   2.138  (defn empathy-demonstration []
   2.139    (let [proprio (atom ())]
   2.140 @@ -479,6 +527,7 @@
   2.141                empathy (mapv phi-space (infer-nils exp-thread))]
   2.142            (println-repl (vector:last-n exp-thread 22))
   2.143            (cond
   2.144 +           (draped? empathy)       (.setText text "Draped")
   2.145             (grand-circle? empathy) (.setText text "Grand Circle")
   2.146             (curled? empathy)       (.setText text "Curled")
   2.147             (wiggling? empathy)     (.setText text "Wiggling")
   2.148 @@ -497,6 +546,11 @@
   2.149        @experiences))
   2.150    (def phi-scan (gen-phi-scan phi-space)))
   2.151  
   2.152 +(defn empathy-experiment-0 [record]
   2.153 +  (.start (worm-world :record record)))
   2.154 +
   2.155 +
   2.156 +
   2.157  (defn empathy-experiment-1 [record]
   2.158    (.start (worm-world :experience-watch (empathy-demonstration)
   2.159                        :record record :worm worm*)))

     3.1 --- a/thesis/Makefile	Mon Apr 28 15:10:59 2014 -0400
     3.2 +++ b/thesis/Makefile	Thu May 01 23:41:41 2014 -0400
     3.3 @@ -14,4 +14,5 @@
     3.4  scp:
     3.5  	scp "r@aurellem.org:/home/r/thesis/$(THESIS_NAME).pdf" .
     3.6  
     3.7 -all: thesis scp
     3.8 +all: thesis
     3.9 +	make scp

     4.1 --- a/thesis/cortex.org	Mon Apr 28 15:10:59 2014 -0400
     4.2 +++ b/thesis/cortex.org	Thu May 01 23:41:41 2014 -0400
     4.3 @@ -513,7 +513,7 @@
     4.4     appears to flow at a constant rate, regardless of how complicated
     4.5     the environment becomes or how many creatures are in the
     4.6     simulation. The cost is that =CORTEX= can sometimes run slower than
     4.7 -   real time. Time dialation works both ways, however --- simulations
     4.8 +   real time. Time dilation works both ways, however --- simulations
     4.9     of very simple creatures in =CORTEX= generally run at 40x real-time
    4.10     on my machine!
    4.11  
    4.12 @@ -565,7 +565,7 @@
    4.13     each sense. 
    4.14  
    4.15     Fortunately this idea is already a well known computer graphics
    4.16 -   technique called /UV-mapping/. In UV-maping, the three-dimensional
    4.17 +   technique called /UV-mapping/. In UV-mapping, the three-dimensional
    4.18     surface of a model is cut and smooshed until it fits on a
    4.19     two-dimensional image. You paint whatever you want on that image,
    4.20     and when the three-dimensional shape is rendered in a game the
    4.21 @@ -2814,7 +2814,7 @@
    4.22  
    4.23     The worm's total life experience is a long looping path through
    4.24     \Phi-space. I will now introduce simple way of taking that
    4.25 -   experiece path and building a function that can infer complete
    4.26 +   experience path and building a function that can infer complete
    4.27     sensory experience given only a stream of proprioceptive data. This
    4.28     /empathy/ function will provide a bridge to use the body centered
    4.29     action predicates on video-like streams of information.
    4.30 @@ -2972,7 +2972,7 @@
    4.31     entries in a proprioceptive bin, because for each element in the
    4.32     starting bin it performs a series of set lookups in the preceding
    4.33     bins. If the total history is limited, then this takes time
    4.34 -   proprotional to a only a constant multiple of the number of entries
    4.35 +   proportional to a only a constant multiple of the number of entries
    4.36     in the starting bin. This analysis also applies, even if the action
    4.37     requires multiple longest chains -- it's still the average number
    4.38     of entries in a proprioceptive bin times the desired chain length.
    4.39 @@ -3125,7 +3125,7 @@
    4.40     the testing environment for the action-predicates, with one major
    4.41     difference : the only sensory information available to the system
    4.42     is proprioception. From just the proprioception data and
    4.43 -   \Phi-space, =longest-thread= synthesises a complete record the last
    4.44 +   \Phi-space, =longest-thread= synthesizes a complete record the last
    4.45     300 sensory experiences of the worm. These synthesized experiences
    4.46     are fed directly into the action predicates =grand-circle?=,
    4.47     =curled?=, =wiggling?=, and =resting?= from before and their output
    4.48 @@ -3365,13 +3365,11 @@
    4.49     [[./images/worm-roll.png]]
    4.50  
    4.51     #+caption: After completing its adventures, the worm now knows 
    4.52 -   #+caption: how its touch sensors are arranged along its skin. These 
    4.53 -   #+caption: are the regions that were deemed important by 
    4.54 +   #+caption: how its touch sensors are arranged along its skin. Each of these six rectangles are touch sensory patterns that were 
    4.55 +   #+caption: deemed important by 
    4.56     #+caption: =learn-touch-regions=. Each white square in the rectangles 
    4.57     #+caption: above is a cluster of ``related" touch nodes as determined 
    4.58 -   #+caption: by the system. Since each square in the ``cross" corresponds 
    4.59 -   #+caption: to a face, the worm has correctly discovered that it has
    4.60 -   #+caption: six faces.
    4.61 +   #+caption: by the system. The worm has correctly discovered that it has six faces, and has partitioned its sensory map into these six faces. 
    4.62     #+name: worm-touch-map
    4.63     #+ATTR_LaTeX: :width 12cm
    4.64     [[./images/touch-learn.png]]
    4.65 @@ -3383,29 +3381,133 @@
    4.66     completely scrambled. The cross shape is just for convenience. This
    4.67     example justifies the use of pre-defined touch regions in =EMPATH=.
    4.68  
    4.69 +** Recognizing an object using embodied representation
    4.70 +
    4.71 +   At the beginning of the thesis, I suggested that we might recognize
    4.72 +   the chair in Figure \ref{hidden-chair} by imagining ourselves in
    4.73 +   the position of the man and realizing that he must be sitting on
    4.74 +   something in order to maintain that position. Here, I present a
    4.75 +   brief elaboration on how to this might be done.
    4.76 +
    4.77 +   First, I need the feeling of leaning or resting /on/ some other
    4.78 +   object that is not the floor. This feeling is easy to describe
    4.79 +   using an embodied representation. 
    4.80 +
    4.81 +  #+caption: Program describing the sense of leaning or resting on something.
    4.82 +  #+caption: This involves a relaxed posture, the feeling of touching something, 
    4.83 +  #+caption: and a period of stability where the worm does not move.
    4.84 +  #+name: draped
    4.85 +  #+begin_listing clojure
    4.86 +  #+begin_src clojure
    4.87 +(defn draped?
    4.88 +  "Is the worm:
    4.89 +    -- not flat (the floor is not a 'chair')
    4.90 +    -- supported (not using its muscles to hold its position)
    4.91 +    -- stable (not changing its position)
    4.92 +    -- touching something (must register contact)"
    4.93 +  [experiences]
    4.94 +  (let [b2-hash (bin 2)
    4.95 +        touch (:touch (peek experiences))
    4.96 +        total-contact
    4.97 +        (reduce
    4.98 +         +
    4.99 +         (map #(contact all-touch-coordinates %)
   4.100 +              (rest touch)))]
   4.101 +    (println total-contact)
   4.102 +    (and (not (resting? experiences))
   4.103 +         (every?
   4.104 +          zero?
   4.105 +          (-> experiences
   4.106 +              (vector:last-n 25)
   4.107 +              (#(map :muscle %))
   4.108 +              (flatten)))
   4.109 +         (-> experiences
   4.110 +             (vector:last-n 20)
   4.111 +             (#(map (comp b2-hash flatten :proprioception) %))
   4.112 +             (set)
   4.113 +             (count) (= 1))
   4.114 +         (< 0.03 total-contact))))
   4.115 +  #+end_src
   4.116 +  #+end_listing
   4.117 +   
   4.118 +   #+caption: The =draped?= predicate detects the presence of the
   4.119 +   #+caption: cube whenever the worm interacts with it. The details of the 
   4.120 +   #+caption: cube are irrelevant; only the way it influences the worm's
   4.121 +   #+caption: body matters.
   4.122 +   #+name: draped-video
   4.123 +   #+ATTR_LaTeX: :width 13cm
   4.124 +   [[./images/draped.png]]
   4.125 +
   4.126 +   Though this is a simple example, using the =draped?= predicate to
   4.127 +   detect the cube has interesting advantages. The =draped?= predicate
   4.128 +   describes the cube not in terms of properties that the cube has,
   4.129 +   but instead in terms of how the worm interacts with it physically.
   4.130 +   This means that the cube can still be detected even if it is not
   4.131 +   visible, as long as its influence on the worm's body is visible.
   4.132 +   
   4.133 +   This system will also see the virtual cube created by a
   4.134 +   ``mimeworm", which uses its muscles in a very controlled way to
   4.135 +   mimic the appearance of leaning on a cube. The system will
   4.136 +   anticipate that there is an actual invisible cube that provides
   4.137 +   support!
   4.138 +
   4.139 +   #+caption: Can you see the thing that this person is leaning on?
   4.140 +   #+caption: What properties does it have, other than how it makes the man's 
   4.141 +   #+caption: elbow and shoulder feel? I wonder if people who can actually 
   4.142 +   #+caption: maintain this pose easily still see the support?
   4.143 +   #+name: mime
   4.144 +   #+ATTR_LaTeX: :width 6cm
   4.145 +   [[./images/pablo-the-mime.png]]
   4.146 +
   4.147 +   This makes me wonder about the psychology of actual mimes. Suppose
   4.148 +   for a moment that people have something analogous to \Phi-space and
   4.149 +   that one of the ways that they find objects in a scene is by their
   4.150 +   relation to other people's bodies. Suppose that a person watches a
   4.151 +   person miming an invisible wall. For a person with no experience
   4.152 +   with miming, their \Phi-space will only have entries that describe
   4.153 +   the scene with the sensation of their hands touching a wall. This
   4.154 +   sensation of touch will create a strong impression of a wall, even
   4.155 +   though the wall would have to be invisible. A person with
   4.156 +   experience in miming however, will have entries in their \Phi-space
   4.157 +   that describe the wall-miming position without a sense of touch. It
   4.158 +   will not seem to such as person that an invisible wall is present,
   4.159 +   but merely that the mime is holding out their hands in a special
   4.160 +   way. Thus, the theory that humans use something like \Phi-space
   4.161 +   weakly predicts that learning how to mime should break the power of
   4.162 +   miming illusions. Most optical illusions still work no matter how
   4.163 +   much you know about them, so this proposal would be quite
   4.164 +   interesting to test, as it predicts a non-standard result!
   4.165 +
   4.166 +
   4.167 +#+BEGIN_LaTeX
   4.168 +\clearpage
   4.169 +#+END_LaTeX
   4.170 +
   4.171  * Contributions
   4.172 +
   4.173 +  The big idea behind this thesis is a new way to represent and
   4.174 +  recognize physical actions, which I call /empathic representation/.
   4.175 +  Actions are represented as predicates which have access to the
   4.176 +  totality of a creature's sensory abilities. To recognize the
   4.177 +  physical actions of another creature similar to yourself, you
   4.178 +  imagine what they would feel by examining the position of their body
   4.179 +  and relating it to your own previous experience.
   4.180    
   4.181 -  The big idea behind this thesis is a new way to represent and
   4.182 -  recognize physical actions -- empathic representation. Actions are
   4.183 -  represented as predicates which have available the totality of a
   4.184 -  creature's sensory abilities. To recognize the physical actions of
   4.185 -  another creature similar to yourself, you imagine what they would
   4.186 -  feel by examining the position of their body and relating it to your
   4.187 -  own previous experience.
   4.188 -  
   4.189 -  Empathic description of physical actions is very robust and general.
   4.190 -  Because the representation is body-centered, it avoids the fragility
   4.191 -  of learning from example videos. Because it relies on all of a
   4.192 +  Empathic representation of physical actions is robust and general.
   4.193 +  Because the representation is body-centered, it avoids baking in a
   4.194 +  particular viewpoint like you might get from learning from example
   4.195 +  videos. Because empathic representation relies on all of a
   4.196    creature's senses, it can describe exactly what an action /feels
   4.197    like/ without getting caught up in irrelevant details such as visual
   4.198    appearance. I think it is important that a correct description of
   4.199 -  jumping (for example) should not waste even a single bit on the
   4.200 -  color of a person's clothes or skin; empathic representation can
   4.201 -  avoid this waste by describing jumping in terms of touch, muscle
   4.202 -  contractions, and the brief feeling of weightlessness. Empathic
   4.203 -  representation is very low-level in that it describes actions using
   4.204 -  concrete sensory data with little abstraction, but it has the
   4.205 -  generality of much more abstract representations!
   4.206 +  jumping (for example) should not include irrelevant details such as
   4.207 +  the color of a person's clothes or skin; empathic representation can
   4.208 +  get right to the heart of what jumping is by describing it in terms
   4.209 +  of touch, muscle contractions, and a brief feeling of
   4.210 +  weightlessness. Empathic representation is very low-level in that it
   4.211 +  describes actions using concrete sensory data with little
   4.212 +  abstraction, but it has the generality of much more abstract
   4.213 +  representations!
   4.214  
   4.215    Another important contribution of this thesis is the development of
   4.216    the =CORTEX= system, a complete environment for creating simulated
   4.217 @@ -3413,29 +3515,31 @@
   4.218    proprioception, hearing, vision, and muscle tension. You have seen
   4.219    how to create new creatures using blender, a 3D modeling tool.
   4.220  
   4.221 -  I hope that =CORTEX= will be useful in further research projects. To
   4.222 -  this end I have included the full source to =CORTEX= along with a
   4.223 -  large suite of tests and examples. I have also created a user guide
   4.224 -  for =CORTEX= which is included in an appendix to this thesis.
   4.225 -
   4.226    As a minor digression, you also saw how I used =CORTEX= to enable a
   4.227    tiny worm to discover the topology of its skin simply by rolling on
   4.228 -  the ground. 
   4.229 -
   4.230 -  In conclusion, the main contributions of this thesis are:
   4.231 -
   4.232 -   - =CORTEX=, a comprehensive platform for embodied AI experiments.
   4.233 -     =CORTEX= supports many features lacking in other systems, such
   4.234 -     proper simulation of hearing. It is easy to create new =CORTEX=
   4.235 -     creatures using Blender, a free 3D modeling program.
   4.236 -
   4.237 -   - =EMPATH=, which uses =CORTEX= to identify the actions of a
   4.238 -     worm-like creature using a computational model of empathy. This
   4.239 -     empathic representation of actions is an important new kind of
   4.240 -     representation for physical actions.
   4.241 +  the ground.  You also saw how to detect objects using only embodied
   4.242 +  predicates. 
   4.243 +
   4.244 +  In conclusion, for this thesis I:
   4.245 +
   4.246 +   - Developed the idea of embodied representation, which describes
   4.247 +     actions that a creature can do in terms of first-person sensory
   4.248 +     data. 
   4.249 +
   4.250 +   - Developed a method of empathic action recognition which uses
   4.251 +     previous embodied experience and embodied representation of
   4.252 +     actions to greatly constrain the possible interpretations of an
   4.253 +     action.
   4.254 +
   4.255 +   - Created =EMPATH=, a program which uses empathic action
   4.256 +     recognition to recognize physical actions in a simple model
   4.257 +     involving segmented worm-like creatures.
   4.258 +
   4.259 +   - Created =CORTEX=, a comprehensive platform for embodied AI
   4.260 +     experiments. It is the base on which =EMPATH= is built.
   4.261  
   4.262  #+BEGIN_LaTeX
   4.263 -\newpage
   4.264 +\clearpage
   4.265  \appendix
   4.266  #+END_LaTeX
   4.267  

     5.1 Binary file thesis/images/draped.png has changed

     6.1 Binary file thesis/images/pablo-the-mime.png has changed

     7.1 --- a/thesis/rlm-cortex-meng.tex	Mon Apr 28 15:10:59 2014 -0400
     7.2 +++ b/thesis/rlm-cortex-meng.tex	Thu May 01 23:41:41 2014 -0400
     7.3 @@ -56,6 +56,7 @@
     7.4  \usepackage{hyperref}
     7.5  \fancyhead[RO,LE]{\bfseries \nouppercase{\rightmark}}
     7.6  \renewcommand{\subsectionmark}[1]{\markright{#1}}
     7.7 +\renewcommand{\sectionmark}[1]{\markright{#1}}
     7.8  
     7.9  \usepackage[backend=bibtex,style=authoryear]{biblatex}
    7.10  %\usepackage[section]{placeins}