view org/worm_learn.clj @ 448:af13fc73e851

completing second part of first chapter.
author Robert McIntyre <rlm@mit.edu>
date Tue, 25 Mar 2014 22:54:41 -0400
parents 47cfbe84f00e
children 09b7c8dd4365
line wrap: on
line source
1 (ns org.aurellem.worm-learn
2 "General worm creation framework."
3 {:author "Robert McIntyre"}
4 (:use (cortex world util import body sense
5 hearing touch vision proprioception movement
6 test))
7 (:import (com.jme3.math ColorRGBA Vector3f))
8 (:import java.io.File)
9 (:import com.jme3.audio.AudioNode)
10 (:import com.aurellem.capture.RatchetTimer)
11 (:import (com.aurellem.capture Capture IsoTimer))
12 (:import (com.jme3.math Vector3f ColorRGBA)))
14 (import org.apache.commons.math3.transform.TransformType)
15 (import org.apache.commons.math3.transform.FastFourierTransformer)
16 (import org.apache.commons.math3.transform.DftNormalization)
18 (use 'clojure.pprint)
19 (use 'clojure.set)
20 (dorun (cortex.import/mega-import-jme3))
21 (rlm.rlm-commands/help)
23 (load-bullet)
25 (def hand "Models/test-creature/hand.blend")
27 (defn worm-model []
28 (load-blender-model "Models/worm/worm.blend"))
30 (def output-base (File. "/home/r/proj/cortex/render/worm-learn/curl"))
33 (defn motor-control-program
34 "Create a function which will execute the motor script"
35 [muscle-labels
36 script]
37 (let [current-frame (atom -1)
38 keyed-script (group-by first script)
39 current-forces (atom {}) ]
40 (fn [effectors]
41 (let [indexed-effectors (vec effectors)]
42 (dorun
43 (for [[_ part force] (keyed-script (swap! current-frame inc))]
44 (swap! current-forces (fn [m] (assoc m part force)))))
45 (doall (map (fn [effector power]
46 (effector (int power)))
47 effectors
48 (map #(@current-forces % 0) muscle-labels)))))))
50 (defn worm-direct-control
51 "Create keybindings and a muscle control program that will enable
52 the user to control the worm via the keyboard."
53 [muscle-labels activation-strength]
54 (let [strengths (mapv (fn [_] (atom 0)) muscle-labels)
55 activator
56 (fn [n]
57 (fn [world pressed?]
58 (let [strength (if pressed? activation-strength 0)]
59 (swap! (nth strengths n) (constantly strength)))))
60 activators
61 (map activator (range (count muscle-labels)))
62 worm-keys
63 ["key-f" "key-r"
64 "key-g" "key-t"
65 "key-h" "key-y"
66 "key-j" "key-u"
67 "key-k" "key-i"
68 "key-l" "key-o"]]
69 {:motor-control
70 (fn [effectors]
71 (doall
72 (map (fn [strength effector]
73 (effector (deref strength)))
74 strengths effectors)))
75 :keybindings
76 ;; assume muscles are listed in pairs and map them to keys.
77 (zipmap worm-keys activators)}))
79 ;; These are scripts that direct the worm to move in two radically
80 ;; different patterns -- a sinusoidal wiggling motion, and a curling
81 ;; motions that causes the worm to form a circle.
83 (def curl-script
84 [[150 :d-flex 40]
85 [250 :d-flex 0]])
87 (def period 18)
89 (def worm-muscle-labels
90 [:base-ex :base-flex
91 :a-ex :a-flex
92 :b-ex :b-flex
93 :c-ex :c-flex
94 :d-ex :d-flex])
96 (defn gen-wiggle [[flexor extensor :as muscle-pair] time-base]
97 (let [period period
98 power 45]
99 [[time-base flexor power]
100 [(+ time-base period) flexor 0]
101 [(+ time-base period 1) extensor power]
102 [(+ time-base (+ (* 2 period) 2)) extensor 0]]))
104 (def wiggle-script
105 (mapcat gen-wiggle (repeat 4000 [:a-ex :a-flex])
106 (range 100 1000000 (+ 3 (* period 2)))))
109 (defn shift-script [shift script]
110 (map (fn [[time label power]] [(+ time shift) label power])
111 script))
113 (def do-all-the-things
114 (concat
115 curl-script
116 [[300 :d-ex 40]
117 [320 :d-ex 0]]
118 (shift-script 280 (take 16 wiggle-script))))
120 ;; Normally, we'd use unsupervised/supervised machine learning to pick
121 ;; out the defining features of the different actions available to the
122 ;; worm. For this project, I am going to explicitely define functions
123 ;; that recognize curling and wiggling respectively. These functions
124 ;; are defined using all the information available from an embodied
125 ;; simulation of the action. Note how much easier they are to define
126 ;; than if I only had vision to work with. Things like scale/position
127 ;; invariance are complete non-issues here. This is the advantage of
128 ;; body-centered action recognition and what I hope to show with this
129 ;; thesis.
132 ;; curled? relies on proprioception, resting? relies on touch,
133 ;; wiggling? relies on a fourier analysis of muscle contraction, and
134 ;; grand-circle? relies on touch and reuses curled? as a gaurd.
136 (defn curled?
137 "Is the worm curled up?"
138 [experiences]
139 (every?
140 (fn [[_ _ bend]]
141 (> (Math/sin bend) 0.64))
142 (:proprioception (peek experiences))))
144 (defn rect-region [[x0 y0] [x1 y1]]
145 (vec
146 (for [x (range x0 (inc x1))
147 y (range y0 (inc y1))]
148 [x y])))
150 (def worm-segment-bottom (rect-region [8 15] [14 22]))
152 (defn contact
153 "Determine how much contact a particular worm segment has with
154 other objects. Returns a value between 0 and 1, where 1 is full
155 contact and 0 is no contact."
156 [touch-region [coords contact :as touch]]
157 (-> (zipmap coords contact)
158 (select-keys touch-region)
159 (vals)
160 (#(map first %))
161 (average)
162 (* 10)
163 (- 1)
164 (Math/abs)))
166 (defn resting?
167 "Is the worm resting on the ground?"
168 [experiences]
169 (every?
170 (fn [touch-data]
171 (< 0.9 (contact worm-segment-bottom touch-data)))
172 (:touch (peek experiences))))
174 (defn vector:last-n [v n]
175 (let [c (count v)]
176 (if (< c n) v
177 (subvec v (- c n) c))))
179 (defn fft [nums]
180 (map
181 #(.getReal %)
182 (.transform
183 (FastFourierTransformer. DftNormalization/STANDARD)
184 (double-array nums) TransformType/FORWARD)))
186 (def indexed (partial map-indexed vector))
188 (defn max-indexed [s]
189 (first (sort-by (comp - second) (indexed s))))
191 (defn wiggling?
192 "Is the worm wiggling?"
193 [experiences]
194 (let [analysis-interval 0x40]
195 (when (> (count experiences) analysis-interval)
196 (let [a-flex 3
197 a-ex 2
198 muscle-activity
199 (map :muscle (vector:last-n experiences analysis-interval))
200 base-activity
201 (map #(- (% a-flex) (% a-ex)) muscle-activity)]
202 (= 2
203 (first
204 (max-indexed
205 (map #(Math/abs %)
206 (take 20 (fft base-activity))))))))))
208 (def worm-segment-bottom-tip (rect-region [15 15] [22 22]))
210 (def worm-segment-top-tip (rect-region [0 15] [7 22]))
212 (defn grand-circle?
213 "Does the worm form a majestic circle (one end touching the other)?"
214 [experiences]
215 (and (curled? experiences)
216 (let [worm-touch (:touch (peek experiences))
217 tail-touch (worm-touch 0)
218 head-touch (worm-touch 4)]
219 (and (< 0.55 (contact worm-segment-bottom-tip tail-touch))
220 (< 0.55 (contact worm-segment-top-tip head-touch))))))
223 (declare phi-space phi-scan)
225 (defn debug-experience
226 [experiences text]
227 (cond
228 (grand-circle? experiences) (.setText text "Grand Circle")
229 (curled? experiences) (.setText text "Curled")
230 (wiggling? experiences) (.setText text "Wiggling")
231 (resting? experiences) (.setText text "Resting")))
234 (def standard-world-view
235 [(Vector3f. 4.207176, -3.7366982, 3.0816958)
236 (Quaternion. 0.11118768, 0.87678415, 0.24434438, -0.3989771)])
238 (def worm-side-view
239 [(Vector3f. 4.207176, -3.7366982, 3.0816958)
240 (Quaternion. -0.11555642, 0.88188726, -0.2854942, -0.3569518)])
242 (def degenerate-worm-view
243 [(Vector3f. -0.0708936, -8.570261, 2.6487997)
244 (Quaternion. -2.318909E-4, 0.9985348, 0.053941682, 0.004291452)])
246 (defn worm-world-defaults []
247 (let [direct-control (worm-direct-control worm-muscle-labels 40)]
248 (merge direct-control
249 {:view worm-side-view
250 :record nil
251 :experiences (atom [])
252 :experience-watch debug-experience
253 :worm-model worm-model
254 :end-frame nil})))
256 (defn dir! [file]
257 (if-not (.exists file)
258 (.mkdir file))
259 file)
261 (defn record-experience! [experiences data]
262 (swap! experiences #(conj % data)))
264 (defn enable-shadows [world]
265 (let [bsr (doto
266 (BasicShadowRenderer. (asset-manager) 512)
267 (.setDirection (.normalizeLocal (Vector3f. 1 -1 -1))))]
268 (.addProcessor (.getViewPort world) bsr)))
270 (defn enable-good-shadows [world]
271 (let [pssm
272 (doto (PssmShadowRenderer. (asset-manager) 1024 3)
273 (.setDirection (.normalizeLocal (Vector3f. -1 -3 -1)))
274 (.setLambda (float 0.55))
275 (.setShadowIntensity (float 0.6))
276 (.setCompareMode PssmShadowRenderer$CompareMode/Software)
277 (.setFilterMode PssmShadowRenderer$FilterMode/Bilinear))]
278 (.addProcessor (.getViewPort world) pssm)))
281 (defn display-text [[x y :as location]]
282 (let []
283 (.setLocalTranslation text 300 (.getLineHeight text) 0)
284 (fn [world]
289 (fn [new-text]
291 (defn worm-world
292 [& {:keys [record motor-control keybindings view experiences
293 worm-model end-frame experience-watch] :as settings}]
294 (let [{:keys [record motor-control keybindings view experiences
295 worm-model end-frame experience-watch]}
296 (merge (worm-world-defaults) settings)
297 worm (doto (worm-model) (body!))
298 touch (touch! worm)
299 prop (proprioception! worm)
300 muscles (movement! worm)
302 touch-display (view-touch)
303 prop-display (view-proprioception)
304 muscle-display (view-movement)
306 floor
307 (box 5 1 5 :position (Vector3f. 0 -10 0)
308 :mass 0
309 :texture "Textures/aurellem.png"
310 :material "Common/MatDefs/Misc/Unshaded.j3md")
311 timer (IsoTimer. 60)
313 font (.loadFont (asset-manager) "Interface/Fonts/Console.fnt")
314 worm-action (doto (BitmapText. font false)
315 (.setSize 35)
316 (.setColor (ColorRGBA/Black)))]
318 (world
319 (nodify [worm floor])
320 (merge standard-debug-controls keybindings)
321 (fn [world]
322 (.setLocalTranslation
323 worm-action 20 470 0)
324 (.attachChild (.getGuiNode world) worm-action)
326 (enable-good-shadows world)
327 (.setShadowMode worm RenderQueue$ShadowMode/CastAndReceive)
328 (.setShadowMode floor RenderQueue$ShadowMode/Receive)
330 (.setBackgroundColor (.getViewPort world) (ColorRGBA/White))
331 (.setDisplayStatView world false)
332 (.setDisplayFps world false)
333 (position-camera world view)
334 (.setTimer world timer)
335 (display-dilated-time world timer)
336 (when record
337 (dir! record)
338 (Capture/captureVideo
339 world
340 (dir! (File. record "main-view"))))
341 (speed-up world)
342 ;;(light-up-everything world)
343 )
344 (fn [world tpf]
345 (if (and end-frame (> (.getTime timer) end-frame))
346 (.stop world))
347 (let [muscle-data (vec (motor-control muscles))
348 proprioception-data (prop)
349 touch-data (mapv #(% (.getRootNode world)) touch)]
350 (when experiences
351 (record-experience!
352 experiences {:touch touch-data
353 :proprioception proprioception-data
354 :muscle muscle-data}))
355 (when experience-watch
356 (experience-watch @experiences worm-action))
357 (muscle-display
358 muscle-data
359 (when record (dir! (File. record "muscle"))))
360 (prop-display
361 proprioception-data
362 (when record (dir! (File. record "proprio"))))
363 (touch-display
364 touch-data
365 (when record (dir! (File. record "touch")))))))))
369 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
370 ;;;;;;;; Phi-Space ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
371 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
373 (defn generate-phi-space []
374 (let [experiences (atom [])]
375 (run-world
376 (apply-map
377 worm-world
378 (merge
379 (worm-world-defaults)
380 {:end-frame 700
381 :motor-control
382 (motor-control-program worm-muscle-labels do-all-the-things)
383 :experiences experiences})))
384 @experiences))
386 (defn bin [digits]
387 (fn [angles]
388 (->> angles
389 (flatten)
390 (map (juxt #(Math/sin %) #(Math/cos %)))
391 (flatten)
392 (mapv #(Math/round (* % (Math/pow 10 (dec digits))))))))
394 ;; k-nearest neighbors with spatial binning. Only returns a result if
395 ;; the propriceptive data is within 10% of a previously recorded
396 ;; result in all dimensions.
397 (defn gen-phi-scan [phi-space]
398 (let [bin-keys (map bin [3 2 1])
399 bin-maps
400 (map (fn [bin-key]
401 (group-by
402 (comp bin-key :proprioception phi-space)
403 (range (count phi-space)))) bin-keys)
404 lookups (map (fn [bin-key bin-map]
405 (fn [proprio] (bin-map (bin-key proprio))))
406 bin-keys bin-maps)]
407 (fn lookup [proprio-data]
408 (set (some #(% proprio-data) lookups)))))
411 (defn longest-thread
412 "Find the longest thread from phi-index-sets. The index sets should
413 be ordered from most recent to least recent."
414 [phi-index-sets]
415 (loop [result '()
416 [thread-bases & remaining :as phi-index-sets] phi-index-sets]
417 (if (empty? phi-index-sets)
418 (vec result)
419 (let [threads
420 (for [thread-base thread-bases]
421 (loop [thread (list thread-base)
422 remaining remaining]
423 (let [next-index (dec (first thread))]
424 (cond (empty? remaining) thread
425 (contains? (first remaining) next-index)
426 (recur
427 (cons next-index thread) (rest remaining))
428 :else thread))))
429 longest-thread
430 (reduce (fn [thread-a thread-b]
431 (if (> (count thread-a) (count thread-b))
432 thread-a thread-b))
433 '(nil)
434 threads)]
435 (recur (concat longest-thread result)
436 (drop (count longest-thread) phi-index-sets))))))
439 (defn init []
440 (def phi-space (generate-phi-space))
441 (def phi-scan (gen-phi-scan phi-space))
442 )
444 ;; (defn infer-nils-dyl [s]
445 ;; (loop [closed ()
446 ;; open s
447 ;; anchor 0]
448 ;; (if-not (empty? open)
449 ;; (recur (conj closed
450 ;; (or (peek open)
451 ;; anchor))
452 ;; (pop open)
453 ;; (or (peek open) anchor))
454 ;; closed)))
456 ;; (defn infer-nils [s]
457 ;; (for [i (range (count s))]
458 ;; (or (get s i)
459 ;; (some (comp not nil?) (vector:last-n (- (count s) i)))
460 ;; 0)))
463 (defn infer-nils
464 "Replace nils with the next available non-nil element in the
465 sequence, or barring that, 0."
466 [s]
467 (loop [i (dec (count s))
468 v (transient s)]
469 (if (zero? i) (persistent! v)
470 (if-let [cur (v i)]
471 (if (get v (dec i) 0)
472 (recur (dec i) v)
473 (recur (dec i) (assoc! v (dec i) cur)))
474 (recur i (assoc! v i 0))))))
476 ;; tests
478 ;;(infer-nils [1 nil 1 1]) [1 1 1 1]
479 ;;(infer-nils [1 1 1 nil]) [1 1 1 0]
480 ;;(infer-nils [nil 2 1 1]) [2 2 1 1]
483 (defn debug-experience-phi []
484 (let [proprio (atom ())]
485 (fn
486 [experiences]
487 (let [phi-indices (phi-scan (:proprioception (peek experiences)))]
488 (swap! proprio (partial cons phi-indices))
489 (let [exp-thread (longest-thread (take 300 @proprio))
490 phi-loop (mapv phi-space (infer-nils exp-thread))]
491 (println-repl (vector:last-n exp-thread 22))
492 (cond
493 (grand-circle? phi-loop) (println "Grand Circle")
494 (curled? phi-loop) (println "Curled")
495 (wiggling? phi-loop) (println "Wiggling")
496 (resting? phi-loop) (println "Resting")
497 :else (println "Unknown")))))))
500 (defn init-interactive []
501 (def phi-space
502 (let [experiences (atom [])]
503 (run-world
504 (apply-map
505 worm-world
506 (merge
507 (worm-world-defaults)
508 {:experiences experiences})))
509 @experiences))
510 (def phi-scan (gen-phi-scan phi-space)))
513 (defn run-experiment-1 []
514 (.start (worm-world :experience-watch (debug-experience-phi))))