Mercurial > cortex

comparison org/integration.org @ 300:c78917fb0615

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
created first version of the video which demonstrates all the senses
author Robert McIntyre <rlm@mit.edu>
date Fri, 17 Feb 2012 11:06:27 -0700
parents 47fe4f7b74b3
children 4203c2140b95
comparison
equal deleted inserted replaced
299:47fe4f7b74b3 300:c78917fb0615
10 10
11 This is the ultimate test which features all of the senses that I've 11 This is the ultimate test which features all of the senses that I've
12 made so far. The blender file for the creature serves as an example of 12 made so far. The blender file for the creature serves as an example of
13 a fully equipped creature in terms of senses. You can find it [[../assets/Models/test-creature/hand.blend][here]]. 13 a fully equipped creature in terms of senses. You can find it [[../assets/Models/test-creature/hand.blend][here]].
14 14
15
16 #+begin_html
17 <div class="figure">
18 <video controls="controls" width="755">
19 <source src="../video/hand.ogg" type="video/ogg"
20 preload="none" poster="../images/aurellem-1280x480.png" />
21 </video>
22 <p>Simulated Senses in a Virtual Environment</p>
23 </div>
24 #+end_html
25
26
27
15 #+name: integration 28 #+name: integration
16 #+begin_src clojure 29 #+begin_src clojure
17 (ns cortex.integration 30 (ns cortex.test.integration
18 "let's play!" 31 "let's play!"
19 {:author "Robert McIntyre"} 32 {:author "Robert McIntyre"}
20 (:use (cortex world util body sense 33 (:use (cortex world util body sense
21 hearing touch vision proprioception movement)) 34 hearing touch vision proprioception movement))
22 (:import (com.jme3.math ColorRGBA Vector3f)) 35 (:import (com.jme3.math ColorRGBA Vector3f))
28 (rlm.rlm-commands/help) 41 (rlm.rlm-commands/help)
29 42
30 (def hand "Models/test-creature/hand.blend") 43 (def hand "Models/test-creature/hand.blend")
31 44
32 (def output-base (File. "/home/r/proj/cortex/render/hand")) 45 (def output-base (File. "/home/r/proj/cortex/render/hand"))
33 46 #+end_src
34 47
35 ;; Let the hand fall palm-up 48 For this demonstration I have to manually drive the muscles of the
36 49 hand. I do this by creating a little mini-language to describe
37 ;; it curls its phalanges, starting with the pinky. 50 simulated muscle contraction.
38 51
39 ;; it lets these fall back down? 52 #+name: integration-2
40 53 #+begin_src clojure
41 ;; block falls down onto the hand, accompanied by a sound. The block
42 ;; can be seen by the hand's eye.
43
44 ;; hand FORCEFULLY catapults the block so that it hits the camera.
45
46
47 (defn motor-control-program 54 (defn motor-control-program
48 "Create a function which will execute the motor script" 55 "Create a function which will execute the motor script"
49 [muscle-positions 56 [muscle-positions
50 script] 57 script]
51 (let [current-frame (atom -1) 58 (let [current-frame (atom -1)
92 :thumb-1-f 99 :thumb-1-f
93 :pinky-1-f 100 :pinky-1-f
94 :pinky-1-e]) 101 :pinky-1-e])
95 102
96 (def full 9001) 103 (def full 9001)
104
105
106 ;; Coreography:
107
108 ;; Let the hand fall palm-up
109
110 ;; it curls its phalanges, starting with the pinky.
111
112 ;; it lets its phalanges fall back down.
113
114 ;; block falls down onto the hand, accompanied by a sound. The block
115 ;; can be seen by the hand's eye.
116
117 ;; hand FORCEFULLY catapults the block so that it hits the camera.
118
119
97 ;; the systax here is [keyframe body-part force] 120 ;; the systax here is [keyframe body-part force]
98 (def wiggle-each-finger-tip 121 (def move-fingers
99 [[300 :pinky-3-f 50] 122 [[300 :pinky-3-f 50]
100 [320 :pinky-2-f 80] 123 [320 :pinky-2-f 80]
101 [340 :pinky-1-f 100] 124 [340 :pinky-1-f 100]
102 125
103 [310 :ring-3-f 100] 126 [310 :ring-3-f 100]
195 (proxy [AbstractControl] [] 218 (proxy [AbstractControl] []
196 (controlUpdate [tpf] 219 (controlUpdate [tpf]
197 (println-repl (.getWorldTranslation brick))) 220 (println-repl (.getWorldTranslation brick)))
198 (controlRender [_ _])))) 221 (controlRender [_ _]))))
199 (add-element world brick)))))) 222 (add-element world brick))))))
200
201
202 (def control-list
203 [
204 0 ;;pointer-21 #<Vector3f (0.99999994, 0.0, 0.0)>
205 0 ;;pointer-21 #<Vector3f (-0.99999994, 0.0, 0.0)>
206 0 ;;thumb-11 #<Vector3f (-0.8802276, -0.39781287, -0.25873658)>
207 0 ;;thumb-11 #<Vector3f (0.8485723, 0.46149826, 0.2587364)>
208 0 ;;pointer-11 #<Vector3f (0.99999994, 0.0, 0.0)>
209 0 ;;pointer-11 #<Vector3f (-0.99999994, 0.0, 0.0)>
210 0 ;;thumb-2.0011 #<Vector3f (-0.71705645, -0.44753736, -0.5343599)>
211 0 ;;thumb-2.0011 #<Vector3f (-0.10567085, 0.83862597, 0.53435963)>
212 0 ;;middle-11 #<Vector3f (0.99999994, 0.0, 0.0)>
213 0 ;;middle-11 #<Vector3f (-0.99999994, 0.0, 0.0)>
214 0 ;;pointer-31 #<Vector3f (-0.99999994, 0.0, 0.0)>
215 0 ;;pointer-31 #<Vector3f (0.99999994, 0.0, 0.0)>
216 0 ;;middle-21 #<Vector3f (0.99999994, 0.0, 0.0)>
217 0 ;;middle-21 #<Vector3f (-0.99999994, 0.0, 0.0)>
218 0 ;;middle-31 #<Vector3f (-0.99999994, 0.0, 0.0)>
219 0 ;;middle-31 #<Vector3f (0.99999994, 0.0, 0.0)>
220 0 ;;pinky-21 #<Vector3f (0.99999994, 0.0, 0.0)>
221 0 ;;pinky-21 #<Vector3f (-0.99999994, 0.0, 0.0)>
222 0 ;;pinky-31 #<Vector3f (-1, 0.0, 0.0)>
223 0 ;;pinky-31 #<Vector3f (1.0, 0.0, 0.0)>
224 0 ;;ring-31 #<Vector3f (0.99999994, 0.0, 0.0)>
225 0 ;;ring-31 #<Vector3f (-0.99999994, 0.0, 0.0)>
226 0 ;;ring-21 #<Vector3f (-0.99999994, 0.0, 0.0)>
227 0 ;;ring-21 #<Vector3f (0.99999994, 0.0, 0.0)>
228 0 ;;ring-11 #<Vector3f (0.99999994, 0.0, 0.0)>
229 0 ;;ring-11 #<Vector3f (-0.99999994, 0.0, 0.0)>
230 0 ;;thumb-11 #<Vector3f (-0.43154645, 0.7302033, 0.5296894)>
231 0 ;;thumb-11 #<Vector3f (-0.8032993, -0.2722854, -0.5296895)>
232 0 ;;pinky-11 #<Vector3f (-0.99999994, 0.0, 0.0)>
233 0 ;;pinky-11 #<Vector3f (0.99999994, 0.0, 0.0)>
234 ])
235 223
236 (import com.aurellem.capture.Capture) 224 (import com.aurellem.capture.Capture)
237 225
238 (defn test-everything! 226 (defn test-everything!
239 ([] (test-everything! false)) 227 ([] (test-everything! false))
259 247
260 prop (proprioception! creature) 248 prop (proprioception! creature)
261 prop-display (view-proprioception) 249 prop-display (view-proprioception)
262 250
263 control-script (motor-control-program 251 control-script (motor-control-program
264 muscle-positions wiggle-each-finger-tip) 252 muscle-positions move-fingers)
265 muscles (movement! creature) 253 muscles (movement! creature)
266 muscle-display (view-movement) 254 muscle-display (view-movement)
267 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 255 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
268 256
269 fix-display (gen-fix-display)] 257 fix-display (gen-fix-display)]
313 301
314 (summon-ball world) 302 (summon-ball world)
315 303
316 (.setLocalTranslation me (.getLocation (.getCamera world))) 304 (.setLocalTranslation me (.getLocation (.getCamera world)))
317 (fix-display world)))))) 305 (fix-display world))))))
318 306 #+end_src
307
308 * Generating the Video
309
310 Just a bunch of calls to imagemagick to arrange the data that
311 =test-everything!= produces.
312
313 #+name: magick-8
314 #+begin_src clojure
315 (ns cortex.video.magick8
316 (:import java.io.File)
317 (:use clojure.contrib.shell-out))
319 318
320 (comment 319 (comment
321 ;; list of touch targets 320 ;; list of touch targets
322 0 middle-11 321 0 middle-11
323 1 middle-21 322 1 middle-21
424 (defn file-groups [& paths] 423 (defn file-groups [& paths]
425 (apply (partial map list ) 424 (apply (partial map list )
426 (map (comp file-names #(File. base %)) 425 (map (comp file-names #(File. base %))
427 paths))) 426 paths)))
428 427
429
430 (defn pinky [] 428 (defn pinky []
431 (file-groups 429 (file-groups
432 "muscle/18" 430 "muscle/18"
433 "muscle/19" 431 "muscle/19"
434 "touch/5" 432 "touch/5"
466 "muscle/14" 464 "muscle/14"
467 "muscle/15" 465 "muscle/15"
468 "touch/2" 466 "touch/2"
469 "proprio/1" 467 "proprio/1"
470 468
469 "muscle/13"
471 "muscle/12" 470 "muscle/12"
472 "muscle/13"
473 "touch/1" 471 "touch/1"
474 "proprio/0" 472 "proprio/0"
475 473
474 "muscle/9"
476 "muscle/8" 475 "muscle/8"
477 "muscle/9"
478 "touch/0" 476 "touch/0"
479 "proprio/9")) 477 "proprio/9"))
480 478
481 (defn pointer [] 479 (defn pointer []
482 (file-groups 480 (file-groups
601 (file-names (File. base "hearing/0"))) 599 (file-names (File. base "hearing/0")))
602 600
603 (defn main [] 601 (defn main []
604 (file-names (File. base "main"))) 602 (file-names (File. base "main")))
605 603
606 (defn targets [] 604 (defn targets [dest max]
607 (map 605 (map
608 (comp #(.getCanonicalPath %) 606 (comp #(.getCanonicalPath %)
609 #(File. (str base "/out/" (format "%07d.png" %)))) 607 #(File. (str base dest (format "%07d.png" %))))
610 (range 0 (count (main))))) 608 (range max)))
611 609
612 610
613 (defn final-image [main [all red green blue] hearing 611 (defn final-image [main [all red green blue] hearing
614 pinky ring middle pointer thumb target] 612 pinky ring middle pointer thumb target]
615 (println target) 613 (println target)
631 main 629 main
632 "-geometry" "+78+202" "-composite" 630 "-geometry" "+78+202" "-composite"
633 631
634 target]))) 632 target])))
635 633
636 (defn convert-files [] 634 (defn combine-files []
637 (dorun 635 (dorun
638 (pmap final-image 636 (pmap final-image
639 (main) 637 (main)
640 (vision) 638 (vision)
641 (hearing) 639 (hearing)
642 (pinky) 640 (pinky)
643 (ring) 641 (ring)
644 (middle) 642 (middle)
645 (pointer) 643 (pointer)
646 (thumb) 644 (thumb)
647 (targets)))) 645 (targets "/out/" (count (main))))))
646
647 (defn subtitles []
648 (file-names (File. base "subs")))
649
650 (defn outs []
651 (file-names (File. base "out")))
652
653
654 (defn mix-subtitles []
655 (let [subs (subtitles)
656 targets (targets "/out-subs/" (count subs))
657 overlay (.getCanonicalPath (File. base "output.png"))]
658 (dorun
659 (pmap
660 (fn [sub target]
661 (sh
662 "convert"
663 overlay
664 sub "-geometry" "+0+0" "-composite"
665 target))
666 subs targets))))
667
668 (defn out-subtitles []
669 (file-names (File. base "out-subs")))
670
671
672 (defn insert-subtitles []
673 (let [subtitles (out-subtitles)
674 outs (outs)
675 targets (targets
676 "/final/"
677 (+ (count outs) (count subtitles)))]
678 (dorun
679 (pmap
680 #(sh "cp" %1 %2)
681 (concat subtitles outs) targets))))
682
683 (defn generate-final []
684 (combine-files)
685 (mix-subtitles)
686 (insert-subtitles))
648 #+end_src 687 #+end_src
649 688
650 689 #+begin_src sh :results silent
651 #+begin_src sh
652 cd /home/r/proj/cortex/render/hand 690 cd /home/r/proj/cortex/render/hand
653 691
654 ffmpeg -r 60 -i out/%07d.png -i main.wav -b:a 128k \ 692 sox --ignore-length main.wav main-delayed.wav delay 24
693
694 mogrify -resize 755x final/*
695
696 ffmpeg -r 60 -i final/%07d.png -i main-delayed.wav -b:a 128k \
655 -b:v 9000k -c:a libvorbis -c:v libtheora hand.ogg 697 -b:v 9000k -c:a libvorbis -c:v libtheora hand.ogg
656 #+end_src 698 #+end_src
657
658 #+results: integration
659 : #'cortex.integration/convert-files
660 699
661 * COMMENT purgatory 700 * COMMENT purgatory
662 #+begin_src clojure 701 #+begin_src clojure
663 (defn bullet-trans* [] 702 (defn bullet-trans* []
664 (let [obj-a (box 1.5 0.5 0.5 :color ColorRGBA/Red 703 (let [obj-a (box 1.5 0.5 0.5 :color ColorRGBA/Red
818 (Vector3f. 1 0 0)))))))) 857 (Vector3f. 1 0 0))))))))
819 #+end_src 858 #+end_src
820 859
821 860
822 * COMMENT generate source 861 * COMMENT generate source
823 #+begin_src clojure :tangle ../src/cortex/integration.clj 862 #+begin_src clojure :tangle ../src/cortex/test/integration.clj
824 <<integration>> 863 <<integration>>
864 <<integration-2>>
825 #+end_src 865 #+end_src
826 866
867 #+begin_src clojure :tangle ../src/cortex/video/magick8.clj
868 <<magick-8>>
869 #+end_src
870
827 871
828 872
829 873