# HG changeset patch # User Robert McIntyre # Date 1318767139 25200 # Node ID 92f8d83b5d0b0b003da06edc834cb65e1f076739 initial import: I've made hearing and vision, and am working on touch. diff -r 000000000000 -r 92f8d83b5d0b .hgignore --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/.hgignore Sun Oct 16 05:12:19 2011 -0700 @@ -0,0 +1,4 @@ +syntax: glob +video* +src* +html* diff -r 000000000000 -r 92f8d83b5d0b assets/Sounds/dream.wav Binary file assets/Sounds/dream.wav has changed diff -r 000000000000 -r 92f8d83b5d0b assets/Sounds/pure.wav Binary file assets/Sounds/pure.wav has changed diff -r 000000000000 -r 92f8d83b5d0b assets/Sounds/silence.wav Binary file assets/Sounds/silence.wav has changed diff -r 000000000000 -r 92f8d83b5d0b assets/Textures/BronzeCopper030.jpg Binary file assets/Textures/BronzeCopper030.jpg has changed diff -r 000000000000 -r 92f8d83b5d0b assets/Textures/PokeCopper.jpg Binary file assets/Textures/PokeCopper.jpg has changed diff -r 000000000000 -r 92f8d83b5d0b assets/Textures/purpleWisp.png Binary file assets/Textures/purpleWisp.png has changed diff -r 000000000000 -r 92f8d83b5d0b assets/Textures/redWisp.png Binary file assets/Textures/redWisp.png has changed diff -r 000000000000 -r 92f8d83b5d0b org/body.org --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/org/body.org Sun Oct 16 05:12:19 2011 -0700 @@ -0,0 +1,37 @@ +#+title: The BODY!!! +#+author: Robert McIntyre +#+email: rlm@mit.edu +#+MATHJAX: align:"left" mathml:t path:"../aurellem/src/MathJax/MathJax.js" +#+STYLE: +#+BABEL: :exports both :noweb yes :cache no :mkdirp yes +#+INCLUDE: ../aurellem/src/templates/level-0.org +#+description: Simulating a body (movement, tough, propioception) in jMonkeyEngine3. + + +* Body ! + +#+srcname: body-main +#+begin_src clojure +(ns body.body) +(use 'cortex.world) +(use 'cortex.import) +(use 'clojure.contrib.def) +(cortex.import/mega-import-jme3) +(rlm.rlm-commands/help) + + + +#+end_src + + + + + + + + +* COMMENT generate Source. +#+begin_src clojure :tangle ../src/body/body.clj +<> +#+end_src + diff -r 000000000000 -r 92f8d83b5d0b org/capture-video.org --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/org/capture-video.org Sun Oct 16 05:12:19 2011 -0700 @@ -0,0 +1,633 @@ +#+title: Capture Live Video Feeds from JMonkeyEngine +#+author: Robert McIntyre +#+email: rlm@mit.edu +#+MATHJAX: align:"left" mathml:t path:"../aurellem/src/MathJax/MathJax.js" +#+STYLE: +#+OPTIONS: H:3 num:t toc:t \n:nil @:t ::t |:t ^:t -:t f:t *:t <:t +#+BABEL: :exports both :noweb yes :cache no :mkdirp yes +#+description: Capture video from a JMonkeyEngine3 Application with Xuggle, and use gstreamer to compress the video to upload to YouTube. +#+keywords: JME3, video, Xuggle, JMonkeyEngine, youtube, capture video, Java +#+INCLUDE: ../aurellem/src/templates/level-0.org +:PROPERTIES: +:EXPORT_FILE_NAME: ../whatever.html +:END: + + +* The Problem +So you've made your cool new JMonkeyEngine3 game and you want to +create a demo video to show off your hard work. Screen capturing is +the most straightforward way to do this, but it can slow down your +game and produce low-quality video as a result. A better way is to +record a video feed directly from the game while it is +running. + +In this post, I'll explain how you can alter your JMonkeyEngine3 game +to output video while it is running. The main trick is to alter the +pace of JMonkeyEngine3's in-game time: we allow the engine as much +time as it needs to compute complicated in-game events and to encode +video frames. As a result, the game appears to speed up and slow down +as the computational demands shift, but the end result is perfectly +smooth video output at a constant framerate. + + +* Game-time vs. User-time vs. Video-time + +A standard JME3 application that extends =SimpleApplication= or +=Application= tries as hard as it can to keep in sync with +/user-time/. If a ball is rolling at 1 game-mile per game-hour in the +game, and you wait for one user-hour as measured by the clock on your +wall, then the ball should have traveled exactly one game-mile. In +order to keep sync with the real world, the game throttles its physics +engine and graphics display. If the computations involved in running +the game are too intense, then the game will first skip frames, then +sacrifice physics accuracy. If there are particuraly demanding +computations, then you may only get 1 fps, and the ball may tunnel +through the floor or obstacles due to inaccurate physics simulation, +but after the end of one user-hour, that ball will have traveled one +game-mile. + +When we're recording video, we don't care if the game-time syncs with +user-time, but instead whether the time in the recorded video +(video-time) syncs with user-time. To continue the analogy, if we +recorded the ball rolling at 1 game-mile per game-hour and watched the +video later, we would want to see 30 fps video of the ball rolling at +1 video-mile per /user-hour/. It doesn't matter how much user-time it +took to simulate that hour of game-time to make the high-quality +recording. + +* COMMENT Two examples to clarify the point: +** Recording from a Simple Simulation + +*** Without a Special Timer +You have a simulation of a ball rolling on an infinite empty plane at +one game-mile per game-hour, and a really good computer. Normally, +JME3 will throttle the physics engine and graphics display to sync the +game-time with user-time. If it takes one-thousandth of a second +user-time to simulate one-sixtieth of a second game time and another +one-thousandth of a second to draw to the screen, then JME3 will just +sit around for the remainder of $\frac{1}{60} - \frac{2}{1000}$ +user-seconds, then calculate the next frame in $\frac{2}{1000}$ +user-seconds, then wait, and so on. For every second of user time that +passes, one second of game-time passes, and the game will run at 60 +frames per user-second. + + +*** With a Special Timer +Then, you change the game's timer so that user-time will be synced to +video-time. Assume that encoding a single frame takes 0 seconds +user-time to complete. + +Now, JME3 takes advantage of all available resources. It still takes +one-thousandth of a second to calculate a physics tick, and another +one-thousandth to render to the screen. Then it takes 0 seconds to +write the video frame to disk and encode the video. In only one second +of user time, JME3 will complete 500 physics-tick/render/encode-video +cycles, and $\frac{500}{60}=8\frac{1}{3}$ seconds of game-time will +have passed. Game-time appears to dilate $8\frac{1}{3}\times$ with +respect to user-time, and in only 7.2 minutes user-time, one hour of +video will have been recorded. The game itself will run at 500 fps. +When someone watches the video, they will see 60 frames per +user-second, and $\frac{1}{60}$ video-seconds will pass each frame. It +will take exactly one hour user-time (and one hour video-time) for the +ball in the video to travel one video-mile. + +** Recording from a Complex Simulation + +*** Without a Special Timer +You have a simulation of a ball rolling on an infinite empty plane at +one game-mile per game-hour accompanied by multiple explosions +involving thousands of nodes, particle effects, and complicated shadow +shaders to create realistic shadows. You also have a slow +laptop. Normally, JME3 must sacrifice rendering and physics simulation +to try to keep up. If it takes $\frac{1}{120}$ of a user-second to +calculate $\frac{1}{60}$ game-seconds, and an additional +$\frac{1}{60}$ of a user-second to render to screen, then JME3 has +it's work cut out for it. In order to render to the screen, it will +first step the game forward by up to four physics ticks before +rendering to the screen. If it still isn't fast enough then it will +decrease the accuracy of the physics engine until game-time and user +time are synched or a certain threshold is reached, at which point the +game visibly slows down. In this case, JME3 continuously repeat a +cycle of two physics ticks, and one screen render. For every +user-second that passes, one game-second will pass, but the game will +run at 30 fps instead of 60 fps like before. + +*** With a Special Timer +Then, you change the game's timer so that user-time will be synced to +video-time. Once again, assume video encoding takes $\frac{1}{60}$ of +a user-second. + +Now, JME3 will spend $\frac{1}{120}$ of a user-second to step the +physics tick $\frac{1}{60}$ game-seconds, $\frac{1}{60}$ to draw to +the screen, and an additional $\frac{1}{60}$ to encode the video and +write the frame to disk. This is a total of $\frac{1}{24}$ +user-seconds for each $\frac{1}{60}$ game-seconds. It will take +$(\frac{60}{24} = 2.5)$ user-hours to record one game-hour and game-time +will appear to flow two-fifths as fast as user time while the game is +running. However, just as in example one, when all is said and done we +will have an hour long video at 60 fps. + + +* COMMENT proposed names for the new timer +# METRONOME +# IsoTimer +# EvenTimer +# PulseTimer +# FixedTimer +# RigidTimer +# FixedTempo +# RegularTimer +# MetronomeTimer +# ConstantTimer +# SteadyTimer + + +* =IsoTimer= records time like a metronome + +The easiest way to achieve this special timing is to create a new +timer that always reports the same framerate to JME3 every time it is +called. + + +=./jme3/src/core/com/jme3/system/IsoTimer.java= +#+include ./jme3/src/core/com/jme3/system/IsoTimer.java src java + +If an Application uses this =IsoTimer= instead of the normal one, we +can be sure that every call to =simpleUpdate=, for example, corresponds +to exactly $(\frac{1}{fps})$ seconds of game-time. + +In order to facilitate setting the =Timer= in user code, I added +getter and setter methods to =Application.java=. + +In =./jme3/src/core/com/jme3/app/Application.java= I added: +#+include ./jme3/src/core/com/jme3/app/Application.java src java :lines "340-356" + +* Encoding to Video + +Now that the issue of time is solved, we just need a function that +writes each frame to a video. We can put this function somewhere +where it will be called exactly one per frame. + +JME3 already provides exactly the class we need: the =SceneProcessor= +class can be attached to any viewport and the methods defined therein +will be called at the appropriate points in the rendering process. + +If you want to generate video from Java, a great option is [[http://www.xuggle.com/][Xuggle]]. It +takes care of everything related to video encoding and decoding and +runs on Windows, Linux and Mac. Out of all the video frameworks for +Java I personally like this one the best. + +Here is a =SceneProcessor= that uses [[http://www.xuggle.com/][Xuggle]] to write each frame to a +video file. + +=./jme3/src/core/com/jme3/app/VideoProcessor.java= +#+include ./jme3/src/core/com/jme3/app/VideoProcessor.java src java + +With this, we are able to record video! + +* Hello Video! + +I've taken [[http://code.google.com/p/jmonkeyengine/source/browse/trunk/engine/src/test/jme3test/helloworld/HelloLoop.java][=./jme3/src/test/jme3test/helloworld/HelloLoop.java=]] and +augmented it with video output as follows: + +=./jme3/src/test/jme3test/helloworld/HelloVideo.java= +#+include ./jme3/src/test/jme3test/helloworld/HelloVideo.java src java + +The videos are created in the =hello-video= directory + +#+begin_src sh :results verbatim +du -h hello-video/* +#+end_src + +#+results: +: 932K hello-video/hello-video-moving.flv +: 640K hello-video/hello-video-static.flv + +And can be immediately uploaded to youtube + +- [[http://www.youtube.com/watch?v=C8gxVAySaPg][hello-video-moving.flv]] +#+BEGIN_HTML + +#+END_HTML +- [[http://www.youtube.com/watch?v=pHcFOtIS07Q][hello-video-static.flv]] +#+BEGIN_HTML + + +#+END_HTML + + + +* Summary +It's quite easy to augment your own application to record video, +almost regardless of how complicated the actual application is. You +can also record from multiple ViewPorts as the above example shows. + +The process for adding video recording to your application is as +follows: + +Assuming you want to record at 30 fps, add: + +#+begin_src java :exports code +this.setTimer(new IsoTimer(30)); +#+end_src + +Somewhere in the initialization of your Application. Right now, you +will have to add the =setTimer= method to =Application=, but hopefully +this method will be included soon by the JMonkeyEngine3 team. + +Then, you create a =VideoProcessor= object and attach it to the +=ViewPort= from which you want to record. + +If you want to record from the game's main =ViewPort= to a file called +=/home/r/record.flv=, then add: + +#+begin_src java :exports code +viewPort.addProcessor(new VideoProcessor(new File("/home/r/record.flv"))); +#+end_src + +Do this for each =ViewPort= from which you want to record. The more +ViewPorts from which you record, the slower the game will run, but +this slowness will not affect the final video output. + +* More Examples +** Hello Physics +=HelloVideo= is boring. Let's add some video capturing to =HelloPhysics= +and create something fun! + +This example is a modified version of =HelloPhysics= that creates four +simultaneous views of the same scene of cannonballs careening into a +brick wall. + +=./jme3/src/test/jme3test/helloworld/HelloPhysicsWithVideo.java= +#+include ./jme3/src/test/jme3test/helloworld/HelloPhysicsWithVideo.java src java + +Running the program outputs four videos into the =./physics-videos= +directory. + +#+begin_src sh :exports both :results verbatim +ls ./physics-videos | grep - +#+end_src + +#+results: +: lower-left.flv +: lower-right.flv +: upper-left.flv +: upper-right.flv + +The videos are fused together with the following =gstreamer= commands: + +#+begin_src sh :results silent +cd physics-videos + +gst-launch-0.10 \ + filesrc location=./upper-right.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=640, height=480, framerate=25/1 ! \ + videobox border-alpha=0 left=-640 ! \ + videomixer name=mix ! ffmpegcolorspace ! videorate ! \ + video/x-raw-yuv, width=1280, height=480, framerate=25/1 ! \ + jpegenc ! avimux ! filesink location=upper.flv \ + \ + filesrc location=./upper-left.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=640, height=480, framerate=25/1 ! \ + videobox right=-640 ! mix. +#+end_src + +#+begin_src sh :results silent +cd physics-videos + +gst-launch-0.10 \ + filesrc location=./lower-left.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=640, height=480, framerate=25/1 ! \ + videobox border-alpha=0 left=-640 ! \ + videomixer name=mix ! ffmpegcolorspace ! videorate ! \ + video/x-raw-yuv, width=1280, height=480, framerate=25/1 ! \ + jpegenc ! avimux ! filesink location=lower.flv \ + \ + filesrc location=./lower-right.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=640, height=480, framerate=25/1 ! \ + videobox right=-640 ! mix. +#+end_src + +#+begin_src sh :results silent +cd physics-videos + +gst-launch-0.10 \ + filesrc location=./upper.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=1280, height=480, framerate=25/1 ! \ + videobox border-alpha=0 bottom=-480 ! \ + videomixer name=mix ! ffmpegcolorspace ! videorate ! \ + video/x-raw-yuv, width=1280, height=960, framerate=25/1 ! \ + jpegenc ! avimux ! filesink location=../youtube/helloPhysics.flv \ + \ + filesrc location=./lower.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=1280, height=480, framerate=25/1 ! \ + videobox top=-480 ! mix. +#+end_src + +#+begin_src sh :results verbatim +du -h youtube/helloPhysics.flv +#+end_src + +#+results: +: 180M physics-videos/helloPhysics.flv + + +Thats a terribly large size! +Let's compress it: + +** Compressing the HelloPhysics Video +First, we'll scale the video, then, we'll decrease it's bitrate. The +end result will be perfect for upload to YouTube. + +#+begin_src sh :results silent +cd youtube + +gst-launch-0.10 \ + filesrc location=./helloPhysics.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + `: # the original size is 1280 by 960` \ + video/x-raw-yuv, width=1280, height=960, framerate=25/1 ! \ + videoscale ! \ + `: # here we scale the video down` \ + video/x-raw-yuv, width=640, height=480, framerate=25/1 ! \ + `: # and here we limit the bitrate` \ + theoraenc bitrate=1024 quality=30 ! \ + oggmux ! progressreport update-freq=1 ! \ + filesink location=./helloPhysics.ogg +#+end_src + +#+begin_src sh :results verbatim +du -h youtube/helloPhysics.ogg +#+end_src + +#+results: +: 13M youtube/helloPhysics.ogg + +[[http://www.youtube.com/watch?v=WIJt9aRGusc][helloPhysics.ogg]] + +#+begin_html + +#+end_html + + +** COMMENT failed attempts +Let's try the [[http://diracvideo.org/][Dirac]] video encoder. + +#+begin_src sh :results verbatim +cd youtube +START=$(date +%s) +gst-launch-0.10 \ + filesrc location=./helloPhysics.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=1280, height=960, framerate=25/1 ! \ + schroenc ! filesink location=./helloPhysics.drc > /dev/null +echo `expr $(( $(date +%s) - $START))` +#+end_src + + +#+results: +: 142 + +That took 142 seconds. Let's see how it does compression-wise: + +#+begin_src sh :results verbatim +du -h ./youtube/helloPhysics.drc +#+end_src + +#+results: +: 22M ./physics-videos/helloPhysics.drc + + +#+begin_src sh :results verbatim +cd youtube +START=$(date +%s) +gst-launch-0.10 \ + filesrc location=./helloPhysics.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=1280, height=960, framerate=25/1 ! \ + theoraenc ! oggmux ! filesink location=./helloPhysics.ogg \ + > /dev/null +echo `expr $(( $(date +%s) - $START))` +#+end_src + +#+results: +: 123 + +#+begin_src sh :results verbatim +du -h youtube/helloPhysics.ogg +#+end_src + +#+results: +: 59M physics-videos/helloPhysics.ogg + + +=*.drc= files can not be uploaded to YouTube, so I'll go for the +avi file. + + +** COMMENT text for videos +Video output from JMonkeyEngine3 (www.jmonkeyengine.org/) using Xuggle +(www.xuggle.com/). Everything is explained at +http://aurellem.org/cortex/capture-video.html. + + +Video output from JMonkeyEngine3 (www.jmonkeyengine.org/) HelloPhysics +demo application using Xuggle (www.xuggle.com/). Everything is +explained at http://aurellem.org/cortex/capture-video.html. Here, +four points of view are simultaneously recorded and then glued +together later. + + JME3 Xuggle Aurellem video capture + + +* Sample Videos +I encoded most of the original JME3 Hello demos for your viewing +pleasure, all using the =VideoProcessor= and =IsoTimer= classes. + +** HelloTerrain +[[http://youtu.be/5_4wyDFwrVQ][HelloTerrain.avi]] + +#+begin_html + +#+end_html + +** HelloAssets +[[http://www.youtube.com/watch?v=oGg-Q6k1BM4][HelloAssets.avi]] + +#+begin_html + +#+end_html + +** HelloEffects +[[http://www.youtube.com/watch?v=TuxlLMe53hA][HelloEffects]] + +#+begin_html + +#+end_html + +** HelloCollision +[[http://www.youtube.com/watch?v=GPlvJkiZfFw][HelloCollision.avi]] + +#+begin_html + +#+end_html + +** HelloAnimation +[[http://www.youtube.com/watch?v=SDCfOSPYUkg][HelloAnimation.avi]] + +#+begin_html + +#+end_html + +** HelloNode +[[http://www.youtube.com/watch?v=pL-0fR0-ilQ][HelloNode.avi]] + +#+begin_html + +#+end_html + +** HelloLoop +[[http://www.youtube.com/watch?v=mosZzzcdE5w][HelloLoop.avi]] + +#+begin_html + +#+end_html + + +*** COMMENT x-form the other stupid +progressreport update-freq=1 + +gst-launch-0.10 \ + filesrc location=./helloPhy ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=1280, height=960, framerate=25/1 ! \ + x264enc ! avimux ! filesink location=helloPhysics.avi \ + + +gst-launch-0.10 \ + filesrc location=./HelloAnimationStatic.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=640, height=480, framerate=25/1 ! \ + videobox border-alpha=0 left=-640 ! \ + videomixer name=mix ! ffmpegcolorspace ! videorate ! \ + video/x-raw-yuv, width=1280, height=480, framerate=25/1 ! \ + x264enc ! avimux ! progressreport update-freq=1 ! \ + filesink location=../youtube/HelloAnimation.avi \ + \ + filesrc location=./HelloAnimationMotion.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=640, height=480, framerate=25/1 ! \ + videobox right=-640 ! mix. + +gst-launch-0.10 \ + filesrc location=./HelloCollisionMotion.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=800, height=600, framerate=25/1 ! \ + x264enc bitrate=1024 ! avimux ! \ + filesink location=../youtube/HelloCollision.avi + +gst-launch-0.10 \ + filesrc location=./HelloEffectsStatic.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=640, height=480, framerate=25/1 ! \ + videobox border-alpha=0 left=-640 ! \ + videomixer name=mix ! ffmpegcolorspace ! videorate ! \ + video/x-raw-yuv, width=1280, height=480, framerate=25/1 ! \ + x264enc bitrate=1024 ! avimux ! progressreport update-freq=1 ! \ + filesink location=../youtube/HelloEffects.avi \ + \ + filesrc location=./HelloEffectsMotion.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=640, height=480, framerate=25/1 ! \ + videobox right=-640 ! mix. + +gst-launch-0.10 \ + filesrc location=./HelloTerrainMotion.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=800, height=600, framerate=25/1 ! \ + x264enc bitrate=1024 ! avimux ! \ + filesink location=../youtube/HelloTerrain.avi + + +gst-launch-0.10 \ + filesrc location=./HelloAssetsStatic.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=640, height=480, framerate=25/1 ! \ + videobox border-alpha=0 left=-640 ! \ + videomixer name=mix ! ffmpegcolorspace ! videorate ! \ + video/x-raw-yuv, width=1280, height=480, framerate=25/1 ! \ + x264enc bitrate=1024 ! avimux ! progressreport update-freq=1 ! \ + filesink location=../youtube/HelloAssets.avi \ + \ + filesrc location=./HelloAssetsMotion.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=640, height=480, framerate=25/1 ! \ + videobox right=-640 ! mix. + + +gst-launch-0.10 \ + filesrc location=./HelloNodeStatic.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=640, height=480, framerate=25/1 ! \ + videobox border-alpha=0 left=-640 ! \ + videomixer name=mix ! ffmpegcolorspace ! videorate ! \ + video/x-raw-yuv, width=1280, height=480, framerate=25/1 ! \ + x264enc bitrate=1024 ! avimux ! progressreport update-freq=1 ! \ + filesink location=../youtube/HelloNode.avi \ + \ + filesrc location=./HelloNodeMotion.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=640, height=480, framerate=25/1 ! \ + videobox right=-640 ! mix. + +gst-launch-0.10 \ + filesrc location=./HelloLoopStatic.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=640, height=480, framerate=25/1 ! \ + videobox border-alpha=0 left=-640 ! \ + videomixer name=mix ! ffmpegcolorspace ! videorate ! \ + video/x-raw-yuv, width=1280, height=480, framerate=25/1 ! \ + x264enc bitrate=1024 ! avimux ! progressreport update-freq=1 ! \ + filesink location=../youtube/HelloLoop.avi \ + \ + filesrc location=./HelloLoopMotion.flv ! decodebin ! \ + videoscale ! ffmpegcolorspace ! \ + video/x-raw-yuv, width=640, height=480, framerate=25/1 ! \ + videobox right=-640 ! mix. + diff -r 000000000000 -r 92f8d83b5d0b org/cortex.org --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/org/cortex.org Sun Oct 16 05:12:19 2011 -0700 @@ -0,0 +1,1738 @@ +#+title: Simulated Senses +#+author: Robert McIntyre +#+email: rlm@mit.edu +#+MATHJAX: align:"left" mathml:t path:"../aurellem/src/MathJax/MathJax.js" +#+STYLE: +#+BABEL: :exports both :noweb yes :cache no :mkdirp yes +#+INCLUDE: ../aurellem/src/templates/level-0.org +#+description: Simulating senses for AI research using JMonkeyEngine3 + +* Background +Artificial Intelligence has tried and failed for more than half a +century to produce programs as flexible, creative, and “intelligent” +as the human mind itself. Clearly, we are still missing some important +ideas concerning intelligent programs or we would have strong AI +already. What idea could be missing? + +When Turing first proposed his famous “Turing Test” in the +groundbreaking paper [[./sources/turing.pdf][/Computing Machines and Intelligence/]], he gave +little importance to how a computer program might interact with the +world: + +#+BEGIN_QUOTE +\ldquo{}We need not be too concerned about the legs, eyes, etc. The example of +Miss Helen Keller shows that education can take place provided that +communication in both directions between teacher and pupil can take +place by some means or other.\rdquo{} +#+END_QUOTE + +And from the example of Hellen Keller he went on to assume that the +only thing a fledgling AI program could need by way of communication +is a teletypewriter. But Hellen Keller did possess vision and hearing +for the first few months of her life, and her tactile sense was far +more rich than any text-stream could hope to achieve. She possessed a +body she could move freely, and had continual access to the real world +to learn from her actions. + +I believe that our programs are suffering from too little sensory +input to become really intelligent. Imagine for a moment that you +lived in a world completely cut off form all sensory stimulation. You +have no eyes to see, no ears to hear, no mouth to speak. No body, no +taste, no feeling whatsoever. The only sense you get at all is a +single point of light, flickering on and off in the void. If this was +your life from birth, you would never learn anything, and could never +become intelligent. Actual humans placed in sensory deprivation +chambers experience hallucinations and can begin to loose their sense +of reality in as little as 15 minutes[sensory-deprivation]. Most of +the time, the programs we write are in exactly this situation. They do +not interface with cameras and microphones, and they do not control a +real or simulated body or interact with any sort of world. + + +* Simulation vs. Reality +I want demonstrate that multiple senses are what enable +intelligence. There are two ways of playing around with senses and +computer programs: + +The first is to go entirely with simulation: virtual world, virtual +character, virtual senses. The advantages are that when everything is +a simulation, experiments in that simulation are absolutely +reproducible. It's also easier to change the character and world to +explore new situations and different sensory combinations. + + +** Issues with Simulation + +If the world is to be simulated on a computer, then not only do you +have to worry about whether the character's senses are rich enough to +learn from the world, but whether the world itself is rendered with +enough detail and realism to give enough working material to the +character's senses. To name just a few difficulties facing modern +physics simulators: destructibility of the environment, simulation of +water/other fluids, large areas, nonrigid bodies, lots of objects, +smoke. I don't know of any computer simulation that would allow a +character to take a rock and grind it into fine dust, then use that +dust to make a clay sculpture, at least not without spending years +calculating the interactions of every single small grain of +dust. Maybe a simulated world with today's limitations doesn't provide +enough richness for real intelligence to evolve. + +** Issues with Reality + +The other approach for playing with senses is to hook your software up +to real cameras, microphones, robots, etc., and let it loose in the +real world. This has the advantage of eliminating concerns about +simulating the world at the expense of increasing the complexity of +implementing the senses. Instead of just grabbing the current rendered +frame for processing, you have to use an actual camera with real +lenses and interact with photons to get an image. It is much harder to +change the character, which is now partly a physical robot of some +sort, since doing so involves changing things around in the real world +instead of modifying lines of code. While the real world is very rich +and definitely provides enough stimulation for intelligence to develop +as evidenced by our own existence, it is also uncontrollable in the +sense that a particular situation cannot be recreated perfectly or +saved for later use. It is harder to conduct science because it is +harder to repeat an experiment. The worst thing about using the real +world instead of a simulation is the matter of time. Instead of +simulated time you get the constant and unstoppable flow of real +time. This severely limits the sorts of software you can use to +program the AI because all sense inputs must be handled in real +time. Complicated ideas may have to be implemented in hardware or may +simply be impossible given the current speed of our +processors. Contrast this with a simulation, in which the flow of time +in the simulated world can be slowed down to accommodate the +limitations of the character's programming. In terms of cost, doing +everything in software is far cheaper than building custom real-time +hardware. All you need is a laptop and some patience. + +* Choose a Simulation Engine + +Mainly because of issues with controlling the flow of time, I chose to +simulate both the world and the character. I set out to make a minimal +world in which I could embed a character with multiple senses. My main +goal is to make an environment where I can perform further experiments +in simulated senses. + +As Carl Sagan once said, "If you wish to make an apple pie from +scratch, you must first invent the universe.” I examined many +different 3D environments to try and find something I would use as the +base for my simulation; eventually the choice came down to three +engines: the Quake II engine, the Source Engine, and jMonkeyEngine. + +** Quake II/Jake2 + +I spent a bit more than a month working with the Quake II Engine from +ID software to see if I could use it for my purposes. All the source +code was released by ID software into the Public Domain several years +ago, and as a result it has been ported and modified for many +different reasons. This engine was famous for its advanced use of +realistic shading and had decent and fast physics +simulation. Researchers at Princeton [[http://www.nature.com/nature/journal/v461/n7266/pdf/nature08499.pdf][used this code]] to study spatial +information encoding in the hippocampal cells of rats. Those +researchers created a special Quake II level that simulated a maze, +and added an interface where a mouse could run around inside a ball in +various directions to move the character in the simulated maze. They +measured hippocampal activity during this exercise to try and tease +out the method in which spatial data was stored in that area of the +brain. I find this promising because if a real living rat can interact +with a computer simulation of a maze in the same way as it interacts +with a real-world maze, then maybe that simulation is close enough to +reality that a simulated sense of vision and motor control interacting +with that simulation could reveal useful information about the real +thing. It happens that there is a Java port of the original C source +code called Jake2. The port demonstrates Java's OpenGL bindings and +runs anywhere from 90% to 105% as fast as the C version. After +reviewing much of the source of Jake2, I eventually rejected it +because the engine is too tied to the concept of a first-person +shooter game. One of the problems I had was that there does not seem +to be any easy way to attach multiple cameras to a single +character. There are also several physics clipping issues that are +corrected in a way that only applies to the main character and does +not apply to arbitrary objects. While there is a large community of +level modders, I couldn't find a community to support using the engine +to make new things. + +** Source Engine + +The Source Engine evolved from the Quake II and Quake I engines and is +used by Valve in the Half-Life series of games. The physics simulation +in the Source Engine is quite accurate and probably the best out of +all the engines I investigated. There is also an extensive community +actively working with the engine. However, applications that use the +Source Engine must be written in C++, the code is not open, it only +runs on Windows, and the tools that come with the SDK to handle models +and textures are complicated and awkward to use. + +** jMonkeyEngine + +jMonkeyEngine is a new library for creating games in Java. It uses +OpenGL to render to the screen and uses screengraphs to avoid drawing +things that do not appear on the screen. It has an active community +and several games in the pipeline. The engine was not built to serve +any particular game but is instead meant to be used for any 3D +game. After experimenting with each of these three engines and a few +others for about 2 months I settled on jMonkeyEngine. I chose it +because it had the most features out of all the open projects I looked +at, and because I could then write my code in Clojure, an +implementation of LISP that runs on the JVM. + +* Setup + +First, I checked out the source to jMonkeyEngine: + +#+srcname: checkout +#+begin_src sh :results verbatim +svn checkout http://jmonkeyengine.googlecode.com/svn/trunk/engine jme3 +#+end_src + +#+results: checkout +: Checked out revision 7975. + + +Building jMonkeyEngine is easy enough: + +#+srcname: build +#+begin_src sh :results verbatim +cd jme3 +ant jar | tail -n 2 +#+end_src + +#+results: build +: BUILD SUCCESSFUL +: Total time: 15 seconds + + +Also build the javadoc: + +#+srcname: javadoc +#+begin_src sh :results verbatim +cd jme3 +ant javadoc | tail -n 2 +#+end_src + +#+results: javadoc +: BUILD SUCCESSFUL +: Total time: 12 seconds + +Now, move the jars from the compilation into the project's lib folder. + +#+srcname: move-jars +#+begin_src sh :results verbatim +mkdir -p lib +mkdir -p src +cp jme3/dist/jMonkeyEngine3.jar lib/ +cp jme3/dist/lib/* lib/ +ls lib +#+end_src + +#+results: move-jars +#+begin_example +eventbus-1.4.jar +jbullet.jar +jheora-jst-debug-0.6.0.jar +jinput.jar +jME3-jbullet.jar +jME3-lwjgl-natives.jar +jME3-testdata.jar +jME3-test.jar +jMonkeyEngine3.jar +j-ogg-oggd.jar +j-ogg-vorbisd.jar +lwjgl.jar +nifty-1.3.jar +nifty-default-controls-1.3.jar +nifty-examples-1.3.jar +nifty-lwjgl-renderer-1.3.jar +nifty-openal-soundsystem-1.0.jar +nifty-style-black-1.3.jar +nifty-style-grey-1.0.jar +noise-0.0.1-SNAPSHOT.jar +stack-alloc.jar +vecmath.jar +xmlpull-xpp3-1.1.4c.jar +#+end_example + +It's good to create a =assets= directory in the style that the +=AssetManager= will like. + +#+srcname: create-assets +#+begin_src sh :results verbatim +mkdir -p assets +mkdir -p assets/Interface +mkdir -p assets/Materials +mkdir -p assets/MatDefs +mkdir -p assets/Models +mkdir -p assets/Scenes +mkdir -p assets/Shaders +mkdir -p assets/Sounds +mkdir -p assets/Textures +tree -L 1 assets +#+end_src + +#+results: create-assets +#+begin_example +assets +|-- Interface +|-- MatDefs +|-- Materials +|-- Models +|-- Scenes +|-- Shaders +|-- Sounds +`-- Textures + +8 directories, 0 files +#+end_example + + +The java classpath should have all the jars contained in the =lib= +directory as well as the src directory. + +For example, here is the file I use to run my REPL for clojure. + +#+include: "~/swank-all" src sh :exports code + +The important thing here is that =cortex/lib/*=, =cortex/src=, and +=cortex/assets= appear on the classpath. (=cortex= is the base +directory of this project.) + +#+srcname: pwd +#+begin_src sh +pwd +#+end_src + +#+results: pwd +: /home/r/cortex + + +* Simulation Base + +** Imports +First, I'll import jme core classes. +#+srcname: import +#+begin_src clojure :results silent +(ns cortex.import + (:require swank.util.class-browse)) + +(defn import-jme3 [] + (import '[com.jme3.system AppSettings JmeSystem]) + (import '[com.jme3.app Application SimpleApplication]) + (import 'com.jme3.material.Material) + (import '[com.jme3.math Vector3f ColorRGBA Quaternion Transform]) + (import '[com.jme3.scene Node Geometry]) + (import '[com.jme3.scene.shape Box Sphere Sphere$TextureMode]) + (import 'com.jme3.font.BitmapText) + (import '[com.jme3.input KeyInput InputManager]) + (import '[com.jme3.input.controls + ActionListener AnalogListener KeyTrigger MouseButtonTrigger]) + (import '[com.jme3.asset AssetManager DesktopAssetManager] ) + (import '[com.jme3.asset.plugins HttpZipLocator ZipLocator]) + (import '[com.jme3.light PointLight DirectionalLight]) + (import '[com.jme3.animation AnimControl Skeleton Bone]) + (import '[com.jme3.bullet.collision.shapes + MeshCollisionShape SphereCollisionShape BoxCollisionShape]) + (import 'com.jme3.renderer.queue.RenderQueue$ShadowMode) + (import 'jme3test.TestChooser) + (import '[com.jme3.bullet PhysicsTickListener PhysicsSpace]) + (import '[com.jme3.bullet.joints SixDofJoint HingeJoint + SliderJoint Point2PointJoint ConeJoint])) + + +(defmacro permissive-import* [class-symbol] + `(try + (import ~class-symbol) + (catch Exception e# + (println "can't import " ~class-symbol)))) + +(defn permissive-import [class-symbol] + (eval (list 'cortex.import/permissive-import* class-symbol))) + +(defn selection-import [selection-fn] + (dorun + (map (comp permissive-import symbol) + (filter selection-fn + (map :name + swank.util.class-browse/available-classes))))) + +(defn mega-import-jme3 + "ALL the jme classes. For REPL use." + [] + (selection-import + #(and + (.startsWith % "com.jme3.") + ;; Don't import the Lwjgl stuff since it can throw exceptions + ;; upon being loaded. + (not (re-matches #".*Lwjgl.*" %))))) +#+end_src + +The =mega-import-jme3= is quite usefull for debugging purposes since +it allows completion for almost all of JME's classes + +** Simplification +*** World + +It is comvienent to wrap the JME elements that deal with creating a +world, creation of basic objects, and Keyboard input with a nicer +interface (at least for my purposes). + +#+srcname: world-inputs +#+begin_src clojure :results silent +(ns cortex.world) +(require 'cortex.import) +(use 'clojure.contrib.def) +(rlm.rlm-commands/help) +(cortex.import/mega-import-jme3) + +(defvar *app-settings* + (doto (AppSettings. true) + (.setFullscreen false) + (.setTitle "Aurellem.") + ;; disable 32 bit stuff for now + ;;(.setAudioRenderer "Send") + ) + "These settings control how the game is displayed on the screen for + debugging purposes. Use binding forms to change this if desired. + Full-screen mode does not work on some computers.") + +(defn asset-manager + "returns a new, configured assetManager" [] + (JmeSystem/newAssetManager + (.getResource + (.getContextClassLoader (Thread/currentThread)) + "com/jme3/asset/Desktop.cfg"))) + +(defmacro no-exceptions + "Sweet relief like I never knew." + [& forms] + `(try ~@forms (catch Exception e# (.printStackTrace e#)))) + +(defn thread-exception-removal [] + (println "removing exceptions from " (Thread/currentThread)) + (.setUncaughtExceptionHandler + (Thread/currentThread) + (proxy [Thread$UncaughtExceptionHandler] [] + (uncaughtException + [thread thrown] + (println "uncaught-exception thrown in " thread) + (println (.getMessage thrown)))))) + +(def println-repl (bound-fn [& args] (apply println args))) + +(use '[pokemon [lpsolve :only [constant-map]]]) + +(defn no-op [& _]) + +(defn all-keys + "Construct a map of strings representing all the manual inputs from + either the keyboard or mouse." + [] + (let [inputs (constant-map KeyInput)] + (assoc + (zipmap (map (fn [field] + (.toLowerCase (re-gsub #"_" "-" field))) (vals inputs)) + (map (fn [val] (KeyTrigger. val)) (keys inputs))) + ;;explicitly add mouse controls + "mouse-left" (MouseButtonTrigger. 0) + "mouse-middle" (MouseButtonTrigger. 2) + "mouse-right" (MouseButtonTrigger. 1)))) + +(defn initialize-inputs + "more java-interop cruft to establish keybindings for a particular virtual world" + [game input-manager key-map] + (doall (map (fn [[name trigger]] + (.addMapping ^InputManager input-manager + name (into-array (class trigger) [trigger]))) key-map)) + (doall (map (fn [name] + (.addListener ^InputManager input-manager game + (into-array String [name]))) (keys key-map)))) + +#+end_src + +These functions are all for debug controlling of the world through +keyboard and mouse. + +We reuse =constant-map= from =pokemon.lpsolve= to get the numerical +values for all the keys defined in the =KeyInput= class. The +documentation for =constant-map= is: + +#+begin_src clojure :results output +(doc pokemon.lpsolve/constant-map) +#+end_src + +#+results: +: ------------------------- +: pokemon.lpsolve/constant-map +: ([class]) +: Takes a class and creates a map of the static constant integer +: fields with their names. This helps with C wrappers where they have +: just defined a bunch of integer constants instead of enums + + +Then, =all-keys= converts the constant names like =KEY_J= to the more +clojure-like =key-j=, and returns a map from these keys to +jMonkeyEngine KeyTrigger objects, the use of which will soon become +apparent. =all-keys= also adds the three mouse button controls to the +map. + +#+srcname: world +#+begin_src clojure :results silent +(in-ns 'cortex.world) + +(defn traverse + "apply f to every non-node, deeply" + [f node] + (if (isa? (class node) Node) + (dorun (map (partial traverse f) (.getChildren node))) + (f node))) + +(def gravity (Vector3f. 0 -9.81 0)) + +(defn world + [root-node key-map setup-fn update-fn] + (let [physics-manager (BulletAppState.) + shadow-renderer (BasicShadowRenderer. (asset-manager) (int 256)) + ;;maybe use a better shadow renderer someday! + ;;shadow-renderer (PssmShadowRenderer. (asset-manager) 256 1) + ] + (doto + (proxy [SimpleApplication ActionListener] [] + (simpleInitApp + [] + (no-exceptions + (.setTimer this (IsoTimer. 60)) + ;; Create key-map. + (.setFrustumFar (.getCamera this) 300) + (initialize-inputs this (.getInputManager this) (all-keys)) + ;; Don't take control of the mouse + (org.lwjgl.input.Mouse/setGrabbed false) + ;; add all objects to the world + (.attachChild (.getRootNode this) root-node) + ;; enable physics + ;; add a physics manager + (.attach (.getStateManager this) physics-manager) + (.setGravity (.getPhysicsSpace physics-manager) gravity) + + + ;; go through every object and add it to the physics manager + ;; if relavant. + (traverse (fn [geom] + (dorun + (for [n (range (.getNumControls geom))] + (do + (println-repl "adding control " (.getControl geom n)) + (.add (.getPhysicsSpace physics-manager) + (.getControl geom n)))))) + (.getRootNode this)) + ;;(.addAll (.getPhysicsSpace physics-manager) (.getRootNode this)) + + (setup-fn this) + (.setDirection shadow-renderer + (.normalizeLocal (Vector3f. -1 -1 -1))) + (.addProcessor (.getViewPort this) shadow-renderer) + (.setShadowMode (.getRootNode this) RenderQueue$ShadowMode/Off) + )) + (simpleUpdate + [tpf] + (no-exceptions + (update-fn this tpf))) + (onAction + [binding value tpf] + ;; whenever a key is pressed, call the function returned from + ;; key-map. + (no-exceptions + (if-let [react (key-map binding)] + (react this value))))) + ;; don't show a menu to change options. + + (.setShowSettings false) + (.setPauseOnLostFocus false) + (.setSettings *app-settings*)))) + +(defn apply-map + "Like apply, but works for maps and functions that expect an implicit map + and nothing else as in (fn [& {}]). +------- Example ------- + (defn jjj [& {:keys [www] :or {www \"oph yeah\"} :as env}] (println www)) + (apply-map jjj {:www \"whatever\"}) + -->\"whatever\"" + [fn m] + (apply fn (reduce #(into %1 %2) [] m))) + +#+end_src + + +=world= is the most important function here. +*** TODO more documentation + +#+srcname: world-shapes +#+begin_src clojure :results silent +(in-ns 'cortex.world) +(defrecord shape-description + [name + color + mass + friction + texture + material + position + rotation + shape + physical?]) + +(def base-shape + (shape-description. + "default-shape" + false + ;;ColorRGBA/Blue + 1.0 ;; mass + 1.0 ;; friction + ;; texture + "Textures/Terrain/BrickWall/BrickWall.jpg" + ;; material + "Common/MatDefs/Misc/Unshaded.j3md" + Vector3f/ZERO + Quaternion/IDENTITY + (Box. Vector3f/ZERO 0.5 0.5 0.5) + true)) + +(defn make-shape + [#^shape-description d] + (let [mat (Material. (asset-manager) (:material d)) + geom (Geometry. (:name d) (:shape d))] + (if (:texture d) + (let [key (TextureKey. (:texture d))] + (.setGenerateMips key true) + (.setTexture mat "ColorMap" (.loadTexture (asset-manager) key)))) + (if (:color d) (.setColor mat "Color" (:color d))) + (.setMaterial geom mat) + (if-let [rotation (:rotation d)] (.rotate geom rotation)) + (.setLocalTranslation geom (:position d)) + (if (:physical? d) + (let [impact-shape (doto (GImpactCollisionShape. (.getMesh geom)) (.setMargin 0)) + physics-control (RigidBodyControl. + impact-shape + (float (:mass d)))] + (.createJmeMesh impact-shape) + (.addControl geom physics-control) + ;;(.setSleepingThresholds physics-control (float 0) (float 0)) + (.setFriction physics-control (:friction d)))) + ;;the default is to keep this node in the physics engine forever. + ;;these commands must come after the control is added to the geometry. + ;; + geom)) + +(defn box + ([l w h & {:as options}] + (let [options (merge base-shape options)] + (make-shape (assoc options + :shape (Box. l w h))))) + ([] (box 0.5 0.5 0.5))) + +(defn sphere + ([r & {:as options}] + (let [options (merge base-shape options)] + (make-shape (assoc options + :shape (Sphere. 32 32 (float r)))))) + ([] (sphere 0.5))) + +(defn add-element [game node] + (.addAll + (.getPhysicsSpace + (.getState + (.getStateManager game) + BulletAppState)) + node) + (.attachChild (.getRootNode game) node)) + +(defn set-gravity* + [game gravity] + (traverse + (fn [geom] + (if-let + [control (.getControl geom RigidBodyControl)] + (do + (.setGravity control gravity) + (.applyImpulse control Vector3f/ZERO Vector3f/ZERO) + ))) + (.getRootNode game))) + +#+end_src + +These are convienence functions for creating JME objects and +manipulating a world. + +#+srcname: world-view +#+begin_src clojure :results silent +(in-ns 'cortex.world) + +(defprotocol Viewable + (view [something])) + +(extend-type com.jme3.scene.Geometry + Viewable + (view [geo] + (view (doto (Node.)(.attachChild geo))))) + +(extend-type com.jme3.scene.Node + Viewable + (view [node] + (.start + (world node + {} + (fn [world] + (.enableDebug + (.getPhysicsSpace + (.getState + (.getStateManager world) + BulletAppState)) + (asset-manager)) + (set-gravity* world Vector3f/ZERO) +;; (set-gravity* world (Vector3f. 0 (float -0.4) 0)) + (let [sun (doto (DirectionalLight.) + (.setDirection (.normalizeLocal (Vector3f. 1 0 -2))) + (.setColor ColorRGBA/White))] + (.addLight (.getRootNode world) sun))) + no-op)))) + +(defn position-camera [game] + (doto (.getCamera game) + (.setLocation (Vector3f. 0 6 6)) + (.lookAt Vector3f/ZERO (Vector3f. 0 1 0)))) + +#+end_src + +Here I make the =Viewable= protocol and extend it to JME's types. Now +hello-world can be written as easily as: + +#+begin_src clojure :results silent +(cortex.world/view (cortex.world/box)) +#+end_src + +** Hello +Here are the jmonkeyengine "Hello" programs translated to clojure. +*** Hello Simple App +Here is the hello world example for jme3 in clojure. +It's a more or less direct translation from the java source +from +http://jmonkeyengine.org/wiki/doku.php/jme3:beginner:hello_simpleapplication. + +Of note is the fact that since we don't have access to the +=AssetManager= via extendig =SimpleApplication=, we have to build one +ourselves. + +#+srcname: hello-simple-app +#+begin_src clojure :results silent +(ns hello.hello-simple-app) +(require 'cortex.import) +(use 'clojure.contrib.def) +(rlm.rlm-commands/help) +(cortex.import/import-jme3) +(use 'cortex.world) + + +(def cube (Box. Vector3f/ZERO 1 1 1)) + +(def geom (Geometry. "Box" cube)) + +(def mat (Material. (asset-manager) "Common/MatDefs/Misc/Unshaded.j3md")) + +(.setColor mat "Color" ColorRGBA/Blue) + +(.setMaterial geom mat) + +(defn simple-app [] + (doto + (proxy [SimpleApplication] [] + (simpleInitApp + [] + ;; Don't take control of the mouse + (org.lwjgl.input.Mouse/setGrabbed false) + (.attachChild (.getRootNode this) geom))) + ;; don't show a menu to change options. + (.setShowSettings false) + (.setPauseOnLostFocus false) + (.setSettings *app-settings*))) +#+end_src + +Running this program will begin a new jMonkeyEngine game which +displays a single blue cube. + +#+begin_src clojure :exports code :results silent +(.start (hello.hello-simple-app/simple-app)) +#+end_src + +#+caption: the simplest JME game. +[[./images/simple-app.jpg]] + + + +*** Hello Physics +From http://jmonkeyengine.org/wiki/doku.php/jme3:beginner:hello_physics + +#+srcname: brick-wall-header +#+begin_src clojure :results silent +(ns hello.brick-wall) +(require 'cortex.import) +(use 'clojure.contrib.def) +(rlm.rlm-commands/help) +(cortex.import/mega-import-jme3) +(use '[pokemon [lpsolve :only [constant-map]]]) +(use 'cortex.world) +#+end_src + +#+srcname: brick-wall-body +#+begin_src clojure :results silent +(in-ns 'hello.brick-wall) + +(defn floor + "make a sturdy, unmovable physical floor" + [] + (box 20 1 20 :mass 0 :color false :position (Vector3f. 0 -2 0))) + +(def brick-length 0.48) +(def brick-width 0.24) +(def brick-height 0.12) + + +(defn brick* [position] + (doto (box brick-length brick-height brick-width + :position position :name "brick" + :material "Common/MatDefs/Misc/Unshaded.j3md" + :texture "Textures/Terrain/BrickWall/BrickWall.jpg" + :mass 36) + (-> + (.getMesh) + (.scaleTextureCoordinates (Vector2f. 1 0.5))) + ;;(.setShadowMode RenderQueue$ShadowMode/CastAndReceive) + ) + ) + +(defn inception-brick-wall + "construct a physical brick wall" + [] + (let [node (Node. "brick-wall")] + (dorun + (map (comp #(.attachChild node %) brick*) + (for + [x (range 15) + y (range 10) + z (range 1)] + (Vector3f. + (* brick-length x 1.03) + (* brick-width y y 10) + (* brick-height z))))) + node)) + +(defn gravity-toggle + [new-value] + (fn [game value] + (println-repl "set gravity to " new-value) + (if value + (set-gravity* game new-value) + (set-gravity* game gravity)))) + +(defn fire-cannon-ball [] + (fn [game value] + (if (not value) + (let [camera (.getCamera game) + cannon-ball + (sphere 0.7 + :material "Common/MatDefs/Misc/Unshaded.j3md" + :texture "Textures/PokeCopper.jpg" + :position + (.add (.getLocation camera) + (.mult (.getDirection camera) (float 1))) + :mass 3)] ;200 0.05 + (.setShadowMode cannon-ball RenderQueue$ShadowMode/CastAndReceive) + (.setLinearVelocity + (.getControl cannon-ball RigidBodyControl) + (.mult (.getDirection camera) (float 50))) ;50 + (add-element game cannon-ball))))) + +(defn floor* [] + (doto (box 10 0.1 5 :name "floor" ;10 0.1 5 ; 240 0.1 240 + :material "Common/MatDefs/Misc/Unshaded.j3md" + :texture "Textures/Terrain/Pond/Pond.png" + :position (Vector3f. 0 -0.1 0 ) + :mass 0) + (-> + (.getMesh) + (.scaleTextureCoordinates (Vector2f. 3 6)));64 64 + (-> + (.getMaterial) + (.getTextureParam "ColorMap") + (.getTextureValue) + (.setWrap Texture$WrapMode/Repeat)) + (.setShadowMode RenderQueue$ShadowMode/Receive) + )) + +(defn brick-wall* [] + (let [node (Node. "brick-wall")] + (dorun + (map + (comp #(.attachChild node %) brick*) + (for [y (range 15) + x (range 4) + z (range 1)] + (Vector3f. + (+ (* 2 x brick-length) + (if (even? (+ y z)) + (/ brick-length 4) (/ brick-length -4))) + (+ (* brick-height (inc (* 2 y)))) + (* 2 z brick-width) )))) + (.setShadowMode node RenderQueue$ShadowMode/CastAndReceive) + node)) + +(defn brick-wall-game-run [] + (doto + (world + (doto (Node.) (.attachChild (floor*)) + (.attachChild (brick-wall*)) + ) + {"key-i" (gravity-toggle (Vector3f. 0 0 -9.81)) + "key-m" (gravity-toggle (Vector3f. 0 0 9.81)) + "key-l" (gravity-toggle (Vector3f. 9.81 0 0)) + "key-j" (gravity-toggle (Vector3f. -9.81 0 0)) + "key-k" (gravity-toggle Vector3f/ZERO) + "key-u" (gravity-toggle (Vector3f. 0 9.81 0)) + "key-o" (gravity-toggle (Vector3f. 0 -9.81 0)) + "key-f" (fn[game value] + (if (not value) (add-element game (brick-wall*)))) + "key-return" (fire-cannon-ball)} + position-camera + (fn [& _])) + (.start))) +#+end_src + +#+begin_src clojure :results silent +(hello.brick-wall/brick-wall-game-run) +#+end_src + +#+caption: the brick wall standing +[[./images/brick-wall-standing.jpg]] + +#+caption: the brick wall after it has been knocked over by a "pok\eacute{}ball" +[[./images/brick-wall-knocked-down.jpg]] + +*** Other Brick Games +#+srcname: other-games +#+begin_src clojure :results silent +(ns cortex.other-games + {:author "Dylan Holmes"}) +(use 'cortex.world) +(use 'hello.brick-wall) +(use 'cortex.import) +(cortex.import/mega-import-jme3) + +(defn scad [position] + (doto (box 0.1 0.1 0.1 + :position position :name "brick" + :material "Common/MatDefs/Misc/Unshaded.j3md" + :texture "Textures/Terrain/BrickWall/BrickWall.jpg" + :mass 20) + (-> + (.getMesh) + (.scaleTextureCoordinates (Vector2f. 1 0.5)) + ) + (-> (.getControl RigidBodyControl) + (.setLinearVelocity (Vector3f. 0 100 0)) + ) + + ;;(.setShadowMode RenderQueue$ShadowMode/Cast) + )) + + +(defn shrapnel [] + (let [node (Node. "explosion-day")] + (dorun + (map + (comp #(.attachChild node %) scad) + (for [y (range 15) + x (range 4) + z (range 1)] + (Vector3f. + (+ (* 2 x brick-height) + (if (even? (+ y z)) (/ brick-height 4) (/ brick-height -4))) + (+ (* brick-height (inc (* 2 y)))) + (* 2 z brick-height) )))) + node)) + + +(def domino-height 0.48) +(def domino-thickness 0.12) +(def domino-width 0.24) + +(def domino-thickness 0.05) +(def domino-width 0.5) +(def domino-height 1) + +(defn domino + ([position] + (domino position (Quaternion/IDENTITY))) + ([position rotation] + (doto (box domino-width domino-height domino-thickness + :position position :name "domino" + :material "Common/MatDefs/Misc/Unshaded.j3md" + :texture "Textures/Terrain/BrickWall/BrickWall.jpg" + :mass 1 + :rotation rotation) + (.setShadowMode RenderQueue$ShadowMode/CastAndReceive) + ))) + + +(defn domino-row [] + (let [node (Node. "domino-row")] + (dorun + (map + (comp #(.attachChild node %) domino) + (for [ + z (range 10) + x (range 5) + ] + (Vector3f. + (+ (* z domino-width) (* x 5 domino-width)) + (/ domino-height 1) + (* -5.5 domino-thickness z) )))) + + node)) + +(defn domino-cycle [] + (let [node (Node. "domino-cycle")] + (dorun + (map + (comp #(.attachChild node %) (partial apply domino) ) + (for [n (range 720)] + (let [space (* domino-height 5.5) + r (fn[n] (* (+ n 3) domino-width 0.5)) + t (fn[n] (reduce + + + (map + (fn dt[n] (/ space (* 2 (Math/PI) (r n)))) + (range n)))) + t (t n) + r (r n) + ct (Math/cos t) + st (Math/sin t) + ] + (list + (Vector3f. + (* -1 r st) + (/ domino-height 1) + (* r ct)) + (.fromAngleAxis (Quaternion.) + (- (/ 3.1415926 2) t) (Vector3f. 0 1 0)) + ))) + )) + node)) + + +(defn domino-game-run [] + (doto + (world + (doto (Node.) (.attachChild (floor*)) + ) + {"key-i" (gravity-toggle (Vector3f. 0 0 -9.81)) + "key-m" (gravity-toggle (Vector3f. 0 0 9.81)) + "key-l" (gravity-toggle (Vector3f. 9.81 0 0)) + "key-j" (gravity-toggle (Vector3f. -9.81 0 0)) + "key-k" (gravity-toggle (Vector3f. 0 9.81 0) ) + "key-u" (fn[g v] ((gravity-toggle (Vector3f. 0 -0 0)) g true)) + "key-o" (gravity-toggle (Vector3f. 0 -9.81 0)) + + "key-space" + (fn[game value] + + (if (not value) + (let [d (domino (Vector3f. 0 (/ domino-height 0.25) 0) + (.fromAngleAxis (Quaternion.) + (/ Math/PI 2) (Vector3f. 0 1 0)))] + (add-element game d)))) + "key-f" + (fn[game value](if (not value) (add-element game (domino-cycle)))) + "key-return" (fire-cannon-ball)} + position-camera + (fn [& _])) + (.start))) +#+end_src + +#+begin_src clojure :results silent +(cortex.other-games/domino-game-run) +#+end_src + +#+caption: floating dominos +[[./images/dominos.jpg]] + +*** Hello Loop +#+srcname: hello-loop +#+begin_src clojure :results silent +(ns hello.loop) +(use 'cortex.world) +(use 'cortex.import) +(cortex.import/mega-import-jme3) +(rlm.rlm-commands/help) + +(defn blue-cube [] + (box 1 1 1 + :color ColorRGBA/Blue + :texture false + :material "Common/MatDefs/Misc/Unshaded.j3md" + :name "blue-cube" + :physical? false)) + +(defn blue-cube-game [] + (let [cube (blue-cube) + root (doto (Node.) (.attachChild cube))] + (world root + {} + no-op + (fn [game tpf] + (.rotate cube 0.0 (* 2 tpf) 0.0))))) +#+end_src + +*** Hello Collision + +#+srcname: hello-collision +#+begin_src clojure :results silent +(ns hello.collision) +(use 'cortex.world) +(use 'cortex.import) +(use 'clojure.contrib.def) + + +(cortex.import/mega-import-jme3) +(rlm.rlm-commands/help) +(use '[hello [brick-wall :only [fire-cannon-ball brick-wall*]]]) + + +(defn environment [] + (let + [scene-model + (doto + (.loadModel + (doto (asset-manager) + (.registerLocator + "/home/r/cortex/assets/zips/town.zip" ZipLocator)) + "main.scene") + (.setLocalScale (float 2.0))) + collision-shape + (CollisionShapeFactory/createMeshShape #^Node scene-model) + landscape (RigidBodyControl. collision-shape 0)] + (.setShadowMode scene-model RenderQueue$ShadowMode/CastAndReceive) + (.addControl scene-model landscape) + scene-model)) + +(defn player-fn [] + (doto + (CharacterControl. + (CapsuleCollisionShape. (float 1.5) (float 6)(float 1)) + (float 0.05)) + (.setJumpSpeed 20) + (.setFallSpeed 30) + (.setGravity 30) ;30 + (.setPhysicsLocation (Vector3f. 0 10 0)))) + +(defn lights [] + [(doto (AmbientLight.) (.setColor (.mult (ColorRGBA. 1 1 1 1) (float 1)))) + (doto (AmbientLight.) (.setColor (.mult (ColorRGBA. 1 0.7 0 1) (float 1)))) + (doto (DirectionalLight.) + (.setColor (.mult ColorRGBA/White (float 0.9) )) + (.setDirection (.normalizeLocal (Vector3f. 2.8 -28 2.8))))]) + +(defn night-lights [] + [(doto (AmbientLight.) (.setColor (.mult (ColorRGBA. 0.275 0.467 0.784 1) (float 0.3)))) + (doto (DirectionalLight.) + (.setColor (.mult ColorRGBA/White (float 0.2) )) + (.setDirection (.normalizeLocal (Vector3f. 2.8 -28 2.8))))]) + +(def player (atom (player-fn))) + +(defn setup-fn [game] + (dorun (map #(.addLight (.getRootNode game) %) (lights))) + ;; set the color of the sky + (.setBackgroundColor (.getViewPort game) (ColorRGBA. 0.3 0.4 0.9 1)) + ;(.setBackgroundColor (.getViewPort game) (ColorRGBA. 0 0 0 1) + (doto (.getFlyByCamera game) + (.setMoveSpeed (float 100)) + (.setRotationSpeed 3)) + (.add + (.getPhysicsSpace + (.getState (.getStateManager game) BulletAppState)) + @player) + + (doto (Node.) (.attachChild (.getRootNode game)) + (.attachChild (brick-wall*)) + ) + +) + + +(def walking-up? (atom false)) +(def walking-down? (atom false)) +(def walking-left? (atom false)) +(def walking-right? (atom false)) + +(defn set-walk [walk-atom game value] + ;;(println-repl "setting stuff to " value) + (reset! walk-atom value)) + +(defn responses [] + {"key-w" (partial set-walk walking-up?) + "key-d" (partial set-walk walking-right?) + "key-s" (partial set-walk walking-down?) + "key-a" (partial set-walk walking-left?) + "key-return" (fire-cannon-ball) + "key-space" (fn [game value] (.jump @player)) + }) + +(defn update-fn + [game tpf] + (let [camera (.getCamera game) + cam-dir (.multLocal + (.clone + (.getDirection camera)) (float 0.6)) + cam-left (.multLocal + (.clone + (.getLeft camera)) (float 0.4)) + walk-direction (Vector3f. 0 0 0)] + + (cond + @walking-up? (.addLocal walk-direction cam-dir) + @walking-right? (.addLocal walk-direction (.negate cam-left)) + @walking-down? (.addLocal walk-direction (.negate cam-dir)) + @walking-left? (.addLocal walk-direction cam-left)) + (.setWalkDirection @player walk-direction) + (.setLocation camera (.getPhysicsLocation @player)))) + +(defn run-game [] + (.start + (world (environment) + (responses) + setup-fn + update-fn))) +#+end_src + +*** Hello Terrain +#+srcname: hello-terrain +#+begin_src clojure :results silent +(ns hello.terrain) +(use 'cortex.world) +(use 'cortex.import) +(use 'clojure.contrib.def) +(import jme3tools.converters.ImageToAwt) + + +(cortex.import/mega-import-jme3) +(rlm.rlm-commands/help) +(use '[hello [brick-wall :only [fire-cannon-ball brick-wall*]]]) + + +(defn setup-fn [type game] + (.setMoveSpeed (.getFlyByCamera game) 50) + (.setFrustumFar (.getCamera game) 10000) + (let [env (environment type) + cameras [(.getCamera game)] + control (TerrainLodControl. env cameras)] + ;;(.addControl env control) + (.attachChild (.getRootNode game) env))) + +(defn environment [type] + (let + [mat_terrain + (Material. (asset-manager) "Common/MatDefs/Terrain/Terrain.j3md") + grass (.loadTexture (asset-manager) "Textures/Terrain/splat/grass.jpg") + dirt (.loadTexture (asset-manager) "Textures/Terrain/splat/dirt.jpg") + rock (.loadTexture (asset-manager) "Textures/Terrain/splat/road.jpg") + heightmap-image (.loadTexture (asset-manager) + ({:mountain "Textures/Terrain/splat/mountains512.png" + :fortress "Textures/Terrain/splat/fortress512.png" + }type)) + heightmap (ImageBasedHeightMap. + (ImageToAwt/convert (.getImage heightmap-image) false true 0)) + terrain (do (.load heightmap) + (TerrainQuad. "my terrain" 65 513 (.getHeightMap heightmap))) + ] + + (dorun (map #(.setWrap % Texture$WrapMode/Repeat) + [grass dirt rock])) + + (doto mat_terrain + (.setTexture "Tex1" grass) + (.setFloat "Tex1Scale" (float 64)) + + (.setTexture "Tex2" dirt) + (.setFloat "Tex2Scale" (float 32)) + + (.setTexture "Tex3" rock) + (.setFloat "Tex3Scale" (float 128)) + + (.setTexture "Alpha" + (.loadTexture + (asset-manager) + ({:mountain "Textures/Terrain/splat/alphamap.png" + :fortress "Textures/Terrain/splat/alphamap2.png"} type)))) + + (doto terrain + (.setMaterial mat_terrain) + (.setLocalTranslation 0 -100 0) + (.setLocalScale 2 1 2)))) + + + +(defn run-terrain-game [type] + (.start + (world + (Node.) + {} + (partial setup-fn type) + no-op))) +#+end_src + + + +#+srcname: hello-animation +#+begin_src clojure :results silent +(ns hello.animation) +(use 'cortex.world) +(use 'cortex.import) +(use 'clojure.contrib.def) +(cortex.import/mega-import-jme3) +(rlm.rlm-commands/help) +(use '[hello [collision :only [lights]]]) + +(defn stand + [channel] + (doto channel + (.setAnim "stand" (float 0.5)) + (.setLoopMode LoopMode/DontLoop) + (.setSpeed (float 1)))) + +(defn anim-listener [] + (proxy [AnimEventListener] [] + (onAnimChange + [control channel animation-name] + (println-repl "RLM --- onAnimChange")) + (onAnimCycleDone + [control channel animation-name] + (if (= animation-name "Walk") + (stand channel) + )))) + +(defn setup-fn [channel game] + (dorun (map #(.addLight (.getRootNode game) %) (lights))) + ;; set the color of the sky + (.setBackgroundColor (.getViewPort game) (ColorRGBA. 0.3 0.4 0.9 1)) + ;(.setBackgroundColor (.getViewPort game) (ColorRGBA. 0 0 0 1) + (.setAnim channel "stand") + (doto (.getFlyByCamera game) + (.setMoveSpeed (float 10)) + (.setRotationSpeed 1))) + +(defn walk [channel] + (println-repl "zzz") + (doto channel + (.setAnim "Walk" (float 0.5)) + (.setLoopMode LoopMode/Loop))) + + +(defn key-map [channel] + {"key-space" (fn [game value] + (if (not value) + (walk channel)))}) + +(defn player [] + (let [model (.loadModel (asset-manager) "Models/Oto/Oto.mesh.xml") + control (.getControl model AnimControl)] + (.setLocalScale model (float 0.5)) + (.clearListeners control) + (.addListener control (anim-control)) + model)) + + + +(defn run-anim-game [] + (let [ninja (player) + control (.getControl ninja AnimControl) + channel (.createChannel control)] + (.start + (world + ninja + (key-map channel) + (partial setup-fn channel) + no-op)))) +#+end_src + +*** Hello Materials +#+srcname: material +#+begin_src clojure :results silent +(ns hello.material) +(use 'cortex.world) +(use 'cortex.import) +(use 'clojure.contrib.def) +(cortex.import/mega-import-jme3) +(rlm.rlm-commands/help) + +(defn simple-cube [] + (box 1 1 1 + :position (Vector3f. -3 1.1 0) + :material "Common/MatDefs/Misc/Unshaded.j3md" + :texture "Interface/Logo/Monkey.jpg" + :physical? false)) + +(defn leaky-box [] + (box 1 1 1 + :position (Vector3f. 3 -1 0) + :material "Common/MatDefs/Misc/ColoredTextured.j3md" + :texture "Textures/ColoredTex/Monkey.png" + :color (ColorRGBA. 1 0 1 1) + :physical? false)) + +(defn transparent-box [] + (doto + (box 1 1 0.1 + :position Vector3f/ZERO + :name "window frame" + :material "Common/MatDefs/Misc/Unshaded.j3md" + :texture "Textures/ColoredTex/Monkey.png" + :physical? false) + (-> (.getMaterial) + (.getAdditionalRenderState) + (.setBlendMode RenderState$BlendMode/Alpha)) + (.setQueueBucket RenderQueue$Bucket/Transparent))) + +(defn bumpy-sphere [] + (doto + (sphere 2 + :position (Vector3f. 0 2 -2) + :name "Shiny rock" + :material "Common/MatDefs/Light/Lighting.j3md" + :texture false + :physical? false) + (-> (.getMesh) + (doto + (.setTextureMode Sphere$TextureMode/Projected) + (TangentBinormalGenerator/generate))) + (-> (.getMaterial) + (doto + (.setTexture "DiffuseMap" (.loadTexture (asset-manager) + "Textures/Terrain/Pond/Pond.png")) + (.setTexture "NormalMap" (.loadTexture (asset-manager) + "Textures/Terrain/Pond/Pond_normal.png")) + (.setFloat "Shininess" (float 5)))) + (.rotate (float 1.6) 0 0))) + + +(defn start-game [] + (.start + (world + (let [root (Node.)] + (dorun (map #(.attachChild root %) + [(simple-cube) (leaky-box) (transparent-box) (bumpy-sphere)])) + root) + {} + (fn [world] + (let [sun (doto (DirectionalLight.) + (.setDirection (.normalizeLocal (Vector3f. 1 0 -2))) + (.setColor ColorRGBA/White))] + (.addLight (.getRootNode world) sun))) + no-op + ))) +#+end_src + + + +* The Body +** Eyes + +Ultimately I want to make creatures with eyes. Each eye can be +independely moved and should see its own version of the world +depending on where it is. +#+srcname: eyes +#+begin_src clojure +(ns body.eye) +(use 'cortex.world) +(use 'cortex.import) +(use 'clojure.contrib.def) +(cortex.import/mega-import-jme3) +(rlm.rlm-commands/help) +(import java.nio.ByteBuffer) +(import java.awt.image.BufferedImage) +(import java.awt.Color) +(import java.awt.Dimension) +(import java.awt.Graphics) +(import java.awt.Graphics2D) +(import java.awt.event.WindowAdapter) +(import java.awt.event.WindowEvent) +(import java.awt.image.BufferedImage) +(import java.nio.ByteBuffer) +(import javax.swing.JFrame) +(import javax.swing.JPanel) +(import javax.swing.SwingUtilities) +(import javax.swing.ImageIcon) +(import javax.swing.JOptionPane) +(import java.awt.image.ImageObserver) + + + +(defn scene-processor + "deals with converting FrameBuffers to BufferedImages so + that the continuation function can be defined only in terms + of what it does with BufferedImages" + [continuation] + (let [byte-buffer (atom nil) + renderer (atom nil) + image (atom nil)] + (proxy [SceneProcessor] [] + (initialize + [renderManager viewPort] + (let [cam (.getCamera viewPort) + width (.getWidth cam) + height (.getHeight cam)] + (reset! renderer (.getRenderer renderManager)) + (reset! byte-buffer + (BufferUtils/createByteBuffer + (* width height 4))) + (reset! image (BufferedImage. width height + BufferedImage/TYPE_4BYTE_ABGR)))) + (isInitialized [] (not (nil? @byte-buffer))) + (reshape [_ _ _]) + (preFrame [_]) + (postQueue [_]) + (postFrame + [#^FrameBuffer fb] + (.clear @byte-buffer) + (.readFrameBuffer @renderer fb @byte-buffer) + (Screenshots/convertScreenShot @byte-buffer @image) + (continuation @image)) + (cleanup [])))) + +(defn add-eye + "Add an eye to the world, and call continuation on + every frame produced" + [world camera continuation] + (let [width (.getWidth camera) + height (.getHeight camera) + render-manager (.getRenderManager world) + viewport (.createMainView render-manager "eye-view" camera)] + (doto viewport + (.setBackgroundColor ColorRGBA/Black) + (.setClearFlags true true true) + (.addProcessor (scene-processor continuation)) + (.attachScene (.getRootNode world))))) + +(defn make-display-frame [display width height] + (SwingUtilities/invokeLater + (fn [] + (.setPreferredSize display (Dimension. width height)) + (doto (JFrame. "Eye Camera!") + (-> (.getContentPane) (.add display)) + (.setDefaultCloseOperation JFrame/DISPOSE_ON_CLOSE) + (.pack) + (.setLocationRelativeTo nil) + (.setResizable false) + (.setVisible true))))) + +(defn image-monitor [#^BufferedImage image] + (proxy [JPanel] [] + (paintComponent + [g] + (proxy-super paintComponent g) + (locking image + (.drawImage g image 0 0 + (proxy [ImageObserver] + [] + (imageUpdate + [] + (proxy-super imageUpdate)))))))) + +(defn movie-image [] + (let [setup + (runonce + (fn [#^BufferedImage image] + (let [width (.getWidth image) + height (.getHeight image) + display (image-monitor image) + frame (make-display-frame display width height)] + display)))] + (fn [#^BufferedImage image] + (.repaint (setup image))))) + + +(defn observer + "place thy eye!" + [world camera] + (let [eye camera + width (.getWidth eye) + height (.getHeight eye)] + (no-exceptions + (add-eye + world + eye + (movie-image))))) +#+end_src + +#+srcname: test-vision +#+begin_src clojure + +(ns test.vision) +(use 'cortex.world) +(use 'cortex.import) +(use 'clojure.contrib.def) +(use 'body.eye) +(cortex.import/mega-import-jme3) +(rlm.rlm-commands/help) +(import java.nio.ByteBuffer) +(import java.awt.image.BufferedImage) +(import java.awt.Color) +(import java.awt.Dimension) +(import java.awt.Graphics) +(import java.awt.Graphics2D) +(import java.awt.event.WindowAdapter) +(import java.awt.event.WindowEvent) +(import java.awt.image.BufferedImage) +(import java.nio.ByteBuffer) +(import javax.swing.JFrame) +(import javax.swing.JPanel) +(import javax.swing.SwingUtilities) +(import javax.swing.ImageIcon) +(import javax.swing.JOptionPane) +(import java.awt.image.ImageObserver) + + +(def width 200) +(def height 200) + +(defn camera [] + (doto (Camera. width height) + (.setFrustumPerspective 45 1 1 1000) + (.setLocation (Vector3f. -3 0 -5)) + (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))) + +(defn camera2 [] + (doto (Camera. width height) + (.setFrustumPerspective 45 1 1 1000) + (.setLocation (Vector3f. 3 0 -5)) + (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))) + +(defn setup-fn [world] + (let [eye (camera) + width (.getWidth eye) + height (.getHeight eye)] + (no-exceptions + (add-eye + world + eye + (runonce visual)) + (add-eye + world + (camera2) + (runonce visual))))) + +(defn spider-eye [position] + (doto (Camera. 200 200 ) + (.setFrustumPerspective 45 1 1 1000) + (.setLocation position) + (.lookAt Vector3f/ZERO Vector3f/UNIT_Y))) + +(defn setup-fn* [world] + (let [eye (camera) + width (.getWidth eye) + height (.getHeight eye)] + ;;(.setClearFlags (.getViewPort world) true true true) + (observer world (.getCamera world)) + (observer world (spider-eye (Vector3f. 3 0 -5))) + ;;(observer world (spider-eye (Vector3f. 0 0 -5))) + ;; (observer world (spider-eye (Vector3f. -3 0 -5))) + ;; (observer world (spider-eye (Vector3f. 0 3 -5))) + ;; (observer world (spider-eye (Vector3f. 0 -3 -5))) + ;; (observer world (spider-eye (Vector3f. 3 3 -5))) + ;; (observer world (spider-eye (Vector3f. -3 3 -5))) + ;; (observer world (spider-eye (Vector3f. 3 -3 -5))) + ;; (observer world (spider-eye (Vector3f. -3 -3 -5))) + + ) + world) + +(defn test-world [] + (let [thing (box 1 1 1 :physical? false)] + (world + (doto (Node.) + (.attachChild thing)) + {} + setup-fn + (fn [world tpf] + (.rotate thing (* tpf 0.2) 0 0) + )))) + + +#+end_src + + +#+results: eyes +: #'body.eye/test-world + +Note the use of continuation passing style for connecting the eye to a +function to process the output. The example code will create two +videos of the same rotating cube from different angles, sutiable for +stereoscopic vision. + + + + + + +* COMMENT code generation +#+begin_src clojure :tangle ../src/cortex/import.clj +<> +#+end_src + +#+begin_src clojure :tangle ../src/hello/brick_wall.clj +<> +<> +#+end_src + +#+begin_src clojure :tangle ../src/hello/hello_simple_app.clj +<> +#+end_src + +#+begin_src clojure :tangle ../src/cortex/world.clj +<> +<> +<> +<> +#+end_src + +#+begin_src clojure :tangle ../src/cortex/other_games.clj +<> +#+end_src + +#+begin_src clojure :tangle ../src/hello/loop.clj +<> +#+end_src + +#+begin_src clojure :tangle ../src/hello/collision.clj +<> +#+end_src + +#+begin_src clojure :tangle ../src/hello/terrain.clj +<> +#+end_src + +#+begin_src clojure :tangle ../src/hello/animation.clj +<> +#+end_src + +#+begin_src clojure :tangle ../src/hello/material.clj +<> +#+end_src + +#+begin_src clojure :tangle ../src/body/eye.clj +<> +#+end_src + +#+begin_src clojure :tangle ../src/test/vision.clj +<> +#+end_src + + + diff -r 000000000000 -r 92f8d83b5d0b org/skin.org --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/org/skin.org Sun Oct 16 05:12:19 2011 -0700 @@ -0,0 +1,322 @@ +#+title: SKIN! +#+author: Robert McIntyre +#+email: rlm@mit.edu +#+MATHJAX: align:"left" mathml:t path:"../aurellem/src/MathJax/MathJax.js" +#+STYLE: +#+BABEL: :exports both :noweb yes :cache no :mkdirp yes +#+INCLUDE: ../aurellem/src/templates/level-0.org +#+description: Simulating touch in JMonkeyEngine + + + + +* skin! + +#+srcname: skin-main +#+begin_src clojure +(ns body.skin) +(use 'cortex.world) +(use 'cortex.import) +(use 'clojure.contrib.def) +(cortex.import/mega-import-jme3) +(rlm.rlm-commands/help) + +(import java.util.logging.Level) +(import java.util.logging.Logger) + + + +;; looks like we can use GhostControls for implementing touch. +;; for example: +(def rc (GhostControl. (BoxCollisionShape. (Vector3f. 2 2 2)))) +(def shifted-sphere + (doto (CompoundCollisionShape.) + (.addChildShape (SphereCollisionShape. 3) (Vector3f. 1 0 0)))) + +(def gc (GhostControl. shifted-sphere)) +(def b (box 1 1 1 :mass 1)) +(.addControl b rc) +(.addControl b gc) + + +(defn looksies! [] + (view b)) + +;; overlapping objects can be gotten via =.getOverlappingCount= and +;; =.getOverlappingObjects= + +;; looks like I might be able to place a small "touch-sphere" whther +;; on every triangle in the object's mesh, or at the verticies, using +;; .getTriangleCount() on the mesh gotten by .getMesh() + +;; this way, I can create a mesh and just divide up it's faces using +;; blender, and this will create the touch sensor distribution. + + +(defn make-touch-sphere [#^Geometry geom] + (let [tri (Triangle.) + mesh (.getMesh geom) + controls! (transient [])] + (dorun + (for [n (range (.getTriangleCount mesh))] + (do + (.getTriangle mesh n tri) + (.calculateCenter tri) + (let [control + (doto + (GhostControl. + (doto (CompoundCollisionShape.) + (.addChildShape + (SphereCollisionShape. (float 0.1)) + (.mult (.getCenter tri) (float 1))) + (.setMargin -0.1))) + (.setApplyPhysicsLocal true))] + + (.addControl geom control) + (conj! controls! control))))) + (persistent! controls!))) + + +(defn make-touch [#^Geometry geom] + (let [tri (Triangle.) + mesh (.getMesh geom) + controls! (transient [])] + (dorun + (for [n (range (.getTriangleCount mesh))] + (do + (.getTriangle mesh n tri) + (.calculateCenter tri) + (.calculateNormal tri) + (println-repl tri) + (println-repl (.get1 tri)) + (println-repl (.get2 tri)) + (println-repl (.get3 tri)) + (println-repl (.getCenter tri)) + (println-repl (.getNormal tri)) + (let [control + (doto + (GhostControl. + + (doto (CompoundCollisionShape.) + (.addChildShape + (SimplexCollisionShape. Vector3f/ZERO) + (.mult (.getCenter tri) (float 1))) + (.setMargin 0) + )) + (.setApplyPhysicsLocal true))] + + (.addControl geom control) + (conj! controls! control))))) + (persistent! controls!))) + +(defn make-fucked-touch [#^Geometry geom] + (let [tri (Triangle.) + mesh (.getMesh geom) + controls! (transient [])] + (dorun + (for [n (range (.getTriangleCount mesh))] + (do + (.getTriangle mesh n tri) + (.calculateCenter tri) + (.calculateNormal tri) + (println-repl tri) + (println-repl (.get1 tri)) + (println-repl (.get2 tri)) + (println-repl (.get3 tri)) + (println-repl (.getCenter tri)) + (println-repl (.getNormal tri)) + (let [control1 + (doto + (GhostControl. + (doto (CompoundCollisionShape.) + (.addChildShape + (SimplexCollisionShape. Vector3f/ZERO) + (.get1 tri)))) + (.setApplyPhysicsLocal true))] + + (.addControl geom control1) + (conj! controls! control1) + ) + + ;; (let [control1 + ;; (doto + ;; (GhostControl. + ;; (doto (CompoundCollisionShape.) + ;; (.addChildShape + ;; (SimplexCollisionShape. Vector3f/ZERO) + ;; (.get2 tri)))) + ;; (.setApplyPhysicsLocal true))] + + ;; (.addControl geom control1) + ;; (conj! controls! control1) + ;; ) + + ;; (let [control1 + ;; (doto + ;; (GhostControl. + ;; (doto (CompoundCollisionShape.) + ;; (.addChildShape + ;; (SimplexCollisionShape. Vector3f/ZERO) + ;; (.get3 tri)))) + ;; (.setApplyPhysicsLocal true))] + + ;; (.addControl geom control1) + ;; (conj! controls! control1) + + ))) + (persistent! controls!))) + + + +(use 'hello.brick-wall) + +(defn touch-reception [controls] + (let [control + (first + (filter + #(< 2 (.getOverlappingCount %)) controls))] + (if (not (nil? control)) + (println + (seq + (.getOverlappingObjects control)))))) + +(defn touch-print [controls] + (println + (map #(count (.getNonGhostOverlappingObjects %)) controls))) + +(defn set-debug-color [#^ColorRGBA color #^GhostControl gc] + (.setColor (.getMaterial (.getChild (.getDebugShape gc) 0)) "Color" color)) + +(defn html-color [html-str] + ((fn [[r g b]] (ColorRGBA. r g b (float 1))) + (map #(float (/ (Integer/parseInt % 16) 255)) + (map (partial apply str) (partition 2 html-str))))) + + +(defn color-touch [controls] + (no-exceptions + (dorun + (map + (fn [control] + (case (count (.getNonGhostOverlappingObjects control)) + 0 (set-debug-color ColorRGBA/Gray control) + 1 (set-debug-color ColorRGBA/Blue control) + 2 (set-debug-color ColorRGBA/Green control) + 3 (set-debug-color ColorRGBA/Yellow control) + 4 (set-debug-color ColorRGBA/Orange control) + 5 (set-debug-color ColorRGBA/Red control) + 6 (set-debug-color ColorRGBA/Magenta control) + 7 (set-debug-color ColorRGBA/Pink control) + 8 (set-debug-color ColorRGBA/White control))) + controls)))) + +(defn enable-debug [world] + (.enableDebug + (.getPhysicsSpace + (.getState + (.getStateManager world) + BulletAppState)) + (asset-manager))) + +(def with-debug + '(1 1 1 1 0 1 2 0 2 0 1 1 1 1 0 2 0 2 2 1 0 0 0 1 0 0 0 0 1 0)) +(def no-debug + '(1 1 1 1 0 1 2 0 2 0 1 1 1 1 0 2 0 2 2 1 0 0 0 1 0 0 0 0 1 0)) + + + +(defn transparent-sphere [] + (doto + (make-shape + (merge base-shape + {:position (Vector3f. 0 2 0) + :name "the blob." + :material "Common/MatDefs/Misc/Unshaded.j3md" + :texture "Textures/purpleWisp.png" + :physical? true + :mass 70 + :color ColorRGBA/Blue + :shape (Sphere. 10 10 1)})) + (-> (.getMaterial) + (.getAdditionalRenderState) + (.setBlendMode RenderState$BlendMode/Alpha)) + (.setQueueBucket RenderQueue$Bucket/Transparent))) + +(defn transparent-box [] + (doto + (make-shape + (merge base-shape + {:position (Vector3f. 0 2 0) + :name "the blob." + :material "Common/MatDefs/Misc/Unshaded.j3md" + :texture "Textures/purpleWisp.png" + :physical? true + :mass 70 + :color ColorRGBA/Blue + :shape (Box. 1 1 1)})) + (-> (.getMaterial) + (.getAdditionalRenderState) + (.setBlendMode RenderState$BlendMode/Alpha)) + (.setQueueBucket RenderQueue$Bucket/Transparent))) + + + +(defn no-logging [] + (.setLevel (Logger/getLogger "com.jme3") Level/OFF)) + +(defn set-accuracy [world new-accuracy] + (let [physics-manager (.getState (.getStateManager world) BulletAppState)] + (.setAccuracy (.getPhysicsSpace physics-manager) (float new-accuracy)))) + +(defn test-skin [] + (let [b + ;;(transparent-box) + (transparent-sphere) + + controls + ;;(make-touch-sphere b) + (make-touch b) + ] + + (world + (doto (Node.) (.attachChild b) + (.attachChild + (doto + (box 5 0.2 5 :mass 0 :position (Vector3f. 0 -2 0) + :material "Common/MatDefs/Misc/Unshaded.j3md" + :texture "Textures/redWisp.png") + (-> (.getMaterial) + (.getAdditionalRenderState) + (.setBlendMode RenderState$BlendMode/Alpha)) + (.setQueueBucket RenderQueue$Bucket/Transparent)))) + + {"key-return" (fire-cannon-ball) + "key-space" (fn [game value] + (touch-print controls))} + (fn [world] + (Capture/SimpleCaptureVideo world (file-str "~/out-temp/blob.avi")) + (no-logging) + (enable-debug world) + (set-accuracy world (/ 1 60)) + ) + + (fn [& _] + (touch-print controls) + (color-touch controls) + )))) + +#+end_src + + + + + + +* COMMENT code generation +#+begin_src clojure :tangle ../src/body/skin.clj +<> +#+end_src + + + +