Mercurial > lasercutter
diff src/clojureDemo/ArchLearning.clj @ 1:6d9bdaf919f7
added clojureDemo source
author | Robert McIntyre <rlm@mit.edu> |
---|---|
date | Fri, 20 Aug 2010 00:32:44 -0400 |
parents | |
children |
line wrap: on
line diff
1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 1.2 +++ b/src/clojureDemo/ArchLearning.clj Fri Aug 20 00:32:44 2010 -0400 1.3 @@ -0,0 +1,562 @@ 1.4 +(ns clojureDemo.ArchLearning 1.5 + (:gen-class 1.6 + :implements [connections.WiredBox] 1.7 + :methods [ [process [Object] void] [setFile [Object] void] ] 1.8 + :post-init register)) 1.9 + 1.10 +(use 'clojure.contrib.import-static) 1.11 +(import '(java.io File)) 1.12 +(import '(org.apache.commons.io FileUtils)) 1.13 +(import '(javax.imageio ImageIO) ) 1.14 +(import '(javax.swing JFrame)) 1.15 +(import '(java.awt Color BorderLayout)) 1.16 +(import '(ij.plugin PlugIn)) 1.17 +(import '(ij ImagePlus IJ)) 1.18 +(import '(java.lang Math)) 1.19 +(import '(java.awt Polygon)) 1.20 +(import '(java.awt.geom Line2D$Double)) 1.21 + 1.22 +(use 'clojureDemo.appeture) 1.23 + 1.24 +(import-static java.lang.Math pow abs) 1.25 + 1.26 +(import '(ij Macro)) 1.27 + 1.28 +(import '(java.io BufferedReader InputStreamReader)) 1.29 +(import '(java.awt.image BufferedImage)) 1.30 +(import '(genesis Genesis)) 1.31 +(import '(utils Mark)) 1.32 +(import '(capenLow StoryProcessor)) 1.33 +(import '(connections Connections WiredBox)) 1.34 +(import '(specialBoxes BasicBox MultiFunctionBox)) 1.35 +(import '(engineering NewHardWiredTranslator)) 1.36 + 1.37 +(import '(java.awt Polygon)) 1.38 +(import '(java.awt.geom Line2D$Double)) 1.39 +(use 'clojure.contrib.str-utils) 1.40 + 1.41 + 1.42 +;genesis imports 1.43 +(import '(http Start)) 1.44 + 1.45 + 1.46 +(use '[clojureDemo.Xuggle :only (video-seq trans-save flash display video-data)]) 1.47 +(use 'clojureDemo.MegaDeath) 1.48 + 1.49 + 1.50 +(use 'clojure.contrib.combinatorics) 1.51 + 1.52 +(use 'clojure.contrib.repl-utils) 1.53 + 1.54 +(use 'clojureDemo.GenesisPlay) 1.55 + 1.56 +(use ['clojureDemo.Defines 1.57 + :only '( 1.58 + lian look getto human0 blow base app0 app1 app2 app3 app4 app5 1.59 + bounce0 bounce1 bounce2 bounce3 bounce4 bounce5 bounce6 1.60 + collide0 collide1 collide2 collide3 collide4 1.61 + give0 give1 give2 give3 give4 target default)]) 1.62 + 1.63 +(defn -register 1.64 + "equivalent to Connections.getPorts(this).addSignalProcessor(\"process\"); in Java" 1.65 + [this] 1.66 + (println "ClojureBox (register) : Register is run 1.67 + only when the object is created, as if it were a constructor.") 1.68 + (. (connections.Connections/getPorts this) addSignalProcessor "process")) 1.69 + 1.70 +(defn -process [ this obj ] 1.71 + (println "ClojureBox (process) : This is a LISP function, 1.72 + being called through Java, through the wiredBox metaphor.") 1.73 + (.transmit (Connections/getPorts this) obj)) 1.74 + 1.75 +(defn -getName 1.76 + "the [_] means that the function gets an explicit 'this' 1.77 + argument, just like python. In this case we don't care about it." 1.78 + [_] "ArchLearning") 1.79 + 1.80 + 1.81 + 1.82 + 1.83 + 1.84 +(def output-base (File. "/home/r/Desktop/output-vision")) 1.85 +(def rsgs (with-meta (take 10 gs) (meta gs))) 1.86 +(def rrsgs (with-meta (take 3 rsgs) (meta gs))) 1.87 +; a concept is going to be derived from Genesis' own xml based representations. 1.88 +; this is an form of archlearning which figures out a function that representes 1.89 +; the concepts. 1.90 + 1.91 + 1.92 +(def black {:r 0 :g 0 :b 0}) 1.93 +(def white {:r 255 :g 255 :b 255}) 1.94 + 1.95 + 1.96 +(defn window-frame 1.97 + "analyzes a frame in terms of lots of tiny windows which 1.98 + each try to find some sort of edge. keeps coordinates." 1.99 + ([x-form frame] 1.100 + (let [lines (frame-windows x-form frame)] 1.101 + (zipmap (for [x lines] (first (rest x))) 1.102 + lines))) 1.103 + ([frame] 1.104 + (window-frame identity frame))) 1.105 + 1.106 + 1.107 +(defn intense-select-x-form 1.108 + "discard silly gray things" 1.109 + [window] 1.110 + (with-meta 1.111 + (zipmap 1.112 + (keys window) 1.113 + (map (fn [rgb] 1.114 + (let [spread (- (max (:r rgb) (:g rgb) (:b rgb)) (min (:r rgb) (:g rgb) (:b rgb)))] 1.115 + (if (> spread 45) 1.116 + rgb 1.117 + {:r 0 :g 0 :b 0}))) (vals window))) (meta window))) 1.118 + 1.119 +(defn edges-x-form 1.120 + [window] 1.121 + (frame-hash (ImagePlus. "sad :(" (overlay-draw blank (frame-windows window))))) 1.122 + 1.123 + 1.124 + 1.125 +(defn rgb-max 1.126 + [rgb1 rgb2] 1.127 + {:r (max (:r rgb1) (:r rgb2)) 1.128 + :g (max (:g rgb1) (:g rgb2)) 1.129 + :b (max (:b rgb1) (:b rgb2))}) 1.130 + 1.131 +(defn frame-hash-add 1.132 + [frame1 frame2] 1.133 + (with-meta 1.134 + (let [indexes (apply hash-set (concat (keys frame1) (keys frame2)))] 1.135 + (zipmap indexes (for [x indexes] (rgb-max (frame1 x black) (frame2 x black))))) (meta frame1))) 1.136 + 1.137 + 1.138 + 1.139 +(defn vid-seq-add 1.140 + "for black and white video-sequences. Just adds them together into one image sequence" 1.141 + [vid-seq1 vid-seq2] 1.142 + (with-meta 1.143 + (map #(ImagePlus. "ADD B&W" (frame-hash->bufferedImage %)) (map frame-hash-add (map frame-hash vid-seq1) (map frame-hash vid-seq2))) 1.144 + (meta vid-seq1))) 1.145 + 1.146 +(defn edges-center-draw 1.147 + ([base edges] 1.148 + (frame-hash-add 1.149 + base 1.150 + (zipmap (keys edges) (repeat white)))) 1.151 + ([edges] 1.152 + (edges-center-draw blank edges))) 1.153 + 1.154 +(defn edge-dot-x-form 1.155 + "gives a new frame-hash with only the edge points, all white." 1.156 + [frame] 1.157 + (edges-center-draw (window-frame frame))) 1.158 + 1.159 + 1.160 +(defn rgb-euclidian 1.161 + [{r1 :r g1 :g b1 :b} {r2 :r g2 :g b2 :b} ] 1.162 + (pow (+ (pow (- r1 r2) 2) 1.163 + (pow (- g1 g2) 2) 1.164 + (pow (- b1 b2) 2)) 0.5)) 1.165 + 1.166 +(defn rgb-sub 1.167 + [tolerance rgb1 rgb2] 1.168 + (if (< (rgb-euclidian rgb1 rgb2) tolerance) black white)) 1.169 + 1.170 + 1.171 + 1.172 +(defn frame-subtract 1.173 + [tolerance frame1 frame2] 1.174 + (with-meta 1.175 + (let [indexes (apply hash-set (concat (keys frame1) (keys frame2)))] 1.176 + (zipmap indexes (for [x indexes] (rgb-sub tolerance (frame1 x) (frame2 x))))) (meta frame1))) 1.177 + 1.178 + 1.179 +(defn image-subtract 1.180 + [tolerance [img1 img2]] 1.181 + (frame-subtract tolerance (frame-hash img1) (frame-hash img2))) 1.182 + 1.183 + 1.184 +(defn motion-detect 1.185 + ([tolerance video-seq] 1.186 + (with-meta 1.187 + (map (partial image-subtract tolerance) (partition 2 1 video-seq)) (meta video-seq))) 1.188 + ([video-seq] 1.189 + (motion-detect 40 video-seq))) 1.190 + 1.191 +(defn motion-x-form 1.192 + ([tolerance video-seq] 1.193 + (with-meta 1.194 + (map #(ImagePlus. "motion-detect!" (frame-hash->bufferedImage %)) (motion-detect tolerance video-seq)) 1.195 + (meta video-seq))) 1.196 + ([video-seq] 1.197 + (motion-x-form 40 video-seq))) 1.198 +;the edge detector is what finds objects. 1.199 +;movement disambiguates between different ways of interperting what objects are there 1.200 +;color / other qualifiers enable focus on a subset of objects, and can give objects names. 1.201 + 1.202 + 1.203 + 1.204 + 1.205 + 1.206 +(defn find-an-object 1.207 + "tries to find a single object from the current sensory-buffer, which 1.208 + is a video-seq for now. My idea here is for this to feed-back on itself, 1.209 + adjusting parameters till it can find it's target, and then using those 1.210 + to construct an representation of the object in terms of how to find it using 1.211 + other visual routines paramaters." 1.212 + [video-seq]) 1.213 + 1.214 + 1.215 + 1.216 +(defn transform 1.217 + [x-form video-seq] 1.218 + (with-meta 1.219 + (map (fn [imgPlus] 1.220 + (let [play (frame-hash imgPlus)] 1.221 + (x-form play))) 1.222 + video-seq) 1.223 + (meta video-seq))) 1.224 + 1.225 + 1.226 +(defn apply-x-form 1.227 + [x-form video-seq] 1.228 + (with-meta 1.229 + (map #(ImagePlus. "transformed!" (frame-hash->bufferedImage %)) 1.230 + (map (fn [imgPlus] 1.231 + (let [play (frame-hash imgPlus)] 1.232 + (x-form play))) 1.233 + video-seq)) 1.234 + (meta video-seq))) 1.235 + 1.236 + 1.237 + 1.238 +(defn only-white 1.239 + "reduce the image to only its white points" 1.240 + [window] 1.241 + (with-meta 1.242 + (let [new-keys 1.243 + (filter #(= white (window %)) (keys window))] 1.244 + (zipmap new-keys (map window new-keys))) (meta window))) 1.245 + 1.246 + 1.247 + 1.248 + 1.249 +(defn white-sum 1.250 + [& rgbs] 1.251 + (let[ wht-map {white 1}] 1.252 + (reduce + (map #(wht-map % 0) rgbs)))) 1.253 + 1.254 +(defn island? 1.255 + "return false if there's nothing around it within a certain radius" 1.256 + [window [x y]] 1.257 + (let [radius 3] 1.258 + (<= (apply white-sum (vals (rectangle-window x y radius radius window))) 1))) 1.259 + 1.260 +(defn white-border 1.261 + "anything that relies on a hack like this to work is wrong" 1.262 + [window] 1.263 + (with-meta 1.264 + (let [info (meta window)] 1.265 + (into window 1.266 + (zipmap 1.267 + (for [x (range (:width info)) y (range (:height info)) 1.268 + :when (or (= (-(:width info) 1) x) (= (- (:height info) 1) y) (= 0 y) (= 0 x))] [x y]) 1.269 + (repeat white))))(meta window))) 1.270 + 1.271 +(defn polygonize 1.272 + "for each edge-point, try to connect it with all the edge points around it, 1.273 + or obliterate it if it doesn't have any edge points close by." 1.274 + [window] 1.275 + (with-meta 1.276 + (let [edges (only-white window)] 1.277 + (let [new-keys 1.278 + (filter (comp not (partial island? window)) (keys window))] 1.279 + (let [ready-points (zipmap new-keys (map window new-keys))] 1.280 + (meta window)))))) 1.281 + 1.282 + 1.283 +(defn connect-the-dots 1.284 + [radius window] 1.285 + (let [edge-points (white-border (only-white window)) 1.286 + image (frame-hash->bufferedImage window) 1.287 + g2 (.getGraphics image)] 1.288 + (doall 1.289 + (for [[x y] (keys edge-points)] 1.290 + 1.291 + (let [points (apply cartesian-product (repeat 2 (keys (only-white (rectangle-window x y radius radius edge-points)))))] 1.292 + (if (not (empty? points)) 1.293 + (doall 1.294 + (for [[[x1 y1][x2 y2]] points] 1.295 + (.drawLine g2 x1 y1 x2 y2))))))) 1.296 + (frame-hash (ImagePlus. "stupid..." image)))) 1.297 + 1.298 + 1.299 +(defn blob-x-form 1.300 + [window] 1.301 + (with-meta 1.302 + ((comp (partial connect-the-dots 4) edge-dot-x-form) window) 1.303 + (meta window))) 1.304 + 1.305 + 1.306 + 1.307 + 1.308 +(defn connect-points 1.309 + [frame-hash overlay] 1.310 + (let [image (frame-hash->bufferedImage frame-hash) 1.311 + g2 (.getGraphics image)] 1.312 + (doall (for [ x overlay] 1.313 + (let [x1 (ffirst x) y1 (second (first x)) x2 (first (last x)) y2 (last (last x))] 1.314 + (.drawLine g2 x1 y1 x2 y2)))) 1.315 + image)) 1.316 + 1.317 + 1.318 +(defn disambiguate-edges 1.319 + "Like in project Prakesh, the thing that lets you discern shapes 1.320 + is watching them *move* coherently. After many months of this 1.321 + motion-boosting, the edge-detector itself becomes good enogh to 1.322 + analyze static pictures without motion. This function takes edges 1.323 + and tries to combine them into lines, dividing the world into 1.324 + polygonal regions. Motion is used to associate two regions together. 1.325 + associated with those points, that information is also used." 1.326 + [edges motion] 1.327 +) 1.328 + 1.329 + 1.330 +(defn triple-seq 1.331 + [triple] 1.332 + (list (.getFirst triple) (.getSecond triple) (.getThird triple))) 1.333 + 1.334 +(defn contains-word? 1.335 + [word triple] 1.336 + (contains? (set (map (fn [s] (re-sub #"-\d+" "" s)) (triple-seq triple))) word)) 1.337 + 1.338 + 1.339 +(defn write 1.340 + [reference] 1.341 + (fn [x] (dosync 1.342 + (println "wrote " " to " "ref.") 1.343 + (ref-set reference x)))) 1.344 + 1.345 + 1.346 +;; (defn join-point-lists 1.347 +;; [pointlist1 pointlist2] 1.348 +;; (for [x :while (not(= x 5))] x))) 1.349 + 1.350 +(defn extract-single-blob 1.351 + "find the biggest blob in an image and return it" 1.352 + [window] 1.353 + ;we're assuming that there are only blobs left -- funning this on an unprocessed 1.354 + ;image will just return the entire image 1.355 + (map list window)) 1.356 + 1.357 + 1.358 + 1.359 + 1.360 +(def gen-out (ref nil)) 1.361 +(def triple (ref nil)) 1.362 + 1.363 + 1.364 + 1.365 +(def gen1 (ref ())) 1.366 +(def gen2 (ref ())) 1.367 +(def gen3 (ref ())) 1.368 +(def gen4 (ref ())) 1.369 +(def gen5 (ref ())) 1.370 +(def gen6 (ref ())) 1.371 +(def gen7 (ref ())) 1.372 +(def gen8 (ref ())) 1.373 + 1.374 + 1.375 +(defn make-color-generator 1.376 + [] 1.377 + (let [r (java.util.Random. 58) 1.378 + g (java.util.Random. 125) 1.379 + b (java.util.Random. 8)] 1.380 + #(hash-map :r (.nextInt r 255) :g (.nextInt r 255) :b (.nextInt r 255)))) 1.381 + 1.382 + 1.383 +;a blob is a collection of: 1.384 +;points, colors 1.385 +;other blobs 1.386 +;so, a window is a blob too. 1.387 + 1.388 + 1.389 + 1.390 + 1.391 + 1.392 + 1.393 +;; (defn blob-color-absob 1.394 +;; [blob1 blob2] 1.395 +;; (if (and (< (rgb-euclidian (color-avg blob1) (color-avg blob2)) 20) (close-together blob1 blob2)) 1.396 +;; (combine blob1 blob2) 1.397 +;; '(blob1 blob2))) 1.398 + 1.399 + 1.400 +(defn make-test-box 1.401 + "stupid." 1.402 + [] 1.403 + (let [box (proxy [MultiFunctionBox] [] (getName [] "test-box [clojure]") 1.404 + (process1 [obj] ((write gen1) obj)) 1.405 + (process2 [obj] ((write gen2) obj)) 1.406 + (process3 [obj] ((write gen3) obj)) 1.407 + (process4 [obj] ((write gen4) obj)) 1.408 + (process5 [obj] ((write gen5) obj)) 1.409 + (process6 [obj] ((write gen6) obj)) 1.410 + (process7 [obj] ((write gen7) obj)) 1.411 + (process8 [obj] ((write gen8) obj)))] 1.412 + 1.413 + (.addSignalProcessor (Connections/getPorts box) "PORT1" "process1") 1.414 + (.addSignalProcessor (Connections/getPorts box) "PORT2" "process2") 1.415 + (.addSignalProcessor (Connections/getPorts box) "PORT3" "process3") 1.416 + (.addSignalProcessor (Connections/getPorts box) "PORT4" "process4") 1.417 + (.addSignalProcessor (Connections/getPorts box) "PORT5" "process5") 1.418 + (.addSignalProcessor (Connections/getPorts box) "PORT6" "process6") 1.419 + (.addSignalProcessor (Connections/getPorts box) "PORT7" "process7") 1.420 + (.addSignalProcessor (Connections/getPorts box) "PORT8" "process8") 1.421 + box)) 1.422 + 1.423 + 1.424 + 1.425 +(defn writer-box 1.426 + [reference] 1.427 + (let [box (proxy [MultiFunctionBox] [] 1.428 + (getName [] "ref-set\n [clojure]") 1.429 + (process1 [obj] ((write reference) obj)))] 1.430 + (.addSignalProcessor (Connections/getPorts box) "process1") 1.431 + box)) 1.432 + 1.433 + 1.434 + 1.435 + 1.436 +(def triples (ref ())) 1.437 +(def parse (ref ())) 1.438 +(def raw (ref ())) 1.439 +(def idioms (ref ())) 1.440 +(def yes-no (ref ())) 1.441 +(def imagine (ref ())) 1.442 +(def traj (ref ())) 1.443 +(def action (ref ())) 1.444 +(def transfer (ref ())) 1.445 +(def pix (ref ())) 1.446 +(def property (ref ())) 1.447 + 1.448 +(use 'clojure.contrib.str-utils) 1.449 + 1.450 +(defn process-video-and-subtitles 1.451 + [this file] 1.452 + ;we're looking for a text file of the same name as the video file. 1.453 + (let [subtitles (File. (.getParent file) (str (last (first (re-seq #"(^.*)\.avi$" (.getName file)))) ".txt"))] 1.454 + (dorun 1.455 + (for [line (re-split #"\n" (slurp (str subtitles)))] 1.456 + (do (println line) 1.457 + (.transmit (Connections/getPorts this) line))))) 1.458 + (display (first (video-seq file)))) 1.459 + 1.460 +(defn process-triple 1.461 + [this triple] 1.462 + (println "RLM [vision-box]: " triple)) 1.463 + 1.464 +(defn visionBox 1.465 + [] 1.466 + (let [box (proxy [MultiFunctionBox] [] 1.467 + (getName [] "VisionBox \n [clojure]") 1.468 + (process1 [obj] (process-video-and-subtitles this obj)) 1.469 + (process2 [obj] (process-triple this obj)))] 1.470 + (.addSignalProcessor (Connections/getPorts box) "video-in" "process1") 1.471 + (.addSignalProcessor (Connections/getPorts box) "triple-in" "process2") 1.472 + 1.473 + (println "the good box") 1.474 + box)) 1.475 + 1.476 + 1.477 + 1.478 + 1.479 + 1.480 +(defn custom-genesis 1.481 + "connects the writer boxes to genesis" 1.482 + [] 1.483 + (Connections/obliterateNetwork) 1.484 + (let [stupid-box (make-test-box) genesis (Genesis.) vis-box (visionBox) ] 1.485 + (Connections/wire "tripple port" (.getStartParser genesis) (writer-box triples)) 1.486 + (Connections/wire "parse" (.getStartParser genesis) (writer-box parse)) 1.487 + (Connections/wire "result" (.getNewSemanticTranslator genesis) (writer-box raw)) 1.488 + (Connections/wire (.getIdiomExpert genesis) (writer-box idioms)) 1.489 + (Connections/wire "yes-no question" (.getCommandExpert genesis) (writer-box yes-no)) 1.490 + (Connections/wire "imagine" (.getCommandExpert genesis) (writer-box imagine)) 1.491 + (Connections/wire "viewer" (.getTrajectoryExpert genesis) (writer-box traj)) 1.492 + (Connections/wire "viewer" (.getActionExpert genesis) (writer-box action)) 1.493 + (Connections/wire "next" (.getTransferExpert genesis) (writer-box transfer)) 1.494 + (Connections/wire (.getRachelsPictureFinder genesis) (writer-box pix)) 1.495 + (Connections/wire "viewer" (.getPropertyExpert genesis) (writer-box property)) 1.496 + (Connections/wire "tripple port" (.getStartParser genesis) "triple-in" vis-box) 1.497 + 1.498 + 1.499 + (Connections/wire (.getArchLearning genesis) "video-in" vis-box) 1.500 + (Connections/wire vis-box "sentence" (.getStartParser genesis)) 1.501 + 1.502 + genesis)) 1.503 + 1.504 + 1.505 +(use 'clojure.contrib.def) 1.506 + 1.507 +(defvar learning-hash {} 1.508 + "Right now this serves as the visual memory. 1.509 + It's full of verbs/objects and the programs 1.510 + that find them.") 1.511 + 1.512 +(def green {:r 0 :g 200 :b 0}) 1.513 +(def blue {:r 0 :g 0 :b 255}) 1.514 +(def red {:r 255 :g 0 :b 0}) 1.515 + 1.516 + 1.517 +(defn color-similar? 1.518 + [threshold window color coord] 1.519 + (< (rgb-euclidian (window coord) color) threshold)) 1.520 +;should also have the same "shape" here 1.521 + 1.522 +(defn color-select 1.523 + [threshold color window] 1.524 + (with-meta 1.525 + (let [new-keys 1.526 + (filter (partial color-similar? threshold window color) (keys window))] 1.527 + (zipmap new-keys (map window new-keys))) 1.528 + (meta window))) 1.529 + 1.530 + 1.531 + 1.532 +(defn object-sequence 1.533 + "get's the largest blob of the given color from a video sequence." 1.534 + [color video-seq] 1.535 + (apply-x-form (comp (partial color-select 135 color) intense-select-x-form) rrsgs)) 1.536 + 1.537 +(defn -setFile 1.538 + [this file] 1.539 + (println "file is " file) 1.540 + (.process this file)) 1.541 + 1.542 + 1.543 + 1.544 + 1.545 +(comment (things you can do that will actually work!) 1.546 + 1.547 +(do (use :reload-all 'clojureDemo.ArchLearning) (in-ns 'clojureDemo.ArchLearning)) 1.548 + 1.549 +(display (overlay-draw (green-select-x-form play) (frame-windows green-select-x-form play ))) 1.550 + 1.551 +;vision stuff 1.552 + 1.553 +(def edgesD (transform window-frame rrsgs)) 1.554 + 1.555 +(doall 1.556 + (def edgesI (apply-x-form edges-x-form rrsgs)) 1.557 + (display (rectangle-window 50 50 50 50 (frame-hash (nth edgesI 1)))) 1.558 + ) 1.559 + 1.560 +(def polyjuice (white-border (only-white (edge-dot-x-form play)))) 1.561 + 1.562 +(count (color-select 135 red (intense-select-x-form (frame-hash (last sg))))) 1.563 +(trans-save (File. output-base "only-red.avi")(apply-x-form (comp (partial color-select 135 red) intense-select-x-form) rrsgs)) 1.564 +) 1.565 +