rlm@1: (ns clojureDemo.ArchLearning rlm@1: (:gen-class rlm@1: :implements [connections.WiredBox] rlm@1: :methods [ [process [Object] void] [setFile [Object] void] ] rlm@1: :post-init register)) rlm@1: rlm@1: (use 'clojure.contrib.import-static) rlm@1: (import '(java.io File)) rlm@1: (import '(org.apache.commons.io FileUtils)) rlm@1: (import '(javax.imageio ImageIO) ) rlm@1: (import '(javax.swing JFrame)) rlm@1: (import '(java.awt Color BorderLayout)) rlm@1: (import '(ij.plugin PlugIn)) rlm@1: (import '(ij ImagePlus IJ)) rlm@1: (import '(java.lang Math)) rlm@1: (import '(java.awt Polygon)) rlm@1: (import '(java.awt.geom Line2D$Double)) rlm@1: rlm@1: (use 'clojureDemo.appeture) rlm@1: rlm@1: (import-static java.lang.Math pow abs) rlm@1: rlm@1: (import '(ij Macro)) rlm@1: rlm@1: (import '(java.io BufferedReader InputStreamReader)) rlm@1: (import '(java.awt.image BufferedImage)) rlm@1: (import '(genesis Genesis)) rlm@1: (import '(utils Mark)) rlm@1: (import '(capenLow StoryProcessor)) rlm@1: (import '(connections Connections WiredBox)) rlm@1: (import '(specialBoxes BasicBox MultiFunctionBox)) rlm@1: (import '(engineering NewHardWiredTranslator)) rlm@1: rlm@1: (import '(java.awt Polygon)) rlm@1: (import '(java.awt.geom Line2D$Double)) rlm@1: (use 'clojure.contrib.str-utils) rlm@1: rlm@1: rlm@1: ;genesis imports rlm@1: (import '(http Start)) rlm@1: rlm@1: rlm@1: (use '[clojureDemo.Xuggle :only (video-seq trans-save flash display video-data)]) rlm@1: (use 'clojureDemo.MegaDeath) rlm@1: rlm@1: rlm@1: (use 'clojure.contrib.combinatorics) rlm@1: rlm@1: (use 'clojure.contrib.repl-utils) rlm@1: rlm@1: (use 'clojureDemo.GenesisPlay) rlm@1: rlm@1: (use ['clojureDemo.Defines rlm@1: :only '( rlm@1: lian look getto human0 blow base app0 app1 app2 app3 app4 app5 rlm@1: bounce0 bounce1 bounce2 bounce3 bounce4 bounce5 bounce6 rlm@1: collide0 collide1 collide2 collide3 collide4 rlm@1: give0 give1 give2 give3 give4 target default)]) rlm@1: rlm@1: (defn -register rlm@1: "equivalent to Connections.getPorts(this).addSignalProcessor(\"process\"); in Java" rlm@1: [this] rlm@1: (println "ClojureBox (register) : Register is run rlm@1: only when the object is created, as if it were a constructor.") rlm@1: (. (connections.Connections/getPorts this) addSignalProcessor "process")) rlm@1: rlm@1: (defn -process [ this obj ] rlm@1: (println "ClojureBox (process) : This is a LISP function, rlm@1: being called through Java, through the wiredBox metaphor.") rlm@1: (.transmit (Connections/getPorts this) obj)) rlm@1: rlm@1: (defn -getName rlm@1: "the [_] means that the function gets an explicit 'this' rlm@1: argument, just like python. In this case we don't care about it." rlm@1: [_] "ArchLearning") rlm@1: rlm@1: rlm@1: rlm@1: rlm@1: rlm@1: (def output-base (File. "/home/r/Desktop/output-vision")) rlm@1: (def rsgs (with-meta (take 10 gs) (meta gs))) rlm@1: (def rrsgs (with-meta (take 3 rsgs) (meta gs))) rlm@1: ; a concept is going to be derived from Genesis' own xml based representations. rlm@1: ; this is an form of archlearning which figures out a function that representes rlm@1: ; the concepts. rlm@1: rlm@1: rlm@1: (def black {:r 0 :g 0 :b 0}) rlm@1: (def white {:r 255 :g 255 :b 255}) rlm@1: rlm@1: rlm@1: (defn window-frame rlm@1: "analyzes a frame in terms of lots of tiny windows which rlm@1: each try to find some sort of edge. keeps coordinates." rlm@1: ([x-form frame] rlm@1: (let [lines (frame-windows x-form frame)] rlm@1: (zipmap (for [x lines] (first (rest x))) rlm@1: lines))) rlm@1: ([frame] rlm@1: (window-frame identity frame))) rlm@1: rlm@1: rlm@1: (defn intense-select-x-form rlm@1: "discard silly gray things" rlm@1: [window] rlm@1: (with-meta rlm@1: (zipmap rlm@1: (keys window) rlm@1: (map (fn [rgb] rlm@1: (let [spread (- (max (:r rgb) (:g rgb) (:b rgb)) (min (:r rgb) (:g rgb) (:b rgb)))] rlm@1: (if (> spread 45) rlm@1: rgb rlm@1: {:r 0 :g 0 :b 0}))) (vals window))) (meta window))) rlm@1: rlm@1: (defn edges-x-form rlm@1: [window] rlm@1: (frame-hash (ImagePlus. "sad :(" (overlay-draw blank (frame-windows window))))) rlm@1: rlm@1: rlm@1: rlm@1: (defn rgb-max rlm@1: [rgb1 rgb2] rlm@1: {:r (max (:r rgb1) (:r rgb2)) rlm@1: :g (max (:g rgb1) (:g rgb2)) rlm@1: :b (max (:b rgb1) (:b rgb2))}) rlm@1: rlm@1: (defn frame-hash-add rlm@1: [frame1 frame2] rlm@1: (with-meta rlm@1: (let [indexes (apply hash-set (concat (keys frame1) (keys frame2)))] rlm@1: (zipmap indexes (for [x indexes] (rgb-max (frame1 x black) (frame2 x black))))) (meta frame1))) rlm@1: rlm@1: rlm@1: rlm@1: (defn vid-seq-add rlm@1: "for black and white video-sequences. Just adds them together into one image sequence" rlm@1: [vid-seq1 vid-seq2] rlm@1: (with-meta rlm@1: (map #(ImagePlus. "ADD B&W" (frame-hash->bufferedImage %)) (map frame-hash-add (map frame-hash vid-seq1) (map frame-hash vid-seq2))) rlm@1: (meta vid-seq1))) rlm@1: rlm@1: (defn edges-center-draw rlm@1: ([base edges] rlm@1: (frame-hash-add rlm@1: base rlm@1: (zipmap (keys edges) (repeat white)))) rlm@1: ([edges] rlm@1: (edges-center-draw blank edges))) rlm@1: rlm@1: (defn edge-dot-x-form rlm@1: "gives a new frame-hash with only the edge points, all white." rlm@1: [frame] rlm@1: (edges-center-draw (window-frame frame))) rlm@1: rlm@1: rlm@1: (defn rgb-euclidian rlm@1: [{r1 :r g1 :g b1 :b} {r2 :r g2 :g b2 :b} ] rlm@1: (pow (+ (pow (- r1 r2) 2) rlm@1: (pow (- g1 g2) 2) rlm@1: (pow (- b1 b2) 2)) 0.5)) rlm@1: rlm@1: (defn rgb-sub rlm@1: [tolerance rgb1 rgb2] rlm@1: (if (< (rgb-euclidian rgb1 rgb2) tolerance) black white)) rlm@1: rlm@1: rlm@1: rlm@1: (defn frame-subtract rlm@1: [tolerance frame1 frame2] rlm@1: (with-meta rlm@1: (let [indexes (apply hash-set (concat (keys frame1) (keys frame2)))] rlm@1: (zipmap indexes (for [x indexes] (rgb-sub tolerance (frame1 x) (frame2 x))))) (meta frame1))) rlm@1: rlm@1: rlm@1: (defn image-subtract rlm@1: [tolerance [img1 img2]] rlm@1: (frame-subtract tolerance (frame-hash img1) (frame-hash img2))) rlm@1: rlm@1: rlm@1: (defn motion-detect rlm@1: ([tolerance video-seq] rlm@1: (with-meta rlm@1: (map (partial image-subtract tolerance) (partition 2 1 video-seq)) (meta video-seq))) rlm@1: ([video-seq] rlm@1: (motion-detect 40 video-seq))) rlm@1: rlm@1: (defn motion-x-form rlm@1: ([tolerance video-seq] rlm@1: (with-meta rlm@1: (map #(ImagePlus. "motion-detect!" (frame-hash->bufferedImage %)) (motion-detect tolerance video-seq)) rlm@1: (meta video-seq))) rlm@1: ([video-seq] rlm@1: (motion-x-form 40 video-seq))) rlm@1: ;the edge detector is what finds objects. rlm@1: ;movement disambiguates between different ways of interperting what objects are there rlm@1: ;color / other qualifiers enable focus on a subset of objects, and can give objects names. rlm@1: rlm@1: rlm@1: rlm@1: rlm@1: rlm@1: (defn find-an-object rlm@1: "tries to find a single object from the current sensory-buffer, which rlm@1: is a video-seq for now. My idea here is for this to feed-back on itself, rlm@1: adjusting parameters till it can find it's target, and then using those rlm@1: to construct an representation of the object in terms of how to find it using rlm@1: other visual routines paramaters." rlm@1: [video-seq]) rlm@1: rlm@1: rlm@1: rlm@1: (defn transform rlm@1: [x-form video-seq] rlm@1: (with-meta rlm@1: (map (fn [imgPlus] rlm@1: (let [play (frame-hash imgPlus)] rlm@1: (x-form play))) rlm@1: video-seq) rlm@1: (meta video-seq))) rlm@1: rlm@1: rlm@1: (defn apply-x-form rlm@1: [x-form video-seq] rlm@1: (with-meta rlm@1: (map #(ImagePlus. "transformed!" (frame-hash->bufferedImage %)) rlm@1: (map (fn [imgPlus] rlm@1: (let [play (frame-hash imgPlus)] rlm@1: (x-form play))) rlm@1: video-seq)) rlm@1: (meta video-seq))) rlm@1: rlm@1: rlm@1: rlm@1: (defn only-white rlm@1: "reduce the image to only its white points" rlm@1: [window] rlm@1: (with-meta rlm@1: (let [new-keys rlm@1: (filter #(= white (window %)) (keys window))] rlm@1: (zipmap new-keys (map window new-keys))) (meta window))) rlm@1: rlm@1: rlm@1: rlm@1: rlm@1: (defn white-sum rlm@1: [& rgbs] rlm@1: (let[ wht-map {white 1}] rlm@1: (reduce + (map #(wht-map % 0) rgbs)))) rlm@1: rlm@1: (defn island? rlm@1: "return false if there's nothing around it within a certain radius" rlm@1: [window [x y]] rlm@1: (let [radius 3] rlm@1: (<= (apply white-sum (vals (rectangle-window x y radius radius window))) 1))) rlm@1: rlm@1: (defn white-border rlm@1: "anything that relies on a hack like this to work is wrong" rlm@1: [window] rlm@1: (with-meta rlm@1: (let [info (meta window)] rlm@1: (into window rlm@1: (zipmap rlm@1: (for [x (range (:width info)) y (range (:height info)) rlm@1: :when (or (= (-(:width info) 1) x) (= (- (:height info) 1) y) (= 0 y) (= 0 x))] [x y]) rlm@1: (repeat white))))(meta window))) rlm@1: rlm@1: (defn polygonize rlm@1: "for each edge-point, try to connect it with all the edge points around it, rlm@1: or obliterate it if it doesn't have any edge points close by." rlm@1: [window] rlm@1: (with-meta rlm@1: (let [edges (only-white window)] rlm@1: (let [new-keys rlm@1: (filter (comp not (partial island? window)) (keys window))] rlm@1: (let [ready-points (zipmap new-keys (map window new-keys))] rlm@1: (meta window)))))) rlm@1: rlm@1: rlm@1: (defn connect-the-dots rlm@1: [radius window] rlm@1: (let [edge-points (white-border (only-white window)) rlm@1: image (frame-hash->bufferedImage window) rlm@1: g2 (.getGraphics image)] rlm@1: (doall rlm@1: (for [[x y] (keys edge-points)] rlm@1: rlm@1: (let [points (apply cartesian-product (repeat 2 (keys (only-white (rectangle-window x y radius radius edge-points)))))] rlm@1: (if (not (empty? points)) rlm@1: (doall rlm@1: (for [[[x1 y1][x2 y2]] points] rlm@1: (.drawLine g2 x1 y1 x2 y2))))))) rlm@1: (frame-hash (ImagePlus. "stupid..." image)))) rlm@1: rlm@1: rlm@1: (defn blob-x-form rlm@1: [window] rlm@1: (with-meta rlm@1: ((comp (partial connect-the-dots 4) edge-dot-x-form) window) rlm@1: (meta window))) rlm@1: rlm@1: rlm@1: rlm@1: rlm@1: (defn connect-points rlm@1: [frame-hash overlay] rlm@1: (let [image (frame-hash->bufferedImage frame-hash) rlm@1: g2 (.getGraphics image)] rlm@1: (doall (for [ x overlay] rlm@1: (let [x1 (ffirst x) y1 (second (first x)) x2 (first (last x)) y2 (last (last x))] rlm@1: (.drawLine g2 x1 y1 x2 y2)))) rlm@1: image)) rlm@1: rlm@1: rlm@1: (defn disambiguate-edges rlm@1: "Like in project Prakesh, the thing that lets you discern shapes rlm@1: is watching them *move* coherently. After many months of this rlm@1: motion-boosting, the edge-detector itself becomes good enogh to rlm@1: analyze static pictures without motion. This function takes edges rlm@1: and tries to combine them into lines, dividing the world into rlm@1: polygonal regions. Motion is used to associate two regions together. rlm@1: associated with those points, that information is also used." rlm@1: [edges motion] rlm@1: ) rlm@1: rlm@1: rlm@1: (defn triple-seq rlm@1: [triple] rlm@1: (list (.getFirst triple) (.getSecond triple) (.getThird triple))) rlm@1: rlm@1: (defn contains-word? rlm@1: [word triple] rlm@1: (contains? (set (map (fn [s] (re-sub #"-\d+" "" s)) (triple-seq triple))) word)) rlm@1: rlm@1: rlm@1: (defn write rlm@1: [reference] rlm@1: (fn [x] (dosync rlm@1: (println "wrote " " to " "ref.") rlm@1: (ref-set reference x)))) rlm@1: rlm@1: rlm@1: ;; (defn join-point-lists rlm@1: ;; [pointlist1 pointlist2] rlm@1: ;; (for [x :while (not(= x 5))] x))) rlm@1: rlm@1: (defn extract-single-blob rlm@1: "find the biggest blob in an image and return it" rlm@1: [window] rlm@1: ;we're assuming that there are only blobs left -- funning this on an unprocessed rlm@1: ;image will just return the entire image rlm@1: (map list window)) rlm@1: rlm@1: rlm@1: rlm@1: rlm@1: (def gen-out (ref nil)) rlm@1: (def triple (ref nil)) rlm@1: rlm@1: rlm@1: rlm@1: (def gen1 (ref ())) rlm@1: (def gen2 (ref ())) rlm@1: (def gen3 (ref ())) rlm@1: (def gen4 (ref ())) rlm@1: (def gen5 (ref ())) rlm@1: (def gen6 (ref ())) rlm@1: (def gen7 (ref ())) rlm@1: (def gen8 (ref ())) rlm@1: rlm@1: rlm@1: (defn make-color-generator rlm@1: [] rlm@1: (let [r (java.util.Random. 58) rlm@1: g (java.util.Random. 125) rlm@1: b (java.util.Random. 8)] rlm@1: #(hash-map :r (.nextInt r 255) :g (.nextInt r 255) :b (.nextInt r 255)))) rlm@1: rlm@1: rlm@1: ;a blob is a collection of: rlm@1: ;points, colors rlm@1: ;other blobs rlm@1: ;so, a window is a blob too. rlm@1: rlm@1: rlm@1: rlm@1: rlm@1: rlm@1: rlm@1: ;; (defn blob-color-absob rlm@1: ;; [blob1 blob2] rlm@1: ;; (if (and (< (rgb-euclidian (color-avg blob1) (color-avg blob2)) 20) (close-together blob1 blob2)) rlm@1: ;; (combine blob1 blob2) rlm@1: ;; '(blob1 blob2))) rlm@1: rlm@1: rlm@1: (defn make-test-box rlm@1: "stupid." rlm@1: [] rlm@1: (let [box (proxy [MultiFunctionBox] [] (getName [] "test-box [clojure]") rlm@1: (process1 [obj] ((write gen1) obj)) rlm@1: (process2 [obj] ((write gen2) obj)) rlm@1: (process3 [obj] ((write gen3) obj)) rlm@1: (process4 [obj] ((write gen4) obj)) rlm@1: (process5 [obj] ((write gen5) obj)) rlm@1: (process6 [obj] ((write gen6) obj)) rlm@1: (process7 [obj] ((write gen7) obj)) rlm@1: (process8 [obj] ((write gen8) obj)))] rlm@1: rlm@1: (.addSignalProcessor (Connections/getPorts box) "PORT1" "process1") rlm@1: (.addSignalProcessor (Connections/getPorts box) "PORT2" "process2") rlm@1: (.addSignalProcessor (Connections/getPorts box) "PORT3" "process3") rlm@1: (.addSignalProcessor (Connections/getPorts box) "PORT4" "process4") rlm@1: (.addSignalProcessor (Connections/getPorts box) "PORT5" "process5") rlm@1: (.addSignalProcessor (Connections/getPorts box) "PORT6" "process6") rlm@1: (.addSignalProcessor (Connections/getPorts box) "PORT7" "process7") rlm@1: (.addSignalProcessor (Connections/getPorts box) "PORT8" "process8") rlm@1: box)) rlm@1: rlm@1: rlm@1: rlm@1: (defn writer-box rlm@1: [reference] rlm@1: (let [box (proxy [MultiFunctionBox] [] rlm@1: (getName [] "ref-set\n [clojure]") rlm@1: (process1 [obj] ((write reference) obj)))] rlm@1: (.addSignalProcessor (Connections/getPorts box) "process1") rlm@1: box)) rlm@1: rlm@1: rlm@1: rlm@1: rlm@1: (def triples (ref ())) rlm@1: (def parse (ref ())) rlm@1: (def raw (ref ())) rlm@1: (def idioms (ref ())) rlm@1: (def yes-no (ref ())) rlm@1: (def imagine (ref ())) rlm@1: (def traj (ref ())) rlm@1: (def action (ref ())) rlm@1: (def transfer (ref ())) rlm@1: (def pix (ref ())) rlm@1: (def property (ref ())) rlm@1: rlm@1: (use 'clojure.contrib.str-utils) rlm@1: rlm@1: (defn process-video-and-subtitles rlm@1: [this file] rlm@1: ;we're looking for a text file of the same name as the video file. rlm@1: (let [subtitles (File. (.getParent file) (str (last (first (re-seq #"(^.*)\.avi$" (.getName file)))) ".txt"))] rlm@1: (dorun rlm@1: (for [line (re-split #"\n" (slurp (str subtitles)))] rlm@1: (do (println line) rlm@1: (.transmit (Connections/getPorts this) line))))) rlm@1: (display (first (video-seq file)))) rlm@1: rlm@1: (defn process-triple rlm@1: [this triple] rlm@1: (println "RLM [vision-box]: " triple)) rlm@1: rlm@1: (defn visionBox rlm@1: [] rlm@1: (let [box (proxy [MultiFunctionBox] [] rlm@1: (getName [] "VisionBox \n [clojure]") rlm@1: (process1 [obj] (process-video-and-subtitles this obj)) rlm@1: (process2 [obj] (process-triple this obj)))] rlm@1: (.addSignalProcessor (Connections/getPorts box) "video-in" "process1") rlm@1: (.addSignalProcessor (Connections/getPorts box) "triple-in" "process2") rlm@1: rlm@1: (println "the good box") rlm@1: box)) rlm@1: rlm@1: rlm@1: rlm@1: rlm@1: rlm@1: (defn custom-genesis rlm@1: "connects the writer boxes to genesis" rlm@1: [] rlm@1: (Connections/obliterateNetwork) rlm@1: (let [stupid-box (make-test-box) genesis (Genesis.) vis-box (visionBox) ] rlm@1: (Connections/wire "tripple port" (.getStartParser genesis) (writer-box triples)) rlm@1: (Connections/wire "parse" (.getStartParser genesis) (writer-box parse)) rlm@1: (Connections/wire "result" (.getNewSemanticTranslator genesis) (writer-box raw)) rlm@1: (Connections/wire (.getIdiomExpert genesis) (writer-box idioms)) rlm@1: (Connections/wire "yes-no question" (.getCommandExpert genesis) (writer-box yes-no)) rlm@1: (Connections/wire "imagine" (.getCommandExpert genesis) (writer-box imagine)) rlm@1: (Connections/wire "viewer" (.getTrajectoryExpert genesis) (writer-box traj)) rlm@1: (Connections/wire "viewer" (.getActionExpert genesis) (writer-box action)) rlm@1: (Connections/wire "next" (.getTransferExpert genesis) (writer-box transfer)) rlm@1: (Connections/wire (.getRachelsPictureFinder genesis) (writer-box pix)) rlm@1: (Connections/wire "viewer" (.getPropertyExpert genesis) (writer-box property)) rlm@1: (Connections/wire "tripple port" (.getStartParser genesis) "triple-in" vis-box) rlm@1: rlm@1: rlm@1: (Connections/wire (.getArchLearning genesis) "video-in" vis-box) rlm@1: (Connections/wire vis-box "sentence" (.getStartParser genesis)) rlm@1: rlm@1: genesis)) rlm@1: rlm@1: rlm@1: (use 'clojure.contrib.def) rlm@1: rlm@1: (defvar learning-hash {} rlm@1: "Right now this serves as the visual memory. rlm@1: It's full of verbs/objects and the programs rlm@1: that find them.") rlm@1: rlm@1: (def green {:r 0 :g 200 :b 0}) rlm@1: (def blue {:r 0 :g 0 :b 255}) rlm@1: (def red {:r 255 :g 0 :b 0}) rlm@1: rlm@1: rlm@1: (defn color-similar? rlm@1: [threshold window color coord] rlm@1: (< (rgb-euclidian (window coord) color) threshold)) rlm@1: ;should also have the same "shape" here rlm@1: rlm@1: (defn color-select rlm@1: [threshold color window] rlm@1: (with-meta rlm@1: (let [new-keys rlm@1: (filter (partial color-similar? threshold window color) (keys window))] rlm@1: (zipmap new-keys (map window new-keys))) rlm@1: (meta window))) rlm@1: rlm@1: rlm@1: rlm@1: (defn object-sequence rlm@1: "get's the largest blob of the given color from a video sequence." rlm@1: [color video-seq] rlm@1: (apply-x-form (comp (partial color-select 135 color) intense-select-x-form) rrsgs)) rlm@1: rlm@1: (defn -setFile rlm@1: [this file] rlm@1: (println "file is " file) rlm@1: (.process this file)) rlm@1: rlm@1: rlm@1: rlm@1: rlm@1: (comment (things you can do that will actually work!) rlm@1: rlm@1: (do (use :reload-all 'clojureDemo.ArchLearning) (in-ns 'clojureDemo.ArchLearning)) rlm@1: rlm@1: (display (overlay-draw (green-select-x-form play) (frame-windows green-select-x-form play ))) rlm@1: rlm@1: ;vision stuff rlm@1: rlm@1: (def edgesD (transform window-frame rrsgs)) rlm@1: rlm@1: (doall rlm@1: (def edgesI (apply-x-form edges-x-form rrsgs)) rlm@1: (display (rectangle-window 50 50 50 50 (frame-hash (nth edgesI 1)))) rlm@1: ) rlm@1: rlm@1: (def polyjuice (white-border (only-white (edge-dot-x-form play)))) rlm@1: rlm@1: (count (color-select 135 red (intense-select-x-form (frame-hash (last sg))))) rlm@1: (trans-save (File. output-base "only-red.avi")(apply-x-form (comp (partial color-select 135 red) intense-select-x-form) rrsgs)) rlm@1: ) rlm@1: