rlm@356: #+title: Fun with Gabor Filters
rlm@356: #+author: Robert McIntyre
rlm@356: #+email: rlm@mit.edu
rlm@356: #+description: gabor filters in clojure with opencv
rlm@356: #+keywords: computer vision, jMonkeyEngine3, clojure, opencv
rlm@356: #+SETUPFILE: ../../aurellem/org/setup.org
rlm@356: #+INCLUDE: ../../aurellem/org/level-0.org
rlm@356: #+babel: :mkdirp yes :noweb yes :exports both
rlm@356:
rlm@356:
rlm@368: Gabor filters were invented by the same guy who invented holograms.
rlm@368:
rlm@368: They work well as edge detectors and are related to the human visual
rlm@368: system.
rlm@356:
rlm@356: #+name: gabor
rlm@356: #+begin_src clojure
rlm@357: (ns cortex.gabor
rlm@357: (:import org.opencv.core.CvType)
rlm@357: (:import java.awt.image.BufferedImage)
rlm@357: (:import ij.ImagePlus)
rlm@358: (:import org.opencv.core.Mat)
rlm@364: (:use (cortex world sense util vision import))
rlm@363: (:import com.jme3.post.SceneProcessor)
rlm@363: (:import (com.jme3.util BufferUtils Screenshots))
rlm@363: (:import java.nio.ByteBuffer)
rlm@363: (:import java.awt.image.BufferedImage)
rlm@363: (:import (com.jme3.renderer ViewPort Camera))
rlm@363: (:import (com.jme3.math ColorRGBA Vector3f Matrix3f Vector2f))
rlm@363: (:import com.jme3.renderer.Renderer)
rlm@363: (:import com.jme3.app.Application)
rlm@363: (:import com.jme3.texture.FrameBuffer)
rlm@363: (:import (com.jme3.scene Node Spatial)))
rlm@363:
rlm@363:
rlm@363: (cortex.import/mega-import-jme3)
rlm@364: (use 'clojure.math.numeric-tower)
rlm@356: (defn load-opencv
rlm@356: "Load the opencv native library. Must be called before any OpenCV
rlm@356: stuff is used."
rlm@356: []
rlm@356: (clojure.lang.RT/loadLibrary "opencv_java249"))
rlm@356:
rlm@364: (load-opencv)
rlm@364:
rlm@364: (defn gabor-kernel
rlm@364: ([sigma wavelength theta]
rlm@364: (gabor-kernel sigma wavelength theta 1 0))
rlm@364: ([sigma wavelength]
rlm@364: (gabor-kernel sigma wavelength 0 1 0))
rlm@364: ([sigma wavelength theta aspect-ratio phase-offset]
rlm@359:
rlm@359: ;; first, find the size of the kernel which is required
rlm@359: (let [square #(expt % 2)
rlm@359: rotated (fn [[x y]]
rlm@359: [(+ (* x (Math/cos theta)) (* y (Math/sin theta)))
rlm@359: (- (* y (Math/cos theta)) (* x (Math/sin theta)))])
rlm@359:
rlm@359: gaussian (fn [[x y]]
rlm@359: (let [[x' y'] (rotated [x y])]
rlm@359: (Math/exp (- (/ (+ (square x')
rlm@359: (square (* aspect-ratio y')))
rlm@359: (* 2 (square sigma)))))))
rlm@359: sinusoid (fn [[x y]]
rlm@359: (let [[x' y'] (rotated [x y])]
rlm@359: (Math/cos
rlm@359: (+ (* 2 Math/PI (/ x' wavelength))
rlm@359: phase-offset))))
rlm@359:
rlm@361: half-width
rlm@361: (let [std-dev-capture 5]
rlm@361: (max
rlm@361: (int (* std-dev-capture (/ sigma aspect-ratio)))
rlm@361: (int (* std-dev-capture sigma))
rlm@361: (int (* std-dev-capture (/ aspect-ratio sigma)))))
rlm@359:
rlm@359: grid (let [axis (range (- half-width) (inc half-width))]
rlm@359: (for [y (reverse axis) x axis] (vector x y)))
rlm@359:
rlm@359: scale (reduce + (map gaussian grid))
rlm@359:
rlm@359: gabor (fn [[x y :as coord]]
rlm@360: (* (sinusoid coord) (gaussian coord) (/ scale)))
rlm@359:
rlm@359: mat-width (+ 1 (* 2 half-width))
rlm@359: mat (Mat. mat-width mat-width CvType/CV_32F)]
rlm@359:
rlm@359: (.put mat 0 0 (float-array (map gabor grid)))
rlm@364: mat)))
rlm@359:
rlm@359:
rlm@361: (defn draw-kernel! [kernel img-path]
rlm@361: (let [output img-path
rlm@360: size (.size kernel)
rlm@360: width (int (.width size))
rlm@360: height (int (.height size))
rlm@360: tmp-array (float-array (* width height))]
rlm@360:
rlm@360: ;; read values from matrix.
rlm@360: (.get kernel 0 0 tmp-array)
rlm@360:
rlm@360: ;; find overall dynamic range of the filter
rlm@360: (let [vals (vec tmp-array)
rlm@360: low (apply min vals)
rlm@360: high (apply max vals)
rlm@360: scaled-vals (map #(* 255 (- % low) (/ (- high low))) vals)
rlm@360: new-mat (Mat. height width CvType/CV_32F)]
rlm@360: (.put new-mat 0 0 (float-array scaled-vals))
rlm@361: (org.opencv.highgui.Highgui/imwrite output new-mat))))
rlm@361:
rlm@364: ;; some cool examples
rlm@364: #+end_src
rlm@364:
rlm@364:
rlm@364:
rlm@364:
rlm@364: [[../images/gabor-50-10.png]]
rlm@364:
rlm@364: #+begin_src clojure
rlm@364: (def img-base "/home/r/proj/cortex/images/")
rlm@364:
rlm@364: (draw-kernel! (gabor-kernel 50 10 0 1 0)
rlm@364: (str img-base "gabor-50-10.png"))
rlm@364: #+end_src
rlm@364:
rlm@364:
rlm@364: [[../images/gabor-50-10-pi-over-4.png]]
rlm@364:
rlm@364: #+begin_src clojure
rlm@364: (draw-kernel! (gabor-kernel 50 10 (/ Math/PI 4) 1 0)
rlm@364: (str img-base "gabor-50-10-pi-over-4.png"))
rlm@364: #+end_src
rlm@364:
rlm@364:
rlm@364: [[../images/gabor-50-10-pi-over-2.png]]
rlm@364:
rlm@364: #+begin_src clojure
rlm@364: (draw-kernel! (gabor-kernel 50 10 (/ Math/PI 2) 1 0)
rlm@364: (str img-base "gabor-50-10-pi-over-2.png"))
rlm@364: #+end_src
rlm@364:
rlm@364:
rlm@364: [[../images/gabor-50-50.png]]
rlm@364:
rlm@364:
rlm@364:
rlm@364: #+begin_src clojure
rlm@364: (draw-kernel! (gabor-kernel 50 50 0 1 0)
rlm@364: (str img-base "gabor-50-50.png"))
rlm@364:
rlm@364: #+end_src
rlm@364:
rlm@364: [[../images/gabor-50-10-0-3.png]]
rlm@364:
rlm@364: #+begin_src clojure
rlm@364: (draw-kernel! (gabor-kernel 50 10 0 3 0)
rlm@364: (str img-base "gabor-50-10-0-3.png"))
rlm@364: #+end_src
rlm@364:
rlm@364:
rlm@364:
rlm@364: [[../images/gabor-50-4-pi-over3-3.png]]
rlm@364: #+begin_src clojure
rlm@364: (draw-kernel! (gabor-kernel 50 4 (/ Math/PI 3) 3 0)
rlm@364: (str img-base "gabor-50-4-pi-over3-3.png"))
rlm@364: #+end_src
rlm@364:
rlm@364:
rlm@364:
rlm@364:
rlm@364:
rlm@364: #:name gabor-tail
rlm@364: #+begin_src clojure
rlm@361: (defn show-kernel [kernel]
rlm@361: (let [img-path "/home/r/proj/cortex/tmp/kernel.png"]
rlm@362: (draw-kernel! kernel img-path)
rlm@364: (view (ImagePlus. img-path))))
rlm@359:
rlm@359: (defn print-kernel [kernel]
rlm@359: (println (.dump kernel)))
rlm@359:
rlm@363:
rlm@363: (def brick-length 0.48)
rlm@363: (def brick-width 0.24)
rlm@363: (def brick-height 0.12)
rlm@363: (def gravity (Vector3f. 0 -9.81 0))
rlm@363:
rlm@363:
rlm@363: (defn brick* [position]
rlm@363: (println "get brick.")
rlm@363: (doto (box brick-length brick-height brick-width
rlm@363: :position position :name "brick"
rlm@363: :material "Common/MatDefs/Misc/Unshaded.j3md"
rlm@363: :texture "Textures/Terrain/BrickWall/BrickWall.jpg"
rlm@363: :mass 34)
rlm@363: (->
rlm@363: (.getMesh)
rlm@363: (.scaleTextureCoordinates (Vector2f. 1 0.5)))
rlm@363: (.setShadowMode RenderQueue$ShadowMode/CastAndReceive)
rlm@363: )
rlm@363: )
rlm@363:
rlm@363:
rlm@363: (defn floor*
rlm@363: "make a sturdy, unmovable physical floor"
rlm@363: []
rlm@366: (box 10 0.1 5 :name "floor" :mass 0
rlm@366: :color ColorRGBA/Gray :position (Vector3f. 0 0 0)))
rlm@363:
rlm@363: (defn floor* []
rlm@363: (doto (box 10 0.1 5 :name "floor" ;10 0.1 5 ; 240 0.1 240
rlm@363: :material "Common/MatDefs/Misc/Unshaded.j3md"
rlm@363: :texture "Textures/BronzeCopper030.jpg"
rlm@363: :position (Vector3f. 0 0 0 )
rlm@363: :mass 0)
rlm@363: (->
rlm@363: (.getMesh)
rlm@363: (.scaleTextureCoordinates (Vector2f. 3 6)));64 64
rlm@363: (->
rlm@363: (.getMaterial)
rlm@363: (.getTextureParam "ColorMap")
rlm@363: (.getTextureValue)
rlm@363: (.setWrap Texture$WrapMode/Repeat))
rlm@363: (.setShadowMode RenderQueue$ShadowMode/Receive)
rlm@363: ))
rlm@363:
rlm@363:
rlm@363: (defn brick-wall* []
rlm@363: (let [node (Node. "brick-wall")]
rlm@363: (dorun
rlm@363: (map
rlm@363: (comp #(.attachChild node %) brick*)
rlm@363: (for [y (range 10)
rlm@363: x (range 4)
rlm@363: z (range 1)]
rlm@363: (Vector3f.
rlm@363: (+ (* 2 x brick-length)
rlm@363: (if (even? (+ y z))
rlm@363: (/ brick-length 4) (/ brick-length -4)))
rlm@363: (+ (* brick-height (inc (* 2 y))))
rlm@363: (* 2 z brick-width) ))))
rlm@363: (.setShadowMode node RenderQueue$ShadowMode/CastAndReceive)
rlm@363: node))
rlm@363:
rlm@363: (import com.aurellem.capture.Capture)
rlm@363:
rlm@363: (import java.io.File)
rlm@363:
rlm@364: (def base "/home/r/proj/cortex/render/gabor-1/")
rlm@363:
rlm@363: (defn brick-wall-game-run [record?]
rlm@364: (let [capture-dir (File. base "main")]
rlm@364:
rlm@364: (.mkdir (File. base "main"))
rlm@364: (doto
rlm@364: (world
rlm@364: (doto (Node.) (.attachChild (floor*))
rlm@364: (.attachChild (brick-wall*))
rlm@364: )
rlm@364: {"key-f" (fn [game value]
rlm@364: (if (not value) (add-element game (brick-wall*))))
rlm@364: "key-space" (fire-cannon-ball )}
rlm@364: (fn [world]
rlm@366: (position-camera
rlm@366: world
rlm@366: (Vector3f. 1.382548, 4.0383573, 5.994235)
rlm@366: (Quaternion. 0.0013082094, 0.98581666,
rlm@366: -0.1676442, 0.0076932586))
rlm@363:
rlm@364: ;;(speed-up world)
rlm@364:
rlm@364: (if record?
rlm@364: (Capture/captureVideo
rlm@364: world capture-dir))
rlm@364:
rlm@363: (add-camera! world (.getCamera world) no-op))
rlm@364: (fn [& _]))
rlm@364: (.start))))
rlm@363:
rlm@364: (defn convolve-preview [kernel]
rlm@363: (let [input "/home/r/proj/cortex/render/gabor-1/main/0000032.png"
rlm@357:
rlm@357:
rlm@357: output "/home/r/ppp.png"
rlm@356:
rlm@357: i (org.opencv.highgui.Highgui/imread input)
rlm@358:
rlm@364: ;;kernel (gabor-kernel 10 1 (/ Math/PI 2) 10 0)
rlm@358:
rlm@358: new-mat (Mat.)
rlm@358:
rlm@357: ]
rlm@356:
rlm@360: (org.opencv.imgproc.Imgproc/filter2D i new-mat CvType/CV_32F kernel)
rlm@358:
rlm@358: (org.opencv.highgui.Highgui/imwrite "/home/r/ppp.png" new-mat)
rlm@358:
rlm@357: (view (ImagePlus. input))
rlm@361: (view (ImagePlus. output))))
rlm@357:
rlm@364: (use 'clojure.java.shell)
rlm@363:
rlm@363:
rlm@364: (defn apply-gabor [kernel source dest]
rlm@364: (let [i (org.opencv.highgui.Highgui/imread source)
rlm@364: new-mat (Mat.)]
rlm@364:
rlm@364: (println dest)
rlm@364: (if (not (.exists (File. dest)))
rlm@364: (do
rlm@364: (org.opencv.imgproc.Imgproc/filter2D i new-mat CvType/CV_32F kernel)
rlm@364: (org.opencv.highgui.Highgui/imwrite dest new-mat)
rlm@364: (println "mogrify" "-modulate" "1000%" dest)
rlm@364: (sh "mogrify" "-modulate" "1000%" dest)))))
rlm@363:
rlm@363:
rlm@364: (import java.io.File)
rlm@363:
rlm@364: (defn images [path]
rlm@364: (sort (rest (file-seq (File. path)))))
rlm@364:
rlm@364:
rlm@364:
rlm@364: (defn pics [file]
rlm@364: (images (str base file)))
rlm@364:
rlm@364: (defn generate-gabor-images [kernel name]
rlm@364: (draw-kernel! kernel (str base name ".png"))
rlm@364:
rlm@364: (.mkdir (File. (str base name)))
rlm@364:
rlm@364: (let [main (map #(.getCanonicalPath %) (pics "main"))
rlm@364: targets (map #(str base name "/" (format "%07d.png" %))
rlm@364: (range 0 (count main)))]
rlm@364: (dorun (pmap (partial apply-gabor kernel) main targets))))
rlm@364:
rlm@364:
rlm@364: (def banks
rlm@364: [[(gabor-kernel 2.8 3.5) "bank-1-1"]
rlm@364: [(gabor-kernel 2.8 3.5 (/ Math/PI 2)) "bank-1-1-rot"]
rlm@364:
rlm@364: ;; [(gabor-kernel 3.6 4.6) "bank-1-2"]
rlm@364: ;; [(gabor-kernel 4.5 5.6) "bank-2-1"]
rlm@364: ;; [(gabor-kernel 6.3 7.9) "bank-3-1"]
rlm@364: ;; [(gabor-kernel 7.3 9.1) "bank-3-2"]
rlm@364:
rlm@364: [(gabor-kernel 12.3 15.4) "bank-6-1"]
rlm@364:
rlm@364:
rlm@364: ;; [(gabor-kernel 17 21.2) "bank-8-1"]
rlm@364: ;; [(gabor-kernel 18.2 22.8) "bank-8-2"]
rlm@364: ])
rlm@364:
rlm@364:
rlm@364: (defn make-all-images []
rlm@364: (dorun (map (partial apply generate-gabor-images) banks)))
rlm@364:
rlm@364:
rlm@364:
rlm@364: (defn compile-left-right []
rlm@364: (.mkdir (File. (str base "left-right")))
rlm@364: (let [main (pics "main")
rlm@364: left (pics "bank-1-1")
rlm@364: right (pics "bank-1-1-rot")
rlm@364: left-kernel (repeat 20000 (File. (str base "bank-1-1.png")))
rlm@364: right-kernel (repeat 20000 (File. (str base "bank-1-1-rot.png")))
rlm@364: targets (map
rlm@364: #(File. (str base "left-right/" (format "%07d.png" %)))
rlm@364: (range 0 (count main)))]
rlm@364:
rlm@364: (dorun
rlm@364: (pmap
rlm@364: (comp
rlm@364: (fn [[main left right left-kernel right-kernel target]]
rlm@364: (println target)
rlm@364: (if (not (.exists (File. target)))
rlm@364: (sh "convert"
rlm@364: "-size" "1940x515" "xc:white"
rlm@364: main "-geometry" "+0+0" "-composite"
rlm@364: left "-geometry" "+650+0" "-composite"
rlm@364: right "-geometry" "+1300+0" "-composite"
rlm@364: left-kernel "-geometry" "+960+485" "-composite"
rlm@364: right-kernel "-geometry" "+1610+485" "-composite"
rlm@364: target)))
rlm@364: (fn [& args] (map #(.getCanonicalPath %) args)))
rlm@364: main left right left-kernel right-kernel targets))))
rlm@364:
rlm@364:
rlm@364: (defn compile-big-small []
rlm@364: (.mkdir (File. (str base "big-small")))
rlm@364: (let [main (pics "main")
rlm@364: left (pics "bank-1-1")
rlm@364: right (pics "bank-6-1")
rlm@364: small-kernel (repeat 20000 (File. (str base "bank-1-1.png")))
rlm@364: big-kernel (repeat 20000 (File. (str base "bank-6-1.png")))
rlm@364: targets (map
rlm@364: #(File. (str base "big-small/" (format "%07d.png" %)))
rlm@364: (range 0 (count main)))]
rlm@364:
rlm@364: (dorun
rlm@364: (pmap
rlm@364: (comp
rlm@364: (fn [[main left right small-kernel big-kernel target]]
rlm@364: (println target)
rlm@364: (if (not (.exists (File. target)))
rlm@364: (sh "convert"
rlm@364: "-size" "1940x610" "xc:white"
rlm@364: main "-geometry" "+0+0" "-composite"
rlm@364: left "-geometry" "+650+0" "-composite"
rlm@364: right "-geometry" "+1300+0" "-composite"
rlm@364: small-kernel "-geometry" "+960+485" "-composite"
rlm@364: big-kernel "-geometry" "+1560+485" "-composite"
rlm@364: target)))
rlm@364: (fn [& args] (map #(.getCanonicalPath %) args)))
rlm@364: main left right small-kernel big-kernel targets))))
rlm@364:
rlm@364:
rlm@364: (defn regen-everything []
rlm@364: (make-all-images)
rlm@364: (compile-left-right)
rlm@364: (compile-big-small))
rlm@364:
rlm@364:
rlm@364: #+end_src
rlm@364:
rlm@364:
rlm@370: #+name: make-videos
rlm@370: #+begin_src makefile
rlm@370: scale:
rlm@370: ffmpeg -framerate 60 -i ./big-small/%07d.png -b:v 9000k\
rlm@370: -c:v theora -r 60 gabor-scale.ogg
rlm@364:
rlm@370: rotation:
rlm@370: ffmpeg -framerate 60 -i ./left-right/%07d.png -b:v 9000k\
rlm@370: -c:v theora -r 60 gabor-rotation.ogg
rlm@370:
rlm@370: all: rotation scale
rlm@370:
rlm@370: clean:
rlm@370: rm gabor-rotation.org gabor-scale.ogg
rlm@364: #+end_src
rlm@364:
rlm@364:
rlm@365: #+begin_html
rlm@365:
rlm@365: #+end_html
rlm@365:
rlm@365:
rlm@365:
rlm@365: #+begin_html
rlm@365:
rlm@365: #+end_html
rlm@365:
rlm@365:
rlm@356:
rlm@356: * COMMENT Generate Source
rlm@356: #+begin_src clojure :tangle ../src/cortex/gabor.clj
rlm@356: <>
rlm@364: <>
rlm@356: #+end_src
rlm@364:
rlm@370: #+begin_src Makefile :tangle ../render/gabor-1/Makefile
rlm@370: <>
rlm@370: #+end_src