Mercurial > cortex

     1 #+title: Fun with Gabor Filters

     2 #+author: Robert McIntyre

     3 #+email: rlm@mit.edu

     4 #+description: gabor filters in clojure with opencv

     5 #+keywords: computer vision, jMonkeyEngine3, clojure, opencv

     6 #+SETUPFILE: ../../aurellem/org/setup.org

     7 #+INCLUDE: ../../aurellem/org/level-0.org

     8 #+babel: :mkdirp yes :noweb yes :exports both

     9 

    10 

    11 Gabor filters were invented by the same guy who invented holograms.

    12 

    13 They work well as edge detectors and are related to the human visual

    14 system. 

    15 

    16 #+name: gabor

    17 #+begin_src clojure

    18 (ns cortex.gabor

    19   (:import org.opencv.core.CvType)

    20   (:import java.awt.image.BufferedImage)

    21   (:import ij.ImagePlus)

    22   (:import org.opencv.core.Mat)

    23   (:use (cortex world sense util vision import))

    24   (:import com.jme3.post.SceneProcessor)

    25   (:import (com.jme3.util BufferUtils Screenshots))

    26   (:import java.nio.ByteBuffer)

    27   (:import java.awt.image.BufferedImage)

    28   (:import (com.jme3.renderer ViewPort Camera))

    29   (:import (com.jme3.math ColorRGBA Vector3f Matrix3f Vector2f))

    30   (:import com.jme3.renderer.Renderer)

    31   (:import com.jme3.app.Application)

    32   (:import com.jme3.texture.FrameBuffer)

    33   (:import (com.jme3.scene Node Spatial)))

    34 

    35 

    36 (cortex.import/mega-import-jme3)

    37 (use  'clojure.math.numeric-tower)

    38 (defn load-opencv

    39   "Load the opencv native library. Must be called before any OpenCV

    40   stuff is used."

    41   []

    42   (clojure.lang.RT/loadLibrary "opencv_java249"))

    43 

    44 (load-opencv)

    45 

    46 (defn gabor-kernel

    47  ([sigma wavelength theta]

    48      (gabor-kernel sigma wavelength theta 1 0))

    49   ([sigma wavelength]

    50      (gabor-kernel sigma wavelength 0 1 0))

    51   ([sigma wavelength theta aspect-ratio phase-offset]

    52   

    53   ;; first, find the size of the kernel which is required

    54   (let [square #(expt % 2)

    55         rotated (fn [[x y]]

    56                   [(+ (* x (Math/cos theta)) (* y (Math/sin theta)))

    57                    (- (* y (Math/cos theta))  (* x (Math/sin theta)))])

    58                              

    59         gaussian (fn [[x y]]

    60                    (let [[x' y'] (rotated [x y])]

    61                      (Math/exp (- (/ (+ (square x')

    62                                         (square (* aspect-ratio y')))

    63                                    (* 2 (square sigma)))))))

    64         sinusoid (fn [[x y]]

    65                    (let [[x' y'] (rotated [x y])]

    66                      (Math/cos

    67                       (+ (* 2 Math/PI (/ x' wavelength))

    68                          phase-offset))))

    69 

    70         half-width

    71         (let [std-dev-capture 5]

    72           (max

    73            (int (* std-dev-capture (/ sigma aspect-ratio)))

    74            (int (* std-dev-capture sigma))

    75            (int (* std-dev-capture (/ aspect-ratio sigma)))))

    76 

    77         grid (let [axis (range (- half-width) (inc half-width))]

    78                (for [y (reverse axis) x axis] (vector x y)))

    79 

    80         scale (reduce + (map gaussian grid))

    81         

    82         gabor (fn [[x y :as coord]]

    83                 (* (sinusoid coord) (gaussian coord) (/ scale)))

    84 

    85         mat-width (+ 1 (* 2 half-width))

    86         mat (Mat. mat-width mat-width CvType/CV_32F)]

    87     

    88     (.put mat 0 0 (float-array (map gabor grid)))

    89     mat)))

    90 

    91 

    92 (defn draw-kernel! [kernel img-path]

    93   (let [output img-path

    94         size (.size kernel)

    95         width (int (.width size))

    96         height (int (.height size))

    97         tmp-array (float-array (* width height))]

    98 

    99     ;; read values from matrix.

   100     (.get kernel 0 0 tmp-array)

   101 

   102     ;; find overall dynamic range of the filter

   103     (let [vals (vec tmp-array)

   104           low (apply min vals)

   105           high (apply max vals)

   106           scaled-vals (map #(* 255 (- % low) (/ (- high low))) vals)

   107           new-mat (Mat. height width CvType/CV_32F)]

   108       (.put new-mat 0 0 (float-array scaled-vals))

   109       (org.opencv.highgui.Highgui/imwrite output new-mat))))

   110 

   111 ;; some cool examples

   112 #+end_src

   113 

   114 

   115 

   116 

   117 [[../images/gabor-50-10.png]]

   118 

   119 #+begin_src clojure

   120 (def img-base "/home/r/proj/cortex/images/")

   121 

   122 (draw-kernel! (gabor-kernel 50 10 0 1 0)

   123               (str img-base "gabor-50-10.png"))

   124 #+end_src

   125 

   126 

   127 [[../images/gabor-50-10-pi-over-4.png]]

   128 

   129 #+begin_src clojure

   130 (draw-kernel! (gabor-kernel 50 10 (/ Math/PI 4) 1 0)

   131               (str img-base "gabor-50-10-pi-over-4.png"))

   132 #+end_src

   133 

   134 

   135 [[../images/gabor-50-10-pi-over-2.png]]

   136 

   137 #+begin_src clojure

   138 (draw-kernel! (gabor-kernel 50 10 (/ Math/PI 2) 1 0)

   139               (str img-base "gabor-50-10-pi-over-2.png"))

   140 #+end_src

   141 

   142 

   143 [[../images/gabor-50-50.png]]

   144 

   145 

   146 

   147 #+begin_src clojure

   148 (draw-kernel! (gabor-kernel 50 50 0 1 0)

   149               (str img-base "gabor-50-50.png"))

   150 

   151 #+end_src

   152 

   153 [[../images/gabor-50-10-0-3.png]]

   154 

   155 #+begin_src clojure

   156 (draw-kernel! (gabor-kernel 50 10 0 3 0)

   157               (str img-base "gabor-50-10-0-3.png"))

   158 #+end_src

   159 

   160 

   161 

   162 [[../images/gabor-50-4-pi-over3-3.png]]

   163 #+begin_src clojure

   164 (draw-kernel! (gabor-kernel 50 4 (/ Math/PI 3) 3 0)

   165               (str img-base "gabor-50-4-pi-over3-3.png"))

   166 #+end_src

   167    

   168 

   169 #+name: gabor-tail

   170 #+begin_src clojure

   171 (defn show-kernel [kernel]

   172   (let [img-path "/home/r/proj/cortex/tmp/kernel.png"]

   173     (draw-kernel! kernel img-path)

   174     (view (ImagePlus. img-path))))

   175 

   176 (defn print-kernel [kernel]

   177   (println (.dump kernel)))

   178                            

   179 

   180 (def brick-length 0.48)

   181 (def brick-width 0.24)

   182 (def brick-height 0.12)

   183 (def gravity (Vector3f. 0 -9.81 0))

   184 

   185 

   186 (defn brick* [position]

   187   (println "get brick.")

   188   (doto (box brick-length brick-height brick-width

   189 	     :position position :name "brick"

   190 	     :material "Common/MatDefs/Misc/Unshaded.j3md"

   191 	     :texture "Textures/Terrain/BrickWall/BrickWall.jpg"

   192 	     :mass 34)

   193     (->

   194      (.getMesh)

   195      (.scaleTextureCoordinates (Vector2f. 1 0.5)))

   196     (.setShadowMode RenderQueue$ShadowMode/CastAndReceive)

   197     )

   198   )

   199 

   200 

   201 (defn floor*

   202   "make a sturdy, unmovable physical floor"

   203   []

   204   (box 10 0.1 5 :name "floor" :mass 0 

   205        :color ColorRGBA/Gray :position (Vector3f. 0 0 0)))

   206 

   207 (defn floor* []

   208   (doto (box 10 0.1 5 :name "floor" ;10 0.1 5 ; 240 0.1 240

   209 	     :material "Common/MatDefs/Misc/Unshaded.j3md"

   210 	     :texture "Textures/BronzeCopper030.jpg"

   211 	     :position (Vector3f. 0 0 0 )

   212 	     :mass 0)

   213     (->

   214      (.getMesh)

   215      (.scaleTextureCoordinates (Vector2f. 3 6)));64 64

   216     (->

   217      (.getMaterial)

   218      (.getTextureParam "ColorMap")

   219      (.getTextureValue)

   220      (.setWrap Texture$WrapMode/Repeat))

   221     (.setShadowMode RenderQueue$ShadowMode/Receive)

   222   ))

   223 

   224 

   225 (defn brick-wall* []

   226   (let [node (Node. "brick-wall")]

   227     (dorun

   228      (map

   229       (comp  #(.attachChild node %) brick*)

   230        (for [y (range 10)

   231 	     x (range 4)

   232 	     z (range 1)]

   233        	    (Vector3f.

   234        	     (+ (* 2 x brick-length)

   235 		(if (even? (+ y z)) 

   236 		  (/ brick-length 4) (/ brick-length -4)))

   237        	     (+ (* brick-height (inc (* 2 y))))

   238 	     (* 2 z brick-width) ))))

   239     (.setShadowMode node RenderQueue$ShadowMode/CastAndReceive)

   240     node))

   241 

   242 (import com.aurellem.capture.Capture)

   243 

   244 (import java.io.File)

   245 

   246 (def base "/home/r/proj/cortex/render/gabor-1/")

   247 

   248 (defn brick-wall-game-run [record?]

   249   (let [capture-dir (File. base "main")]

   250     

   251     (.mkdir (File. base "main"))

   252     (doto

   253         (world

   254          (doto (Node.) (.attachChild (floor*))

   255                (.attachChild (brick-wall*))

   256                )

   257          {"key-f" (fn [game value] 

   258                     (if (not value) (add-element game (brick-wall*))))

   259           "key-space" (fire-cannon-ball )}

   260          (fn [world]

   261            (position-camera

   262             world

   263             (Vector3f. 1.382548, 4.0383573, 5.994235)

   264             (Quaternion. 0.0013082094, 0.98581666,

   265                          -0.1676442, 0.0076932586))

   266 

   267            ;;(speed-up world)

   268            

   269            (if record?

   270              (Capture/captureVideo

   271               world capture-dir))

   272               

   273            (add-camera! world (.getCamera world) no-op))

   274          (fn [& _]))     

   275       (.start))))

   276 

   277 (defn convolve-preview [kernel]

   278   (let [input "/home/r/proj/cortex/render/gabor-1/main/0000032.png"

   279         

   280         

   281         output "/home/r/ppp.png"

   282 

   283         i (org.opencv.highgui.Highgui/imread input)

   284 

   285         ;;kernel (gabor-kernel 10 1 (/ Math/PI 2) 10 0)

   286 

   287         new-mat (Mat.)

   288                 

   289         ]

   290 

   291     (org.opencv.imgproc.Imgproc/filter2D

   292      i new-mat CvType/CV_32F kernel)

   293     

   294     (org.opencv.highgui.Highgui/imwrite "/home/r/ppp.png" new-mat)

   295     

   296     (view (ImagePlus. input))

   297     (view (ImagePlus. output))))

   298 

   299 (use 'clojure.java.shell)

   300 

   301 

   302 (defn apply-gabor [kernel source dest]

   303     (let [i (org.opencv.highgui.Highgui/imread source)

   304           new-mat (Mat.)]

   305       

   306       (println dest)

   307       (if (not (.exists (File. dest)))

   308         (do

   309           (org.opencv.imgproc.Imgproc/filter2D

   310            i new-mat CvType/CV_32F kernel)

   311           (org.opencv.highgui.Highgui/imwrite dest new-mat)

   312           (println "mogrify" "-modulate" "1000%" dest)

   313           (sh "mogrify" "-modulate" "1000%" dest)))))

   314 

   315 

   316 (import java.io.File)

   317 

   318 (defn images [path]

   319   (sort (rest (file-seq (File. path)))))

   320 

   321 

   322 

   323 (defn pics [file]

   324   (images (str base file)))

   325 

   326 (defn generate-gabor-images [kernel name]

   327   (draw-kernel! kernel (str base name ".png"))

   328 

   329   (.mkdir (File. (str base name)))

   330   

   331   (let [main (map #(.getCanonicalPath %) (pics "main"))

   332         targets (map #(str base name "/" (format "%07d.png" %))

   333                      (range 0 (count main)))]

   334     (dorun (pmap (partial apply-gabor kernel) main targets))))

   335 

   336 

   337 (def banks

   338   [[(gabor-kernel 2.8 3.5) "bank-1-1"]

   339    [(gabor-kernel 2.8 3.5 (/ Math/PI 2)) "bank-1-1-rot"]

   340    

   341 ;;   [(gabor-kernel 3.6 4.6) "bank-1-2"]

   342 ;;   [(gabor-kernel 4.5 5.6) "bank-2-1"]

   343 ;;   [(gabor-kernel 6.3 7.9) "bank-3-1"]

   344 ;;   [(gabor-kernel 7.3 9.1) "bank-3-2"]

   345 

   346    [(gabor-kernel 12.3 15.4) "bank-6-1"]

   347    

   348 

   349 ;;   [(gabor-kernel 17 21.2) "bank-8-1"]

   350 ;;   [(gabor-kernel 18.2 22.8) "bank-8-2"]

   351    ])

   352    

   353 

   354 (defn make-all-images []

   355   (dorun (map (partial apply generate-gabor-images) banks)))

   356 

   357 

   358 

   359 (defn compile-left-right []

   360   (.mkdir (File. (str base "left-right")))

   361   (let [main (pics "main")

   362         left (pics "bank-1-1")

   363         right (pics "bank-1-1-rot")

   364         left-kernel (repeat 20000 (File. (str base "bank-1-1.png")))

   365         right-kernel (repeat 20000 (File. (str base "bank-1-1-rot.png")))

   366         targets (map

   367                  #(File. (str base "left-right/" (format "%07d.png" %)))

   368                  (range 0 (count main)))]

   369     

   370     (dorun

   371      (pmap

   372       (comp

   373        (fn [[main left right left-kernel right-kernel target]]

   374          (println target)

   375          (if (not (.exists (File. target)))

   376            (sh "convert"

   377                "-size" "1940x515" "xc:white"

   378                main      "-geometry" "+0+0"    "-composite"

   379                left "-geometry" "+650+0"   "-composite"

   380                right     "-geometry" "+1300+0"  "-composite"

   381                left-kernel      "-geometry" "+960+485"  "-composite"

   382                right-kernel      "-geometry" "+1610+485"  "-composite"

   383                target)))

   384        (fn [& args] (map #(.getCanonicalPath %) args)))

   385       main left right left-kernel right-kernel targets))))

   386 

   387 

   388 (defn compile-big-small []

   389   (.mkdir (File. (str base "big-small")))

   390   (let [main (pics "main")

   391         left (pics "bank-1-1")

   392         right (pics "bank-6-1")

   393         small-kernel (repeat 20000 (File. (str base "bank-1-1.png")))

   394         big-kernel (repeat 20000 (File. (str base "bank-6-1.png")))

   395         targets (map

   396                  #(File. (str base "big-small/" (format "%07d.png" %)))

   397                  (range 0 (count main)))]

   398     

   399     (dorun

   400      (pmap

   401       (comp

   402        (fn [[main left right small-kernel big-kernel target]]

   403          (println target)

   404          (if (not (.exists (File. target)))

   405            (sh "convert"

   406                "-size" "1940x610" "xc:white"

   407                main      "-geometry" "+0+0"    "-composite"

   408                left "-geometry" "+650+0"   "-composite"

   409                right     "-geometry" "+1300+0"  "-composite"

   410                small-kernel      "-geometry" "+960+485"  "-composite"

   411                big-kernel      "-geometry" "+1560+485"  "-composite"

   412                target)))

   413        (fn [& args] (map #(.getCanonicalPath %) args)))

   414       main left right small-kernel big-kernel targets))))

   415 

   416 

   417 (defn regen-everything []

   418   (make-all-images)

   419   (compile-left-right)

   420   (compile-big-small))

   421 

   422       

   423 #+end_src

   424 

   425 

   426 #+name: make-videos

   427 #+begin_src makefile

   428 scale: 

   429 	ffmpeg -framerate 60 -i ./big-small/%07d.png -b:v 9000k\

   430             -c:v theora -r 60 gabor-scale.ogg

   431 

   432 rotation: 

   433 	ffmpeg -framerate 60 -i ./left-right/%07d.png -b:v 9000k\

   434             -c:v theora -r 60 gabor-rotation.ogg

   435 

   436 all: rotation scale

   437 

   438 clean: 

   439 	rm gabor-rotation.org gabor-scale.ogg

   440 #+end_src

   441 

   442 

   443 #+begin_html

   444 <div class="figure">

   445 <video controls="controls">

   446   <source src="../video/gabor-rotation.ogg" type="video/ogg"

   447 	  preload="none" poster="../images/aurellem-1280x480.png" />

   448 </video>

   449  <br> <a href="http://youtu.be/bxujjO97B-U"> YouTube </a>

   450 <p>Two gabor filters with different values of theta are compared. The

   451 horizontally aligned one does better in this example.</p>

   452 </div>

   453 #+end_html

   454 

   455 

   456 

   457 #+begin_html

   458 <div class="figure">

   459 <video controls="controls">

   460   <source src="../video/gabor-scale.ogg" type="video/ogg"

   461 	  preload="none" poster="../images/aurellem-1280x480.png" />

   462 </video>

   463  <br> <a href="http://youtu.be/-EsfA2ceWUk"> YouTube </a>

   464 <p>Here we compare a gabor filter from bank 1 with one from bank 6.</p>

   465 </div>

   466 #+end_html

   467 

   468 * Source Listing

   469   - [[../src/cortex/gabor.clj][cortex.gabor]]

   470 #+html: <ul> <li> <a href="../org/gabor.org">This org file</a> </li> </ul>

   471   - [[http://hg.bortreb.com ][source-repository]]

   472 

   473 

   474 

   475 * COMMENT Generate Source

   476 #+begin_src clojure :tangle ../src/cortex/gabor.clj

   477 <<gabor>>

   478 <<gabor-tail>>

   479 #+end_src

   480 

   481 #+begin_src Makefile :tangle ../render/gabor-1/Makefile

   482 <<make-videos>>

   483 #+end_src
author	Robert McIntyre <rlm@mit.edu>
date	Thu, 27 Mar 2014 17:56:25 -0400
parents	057d47fc4789
children	42ddfe406c0a