Mercurial > cortex

     1 #+title: Fun with Gabor Filters

     2 #+author: Robert McIntyre

     3 #+email: rlm@mit.edu

     4 #+description: gabor filters in clojure with opencv

     5 #+keywords: computer vision, jMonkeyEngine3, clojure, opencv

     6 #+SETUPFILE: ../../aurellem/org/setup.org

     7 #+INCLUDE: ../../aurellem/org/level-0.org

     8 #+babel: :mkdirp yes :noweb yes :exports both

     9 

    10 

    11 

    12 #+name: gabor

    13 #+begin_src clojure

    14 (ns cortex.gabor

    15   (:import org.opencv.core.CvType)

    16   (:import java.awt.image.BufferedImage)

    17   (:import ij.ImagePlus)

    18   (:import org.opencv.core.Mat)

    19   (:use (cortex world sense util vision import))

    20   (:import com.jme3.post.SceneProcessor)

    21   (:import (com.jme3.util BufferUtils Screenshots))

    22   (:import java.nio.ByteBuffer)

    23   (:import java.awt.image.BufferedImage)

    24   (:import (com.jme3.renderer ViewPort Camera))

    25   (:import (com.jme3.math ColorRGBA Vector3f Matrix3f Vector2f))

    26   (:import com.jme3.renderer.Renderer)

    27   (:import com.jme3.app.Application)

    28   (:import com.jme3.texture.FrameBuffer)

    29   (:import (com.jme3.scene Node Spatial)))

    30 

    31 

    32 (cortex.import/mega-import-jme3)

    33 (use  'clojure.math.numeric-tower)

    34 (defn load-opencv

    35   "Load the opencv native library. Must be called before any OpenCV

    36   stuff is used."

    37   []

    38   (clojure.lang.RT/loadLibrary "opencv_java249"))

    39 

    40 (load-opencv)

    41 

    42 (defn gabor-kernel

    43  ([sigma wavelength theta]

    44      (gabor-kernel sigma wavelength theta 1 0))

    45   ([sigma wavelength]

    46      (gabor-kernel sigma wavelength 0 1 0))

    47   ([sigma wavelength theta aspect-ratio phase-offset]

    48   

    49   ;; first, find the size of the kernel which is required

    50   (let [square #(expt % 2)

    51         rotated (fn [[x y]]

    52                   [(+ (* x (Math/cos theta)) (* y (Math/sin theta)))

    53                    (- (* y (Math/cos theta))  (* x (Math/sin theta)))])

    54                              

    55         gaussian (fn [[x y]]

    56                    (let [[x' y'] (rotated [x y])]

    57                      (Math/exp (- (/ (+ (square x')

    58                                         (square (* aspect-ratio y')))

    59                                    (* 2 (square sigma)))))))

    60         sinusoid (fn [[x y]]

    61                    (let [[x' y'] (rotated [x y])]

    62                      (Math/cos

    63                       (+ (* 2 Math/PI (/ x' wavelength))

    64                          phase-offset))))

    65 

    66         half-width

    67         (let [std-dev-capture 5]

    68           (max

    69            (int (* std-dev-capture (/ sigma aspect-ratio)))

    70            (int (* std-dev-capture sigma))

    71            (int (* std-dev-capture (/ aspect-ratio sigma)))))

    72 

    73         grid (let [axis (range (- half-width) (inc half-width))]

    74                (for [y (reverse axis) x axis] (vector x y)))

    75 

    76         scale (reduce + (map gaussian grid))

    77         

    78         gabor (fn [[x y :as coord]]

    79                 (* (sinusoid coord) (gaussian coord) (/ scale)))

    80 

    81         mat-width (+ 1 (* 2 half-width))

    82         mat (Mat. mat-width mat-width CvType/CV_32F)]

    83     

    84     (.put mat 0 0 (float-array (map gabor grid)))

    85     mat)))

    86 

    87 

    88 (defn draw-kernel! [kernel img-path]

    89   (let [output img-path

    90         size (.size kernel)

    91         width (int (.width size))

    92         height (int (.height size))

    93         tmp-array (float-array (* width height))]

    94 

    95     ;; read values from matrix.

    96     (.get kernel 0 0 tmp-array)

    97 

    98     ;; find overall dynamic range of the filter

    99     (let [vals (vec tmp-array)

   100           low (apply min vals)

   101           high (apply max vals)

   102           scaled-vals (map #(* 255 (- % low) (/ (- high low))) vals)

   103           new-mat (Mat. height width CvType/CV_32F)]

   104       (.put new-mat 0 0 (float-array scaled-vals))

   105       (org.opencv.highgui.Highgui/imwrite output new-mat))))

   106 

   107 ;; some cool examples

   108 #+end_src

   109 

   110 

   111 

   112 

   113 [[../images/gabor-50-10.png]]

   114 

   115 #+begin_src clojure

   116 (def img-base "/home/r/proj/cortex/images/")

   117 

   118 (draw-kernel! (gabor-kernel 50 10 0 1 0)

   119               (str img-base "gabor-50-10.png"))

   120 #+end_src

   121 

   122 

   123 [[../images/gabor-50-10-pi-over-4.png]]

   124 

   125 #+begin_src clojure

   126 (draw-kernel! (gabor-kernel 50 10 (/ Math/PI 4) 1 0)

   127               (str img-base "gabor-50-10-pi-over-4.png"))

   128 #+end_src

   129 

   130 

   131 [[../images/gabor-50-10-pi-over-2.png]]

   132 

   133 #+begin_src clojure

   134 (draw-kernel! (gabor-kernel 50 10 (/ Math/PI 2) 1 0)

   135               (str img-base "gabor-50-10-pi-over-2.png"))

   136 #+end_src

   137 

   138 

   139 [[../images/gabor-50-50.png]]

   140 

   141 

   142 

   143 #+begin_src clojure

   144 (draw-kernel! (gabor-kernel 50 50 0 1 0)

   145               (str img-base "gabor-50-50.png"))

   146 

   147 #+end_src

   148 

   149 [[../images/gabor-50-10-0-3.png]]

   150 

   151 #+begin_src clojure

   152 (draw-kernel! (gabor-kernel 50 10 0 3 0)

   153               (str img-base "gabor-50-10-0-3.png"))

   154 #+end_src

   155 

   156 

   157 

   158 [[../images/gabor-50-4-pi-over3-3.png]]

   159 #+begin_src clojure

   160 (draw-kernel! (gabor-kernel 50 4 (/ Math/PI 3) 3 0)

   161               (str img-base "gabor-50-4-pi-over3-3.png"))

   162 #+end_src

   163 

   164 

   165 

   166      

   167 

   168 #:name gabor-tail

   169 #+begin_src clojure

   170 (defn show-kernel [kernel]

   171   (let [img-path "/home/r/proj/cortex/tmp/kernel.png"]

   172     (draw-kernel! kernel img-path)

   173     (view (ImagePlus. img-path))))

   174 

   175 (defn print-kernel [kernel]

   176   (println (.dump kernel)))

   177                            

   178 

   179 (def brick-length 0.48)

   180 (def brick-width 0.24)

   181 (def brick-height 0.12)

   182 (def gravity (Vector3f. 0 -9.81 0))

   183 

   184 

   185 (defn brick* [position]

   186   (println "get brick.")

   187   (doto (box brick-length brick-height brick-width

   188 	     :position position :name "brick"

   189 	     :material "Common/MatDefs/Misc/Unshaded.j3md"

   190 	     :texture "Textures/Terrain/BrickWall/BrickWall.jpg"

   191 	     :mass 34)

   192     (->

   193      (.getMesh)

   194      (.scaleTextureCoordinates (Vector2f. 1 0.5)))

   195     (.setShadowMode RenderQueue$ShadowMode/CastAndReceive)

   196     )

   197   )

   198 

   199 

   200 (defn floor*

   201   "make a sturdy, unmovable physical floor"

   202   []

   203   (box 10 0.1 5 :name "floor" :mass 0 

   204        :color ColorRGBA/Gray :position (Vector3f. 0 0 0)))

   205 

   206 (defn floor* []

   207   (doto (box 10 0.1 5 :name "floor" ;10 0.1 5 ; 240 0.1 240

   208 	     :material "Common/MatDefs/Misc/Unshaded.j3md"

   209 	     :texture "Textures/BronzeCopper030.jpg"

   210 	     :position (Vector3f. 0 0 0 )

   211 	     :mass 0)

   212     (->

   213      (.getMesh)

   214      (.scaleTextureCoordinates (Vector2f. 3 6)));64 64

   215     (->

   216      (.getMaterial)

   217      (.getTextureParam "ColorMap")

   218      (.getTextureValue)

   219      (.setWrap Texture$WrapMode/Repeat))

   220     (.setShadowMode RenderQueue$ShadowMode/Receive)

   221   ))

   222 

   223 

   224 (defn brick-wall* []

   225   (let [node (Node. "brick-wall")]

   226     (dorun

   227      (map

   228       (comp  #(.attachChild node %) brick*)

   229        (for [y (range 10)

   230 	     x (range 4)

   231 	     z (range 1)]

   232        	    (Vector3f.

   233        	     (+ (* 2 x brick-length)

   234 		(if (even? (+ y z)) 

   235 		  (/ brick-length 4) (/ brick-length -4)))

   236        	     (+ (* brick-height (inc (* 2 y))))

   237 	     (* 2 z brick-width) ))))

   238     (.setShadowMode node RenderQueue$ShadowMode/CastAndReceive)

   239     node))

   240 

   241 (import com.aurellem.capture.Capture)

   242 

   243 (import java.io.File)

   244 

   245 (def base "/home/r/proj/cortex/render/gabor-1/")

   246 

   247 (defn brick-wall-game-run [record?]

   248   (let [capture-dir (File. base "main")]

   249     

   250     (.mkdir (File. base "main"))

   251     (doto

   252         (world

   253          (doto (Node.) (.attachChild (floor*))

   254                (.attachChild (brick-wall*))

   255                )

   256          {"key-f" (fn [game value] 

   257                     (if (not value) (add-element game (brick-wall*))))

   258           "key-space" (fire-cannon-ball )}

   259          (fn [world]

   260            (position-camera

   261             world

   262             (Vector3f. 1.382548, 4.0383573, 5.994235)

   263             (Quaternion. 0.0013082094, 0.98581666,

   264                          -0.1676442, 0.0076932586))

   265 

   266            ;;(speed-up world)

   267            

   268            (if record?

   269              (Capture/captureVideo

   270               world capture-dir))

   271               

   272            (add-camera! world (.getCamera world) no-op))

   273          (fn [& _]))     

   274       (.start))))

   275 

   276 (defn convolve-preview [kernel]

   277   (let [input "/home/r/proj/cortex/render/gabor-1/main/0000032.png"

   278         

   279         

   280         output "/home/r/ppp.png"

   281 

   282         i (org.opencv.highgui.Highgui/imread input)

   283 

   284         ;;kernel (gabor-kernel 10 1 (/ Math/PI 2) 10 0)

   285 

   286         new-mat (Mat.)

   287                 

   288         ]

   289 

   290     (org.opencv.imgproc.Imgproc/filter2D  i new-mat CvType/CV_32F kernel)

   291     

   292     (org.opencv.highgui.Highgui/imwrite "/home/r/ppp.png" new-mat)

   293     

   294     (view (ImagePlus. input))

   295     (view (ImagePlus. output))))

   296 

   297 (use 'clojure.java.shell)

   298 

   299 

   300 (defn apply-gabor [kernel source dest]

   301     (let [i (org.opencv.highgui.Highgui/imread source)

   302           new-mat (Mat.)]

   303       

   304       (println dest)

   305       (if (not (.exists (File. dest)))

   306         (do

   307           (org.opencv.imgproc.Imgproc/filter2D  i new-mat CvType/CV_32F kernel)

   308           (org.opencv.highgui.Highgui/imwrite dest new-mat)

   309           (println "mogrify" "-modulate" "1000%" dest)

   310           (sh "mogrify" "-modulate" "1000%" dest)))))

   311 

   312 

   313 (import java.io.File)

   314 

   315 (defn images [path]

   316   (sort (rest (file-seq (File. path)))))

   317 

   318 

   319 

   320 (defn pics [file]

   321   (images (str base file)))

   322 

   323 (defn generate-gabor-images [kernel name]

   324   (draw-kernel! kernel (str base name ".png"))

   325 

   326   (.mkdir (File. (str base name)))

   327   

   328   (let [main (map #(.getCanonicalPath %) (pics "main"))

   329         targets (map #(str base name "/" (format "%07d.png" %))

   330                      (range 0 (count main)))]

   331     (dorun (pmap (partial apply-gabor kernel) main targets))))

   332 

   333 

   334 (def banks

   335   [[(gabor-kernel 2.8 3.5) "bank-1-1"]

   336    [(gabor-kernel 2.8 3.5 (/ Math/PI 2)) "bank-1-1-rot"]

   337    

   338 ;;   [(gabor-kernel 3.6 4.6) "bank-1-2"]

   339 ;;   [(gabor-kernel 4.5 5.6) "bank-2-1"]

   340 ;;   [(gabor-kernel 6.3 7.9) "bank-3-1"]

   341 ;;   [(gabor-kernel 7.3 9.1) "bank-3-2"]

   342 

   343    [(gabor-kernel 12.3 15.4) "bank-6-1"]

   344    

   345 

   346 ;;   [(gabor-kernel 17 21.2) "bank-8-1"]

   347 ;;   [(gabor-kernel 18.2 22.8) "bank-8-2"]

   348    ])

   349    

   350 

   351 (defn make-all-images []

   352   (dorun (map (partial apply generate-gabor-images) banks)))

   353 

   354 

   355 

   356 (defn compile-left-right []

   357   (.mkdir (File. (str base "left-right")))

   358   (let [main (pics "main")

   359         left (pics "bank-1-1")

   360         right (pics "bank-1-1-rot")

   361         left-kernel (repeat 20000 (File. (str base "bank-1-1.png")))

   362         right-kernel (repeat 20000 (File. (str base "bank-1-1-rot.png")))

   363         targets (map

   364                  #(File. (str base "left-right/" (format "%07d.png" %)))

   365                  (range 0 (count main)))]

   366     

   367     (dorun

   368      (pmap

   369       (comp

   370        (fn [[main left right left-kernel right-kernel target]]

   371          (println target)

   372          (if (not (.exists (File. target)))

   373            (sh "convert"

   374                "-size" "1940x515" "xc:white"

   375                main      "-geometry" "+0+0"    "-composite"

   376                left "-geometry" "+650+0"   "-composite"

   377                right     "-geometry" "+1300+0"  "-composite"

   378                left-kernel      "-geometry" "+960+485"  "-composite"

   379                right-kernel      "-geometry" "+1610+485"  "-composite"

   380                target)))

   381        (fn [& args] (map #(.getCanonicalPath %) args)))

   382       main left right left-kernel right-kernel targets))))

   383 

   384 

   385 (defn compile-big-small []

   386   (.mkdir (File. (str base "big-small")))

   387   (let [main (pics "main")

   388         left (pics "bank-1-1")

   389         right (pics "bank-6-1")

   390         small-kernel (repeat 20000 (File. (str base "bank-1-1.png")))

   391         big-kernel (repeat 20000 (File. (str base "bank-6-1.png")))

   392         targets (map

   393                  #(File. (str base "big-small/" (format "%07d.png" %)))

   394                  (range 0 (count main)))]

   395     

   396     (dorun

   397      (pmap

   398       (comp

   399        (fn [[main left right small-kernel big-kernel target]]

   400          (println target)

   401          (if (not (.exists (File. target)))

   402            (sh "convert"

   403                "-size" "1940x610" "xc:white"

   404                main      "-geometry" "+0+0"    "-composite"

   405                left "-geometry" "+650+0"   "-composite"

   406                right     "-geometry" "+1300+0"  "-composite"

   407                small-kernel      "-geometry" "+960+485"  "-composite"

   408                big-kernel      "-geometry" "+1560+485"  "-composite"

   409                target)))

   410        (fn [& args] (map #(.getCanonicalPath %) args)))

   411       main left right small-kernel big-kernel targets))))

   412 

   413 

   414 (defn regen-everything []

   415   (make-all-images)

   416   (compile-left-right)

   417   (compile-big-small))

   418 

   419       

   420 #+end_src

   421 

   422 #+name: make-left-right

   423 #+begin_src sh

   424 #!/bin/sh

   425 

   426 ffmpeg -framerate 60 -i ./left-right/%07d.png -b:v 9000k\

   427      -c:v mpeg4 -r 60 gabor-rotation.mp4

   428 

   429 #+end_src

   430 

   431 

   432 #+name: make-big-small

   433 #+begin_src sh

   434 #!/bin/sh

   435 

   436 ffmpeg -framerate 60 -i ./big-small/%07d.png -b:v 9000k\

   437      -c:v mpeg4 -r 60 gabor-big-small.mp4

   438 

   439 #+end_src

   440 

   441 #+begin_html

   442 <div class="figure">

   443 <video controls="controls" width="755">

   444   <source src="../video/gabor-rotation.mp4" type="video/mp4"

   445 	  preload="none" poster="../images/aurellem-1280x480.png" />

   446 </video>

   447  <br> <a href="http://youtu.be/r5Bn2aG7MO0"> YouTube </a>

   448 <p>Two gabor filters with different values of theta are compared. The

   449 horizontally aligned one does better in this example.</p>

   450 </div>

   451 #+end_html

   452 

   453 

   454 

   455 #+begin_html

   456 <div class="figure">

   457 <video controls="controls" width="755">

   458   <source src="../video/gabor-big-small.mp4" type="video/mp4"

   459 	  preload="none" poster="../images/aurellem-1280x480.png" />

   460 </video>

   461  <br> <a href="http://youtu.be/r5Bn2aG7MO0"> YouTube </a>

   462 <p>Here we compare a gabor filter from bank 1 with one from bank 6.</p>

   463 </div>

   464 #+end_html

   465 

   466 

   467 

   468 

   469 * COMMENT Generate Source

   470 #+begin_src clojure :tangle ../src/cortex/gabor.clj

   471 <<gabor>>

   472 <<gabor-tail>>

   473 #+end_src

   474 

   475 #+begin_src clojure :tangle ../render/gabor-1/make-rotation.sh

   476 <<make-left-right>>

   477 #+end_src

   478 

   479 #+begin_src clojure :tangle ../render/gabor-1/make-big-small.sh

   480 <<make-big-small>>

   481 #+end_src
author	Robert McIntyre <rlm@mit.edu>
date	Thu, 07 Mar 2013 07:50:28 +0000
parents	1727b68634ea
children	078600644d94