view org/gabor.org @ 368:7a90d37c84b0

intro.
author Robert McIntyre <rlm@mit.edu>
date Thu, 07 Mar 2013 07:54:38 +0000
parents 078600644d94
children 44fe96a568b9
line wrap: on
line source
1 #+title: Fun with Gabor Filters
2 #+author: Robert McIntyre
3 #+email: rlm@mit.edu
4 #+description: gabor filters in clojure with opencv
5 #+keywords: computer vision, jMonkeyEngine3, clojure, opencv
6 #+SETUPFILE: ../../aurellem/org/setup.org
7 #+INCLUDE: ../../aurellem/org/level-0.org
8 #+babel: :mkdirp yes :noweb yes :exports both
11 Gabor filters were invented by the same guy who invented holograms.
13 They work well as edge detectors and are related to the human visual
14 system.
16 #+name: gabor
17 #+begin_src clojure
18 (ns cortex.gabor
19 (:import org.opencv.core.CvType)
20 (:import java.awt.image.BufferedImage)
21 (:import ij.ImagePlus)
22 (:import org.opencv.core.Mat)
23 (:use (cortex world sense util vision import))
24 (:import com.jme3.post.SceneProcessor)
25 (:import (com.jme3.util BufferUtils Screenshots))
26 (:import java.nio.ByteBuffer)
27 (:import java.awt.image.BufferedImage)
28 (:import (com.jme3.renderer ViewPort Camera))
29 (:import (com.jme3.math ColorRGBA Vector3f Matrix3f Vector2f))
30 (:import com.jme3.renderer.Renderer)
31 (:import com.jme3.app.Application)
32 (:import com.jme3.texture.FrameBuffer)
33 (:import (com.jme3.scene Node Spatial)))
36 (cortex.import/mega-import-jme3)
37 (use 'clojure.math.numeric-tower)
38 (defn load-opencv
39 "Load the opencv native library. Must be called before any OpenCV
40 stuff is used."
41 []
42 (clojure.lang.RT/loadLibrary "opencv_java249"))
44 (load-opencv)
46 (defn gabor-kernel
47 ([sigma wavelength theta]
48 (gabor-kernel sigma wavelength theta 1 0))
49 ([sigma wavelength]
50 (gabor-kernel sigma wavelength 0 1 0))
51 ([sigma wavelength theta aspect-ratio phase-offset]
53 ;; first, find the size of the kernel which is required
54 (let [square #(expt % 2)
55 rotated (fn [[x y]]
56 [(+ (* x (Math/cos theta)) (* y (Math/sin theta)))
57 (- (* y (Math/cos theta)) (* x (Math/sin theta)))])
59 gaussian (fn [[x y]]
60 (let [[x' y'] (rotated [x y])]
61 (Math/exp (- (/ (+ (square x')
62 (square (* aspect-ratio y')))
63 (* 2 (square sigma)))))))
64 sinusoid (fn [[x y]]
65 (let [[x' y'] (rotated [x y])]
66 (Math/cos
67 (+ (* 2 Math/PI (/ x' wavelength))
68 phase-offset))))
70 half-width
71 (let [std-dev-capture 5]
72 (max
73 (int (* std-dev-capture (/ sigma aspect-ratio)))
74 (int (* std-dev-capture sigma))
75 (int (* std-dev-capture (/ aspect-ratio sigma)))))
77 grid (let [axis (range (- half-width) (inc half-width))]
78 (for [y (reverse axis) x axis] (vector x y)))
80 scale (reduce + (map gaussian grid))
82 gabor (fn [[x y :as coord]]
83 (* (sinusoid coord) (gaussian coord) (/ scale)))
85 mat-width (+ 1 (* 2 half-width))
86 mat (Mat. mat-width mat-width CvType/CV_32F)]
88 (.put mat 0 0 (float-array (map gabor grid)))
89 mat)))
92 (defn draw-kernel! [kernel img-path]
93 (let [output img-path
94 size (.size kernel)
95 width (int (.width size))
96 height (int (.height size))
97 tmp-array (float-array (* width height))]
99 ;; read values from matrix.
100 (.get kernel 0 0 tmp-array)
102 ;; find overall dynamic range of the filter
103 (let [vals (vec tmp-array)
104 low (apply min vals)
105 high (apply max vals)
106 scaled-vals (map #(* 255 (- % low) (/ (- high low))) vals)
107 new-mat (Mat. height width CvType/CV_32F)]
108 (.put new-mat 0 0 (float-array scaled-vals))
109 (org.opencv.highgui.Highgui/imwrite output new-mat))))
111 ;; some cool examples
112 #+end_src
117 [[../images/gabor-50-10.png]]
119 #+begin_src clojure
120 (def img-base "/home/r/proj/cortex/images/")
122 (draw-kernel! (gabor-kernel 50 10 0 1 0)
123 (str img-base "gabor-50-10.png"))
124 #+end_src
127 [[../images/gabor-50-10-pi-over-4.png]]
129 #+begin_src clojure
130 (draw-kernel! (gabor-kernel 50 10 (/ Math/PI 4) 1 0)
131 (str img-base "gabor-50-10-pi-over-4.png"))
132 #+end_src
135 [[../images/gabor-50-10-pi-over-2.png]]
137 #+begin_src clojure
138 (draw-kernel! (gabor-kernel 50 10 (/ Math/PI 2) 1 0)
139 (str img-base "gabor-50-10-pi-over-2.png"))
140 #+end_src
143 [[../images/gabor-50-50.png]]
147 #+begin_src clojure
148 (draw-kernel! (gabor-kernel 50 50 0 1 0)
149 (str img-base "gabor-50-50.png"))
151 #+end_src
153 [[../images/gabor-50-10-0-3.png]]
155 #+begin_src clojure
156 (draw-kernel! (gabor-kernel 50 10 0 3 0)
157 (str img-base "gabor-50-10-0-3.png"))
158 #+end_src
162 [[../images/gabor-50-4-pi-over3-3.png]]
163 #+begin_src clojure
164 (draw-kernel! (gabor-kernel 50 4 (/ Math/PI 3) 3 0)
165 (str img-base "gabor-50-4-pi-over3-3.png"))
166 #+end_src
172 #:name gabor-tail
173 #+begin_src clojure
174 (defn show-kernel [kernel]
175 (let [img-path "/home/r/proj/cortex/tmp/kernel.png"]
176 (draw-kernel! kernel img-path)
177 (view (ImagePlus. img-path))))
179 (defn print-kernel [kernel]
180 (println (.dump kernel)))
183 (def brick-length 0.48)
184 (def brick-width 0.24)
185 (def brick-height 0.12)
186 (def gravity (Vector3f. 0 -9.81 0))
189 (defn brick* [position]
190 (println "get brick.")
191 (doto (box brick-length brick-height brick-width
192 :position position :name "brick"
193 :material "Common/MatDefs/Misc/Unshaded.j3md"
194 :texture "Textures/Terrain/BrickWall/BrickWall.jpg"
195 :mass 34)
196 (->
197 (.getMesh)
198 (.scaleTextureCoordinates (Vector2f. 1 0.5)))
199 (.setShadowMode RenderQueue$ShadowMode/CastAndReceive)
200 )
201 )
204 (defn floor*
205 "make a sturdy, unmovable physical floor"
206 []
207 (box 10 0.1 5 :name "floor" :mass 0
208 :color ColorRGBA/Gray :position (Vector3f. 0 0 0)))
210 (defn floor* []
211 (doto (box 10 0.1 5 :name "floor" ;10 0.1 5 ; 240 0.1 240
212 :material "Common/MatDefs/Misc/Unshaded.j3md"
213 :texture "Textures/BronzeCopper030.jpg"
214 :position (Vector3f. 0 0 0 )
215 :mass 0)
216 (->
217 (.getMesh)
218 (.scaleTextureCoordinates (Vector2f. 3 6)));64 64
219 (->
220 (.getMaterial)
221 (.getTextureParam "ColorMap")
222 (.getTextureValue)
223 (.setWrap Texture$WrapMode/Repeat))
224 (.setShadowMode RenderQueue$ShadowMode/Receive)
225 ))
228 (defn brick-wall* []
229 (let [node (Node. "brick-wall")]
230 (dorun
231 (map
232 (comp #(.attachChild node %) brick*)
233 (for [y (range 10)
234 x (range 4)
235 z (range 1)]
236 (Vector3f.
237 (+ (* 2 x brick-length)
238 (if (even? (+ y z))
239 (/ brick-length 4) (/ brick-length -4)))
240 (+ (* brick-height (inc (* 2 y))))
241 (* 2 z brick-width) ))))
242 (.setShadowMode node RenderQueue$ShadowMode/CastAndReceive)
243 node))
245 (import com.aurellem.capture.Capture)
247 (import java.io.File)
249 (def base "/home/r/proj/cortex/render/gabor-1/")
251 (defn brick-wall-game-run [record?]
252 (let [capture-dir (File. base "main")]
254 (.mkdir (File. base "main"))
255 (doto
256 (world
257 (doto (Node.) (.attachChild (floor*))
258 (.attachChild (brick-wall*))
259 )
260 {"key-f" (fn [game value]
261 (if (not value) (add-element game (brick-wall*))))
262 "key-space" (fire-cannon-ball )}
263 (fn [world]
264 (position-camera
265 world
266 (Vector3f. 1.382548, 4.0383573, 5.994235)
267 (Quaternion. 0.0013082094, 0.98581666,
268 -0.1676442, 0.0076932586))
270 ;;(speed-up world)
272 (if record?
273 (Capture/captureVideo
274 world capture-dir))
276 (add-camera! world (.getCamera world) no-op))
277 (fn [& _]))
278 (.start))))
280 (defn convolve-preview [kernel]
281 (let [input "/home/r/proj/cortex/render/gabor-1/main/0000032.png"
284 output "/home/r/ppp.png"
286 i (org.opencv.highgui.Highgui/imread input)
288 ;;kernel (gabor-kernel 10 1 (/ Math/PI 2) 10 0)
290 new-mat (Mat.)
292 ]
294 (org.opencv.imgproc.Imgproc/filter2D i new-mat CvType/CV_32F kernel)
296 (org.opencv.highgui.Highgui/imwrite "/home/r/ppp.png" new-mat)
298 (view (ImagePlus. input))
299 (view (ImagePlus. output))))
301 (use 'clojure.java.shell)
304 (defn apply-gabor [kernel source dest]
305 (let [i (org.opencv.highgui.Highgui/imread source)
306 new-mat (Mat.)]
308 (println dest)
309 (if (not (.exists (File. dest)))
310 (do
311 (org.opencv.imgproc.Imgproc/filter2D i new-mat CvType/CV_32F kernel)
312 (org.opencv.highgui.Highgui/imwrite dest new-mat)
313 (println "mogrify" "-modulate" "1000%" dest)
314 (sh "mogrify" "-modulate" "1000%" dest)))))
317 (import java.io.File)
319 (defn images [path]
320 (sort (rest (file-seq (File. path)))))
324 (defn pics [file]
325 (images (str base file)))
327 (defn generate-gabor-images [kernel name]
328 (draw-kernel! kernel (str base name ".png"))
330 (.mkdir (File. (str base name)))
332 (let [main (map #(.getCanonicalPath %) (pics "main"))
333 targets (map #(str base name "/" (format "%07d.png" %))
334 (range 0 (count main)))]
335 (dorun (pmap (partial apply-gabor kernel) main targets))))
338 (def banks
339 [[(gabor-kernel 2.8 3.5) "bank-1-1"]
340 [(gabor-kernel 2.8 3.5 (/ Math/PI 2)) "bank-1-1-rot"]
342 ;; [(gabor-kernel 3.6 4.6) "bank-1-2"]
343 ;; [(gabor-kernel 4.5 5.6) "bank-2-1"]
344 ;; [(gabor-kernel 6.3 7.9) "bank-3-1"]
345 ;; [(gabor-kernel 7.3 9.1) "bank-3-2"]
347 [(gabor-kernel 12.3 15.4) "bank-6-1"]
350 ;; [(gabor-kernel 17 21.2) "bank-8-1"]
351 ;; [(gabor-kernel 18.2 22.8) "bank-8-2"]
352 ])
355 (defn make-all-images []
356 (dorun (map (partial apply generate-gabor-images) banks)))
360 (defn compile-left-right []
361 (.mkdir (File. (str base "left-right")))
362 (let [main (pics "main")
363 left (pics "bank-1-1")
364 right (pics "bank-1-1-rot")
365 left-kernel (repeat 20000 (File. (str base "bank-1-1.png")))
366 right-kernel (repeat 20000 (File. (str base "bank-1-1-rot.png")))
367 targets (map
368 #(File. (str base "left-right/" (format "%07d.png" %)))
369 (range 0 (count main)))]
371 (dorun
372 (pmap
373 (comp
374 (fn [[main left right left-kernel right-kernel target]]
375 (println target)
376 (if (not (.exists (File. target)))
377 (sh "convert"
378 "-size" "1940x515" "xc:white"
379 main "-geometry" "+0+0" "-composite"
380 left "-geometry" "+650+0" "-composite"
381 right "-geometry" "+1300+0" "-composite"
382 left-kernel "-geometry" "+960+485" "-composite"
383 right-kernel "-geometry" "+1610+485" "-composite"
384 target)))
385 (fn [& args] (map #(.getCanonicalPath %) args)))
386 main left right left-kernel right-kernel targets))))
389 (defn compile-big-small []
390 (.mkdir (File. (str base "big-small")))
391 (let [main (pics "main")
392 left (pics "bank-1-1")
393 right (pics "bank-6-1")
394 small-kernel (repeat 20000 (File. (str base "bank-1-1.png")))
395 big-kernel (repeat 20000 (File. (str base "bank-6-1.png")))
396 targets (map
397 #(File. (str base "big-small/" (format "%07d.png" %)))
398 (range 0 (count main)))]
400 (dorun
401 (pmap
402 (comp
403 (fn [[main left right small-kernel big-kernel target]]
404 (println target)
405 (if (not (.exists (File. target)))
406 (sh "convert"
407 "-size" "1940x610" "xc:white"
408 main "-geometry" "+0+0" "-composite"
409 left "-geometry" "+650+0" "-composite"
410 right "-geometry" "+1300+0" "-composite"
411 small-kernel "-geometry" "+960+485" "-composite"
412 big-kernel "-geometry" "+1560+485" "-composite"
413 target)))
414 (fn [& args] (map #(.getCanonicalPath %) args)))
415 main left right small-kernel big-kernel targets))))
418 (defn regen-everything []
419 (make-all-images)
420 (compile-left-right)
421 (compile-big-small))
424 #+end_src
426 #+name: make-left-right
427 #+begin_src sh
428 #!/bin/sh
430 ffmpeg -framerate 60 -i ./left-right/%07d.png -b:v 9000k\
431 -c:v mpeg4 -r 60 gabor-rotation.mp4
433 #+end_src
436 #+name: make-big-small
437 #+begin_src sh
438 #!/bin/sh
440 ffmpeg -framerate 60 -i ./big-small/%07d.png -b:v 9000k\
441 -c:v mpeg4 -r 60 gabor-big-small.mp4
443 #+end_src
445 #+begin_html
446 <div class="figure">
447 <video controls="controls" width="755">
448 <source src="../video/gabor-rotation.mp4" type="video/mp4"
449 preload="none" poster="../images/aurellem-1280x480.png" />
450 </video>
451 <br> <a href="http://youtu.be/bxujjO97B-U"> YouTube </a>
452 <p>Two gabor filters with different values of theta are compared. The
453 horizontally aligned one does better in this example.</p>
454 </div>
455 #+end_html
459 #+begin_html
460 <div class="figure">
461 <video controls="controls" width="755">
462 <source src="../video/gabor-big-small.mp4" type="video/mp4"
463 preload="none" poster="../images/aurellem-1280x480.png" />
464 </video>
465 <br> <a href="http://youtu.be/-EsfA2ceWUk"> YouTube </a>
466 <p>Here we compare a gabor filter from bank 1 with one from bank 6.</p>
467 </div>
468 #+end_html
473 * COMMENT Generate Source
474 #+begin_src clojure :tangle ../src/cortex/gabor.clj
475 <<gabor>>
476 <<gabor-tail>>
477 #+end_src
479 #+begin_src clojure :tangle ../render/gabor-1/make-rotation.sh
480 <<make-left-right>>
481 #+end_src
483 #+begin_src clojure :tangle ../render/gabor-1/make-big-small.sh
484 <<make-big-small>>
485 #+end_src