view org/gabor.org @ 373:6205677359e9

minor formatting.
author Robert McIntyre <rlm@mit.edu>
date Tue, 12 Mar 2013 04:22:40 +0000
parents 182f890a9a9d
children 31f15781db36
line wrap: on
line source
1 #+title: Fun with Gabor Filters
2 #+author: Robert McIntyre
3 #+email: rlm@mit.edu
4 #+description: gabor filters in clojure with opencv
5 #+keywords: computer vision, jMonkeyEngine3, clojure, opencv
6 #+SETUPFILE: ../../aurellem/org/setup.org
7 #+INCLUDE: ../../aurellem/org/level-0.org
8 #+babel: :mkdirp yes :noweb yes :exports both
11 Gabor filters were invented by the same guy who invented holograms.
13 They work well as edge detectors and are related to the human visual
14 system.
16 #+name: gabor
17 #+begin_src clojure
18 (ns cortex.gabor
19 (:import org.opencv.core.CvType)
20 (:import java.awt.image.BufferedImage)
21 (:import ij.ImagePlus)
22 (:import org.opencv.core.Mat)
23 (:use (cortex world sense util vision import))
24 (:import com.jme3.post.SceneProcessor)
25 (:import (com.jme3.util BufferUtils Screenshots))
26 (:import java.nio.ByteBuffer)
27 (:import java.awt.image.BufferedImage)
28 (:import (com.jme3.renderer ViewPort Camera))
29 (:import (com.jme3.math ColorRGBA Vector3f Matrix3f Vector2f))
30 (:import com.jme3.renderer.Renderer)
31 (:import com.jme3.app.Application)
32 (:import com.jme3.texture.FrameBuffer)
33 (:import (com.jme3.scene Node Spatial)))
36 (cortex.import/mega-import-jme3)
37 (use 'clojure.math.numeric-tower)
38 (defn load-opencv
39 "Load the opencv native library. Must be called before any OpenCV
40 stuff is used."
41 []
42 (clojure.lang.RT/loadLibrary "opencv_java249"))
44 (load-opencv)
46 (defn gabor-kernel
47 ([sigma wavelength theta]
48 (gabor-kernel sigma wavelength theta 1 0))
49 ([sigma wavelength]
50 (gabor-kernel sigma wavelength 0 1 0))
51 ([sigma wavelength theta aspect-ratio phase-offset]
53 ;; first, find the size of the kernel which is required
54 (let [square #(expt % 2)
55 rotated (fn [[x y]]
56 [(+ (* x (Math/cos theta)) (* y (Math/sin theta)))
57 (- (* y (Math/cos theta)) (* x (Math/sin theta)))])
59 gaussian (fn [[x y]]
60 (let [[x' y'] (rotated [x y])]
61 (Math/exp (- (/ (+ (square x')
62 (square (* aspect-ratio y')))
63 (* 2 (square sigma)))))))
64 sinusoid (fn [[x y]]
65 (let [[x' y'] (rotated [x y])]
66 (Math/cos
67 (+ (* 2 Math/PI (/ x' wavelength))
68 phase-offset))))
70 half-width
71 (let [std-dev-capture 5]
72 (max
73 (int (* std-dev-capture (/ sigma aspect-ratio)))
74 (int (* std-dev-capture sigma))
75 (int (* std-dev-capture (/ aspect-ratio sigma)))))
77 grid (let [axis (range (- half-width) (inc half-width))]
78 (for [y (reverse axis) x axis] (vector x y)))
80 scale (reduce + (map gaussian grid))
82 gabor (fn [[x y :as coord]]
83 (* (sinusoid coord) (gaussian coord) (/ scale)))
85 mat-width (+ 1 (* 2 half-width))
86 mat (Mat. mat-width mat-width CvType/CV_32F)]
88 (.put mat 0 0 (float-array (map gabor grid)))
89 mat)))
92 (defn draw-kernel! [kernel img-path]
93 (let [output img-path
94 size (.size kernel)
95 width (int (.width size))
96 height (int (.height size))
97 tmp-array (float-array (* width height))]
99 ;; read values from matrix.
100 (.get kernel 0 0 tmp-array)
102 ;; find overall dynamic range of the filter
103 (let [vals (vec tmp-array)
104 low (apply min vals)
105 high (apply max vals)
106 scaled-vals (map #(* 255 (- % low) (/ (- high low))) vals)
107 new-mat (Mat. height width CvType/CV_32F)]
108 (.put new-mat 0 0 (float-array scaled-vals))
109 (org.opencv.highgui.Highgui/imwrite output new-mat))))
111 ;; some cool examples
112 #+end_src
117 [[../images/gabor-50-10.png]]
119 #+begin_src clojure
120 (def img-base "/home/r/proj/cortex/images/")
122 (draw-kernel! (gabor-kernel 50 10 0 1 0)
123 (str img-base "gabor-50-10.png"))
124 #+end_src
127 [[../images/gabor-50-10-pi-over-4.png]]
129 #+begin_src clojure
130 (draw-kernel! (gabor-kernel 50 10 (/ Math/PI 4) 1 0)
131 (str img-base "gabor-50-10-pi-over-4.png"))
132 #+end_src
135 [[../images/gabor-50-10-pi-over-2.png]]
137 #+begin_src clojure
138 (draw-kernel! (gabor-kernel 50 10 (/ Math/PI 2) 1 0)
139 (str img-base "gabor-50-10-pi-over-2.png"))
140 #+end_src
143 [[../images/gabor-50-50.png]]
147 #+begin_src clojure
148 (draw-kernel! (gabor-kernel 50 50 0 1 0)
149 (str img-base "gabor-50-50.png"))
151 #+end_src
153 [[../images/gabor-50-10-0-3.png]]
155 #+begin_src clojure
156 (draw-kernel! (gabor-kernel 50 10 0 3 0)
157 (str img-base "gabor-50-10-0-3.png"))
158 #+end_src
162 [[../images/gabor-50-4-pi-over3-3.png]]
163 #+begin_src clojure
164 (draw-kernel! (gabor-kernel 50 4 (/ Math/PI 3) 3 0)
165 (str img-base "gabor-50-4-pi-over3-3.png"))
166 #+end_src
172 #:name gabor-tail
173 #+begin_src clojure
174 (defn show-kernel [kernel]
175 (let [img-path "/home/r/proj/cortex/tmp/kernel.png"]
176 (draw-kernel! kernel img-path)
177 (view (ImagePlus. img-path))))
179 (defn print-kernel [kernel]
180 (println (.dump kernel)))
183 (def brick-length 0.48)
184 (def brick-width 0.24)
185 (def brick-height 0.12)
186 (def gravity (Vector3f. 0 -9.81 0))
189 (defn brick* [position]
190 (println "get brick.")
191 (doto (box brick-length brick-height brick-width
192 :position position :name "brick"
193 :material "Common/MatDefs/Misc/Unshaded.j3md"
194 :texture "Textures/Terrain/BrickWall/BrickWall.jpg"
195 :mass 34)
196 (->
197 (.getMesh)
198 (.scaleTextureCoordinates (Vector2f. 1 0.5)))
199 (.setShadowMode RenderQueue$ShadowMode/CastAndReceive)
200 )
201 )
204 (defn floor*
205 "make a sturdy, unmovable physical floor"
206 []
207 (box 10 0.1 5 :name "floor" :mass 0
208 :color ColorRGBA/Gray :position (Vector3f. 0 0 0)))
210 (defn floor* []
211 (doto (box 10 0.1 5 :name "floor" ;10 0.1 5 ; 240 0.1 240
212 :material "Common/MatDefs/Misc/Unshaded.j3md"
213 :texture "Textures/BronzeCopper030.jpg"
214 :position (Vector3f. 0 0 0 )
215 :mass 0)
216 (->
217 (.getMesh)
218 (.scaleTextureCoordinates (Vector2f. 3 6)));64 64
219 (->
220 (.getMaterial)
221 (.getTextureParam "ColorMap")
222 (.getTextureValue)
223 (.setWrap Texture$WrapMode/Repeat))
224 (.setShadowMode RenderQueue$ShadowMode/Receive)
225 ))
228 (defn brick-wall* []
229 (let [node (Node. "brick-wall")]
230 (dorun
231 (map
232 (comp #(.attachChild node %) brick*)
233 (for [y (range 10)
234 x (range 4)
235 z (range 1)]
236 (Vector3f.
237 (+ (* 2 x brick-length)
238 (if (even? (+ y z))
239 (/ brick-length 4) (/ brick-length -4)))
240 (+ (* brick-height (inc (* 2 y))))
241 (* 2 z brick-width) ))))
242 (.setShadowMode node RenderQueue$ShadowMode/CastAndReceive)
243 node))
245 (import com.aurellem.capture.Capture)
247 (import java.io.File)
249 (def base "/home/r/proj/cortex/render/gabor-1/")
251 (defn brick-wall-game-run [record?]
252 (let [capture-dir (File. base "main")]
254 (.mkdir (File. base "main"))
255 (doto
256 (world
257 (doto (Node.) (.attachChild (floor*))
258 (.attachChild (brick-wall*))
259 )
260 {"key-f" (fn [game value]
261 (if (not value) (add-element game (brick-wall*))))
262 "key-space" (fire-cannon-ball )}
263 (fn [world]
264 (position-camera
265 world
266 (Vector3f. 1.382548, 4.0383573, 5.994235)
267 (Quaternion. 0.0013082094, 0.98581666,
268 -0.1676442, 0.0076932586))
270 ;;(speed-up world)
272 (if record?
273 (Capture/captureVideo
274 world capture-dir))
276 (add-camera! world (.getCamera world) no-op))
277 (fn [& _]))
278 (.start))))
280 (defn convolve-preview [kernel]
281 (let [input "/home/r/proj/cortex/render/gabor-1/main/0000032.png"
284 output "/home/r/ppp.png"
286 i (org.opencv.highgui.Highgui/imread input)
288 ;;kernel (gabor-kernel 10 1 (/ Math/PI 2) 10 0)
290 new-mat (Mat.)
292 ]
294 (org.opencv.imgproc.Imgproc/filter2D
295 i new-mat CvType/CV_32F kernel)
297 (org.opencv.highgui.Highgui/imwrite "/home/r/ppp.png" new-mat)
299 (view (ImagePlus. input))
300 (view (ImagePlus. output))))
302 (use 'clojure.java.shell)
305 (defn apply-gabor [kernel source dest]
306 (let [i (org.opencv.highgui.Highgui/imread source)
307 new-mat (Mat.)]
309 (println dest)
310 (if (not (.exists (File. dest)))
311 (do
312 (org.opencv.imgproc.Imgproc/filter2D
313 i new-mat CvType/CV_32F kernel)
314 (org.opencv.highgui.Highgui/imwrite dest new-mat)
315 (println "mogrify" "-modulate" "1000%" dest)
316 (sh "mogrify" "-modulate" "1000%" dest)))))
319 (import java.io.File)
321 (defn images [path]
322 (sort (rest (file-seq (File. path)))))
326 (defn pics [file]
327 (images (str base file)))
329 (defn generate-gabor-images [kernel name]
330 (draw-kernel! kernel (str base name ".png"))
332 (.mkdir (File. (str base name)))
334 (let [main (map #(.getCanonicalPath %) (pics "main"))
335 targets (map #(str base name "/" (format "%07d.png" %))
336 (range 0 (count main)))]
337 (dorun (pmap (partial apply-gabor kernel) main targets))))
340 (def banks
341 [[(gabor-kernel 2.8 3.5) "bank-1-1"]
342 [(gabor-kernel 2.8 3.5 (/ Math/PI 2)) "bank-1-1-rot"]
344 ;; [(gabor-kernel 3.6 4.6) "bank-1-2"]
345 ;; [(gabor-kernel 4.5 5.6) "bank-2-1"]
346 ;; [(gabor-kernel 6.3 7.9) "bank-3-1"]
347 ;; [(gabor-kernel 7.3 9.1) "bank-3-2"]
349 [(gabor-kernel 12.3 15.4) "bank-6-1"]
352 ;; [(gabor-kernel 17 21.2) "bank-8-1"]
353 ;; [(gabor-kernel 18.2 22.8) "bank-8-2"]
354 ])
357 (defn make-all-images []
358 (dorun (map (partial apply generate-gabor-images) banks)))
362 (defn compile-left-right []
363 (.mkdir (File. (str base "left-right")))
364 (let [main (pics "main")
365 left (pics "bank-1-1")
366 right (pics "bank-1-1-rot")
367 left-kernel (repeat 20000 (File. (str base "bank-1-1.png")))
368 right-kernel (repeat 20000 (File. (str base "bank-1-1-rot.png")))
369 targets (map
370 #(File. (str base "left-right/" (format "%07d.png" %)))
371 (range 0 (count main)))]
373 (dorun
374 (pmap
375 (comp
376 (fn [[main left right left-kernel right-kernel target]]
377 (println target)
378 (if (not (.exists (File. target)))
379 (sh "convert"
380 "-size" "1940x515" "xc:white"
381 main "-geometry" "+0+0" "-composite"
382 left "-geometry" "+650+0" "-composite"
383 right "-geometry" "+1300+0" "-composite"
384 left-kernel "-geometry" "+960+485" "-composite"
385 right-kernel "-geometry" "+1610+485" "-composite"
386 target)))
387 (fn [& args] (map #(.getCanonicalPath %) args)))
388 main left right left-kernel right-kernel targets))))
391 (defn compile-big-small []
392 (.mkdir (File. (str base "big-small")))
393 (let [main (pics "main")
394 left (pics "bank-1-1")
395 right (pics "bank-6-1")
396 small-kernel (repeat 20000 (File. (str base "bank-1-1.png")))
397 big-kernel (repeat 20000 (File. (str base "bank-6-1.png")))
398 targets (map
399 #(File. (str base "big-small/" (format "%07d.png" %)))
400 (range 0 (count main)))]
402 (dorun
403 (pmap
404 (comp
405 (fn [[main left right small-kernel big-kernel target]]
406 (println target)
407 (if (not (.exists (File. target)))
408 (sh "convert"
409 "-size" "1940x610" "xc:white"
410 main "-geometry" "+0+0" "-composite"
411 left "-geometry" "+650+0" "-composite"
412 right "-geometry" "+1300+0" "-composite"
413 small-kernel "-geometry" "+960+485" "-composite"
414 big-kernel "-geometry" "+1560+485" "-composite"
415 target)))
416 (fn [& args] (map #(.getCanonicalPath %) args)))
417 main left right small-kernel big-kernel targets))))
420 (defn regen-everything []
421 (make-all-images)
422 (compile-left-right)
423 (compile-big-small))
426 #+end_src
429 #+name: make-videos
430 #+begin_src makefile
431 scale:
432 ffmpeg -framerate 60 -i ./big-small/%07d.png -b:v 9000k\
433 -c:v theora -r 60 gabor-scale.ogg
435 rotation:
436 ffmpeg -framerate 60 -i ./left-right/%07d.png -b:v 9000k\
437 -c:v theora -r 60 gabor-rotation.ogg
439 all: rotation scale
441 clean:
442 rm gabor-rotation.org gabor-scale.ogg
443 #+end_src
446 #+begin_html
447 <div class="figure">
448 <video controls="controls">
449 <source src="../video/gabor-rotation.ogg" type="video/ogg"
450 preload="none" poster="../images/aurellem-1280x480.png" />
451 </video>
452 <br> <a href="http://youtu.be/bxujjO97B-U"> YouTube </a>
453 <p>Two gabor filters with different values of theta are compared. The
454 horizontally aligned one does better in this example.</p>
455 </div>
456 #+end_html
460 #+begin_html
461 <div class="figure">
462 <video controls="controls">
463 <source src="../video/gabor-scale.ogg" type="video/ogg"
464 preload="none" poster="../images/aurellem-1280x480.png" />
465 </video>
466 <br> <a href="http://youtu.be/-EsfA2ceWUk"> YouTube </a>
467 <p>Here we compare a gabor filter from bank 1 with one from bank 6.</p>
468 </div>
469 #+end_html
473 * COMMENT Generate Source
474 #+begin_src clojure :tangle ../src/cortex/gabor.clj
475 <<gabor>>
476 <<gabor-tail>>
477 #+end_src
479 #+begin_src Makefile :tangle ../render/gabor-1/Makefile
480 <<make-videos>>
481 #+end_src