view org/gabor.org @ 365:1727b68634ea

add videos.
author Robert McIntyre <rlm@mit.edu>
date Thu, 07 Mar 2013 07:49:55 +0000
parents b599a189433b
children 871882350c83
line wrap: on
line source
1 #+title: Fun with Gabor Filters
2 #+author: Robert McIntyre
3 #+email: rlm@mit.edu
4 #+description: gabor filters in clojure with opencv
5 #+keywords: computer vision, jMonkeyEngine3, clojure, opencv
6 #+SETUPFILE: ../../aurellem/org/setup.org
7 #+INCLUDE: ../../aurellem/org/level-0.org
8 #+babel: :mkdirp yes :noweb yes :exports both
12 #+name: gabor
13 #+begin_src clojure
14 (ns cortex.gabor
15 (:import org.opencv.core.CvType)
16 (:import java.awt.image.BufferedImage)
17 (:import ij.ImagePlus)
18 (:import org.opencv.core.Mat)
19 (:use (cortex world sense util vision import))
20 (:import com.jme3.post.SceneProcessor)
21 (:import (com.jme3.util BufferUtils Screenshots))
22 (:import java.nio.ByteBuffer)
23 (:import java.awt.image.BufferedImage)
24 (:import (com.jme3.renderer ViewPort Camera))
25 (:import (com.jme3.math ColorRGBA Vector3f Matrix3f Vector2f))
26 (:import com.jme3.renderer.Renderer)
27 (:import com.jme3.app.Application)
28 (:import com.jme3.texture.FrameBuffer)
29 (:import (com.jme3.scene Node Spatial)))
32 (cortex.import/mega-import-jme3)
33 (use 'clojure.math.numeric-tower)
34 (defn load-opencv
35 "Load the opencv native library. Must be called before any OpenCV
36 stuff is used."
37 []
38 (clojure.lang.RT/loadLibrary "opencv_java249"))
40 (load-opencv)
42 (defn gabor-kernel
43 ([sigma wavelength theta]
44 (gabor-kernel sigma wavelength theta 1 0))
45 ([sigma wavelength]
46 (gabor-kernel sigma wavelength 0 1 0))
47 ([sigma wavelength theta aspect-ratio phase-offset]
49 ;; first, find the size of the kernel which is required
50 (let [square #(expt % 2)
51 rotated (fn [[x y]]
52 [(+ (* x (Math/cos theta)) (* y (Math/sin theta)))
53 (- (* y (Math/cos theta)) (* x (Math/sin theta)))])
55 gaussian (fn [[x y]]
56 (let [[x' y'] (rotated [x y])]
57 (Math/exp (- (/ (+ (square x')
58 (square (* aspect-ratio y')))
59 (* 2 (square sigma)))))))
60 sinusoid (fn [[x y]]
61 (let [[x' y'] (rotated [x y])]
62 (Math/cos
63 (+ (* 2 Math/PI (/ x' wavelength))
64 phase-offset))))
66 half-width
67 (let [std-dev-capture 5]
68 (max
69 (int (* std-dev-capture (/ sigma aspect-ratio)))
70 (int (* std-dev-capture sigma))
71 (int (* std-dev-capture (/ aspect-ratio sigma)))))
73 grid (let [axis (range (- half-width) (inc half-width))]
74 (for [y (reverse axis) x axis] (vector x y)))
76 scale (reduce + (map gaussian grid))
78 gabor (fn [[x y :as coord]]
79 (* (sinusoid coord) (gaussian coord) (/ scale)))
81 mat-width (+ 1 (* 2 half-width))
82 mat (Mat. mat-width mat-width CvType/CV_32F)]
84 (.put mat 0 0 (float-array (map gabor grid)))
85 mat)))
88 (defn draw-kernel! [kernel img-path]
89 (let [output img-path
90 size (.size kernel)
91 width (int (.width size))
92 height (int (.height size))
93 tmp-array (float-array (* width height))]
95 ;; read values from matrix.
96 (.get kernel 0 0 tmp-array)
98 ;; find overall dynamic range of the filter
99 (let [vals (vec tmp-array)
100 low (apply min vals)
101 high (apply max vals)
102 scaled-vals (map #(* 255 (- % low) (/ (- high low))) vals)
103 new-mat (Mat. height width CvType/CV_32F)]
104 (.put new-mat 0 0 (float-array scaled-vals))
105 (org.opencv.highgui.Highgui/imwrite output new-mat))))
107 ;; some cool examples
108 #+end_src
113 [[../images/gabor-50-10.png]]
115 #+begin_src clojure
116 (def img-base "/home/r/proj/cortex/images/")
118 (draw-kernel! (gabor-kernel 50 10 0 1 0)
119 (str img-base "gabor-50-10.png"))
120 #+end_src
123 [[../images/gabor-50-10-pi-over-4.png]]
125 #+begin_src clojure
126 (draw-kernel! (gabor-kernel 50 10 (/ Math/PI 4) 1 0)
127 (str img-base "gabor-50-10-pi-over-4.png"))
128 #+end_src
131 [[../images/gabor-50-10-pi-over-2.png]]
133 #+begin_src clojure
134 (draw-kernel! (gabor-kernel 50 10 (/ Math/PI 2) 1 0)
135 (str img-base "gabor-50-10-pi-over-2.png"))
136 #+end_src
139 [[../images/gabor-50-50.png]]
143 #+begin_src clojure
144 (draw-kernel! (gabor-kernel 50 50 0 1 0)
145 (str img-base "gabor-50-50.png"))
147 #+end_src
149 [[../images/gabor-50-10-0-3.png]]
151 #+begin_src clojure
152 (draw-kernel! (gabor-kernel 50 10 0 3 0)
153 (str img-base "gabor-50-10-0-3.png"))
154 #+end_src
158 [[../images/gabor-50-4-pi-over3-3.png]]
159 #+begin_src clojure
160 (draw-kernel! (gabor-kernel 50 4 (/ Math/PI 3) 3 0)
161 (str img-base "gabor-50-4-pi-over3-3.png"))
162 #+end_src
168 #:name gabor-tail
169 #+begin_src clojure
170 (defn show-kernel [kernel]
171 (let [img-path "/home/r/proj/cortex/tmp/kernel.png"]
172 (draw-kernel! kernel img-path)
173 (view (ImagePlus. img-path))))
175 (defn print-kernel [kernel]
176 (println (.dump kernel)))
179 (def brick-length 0.48)
180 (def brick-width 0.24)
181 (def brick-height 0.12)
182 (def gravity (Vector3f. 0 -9.81 0))
185 (defn brick* [position]
186 (println "get brick.")
187 (doto (box brick-length brick-height brick-width
188 :position position :name "brick"
189 :material "Common/MatDefs/Misc/Unshaded.j3md"
190 :texture "Textures/Terrain/BrickWall/BrickWall.jpg"
191 :mass 34)
192 (->
193 (.getMesh)
194 (.scaleTextureCoordinates (Vector2f. 1 0.5)))
195 (.setShadowMode RenderQueue$ShadowMode/CastAndReceive)
196 )
197 )
200 (defn floor*
201 "make a sturdy, unmovable physical floor"
202 []
203 (box 10 0.1 5 :name "floor" :mass 0 :color ColorRGBA/Gray :position (Vector3f. 0 0 0)))
205 (defn floor* []
206 (doto (box 10 0.1 5 :name "floor" ;10 0.1 5 ; 240 0.1 240
207 :material "Common/MatDefs/Misc/Unshaded.j3md"
208 :texture "Textures/BronzeCopper030.jpg"
209 :position (Vector3f. 0 0 0 )
210 :mass 0)
211 (->
212 (.getMesh)
213 (.scaleTextureCoordinates (Vector2f. 3 6)));64 64
214 (->
215 (.getMaterial)
216 (.getTextureParam "ColorMap")
217 (.getTextureValue)
218 (.setWrap Texture$WrapMode/Repeat))
219 (.setShadowMode RenderQueue$ShadowMode/Receive)
220 ))
223 (defn brick-wall* []
224 (let [node (Node. "brick-wall")]
225 (dorun
226 (map
227 (comp #(.attachChild node %) brick*)
228 (for [y (range 10)
229 x (range 4)
230 z (range 1)]
231 (Vector3f.
232 (+ (* 2 x brick-length)
233 (if (even? (+ y z))
234 (/ brick-length 4) (/ brick-length -4)))
235 (+ (* brick-height (inc (* 2 y))))
236 (* 2 z brick-width) ))))
237 (.setShadowMode node RenderQueue$ShadowMode/CastAndReceive)
238 node))
240 (import com.aurellem.capture.Capture)
242 (import java.io.File)
244 (def base "/home/r/proj/cortex/render/gabor-1/")
246 (defn brick-wall-game-run [record?]
247 (let [capture-dir (File. base "main")]
249 (.mkdir (File. base "main"))
250 (doto
251 (world
252 (doto (Node.) (.attachChild (floor*))
253 (.attachChild (brick-wall*))
254 )
255 {"key-f" (fn [game value]
256 (if (not value) (add-element game (brick-wall*))))
257 "key-space" (fire-cannon-ball )}
258 (fn [world]
259 (position-camera world
260 (Vector3f. 1.382548, 4.0383573, 5.994235)
261 (Quaternion. 0.0013082094, 0.98581666, -0.1676442, 0.0076932586))
263 ;;(speed-up world)
265 (if record?
266 (Capture/captureVideo
267 world capture-dir))
269 (add-camera! world (.getCamera world) no-op))
270 (fn [& _]))
271 (.start))))
273 (defn convolve-preview [kernel]
274 (let [input "/home/r/proj/cortex/render/gabor-1/main/0000032.png"
277 output "/home/r/ppp.png"
279 i (org.opencv.highgui.Highgui/imread input)
281 ;;kernel (gabor-kernel 10 1 (/ Math/PI 2) 10 0)
283 new-mat (Mat.)
285 ]
287 (org.opencv.imgproc.Imgproc/filter2D i new-mat CvType/CV_32F kernel)
289 (org.opencv.highgui.Highgui/imwrite "/home/r/ppp.png" new-mat)
291 (view (ImagePlus. input))
292 (view (ImagePlus. output))))
294 (use 'clojure.java.shell)
297 (defn apply-gabor [kernel source dest]
298 (let [i (org.opencv.highgui.Highgui/imread source)
299 new-mat (Mat.)]
301 (println dest)
302 (if (not (.exists (File. dest)))
303 (do
304 (org.opencv.imgproc.Imgproc/filter2D i new-mat CvType/CV_32F kernel)
305 (org.opencv.highgui.Highgui/imwrite dest new-mat)
306 (println "mogrify" "-modulate" "1000%" dest)
307 (sh "mogrify" "-modulate" "1000%" dest)))))
310 (import java.io.File)
312 (defn images [path]
313 (sort (rest (file-seq (File. path)))))
317 (defn pics [file]
318 (images (str base file)))
320 (defn generate-gabor-images [kernel name]
321 (draw-kernel! kernel (str base name ".png"))
323 (.mkdir (File. (str base name)))
325 (let [main (map #(.getCanonicalPath %) (pics "main"))
326 targets (map #(str base name "/" (format "%07d.png" %))
327 (range 0 (count main)))]
328 (dorun (pmap (partial apply-gabor kernel) main targets))))
331 (def banks
332 [[(gabor-kernel 2.8 3.5) "bank-1-1"]
333 [(gabor-kernel 2.8 3.5 (/ Math/PI 2)) "bank-1-1-rot"]
335 ;; [(gabor-kernel 3.6 4.6) "bank-1-2"]
336 ;; [(gabor-kernel 4.5 5.6) "bank-2-1"]
337 ;; [(gabor-kernel 6.3 7.9) "bank-3-1"]
338 ;; [(gabor-kernel 7.3 9.1) "bank-3-2"]
340 [(gabor-kernel 12.3 15.4) "bank-6-1"]
343 ;; [(gabor-kernel 17 21.2) "bank-8-1"]
344 ;; [(gabor-kernel 18.2 22.8) "bank-8-2"]
345 ])
348 (defn make-all-images []
349 (dorun (map (partial apply generate-gabor-images) banks)))
353 (defn compile-left-right []
354 (.mkdir (File. (str base "left-right")))
355 (let [main (pics "main")
356 left (pics "bank-1-1")
357 right (pics "bank-1-1-rot")
358 left-kernel (repeat 20000 (File. (str base "bank-1-1.png")))
359 right-kernel (repeat 20000 (File. (str base "bank-1-1-rot.png")))
360 targets (map
361 #(File. (str base "left-right/" (format "%07d.png" %)))
362 (range 0 (count main)))]
364 (dorun
365 (pmap
366 (comp
367 (fn [[main left right left-kernel right-kernel target]]
368 (println target)
369 (if (not (.exists (File. target)))
370 (sh "convert"
371 "-size" "1940x515" "xc:white"
372 main "-geometry" "+0+0" "-composite"
373 left "-geometry" "+650+0" "-composite"
374 right "-geometry" "+1300+0" "-composite"
375 left-kernel "-geometry" "+960+485" "-composite"
376 right-kernel "-geometry" "+1610+485" "-composite"
377 target)))
378 (fn [& args] (map #(.getCanonicalPath %) args)))
379 main left right left-kernel right-kernel targets))))
382 (defn compile-big-small []
383 (.mkdir (File. (str base "big-small")))
384 (let [main (pics "main")
385 left (pics "bank-1-1")
386 right (pics "bank-6-1")
387 small-kernel (repeat 20000 (File. (str base "bank-1-1.png")))
388 big-kernel (repeat 20000 (File. (str base "bank-6-1.png")))
389 targets (map
390 #(File. (str base "big-small/" (format "%07d.png" %)))
391 (range 0 (count main)))]
393 (dorun
394 (pmap
395 (comp
396 (fn [[main left right small-kernel big-kernel target]]
397 (println target)
398 (if (not (.exists (File. target)))
399 (sh "convert"
400 "-size" "1940x610" "xc:white"
401 main "-geometry" "+0+0" "-composite"
402 left "-geometry" "+650+0" "-composite"
403 right "-geometry" "+1300+0" "-composite"
404 small-kernel "-geometry" "+960+485" "-composite"
405 big-kernel "-geometry" "+1560+485" "-composite"
406 target)))
407 (fn [& args] (map #(.getCanonicalPath %) args)))
408 main left right small-kernel big-kernel targets))))
411 (defn regen-everything []
412 (make-all-images)
413 (compile-left-right)
414 (compile-big-small))
417 #+end_src
419 #+name: make-left-right
420 #+begin_src sh
421 #!/bin/sh
423 ffmpeg -framerate 60 -i ./left-right/%07d.png -b:v 9000k\
424 -c:v mpeg4 -r 60 gabor-rotation.mp4
426 #+end_src
429 #+name: make-big-small
430 #+begin_src sh
431 #!/bin/sh
433 ffmpeg -framerate 60 -i ./big-small/%07d.png -b:v 9000k\
434 -c:v mpeg4 -r 60 gabor-big-small.mp4
436 #+end_src
438 #+begin_html
439 <div class="figure">
440 <video controls="controls" width="755">
441 <source src="../video/gabor-rotation.mp4" type="video/mp4"
442 preload="none" poster="../images/aurellem-1280x480.png" />
443 </video>
444 <br> <a href="http://youtu.be/r5Bn2aG7MO0"> YouTube </a>
445 <p>Two gabor filters with different values of theta are compared. The
446 horizontally aligned one does better in this example.</p>
447 </div>
448 #+end_html
452 #+begin_html
453 <div class="figure">
454 <video controls="controls" width="755">
455 <source src="../video/gabor-big-small.mp4" type="video/mp4"
456 preload="none" poster="../images/aurellem-1280x480.png" />
457 </video>
458 <br> <a href="http://youtu.be/r5Bn2aG7MO0"> YouTube </a>
459 <p>Here we compare a gabor filter from bank 1 with one from bank 6.</p>
460 </div>
461 #+end_html
466 * COMMENT Generate Source
467 #+begin_src clojure :tangle ../src/cortex/gabor.clj
468 <<gabor>>
469 <<gabor-tail>>
470 #+end_src
472 #+begin_src clojure :tangle ../render/gabor-1/make-rotation.sh
473 <<make-left-right>>
474 #+end_src
476 #+begin_src clojure :tangle ../render/gabor-1/make-big-small.sh
477 <<make-big-small>>
478 #+end_src