view org/gabor.org @ 366:871882350c83

cosmetic improvement.
author Robert McIntyre <rlm@mit.edu>
date Thu, 07 Mar 2013 07:50:28 +0000
parents 1727b68634ea
children 078600644d94
line wrap: on
line source
1 #+title: Fun with Gabor Filters
2 #+author: Robert McIntyre
3 #+email: rlm@mit.edu
4 #+description: gabor filters in clojure with opencv
5 #+keywords: computer vision, jMonkeyEngine3, clojure, opencv
6 #+SETUPFILE: ../../aurellem/org/setup.org
7 #+INCLUDE: ../../aurellem/org/level-0.org
8 #+babel: :mkdirp yes :noweb yes :exports both
12 #+name: gabor
13 #+begin_src clojure
14 (ns cortex.gabor
15 (:import org.opencv.core.CvType)
16 (:import java.awt.image.BufferedImage)
17 (:import ij.ImagePlus)
18 (:import org.opencv.core.Mat)
19 (:use (cortex world sense util vision import))
20 (:import com.jme3.post.SceneProcessor)
21 (:import (com.jme3.util BufferUtils Screenshots))
22 (:import java.nio.ByteBuffer)
23 (:import java.awt.image.BufferedImage)
24 (:import (com.jme3.renderer ViewPort Camera))
25 (:import (com.jme3.math ColorRGBA Vector3f Matrix3f Vector2f))
26 (:import com.jme3.renderer.Renderer)
27 (:import com.jme3.app.Application)
28 (:import com.jme3.texture.FrameBuffer)
29 (:import (com.jme3.scene Node Spatial)))
32 (cortex.import/mega-import-jme3)
33 (use 'clojure.math.numeric-tower)
34 (defn load-opencv
35 "Load the opencv native library. Must be called before any OpenCV
36 stuff is used."
37 []
38 (clojure.lang.RT/loadLibrary "opencv_java249"))
40 (load-opencv)
42 (defn gabor-kernel
43 ([sigma wavelength theta]
44 (gabor-kernel sigma wavelength theta 1 0))
45 ([sigma wavelength]
46 (gabor-kernel sigma wavelength 0 1 0))
47 ([sigma wavelength theta aspect-ratio phase-offset]
49 ;; first, find the size of the kernel which is required
50 (let [square #(expt % 2)
51 rotated (fn [[x y]]
52 [(+ (* x (Math/cos theta)) (* y (Math/sin theta)))
53 (- (* y (Math/cos theta)) (* x (Math/sin theta)))])
55 gaussian (fn [[x y]]
56 (let [[x' y'] (rotated [x y])]
57 (Math/exp (- (/ (+ (square x')
58 (square (* aspect-ratio y')))
59 (* 2 (square sigma)))))))
60 sinusoid (fn [[x y]]
61 (let [[x' y'] (rotated [x y])]
62 (Math/cos
63 (+ (* 2 Math/PI (/ x' wavelength))
64 phase-offset))))
66 half-width
67 (let [std-dev-capture 5]
68 (max
69 (int (* std-dev-capture (/ sigma aspect-ratio)))
70 (int (* std-dev-capture sigma))
71 (int (* std-dev-capture (/ aspect-ratio sigma)))))
73 grid (let [axis (range (- half-width) (inc half-width))]
74 (for [y (reverse axis) x axis] (vector x y)))
76 scale (reduce + (map gaussian grid))
78 gabor (fn [[x y :as coord]]
79 (* (sinusoid coord) (gaussian coord) (/ scale)))
81 mat-width (+ 1 (* 2 half-width))
82 mat (Mat. mat-width mat-width CvType/CV_32F)]
84 (.put mat 0 0 (float-array (map gabor grid)))
85 mat)))
88 (defn draw-kernel! [kernel img-path]
89 (let [output img-path
90 size (.size kernel)
91 width (int (.width size))
92 height (int (.height size))
93 tmp-array (float-array (* width height))]
95 ;; read values from matrix.
96 (.get kernel 0 0 tmp-array)
98 ;; find overall dynamic range of the filter
99 (let [vals (vec tmp-array)
100 low (apply min vals)
101 high (apply max vals)
102 scaled-vals (map #(* 255 (- % low) (/ (- high low))) vals)
103 new-mat (Mat. height width CvType/CV_32F)]
104 (.put new-mat 0 0 (float-array scaled-vals))
105 (org.opencv.highgui.Highgui/imwrite output new-mat))))
107 ;; some cool examples
108 #+end_src
113 [[../images/gabor-50-10.png]]
115 #+begin_src clojure
116 (def img-base "/home/r/proj/cortex/images/")
118 (draw-kernel! (gabor-kernel 50 10 0 1 0)
119 (str img-base "gabor-50-10.png"))
120 #+end_src
123 [[../images/gabor-50-10-pi-over-4.png]]
125 #+begin_src clojure
126 (draw-kernel! (gabor-kernel 50 10 (/ Math/PI 4) 1 0)
127 (str img-base "gabor-50-10-pi-over-4.png"))
128 #+end_src
131 [[../images/gabor-50-10-pi-over-2.png]]
133 #+begin_src clojure
134 (draw-kernel! (gabor-kernel 50 10 (/ Math/PI 2) 1 0)
135 (str img-base "gabor-50-10-pi-over-2.png"))
136 #+end_src
139 [[../images/gabor-50-50.png]]
143 #+begin_src clojure
144 (draw-kernel! (gabor-kernel 50 50 0 1 0)
145 (str img-base "gabor-50-50.png"))
147 #+end_src
149 [[../images/gabor-50-10-0-3.png]]
151 #+begin_src clojure
152 (draw-kernel! (gabor-kernel 50 10 0 3 0)
153 (str img-base "gabor-50-10-0-3.png"))
154 #+end_src
158 [[../images/gabor-50-4-pi-over3-3.png]]
159 #+begin_src clojure
160 (draw-kernel! (gabor-kernel 50 4 (/ Math/PI 3) 3 0)
161 (str img-base "gabor-50-4-pi-over3-3.png"))
162 #+end_src
168 #:name gabor-tail
169 #+begin_src clojure
170 (defn show-kernel [kernel]
171 (let [img-path "/home/r/proj/cortex/tmp/kernel.png"]
172 (draw-kernel! kernel img-path)
173 (view (ImagePlus. img-path))))
175 (defn print-kernel [kernel]
176 (println (.dump kernel)))
179 (def brick-length 0.48)
180 (def brick-width 0.24)
181 (def brick-height 0.12)
182 (def gravity (Vector3f. 0 -9.81 0))
185 (defn brick* [position]
186 (println "get brick.")
187 (doto (box brick-length brick-height brick-width
188 :position position :name "brick"
189 :material "Common/MatDefs/Misc/Unshaded.j3md"
190 :texture "Textures/Terrain/BrickWall/BrickWall.jpg"
191 :mass 34)
192 (->
193 (.getMesh)
194 (.scaleTextureCoordinates (Vector2f. 1 0.5)))
195 (.setShadowMode RenderQueue$ShadowMode/CastAndReceive)
196 )
197 )
200 (defn floor*
201 "make a sturdy, unmovable physical floor"
202 []
203 (box 10 0.1 5 :name "floor" :mass 0
204 :color ColorRGBA/Gray :position (Vector3f. 0 0 0)))
206 (defn floor* []
207 (doto (box 10 0.1 5 :name "floor" ;10 0.1 5 ; 240 0.1 240
208 :material "Common/MatDefs/Misc/Unshaded.j3md"
209 :texture "Textures/BronzeCopper030.jpg"
210 :position (Vector3f. 0 0 0 )
211 :mass 0)
212 (->
213 (.getMesh)
214 (.scaleTextureCoordinates (Vector2f. 3 6)));64 64
215 (->
216 (.getMaterial)
217 (.getTextureParam "ColorMap")
218 (.getTextureValue)
219 (.setWrap Texture$WrapMode/Repeat))
220 (.setShadowMode RenderQueue$ShadowMode/Receive)
221 ))
224 (defn brick-wall* []
225 (let [node (Node. "brick-wall")]
226 (dorun
227 (map
228 (comp #(.attachChild node %) brick*)
229 (for [y (range 10)
230 x (range 4)
231 z (range 1)]
232 (Vector3f.
233 (+ (* 2 x brick-length)
234 (if (even? (+ y z))
235 (/ brick-length 4) (/ brick-length -4)))
236 (+ (* brick-height (inc (* 2 y))))
237 (* 2 z brick-width) ))))
238 (.setShadowMode node RenderQueue$ShadowMode/CastAndReceive)
239 node))
241 (import com.aurellem.capture.Capture)
243 (import java.io.File)
245 (def base "/home/r/proj/cortex/render/gabor-1/")
247 (defn brick-wall-game-run [record?]
248 (let [capture-dir (File. base "main")]
250 (.mkdir (File. base "main"))
251 (doto
252 (world
253 (doto (Node.) (.attachChild (floor*))
254 (.attachChild (brick-wall*))
255 )
256 {"key-f" (fn [game value]
257 (if (not value) (add-element game (brick-wall*))))
258 "key-space" (fire-cannon-ball )}
259 (fn [world]
260 (position-camera
261 world
262 (Vector3f. 1.382548, 4.0383573, 5.994235)
263 (Quaternion. 0.0013082094, 0.98581666,
264 -0.1676442, 0.0076932586))
266 ;;(speed-up world)
268 (if record?
269 (Capture/captureVideo
270 world capture-dir))
272 (add-camera! world (.getCamera world) no-op))
273 (fn [& _]))
274 (.start))))
276 (defn convolve-preview [kernel]
277 (let [input "/home/r/proj/cortex/render/gabor-1/main/0000032.png"
280 output "/home/r/ppp.png"
282 i (org.opencv.highgui.Highgui/imread input)
284 ;;kernel (gabor-kernel 10 1 (/ Math/PI 2) 10 0)
286 new-mat (Mat.)
288 ]
290 (org.opencv.imgproc.Imgproc/filter2D i new-mat CvType/CV_32F kernel)
292 (org.opencv.highgui.Highgui/imwrite "/home/r/ppp.png" new-mat)
294 (view (ImagePlus. input))
295 (view (ImagePlus. output))))
297 (use 'clojure.java.shell)
300 (defn apply-gabor [kernel source dest]
301 (let [i (org.opencv.highgui.Highgui/imread source)
302 new-mat (Mat.)]
304 (println dest)
305 (if (not (.exists (File. dest)))
306 (do
307 (org.opencv.imgproc.Imgproc/filter2D i new-mat CvType/CV_32F kernel)
308 (org.opencv.highgui.Highgui/imwrite dest new-mat)
309 (println "mogrify" "-modulate" "1000%" dest)
310 (sh "mogrify" "-modulate" "1000%" dest)))))
313 (import java.io.File)
315 (defn images [path]
316 (sort (rest (file-seq (File. path)))))
320 (defn pics [file]
321 (images (str base file)))
323 (defn generate-gabor-images [kernel name]
324 (draw-kernel! kernel (str base name ".png"))
326 (.mkdir (File. (str base name)))
328 (let [main (map #(.getCanonicalPath %) (pics "main"))
329 targets (map #(str base name "/" (format "%07d.png" %))
330 (range 0 (count main)))]
331 (dorun (pmap (partial apply-gabor kernel) main targets))))
334 (def banks
335 [[(gabor-kernel 2.8 3.5) "bank-1-1"]
336 [(gabor-kernel 2.8 3.5 (/ Math/PI 2)) "bank-1-1-rot"]
338 ;; [(gabor-kernel 3.6 4.6) "bank-1-2"]
339 ;; [(gabor-kernel 4.5 5.6) "bank-2-1"]
340 ;; [(gabor-kernel 6.3 7.9) "bank-3-1"]
341 ;; [(gabor-kernel 7.3 9.1) "bank-3-2"]
343 [(gabor-kernel 12.3 15.4) "bank-6-1"]
346 ;; [(gabor-kernel 17 21.2) "bank-8-1"]
347 ;; [(gabor-kernel 18.2 22.8) "bank-8-2"]
348 ])
351 (defn make-all-images []
352 (dorun (map (partial apply generate-gabor-images) banks)))
356 (defn compile-left-right []
357 (.mkdir (File. (str base "left-right")))
358 (let [main (pics "main")
359 left (pics "bank-1-1")
360 right (pics "bank-1-1-rot")
361 left-kernel (repeat 20000 (File. (str base "bank-1-1.png")))
362 right-kernel (repeat 20000 (File. (str base "bank-1-1-rot.png")))
363 targets (map
364 #(File. (str base "left-right/" (format "%07d.png" %)))
365 (range 0 (count main)))]
367 (dorun
368 (pmap
369 (comp
370 (fn [[main left right left-kernel right-kernel target]]
371 (println target)
372 (if (not (.exists (File. target)))
373 (sh "convert"
374 "-size" "1940x515" "xc:white"
375 main "-geometry" "+0+0" "-composite"
376 left "-geometry" "+650+0" "-composite"
377 right "-geometry" "+1300+0" "-composite"
378 left-kernel "-geometry" "+960+485" "-composite"
379 right-kernel "-geometry" "+1610+485" "-composite"
380 target)))
381 (fn [& args] (map #(.getCanonicalPath %) args)))
382 main left right left-kernel right-kernel targets))))
385 (defn compile-big-small []
386 (.mkdir (File. (str base "big-small")))
387 (let [main (pics "main")
388 left (pics "bank-1-1")
389 right (pics "bank-6-1")
390 small-kernel (repeat 20000 (File. (str base "bank-1-1.png")))
391 big-kernel (repeat 20000 (File. (str base "bank-6-1.png")))
392 targets (map
393 #(File. (str base "big-small/" (format "%07d.png" %)))
394 (range 0 (count main)))]
396 (dorun
397 (pmap
398 (comp
399 (fn [[main left right small-kernel big-kernel target]]
400 (println target)
401 (if (not (.exists (File. target)))
402 (sh "convert"
403 "-size" "1940x610" "xc:white"
404 main "-geometry" "+0+0" "-composite"
405 left "-geometry" "+650+0" "-composite"
406 right "-geometry" "+1300+0" "-composite"
407 small-kernel "-geometry" "+960+485" "-composite"
408 big-kernel "-geometry" "+1560+485" "-composite"
409 target)))
410 (fn [& args] (map #(.getCanonicalPath %) args)))
411 main left right small-kernel big-kernel targets))))
414 (defn regen-everything []
415 (make-all-images)
416 (compile-left-right)
417 (compile-big-small))
420 #+end_src
422 #+name: make-left-right
423 #+begin_src sh
424 #!/bin/sh
426 ffmpeg -framerate 60 -i ./left-right/%07d.png -b:v 9000k\
427 -c:v mpeg4 -r 60 gabor-rotation.mp4
429 #+end_src
432 #+name: make-big-small
433 #+begin_src sh
434 #!/bin/sh
436 ffmpeg -framerate 60 -i ./big-small/%07d.png -b:v 9000k\
437 -c:v mpeg4 -r 60 gabor-big-small.mp4
439 #+end_src
441 #+begin_html
442 <div class="figure">
443 <video controls="controls" width="755">
444 <source src="../video/gabor-rotation.mp4" type="video/mp4"
445 preload="none" poster="../images/aurellem-1280x480.png" />
446 </video>
447 <br> <a href="http://youtu.be/r5Bn2aG7MO0"> YouTube </a>
448 <p>Two gabor filters with different values of theta are compared. The
449 horizontally aligned one does better in this example.</p>
450 </div>
451 #+end_html
455 #+begin_html
456 <div class="figure">
457 <video controls="controls" width="755">
458 <source src="../video/gabor-big-small.mp4" type="video/mp4"
459 preload="none" poster="../images/aurellem-1280x480.png" />
460 </video>
461 <br> <a href="http://youtu.be/r5Bn2aG7MO0"> YouTube </a>
462 <p>Here we compare a gabor filter from bank 1 with one from bank 6.</p>
463 </div>
464 #+end_html
469 * COMMENT Generate Source
470 #+begin_src clojure :tangle ../src/cortex/gabor.clj
471 <<gabor>>
472 <<gabor-tail>>
473 #+end_src
475 #+begin_src clojure :tangle ../render/gabor-1/make-rotation.sh
476 <<make-left-right>>
477 #+end_src
479 #+begin_src clojure :tangle ../render/gabor-1/make-big-small.sh
480 <<make-big-small>>
481 #+end_src