view org/gabor.org @ 429:b5d0f0adf19f

improvements by Dylan!
author Robert McIntyre <rlm@mit.edu>
date Fri, 21 Mar 2014 20:56:56 -0400
parents 057d47fc4789
children 42ddfe406c0a
line wrap: on
line source
1 #+title: Fun with Gabor Filters
2 #+author: Robert McIntyre
3 #+email: rlm@mit.edu
4 #+description: gabor filters in clojure with opencv
5 #+keywords: computer vision, jMonkeyEngine3, clojure, opencv
6 #+SETUPFILE: ../../aurellem/org/setup.org
7 #+INCLUDE: ../../aurellem/org/level-0.org
8 #+babel: :mkdirp yes :noweb yes :exports both
11 Gabor filters were invented by the same guy who invented holograms.
13 They work well as edge detectors and are related to the human visual
14 system.
16 #+name: gabor
17 #+begin_src clojure
18 (ns cortex.gabor
19 (:import org.opencv.core.CvType)
20 (:import java.awt.image.BufferedImage)
21 (:import ij.ImagePlus)
22 (:import org.opencv.core.Mat)
23 (:use (cortex world sense util vision import))
24 (:import com.jme3.post.SceneProcessor)
25 (:import (com.jme3.util BufferUtils Screenshots))
26 (:import java.nio.ByteBuffer)
27 (:import java.awt.image.BufferedImage)
28 (:import (com.jme3.renderer ViewPort Camera))
29 (:import (com.jme3.math ColorRGBA Vector3f Matrix3f Vector2f))
30 (:import com.jme3.renderer.Renderer)
31 (:import com.jme3.app.Application)
32 (:import com.jme3.texture.FrameBuffer)
33 (:import (com.jme3.scene Node Spatial)))
36 (cortex.import/mega-import-jme3)
37 (use 'clojure.math.numeric-tower)
38 (defn load-opencv
39 "Load the opencv native library. Must be called before any OpenCV
40 stuff is used."
41 []
42 (clojure.lang.RT/loadLibrary "opencv_java249"))
44 (load-opencv)
46 (defn gabor-kernel
47 ([sigma wavelength theta]
48 (gabor-kernel sigma wavelength theta 1 0))
49 ([sigma wavelength]
50 (gabor-kernel sigma wavelength 0 1 0))
51 ([sigma wavelength theta aspect-ratio phase-offset]
53 ;; first, find the size of the kernel which is required
54 (let [square #(expt % 2)
55 rotated (fn [[x y]]
56 [(+ (* x (Math/cos theta)) (* y (Math/sin theta)))
57 (- (* y (Math/cos theta)) (* x (Math/sin theta)))])
59 gaussian (fn [[x y]]
60 (let [[x' y'] (rotated [x y])]
61 (Math/exp (- (/ (+ (square x')
62 (square (* aspect-ratio y')))
63 (* 2 (square sigma)))))))
64 sinusoid (fn [[x y]]
65 (let [[x' y'] (rotated [x y])]
66 (Math/cos
67 (+ (* 2 Math/PI (/ x' wavelength))
68 phase-offset))))
70 half-width
71 (let [std-dev-capture 5]
72 (max
73 (int (* std-dev-capture (/ sigma aspect-ratio)))
74 (int (* std-dev-capture sigma))
75 (int (* std-dev-capture (/ aspect-ratio sigma)))))
77 grid (let [axis (range (- half-width) (inc half-width))]
78 (for [y (reverse axis) x axis] (vector x y)))
80 scale (reduce + (map gaussian grid))
82 gabor (fn [[x y :as coord]]
83 (* (sinusoid coord) (gaussian coord) (/ scale)))
85 mat-width (+ 1 (* 2 half-width))
86 mat (Mat. mat-width mat-width CvType/CV_32F)]
88 (.put mat 0 0 (float-array (map gabor grid)))
89 mat)))
92 (defn draw-kernel! [kernel img-path]
93 (let [output img-path
94 size (.size kernel)
95 width (int (.width size))
96 height (int (.height size))
97 tmp-array (float-array (* width height))]
99 ;; read values from matrix.
100 (.get kernel 0 0 tmp-array)
102 ;; find overall dynamic range of the filter
103 (let [vals (vec tmp-array)
104 low (apply min vals)
105 high (apply max vals)
106 scaled-vals (map #(* 255 (- % low) (/ (- high low))) vals)
107 new-mat (Mat. height width CvType/CV_32F)]
108 (.put new-mat 0 0 (float-array scaled-vals))
109 (org.opencv.highgui.Highgui/imwrite output new-mat))))
111 ;; some cool examples
112 #+end_src
117 [[../images/gabor-50-10.png]]
119 #+begin_src clojure
120 (def img-base "/home/r/proj/cortex/images/")
122 (draw-kernel! (gabor-kernel 50 10 0 1 0)
123 (str img-base "gabor-50-10.png"))
124 #+end_src
127 [[../images/gabor-50-10-pi-over-4.png]]
129 #+begin_src clojure
130 (draw-kernel! (gabor-kernel 50 10 (/ Math/PI 4) 1 0)
131 (str img-base "gabor-50-10-pi-over-4.png"))
132 #+end_src
135 [[../images/gabor-50-10-pi-over-2.png]]
137 #+begin_src clojure
138 (draw-kernel! (gabor-kernel 50 10 (/ Math/PI 2) 1 0)
139 (str img-base "gabor-50-10-pi-over-2.png"))
140 #+end_src
143 [[../images/gabor-50-50.png]]
147 #+begin_src clojure
148 (draw-kernel! (gabor-kernel 50 50 0 1 0)
149 (str img-base "gabor-50-50.png"))
151 #+end_src
153 [[../images/gabor-50-10-0-3.png]]
155 #+begin_src clojure
156 (draw-kernel! (gabor-kernel 50 10 0 3 0)
157 (str img-base "gabor-50-10-0-3.png"))
158 #+end_src
162 [[../images/gabor-50-4-pi-over3-3.png]]
163 #+begin_src clojure
164 (draw-kernel! (gabor-kernel 50 4 (/ Math/PI 3) 3 0)
165 (str img-base "gabor-50-4-pi-over3-3.png"))
166 #+end_src
169 #+name: gabor-tail
170 #+begin_src clojure
171 (defn show-kernel [kernel]
172 (let [img-path "/home/r/proj/cortex/tmp/kernel.png"]
173 (draw-kernel! kernel img-path)
174 (view (ImagePlus. img-path))))
176 (defn print-kernel [kernel]
177 (println (.dump kernel)))
180 (def brick-length 0.48)
181 (def brick-width 0.24)
182 (def brick-height 0.12)
183 (def gravity (Vector3f. 0 -9.81 0))
186 (defn brick* [position]
187 (println "get brick.")
188 (doto (box brick-length brick-height brick-width
189 :position position :name "brick"
190 :material "Common/MatDefs/Misc/Unshaded.j3md"
191 :texture "Textures/Terrain/BrickWall/BrickWall.jpg"
192 :mass 34)
193 (->
194 (.getMesh)
195 (.scaleTextureCoordinates (Vector2f. 1 0.5)))
196 (.setShadowMode RenderQueue$ShadowMode/CastAndReceive)
197 )
198 )
201 (defn floor*
202 "make a sturdy, unmovable physical floor"
203 []
204 (box 10 0.1 5 :name "floor" :mass 0
205 :color ColorRGBA/Gray :position (Vector3f. 0 0 0)))
207 (defn floor* []
208 (doto (box 10 0.1 5 :name "floor" ;10 0.1 5 ; 240 0.1 240
209 :material "Common/MatDefs/Misc/Unshaded.j3md"
210 :texture "Textures/BronzeCopper030.jpg"
211 :position (Vector3f. 0 0 0 )
212 :mass 0)
213 (->
214 (.getMesh)
215 (.scaleTextureCoordinates (Vector2f. 3 6)));64 64
216 (->
217 (.getMaterial)
218 (.getTextureParam "ColorMap")
219 (.getTextureValue)
220 (.setWrap Texture$WrapMode/Repeat))
221 (.setShadowMode RenderQueue$ShadowMode/Receive)
222 ))
225 (defn brick-wall* []
226 (let [node (Node. "brick-wall")]
227 (dorun
228 (map
229 (comp #(.attachChild node %) brick*)
230 (for [y (range 10)
231 x (range 4)
232 z (range 1)]
233 (Vector3f.
234 (+ (* 2 x brick-length)
235 (if (even? (+ y z))
236 (/ brick-length 4) (/ brick-length -4)))
237 (+ (* brick-height (inc (* 2 y))))
238 (* 2 z brick-width) ))))
239 (.setShadowMode node RenderQueue$ShadowMode/CastAndReceive)
240 node))
242 (import com.aurellem.capture.Capture)
244 (import java.io.File)
246 (def base "/home/r/proj/cortex/render/gabor-1/")
248 (defn brick-wall-game-run [record?]
249 (let [capture-dir (File. base "main")]
251 (.mkdir (File. base "main"))
252 (doto
253 (world
254 (doto (Node.) (.attachChild (floor*))
255 (.attachChild (brick-wall*))
256 )
257 {"key-f" (fn [game value]
258 (if (not value) (add-element game (brick-wall*))))
259 "key-space" (fire-cannon-ball )}
260 (fn [world]
261 (position-camera
262 world
263 (Vector3f. 1.382548, 4.0383573, 5.994235)
264 (Quaternion. 0.0013082094, 0.98581666,
265 -0.1676442, 0.0076932586))
267 ;;(speed-up world)
269 (if record?
270 (Capture/captureVideo
271 world capture-dir))
273 (add-camera! world (.getCamera world) no-op))
274 (fn [& _]))
275 (.start))))
277 (defn convolve-preview [kernel]
278 (let [input "/home/r/proj/cortex/render/gabor-1/main/0000032.png"
281 output "/home/r/ppp.png"
283 i (org.opencv.highgui.Highgui/imread input)
285 ;;kernel (gabor-kernel 10 1 (/ Math/PI 2) 10 0)
287 new-mat (Mat.)
289 ]
291 (org.opencv.imgproc.Imgproc/filter2D
292 i new-mat CvType/CV_32F kernel)
294 (org.opencv.highgui.Highgui/imwrite "/home/r/ppp.png" new-mat)
296 (view (ImagePlus. input))
297 (view (ImagePlus. output))))
299 (use 'clojure.java.shell)
302 (defn apply-gabor [kernel source dest]
303 (let [i (org.opencv.highgui.Highgui/imread source)
304 new-mat (Mat.)]
306 (println dest)
307 (if (not (.exists (File. dest)))
308 (do
309 (org.opencv.imgproc.Imgproc/filter2D
310 i new-mat CvType/CV_32F kernel)
311 (org.opencv.highgui.Highgui/imwrite dest new-mat)
312 (println "mogrify" "-modulate" "1000%" dest)
313 (sh "mogrify" "-modulate" "1000%" dest)))))
316 (import java.io.File)
318 (defn images [path]
319 (sort (rest (file-seq (File. path)))))
323 (defn pics [file]
324 (images (str base file)))
326 (defn generate-gabor-images [kernel name]
327 (draw-kernel! kernel (str base name ".png"))
329 (.mkdir (File. (str base name)))
331 (let [main (map #(.getCanonicalPath %) (pics "main"))
332 targets (map #(str base name "/" (format "%07d.png" %))
333 (range 0 (count main)))]
334 (dorun (pmap (partial apply-gabor kernel) main targets))))
337 (def banks
338 [[(gabor-kernel 2.8 3.5) "bank-1-1"]
339 [(gabor-kernel 2.8 3.5 (/ Math/PI 2)) "bank-1-1-rot"]
341 ;; [(gabor-kernel 3.6 4.6) "bank-1-2"]
342 ;; [(gabor-kernel 4.5 5.6) "bank-2-1"]
343 ;; [(gabor-kernel 6.3 7.9) "bank-3-1"]
344 ;; [(gabor-kernel 7.3 9.1) "bank-3-2"]
346 [(gabor-kernel 12.3 15.4) "bank-6-1"]
349 ;; [(gabor-kernel 17 21.2) "bank-8-1"]
350 ;; [(gabor-kernel 18.2 22.8) "bank-8-2"]
351 ])
354 (defn make-all-images []
355 (dorun (map (partial apply generate-gabor-images) banks)))
359 (defn compile-left-right []
360 (.mkdir (File. (str base "left-right")))
361 (let [main (pics "main")
362 left (pics "bank-1-1")
363 right (pics "bank-1-1-rot")
364 left-kernel (repeat 20000 (File. (str base "bank-1-1.png")))
365 right-kernel (repeat 20000 (File. (str base "bank-1-1-rot.png")))
366 targets (map
367 #(File. (str base "left-right/" (format "%07d.png" %)))
368 (range 0 (count main)))]
370 (dorun
371 (pmap
372 (comp
373 (fn [[main left right left-kernel right-kernel target]]
374 (println target)
375 (if (not (.exists (File. target)))
376 (sh "convert"
377 "-size" "1940x515" "xc:white"
378 main "-geometry" "+0+0" "-composite"
379 left "-geometry" "+650+0" "-composite"
380 right "-geometry" "+1300+0" "-composite"
381 left-kernel "-geometry" "+960+485" "-composite"
382 right-kernel "-geometry" "+1610+485" "-composite"
383 target)))
384 (fn [& args] (map #(.getCanonicalPath %) args)))
385 main left right left-kernel right-kernel targets))))
388 (defn compile-big-small []
389 (.mkdir (File. (str base "big-small")))
390 (let [main (pics "main")
391 left (pics "bank-1-1")
392 right (pics "bank-6-1")
393 small-kernel (repeat 20000 (File. (str base "bank-1-1.png")))
394 big-kernel (repeat 20000 (File. (str base "bank-6-1.png")))
395 targets (map
396 #(File. (str base "big-small/" (format "%07d.png" %)))
397 (range 0 (count main)))]
399 (dorun
400 (pmap
401 (comp
402 (fn [[main left right small-kernel big-kernel target]]
403 (println target)
404 (if (not (.exists (File. target)))
405 (sh "convert"
406 "-size" "1940x610" "xc:white"
407 main "-geometry" "+0+0" "-composite"
408 left "-geometry" "+650+0" "-composite"
409 right "-geometry" "+1300+0" "-composite"
410 small-kernel "-geometry" "+960+485" "-composite"
411 big-kernel "-geometry" "+1560+485" "-composite"
412 target)))
413 (fn [& args] (map #(.getCanonicalPath %) args)))
414 main left right small-kernel big-kernel targets))))
417 (defn regen-everything []
418 (make-all-images)
419 (compile-left-right)
420 (compile-big-small))
423 #+end_src
426 #+name: make-videos
427 #+begin_src makefile
428 scale:
429 ffmpeg -framerate 60 -i ./big-small/%07d.png -b:v 9000k\
430 -c:v theora -r 60 gabor-scale.ogg
432 rotation:
433 ffmpeg -framerate 60 -i ./left-right/%07d.png -b:v 9000k\
434 -c:v theora -r 60 gabor-rotation.ogg
436 all: rotation scale
438 clean:
439 rm gabor-rotation.org gabor-scale.ogg
440 #+end_src
443 #+begin_html
444 <div class="figure">
445 <video controls="controls">
446 <source src="../video/gabor-rotation.ogg" type="video/ogg"
447 preload="none" poster="../images/aurellem-1280x480.png" />
448 </video>
449 <br> <a href="http://youtu.be/bxujjO97B-U"> YouTube </a>
450 <p>Two gabor filters with different values of theta are compared. The
451 horizontally aligned one does better in this example.</p>
452 </div>
453 #+end_html
457 #+begin_html
458 <div class="figure">
459 <video controls="controls">
460 <source src="../video/gabor-scale.ogg" type="video/ogg"
461 preload="none" poster="../images/aurellem-1280x480.png" />
462 </video>
463 <br> <a href="http://youtu.be/-EsfA2ceWUk"> YouTube </a>
464 <p>Here we compare a gabor filter from bank 1 with one from bank 6.</p>
465 </div>
466 #+end_html
468 * Source Listing
469 - [[../src/cortex/gabor.clj][cortex.gabor]]
470 #+html: <ul> <li> <a href="../org/gabor.org">This org file</a> </li> </ul>
471 - [[http://hg.bortreb.com ][source-repository]]
475 * COMMENT Generate Source
476 #+begin_src clojure :tangle ../src/cortex/gabor.clj
477 <<gabor>>
478 <<gabor-tail>>
479 #+end_src
481 #+begin_src Makefile :tangle ../render/gabor-1/Makefile
482 <<make-videos>>
483 #+end_src