Mercurial > cortex

annotate org/gabor.org @ 366:871882350c83

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
cosmetic improvement.
author Robert McIntyre <rlm@mit.edu>
date Thu, 07 Mar 2013 07:50:28 +0000
parents 1727b68634ea
children 078600644d94
rev   line source
rlm@356 1 #+title: Fun with Gabor Filters
rlm@356 2 #+author: Robert McIntyre
rlm@356 3 #+email: rlm@mit.edu
rlm@356 4 #+description: gabor filters in clojure with opencv
rlm@356 5 #+keywords: computer vision, jMonkeyEngine3, clojure, opencv
rlm@356 6 #+SETUPFILE: ../../aurellem/org/setup.org
rlm@356 7 #+INCLUDE: ../../aurellem/org/level-0.org
rlm@356 8 #+babel: :mkdirp yes :noweb yes :exports both
rlm@356 9
rlm@356 10
rlm@356 11
rlm@356 12 #+name: gabor
rlm@356 13 #+begin_src clojure
rlm@357 14 (ns cortex.gabor
rlm@357 15 (:import org.opencv.core.CvType)
rlm@357 16 (:import java.awt.image.BufferedImage)
rlm@357 17 (:import ij.ImagePlus)
rlm@358 18 (:import org.opencv.core.Mat)
rlm@364 19 (:use (cortex world sense util vision import))
rlm@363 20 (:import com.jme3.post.SceneProcessor)
rlm@363 21 (:import (com.jme3.util BufferUtils Screenshots))
rlm@363 22 (:import java.nio.ByteBuffer)
rlm@363 23 (:import java.awt.image.BufferedImage)
rlm@363 24 (:import (com.jme3.renderer ViewPort Camera))
rlm@363 25 (:import (com.jme3.math ColorRGBA Vector3f Matrix3f Vector2f))
rlm@363 26 (:import com.jme3.renderer.Renderer)
rlm@363 27 (:import com.jme3.app.Application)
rlm@363 28 (:import com.jme3.texture.FrameBuffer)
rlm@363 29 (:import (com.jme3.scene Node Spatial)))
rlm@363 30
rlm@363 31
rlm@363 32 (cortex.import/mega-import-jme3)
rlm@364 33 (use 'clojure.math.numeric-tower)
rlm@356 34 (defn load-opencv
rlm@356 35 "Load the opencv native library. Must be called before any OpenCV
rlm@356 36 stuff is used."
rlm@356 37 []
rlm@356 38 (clojure.lang.RT/loadLibrary "opencv_java249"))
rlm@356 39
rlm@364 40 (load-opencv)
rlm@364 41
rlm@364 42 (defn gabor-kernel
rlm@364 43 ([sigma wavelength theta]
rlm@364 44 (gabor-kernel sigma wavelength theta 1 0))
rlm@364 45 ([sigma wavelength]
rlm@364 46 (gabor-kernel sigma wavelength 0 1 0))
rlm@364 47 ([sigma wavelength theta aspect-ratio phase-offset]
rlm@359 48
rlm@359 49 ;; first, find the size of the kernel which is required
rlm@359 50 (let [square #(expt % 2)
rlm@359 51 rotated (fn [[x y]]
rlm@359 52 [(+ (* x (Math/cos theta)) (* y (Math/sin theta)))
rlm@359 53 (- (* y (Math/cos theta)) (* x (Math/sin theta)))])
rlm@359 54
rlm@359 55 gaussian (fn [[x y]]
rlm@359 56 (let [[x' y'] (rotated [x y])]
rlm@359 57 (Math/exp (- (/ (+ (square x')
rlm@359 58 (square (* aspect-ratio y')))
rlm@359 59 (* 2 (square sigma)))))))
rlm@359 60 sinusoid (fn [[x y]]
rlm@359 61 (let [[x' y'] (rotated [x y])]
rlm@359 62 (Math/cos
rlm@359 63 (+ (* 2 Math/PI (/ x' wavelength))
rlm@359 64 phase-offset))))
rlm@359 65
rlm@361 66 half-width
rlm@361 67 (let [std-dev-capture 5]
rlm@361 68 (max
rlm@361 69 (int (* std-dev-capture (/ sigma aspect-ratio)))
rlm@361 70 (int (* std-dev-capture sigma))
rlm@361 71 (int (* std-dev-capture (/ aspect-ratio sigma)))))
rlm@359 72
rlm@359 73 grid (let [axis (range (- half-width) (inc half-width))]
rlm@359 74 (for [y (reverse axis) x axis] (vector x y)))
rlm@359 75
rlm@359 76 scale (reduce + (map gaussian grid))
rlm@359 77
rlm@359 78 gabor (fn [[x y :as coord]]
rlm@360 79 (* (sinusoid coord) (gaussian coord) (/ scale)))
rlm@359 80
rlm@359 81 mat-width (+ 1 (* 2 half-width))
rlm@359 82 mat (Mat. mat-width mat-width CvType/CV_32F)]
rlm@359 83
rlm@359 84 (.put mat 0 0 (float-array (map gabor grid)))
rlm@364 85 mat)))
rlm@359 86
rlm@359 87
rlm@361 88 (defn draw-kernel! [kernel img-path]
rlm@361 89 (let [output img-path
rlm@360 90 size (.size kernel)
rlm@360 91 width (int (.width size))
rlm@360 92 height (int (.height size))
rlm@360 93 tmp-array (float-array (* width height))]
rlm@360 94
rlm@360 95 ;; read values from matrix.
rlm@360 96 (.get kernel 0 0 tmp-array)
rlm@360 97
rlm@360 98 ;; find overall dynamic range of the filter
rlm@360 99 (let [vals (vec tmp-array)
rlm@360 100 low (apply min vals)
rlm@360 101 high (apply max vals)
rlm@360 102 scaled-vals (map #(* 255 (- % low) (/ (- high low))) vals)
rlm@360 103 new-mat (Mat. height width CvType/CV_32F)]
rlm@360 104 (.put new-mat 0 0 (float-array scaled-vals))
rlm@361 105 (org.opencv.highgui.Highgui/imwrite output new-mat))))
rlm@361 106
rlm@364 107 ;; some cool examples
rlm@364 108 #+end_src
rlm@364 109
rlm@364 110
rlm@364 111
rlm@364 112
rlm@364 113 [[../images/gabor-50-10.png]]
rlm@364 114
rlm@364 115 #+begin_src clojure
rlm@364 116 (def img-base "/home/r/proj/cortex/images/")
rlm@364 117
rlm@364 118 (draw-kernel! (gabor-kernel 50 10 0 1 0)
rlm@364 119 (str img-base "gabor-50-10.png"))
rlm@364 120 #+end_src
rlm@364 121
rlm@364 122
rlm@364 123 [[../images/gabor-50-10-pi-over-4.png]]
rlm@364 124
rlm@364 125 #+begin_src clojure
rlm@364 126 (draw-kernel! (gabor-kernel 50 10 (/ Math/PI 4) 1 0)
rlm@364 127 (str img-base "gabor-50-10-pi-over-4.png"))
rlm@364 128 #+end_src
rlm@364 129
rlm@364 130
rlm@364 131 [[../images/gabor-50-10-pi-over-2.png]]
rlm@364 132
rlm@364 133 #+begin_src clojure
rlm@364 134 (draw-kernel! (gabor-kernel 50 10 (/ Math/PI 2) 1 0)
rlm@364 135 (str img-base "gabor-50-10-pi-over-2.png"))
rlm@364 136 #+end_src
rlm@364 137
rlm@364 138
rlm@364 139 [[../images/gabor-50-50.png]]
rlm@364 140
rlm@364 141
rlm@364 142
rlm@364 143 #+begin_src clojure
rlm@364 144 (draw-kernel! (gabor-kernel 50 50 0 1 0)
rlm@364 145 (str img-base "gabor-50-50.png"))
rlm@364 146
rlm@364 147 #+end_src
rlm@364 148
rlm@364 149 [[../images/gabor-50-10-0-3.png]]
rlm@364 150
rlm@364 151 #+begin_src clojure
rlm@364 152 (draw-kernel! (gabor-kernel 50 10 0 3 0)
rlm@364 153 (str img-base "gabor-50-10-0-3.png"))
rlm@364 154 #+end_src
rlm@364 155
rlm@364 156
rlm@364 157
rlm@364 158 [[../images/gabor-50-4-pi-over3-3.png]]
rlm@364 159 #+begin_src clojure
rlm@364 160 (draw-kernel! (gabor-kernel 50 4 (/ Math/PI 3) 3 0)
rlm@364 161 (str img-base "gabor-50-4-pi-over3-3.png"))
rlm@364 162 #+end_src
rlm@364 163
rlm@364 164
rlm@364 165
rlm@364 166
rlm@364 167
rlm@364 168 #:name gabor-tail
rlm@364 169 #+begin_src clojure
rlm@361 170 (defn show-kernel [kernel]
rlm@361 171 (let [img-path "/home/r/proj/cortex/tmp/kernel.png"]
rlm@362 172 (draw-kernel! kernel img-path)
rlm@364 173 (view (ImagePlus. img-path))))
rlm@359 174
rlm@359 175 (defn print-kernel [kernel]
rlm@359 176 (println (.dump kernel)))
rlm@359 177
rlm@363 178
rlm@363 179 (def brick-length 0.48)
rlm@363 180 (def brick-width 0.24)
rlm@363 181 (def brick-height 0.12)
rlm@363 182 (def gravity (Vector3f. 0 -9.81 0))
rlm@363 183
rlm@363 184
rlm@363 185 (defn brick* [position]
rlm@363 186 (println "get brick.")
rlm@363 187 (doto (box brick-length brick-height brick-width
rlm@363 188 :position position :name "brick"
rlm@363 189 :material "Common/MatDefs/Misc/Unshaded.j3md"
rlm@363 190 :texture "Textures/Terrain/BrickWall/BrickWall.jpg"
rlm@363 191 :mass 34)
rlm@363 192 (->
rlm@363 193 (.getMesh)
rlm@363 194 (.scaleTextureCoordinates (Vector2f. 1 0.5)))
rlm@363 195 (.setShadowMode RenderQueue$ShadowMode/CastAndReceive)
rlm@363 196 )
rlm@363 197 )
rlm@363 198
rlm@363 199
rlm@363 200 (defn floor*
rlm@363 201 "make a sturdy, unmovable physical floor"
rlm@363 202 []
rlm@366 203 (box 10 0.1 5 :name "floor" :mass 0
rlm@366 204 :color ColorRGBA/Gray :position (Vector3f. 0 0 0)))
rlm@363 205
rlm@363 206 (defn floor* []
rlm@363 207 (doto (box 10 0.1 5 :name "floor" ;10 0.1 5 ; 240 0.1 240
rlm@363 208 :material "Common/MatDefs/Misc/Unshaded.j3md"
rlm@363 209 :texture "Textures/BronzeCopper030.jpg"
rlm@363 210 :position (Vector3f. 0 0 0 )
rlm@363 211 :mass 0)
rlm@363 212 (->
rlm@363 213 (.getMesh)
rlm@363 214 (.scaleTextureCoordinates (Vector2f. 3 6)));64 64
rlm@363 215 (->
rlm@363 216 (.getMaterial)
rlm@363 217 (.getTextureParam "ColorMap")
rlm@363 218 (.getTextureValue)
rlm@363 219 (.setWrap Texture$WrapMode/Repeat))
rlm@363 220 (.setShadowMode RenderQueue$ShadowMode/Receive)
rlm@363 221 ))
rlm@363 222
rlm@363 223
rlm@363 224 (defn brick-wall* []
rlm@363 225 (let [node (Node. "brick-wall")]
rlm@363 226 (dorun
rlm@363 227 (map
rlm@363 228 (comp #(.attachChild node %) brick*)
rlm@363 229 (for [y (range 10)
rlm@363 230 x (range 4)
rlm@363 231 z (range 1)]
rlm@363 232 (Vector3f.
rlm@363 233 (+ (* 2 x brick-length)
rlm@363 234 (if (even? (+ y z))
rlm@363 235 (/ brick-length 4) (/ brick-length -4)))
rlm@363 236 (+ (* brick-height (inc (* 2 y))))
rlm@363 237 (* 2 z brick-width) ))))
rlm@363 238 (.setShadowMode node RenderQueue$ShadowMode/CastAndReceive)
rlm@363 239 node))
rlm@363 240
rlm@363 241 (import com.aurellem.capture.Capture)
rlm@363 242
rlm@363 243 (import java.io.File)
rlm@363 244
rlm@364 245 (def base "/home/r/proj/cortex/render/gabor-1/")
rlm@363 246
rlm@363 247 (defn brick-wall-game-run [record?]
rlm@364 248 (let [capture-dir (File. base "main")]
rlm@364 249
rlm@364 250 (.mkdir (File. base "main"))
rlm@364 251 (doto
rlm@364 252 (world
rlm@364 253 (doto (Node.) (.attachChild (floor*))
rlm@364 254 (.attachChild (brick-wall*))
rlm@364 255 )
rlm@364 256 {"key-f" (fn [game value]
rlm@364 257 (if (not value) (add-element game (brick-wall*))))
rlm@364 258 "key-space" (fire-cannon-ball )}
rlm@364 259 (fn [world]
rlm@366 260 (position-camera
rlm@366 261 world
rlm@366 262 (Vector3f. 1.382548, 4.0383573, 5.994235)
rlm@366 263 (Quaternion. 0.0013082094, 0.98581666,
rlm@366 264 -0.1676442, 0.0076932586))
rlm@363 265
rlm@364 266 ;;(speed-up world)
rlm@364 267
rlm@364 268 (if record?
rlm@364 269 (Capture/captureVideo
rlm@364 270 world capture-dir))
rlm@364 271
rlm@363 272 (add-camera! world (.getCamera world) no-op))
rlm@364 273 (fn [& _]))
rlm@364 274 (.start))))
rlm@363 275
rlm@364 276 (defn convolve-preview [kernel]
rlm@363 277 (let [input "/home/r/proj/cortex/render/gabor-1/main/0000032.png"
rlm@357 278
rlm@357 279
rlm@357 280 output "/home/r/ppp.png"
rlm@356 281
rlm@357 282 i (org.opencv.highgui.Highgui/imread input)
rlm@358 283
rlm@364 284 ;;kernel (gabor-kernel 10 1 (/ Math/PI 2) 10 0)
rlm@358 285
rlm@358 286 new-mat (Mat.)
rlm@358 287
rlm@357 288 ]
rlm@356 289
rlm@360 290 (org.opencv.imgproc.Imgproc/filter2D i new-mat CvType/CV_32F kernel)
rlm@358 291
rlm@358 292 (org.opencv.highgui.Highgui/imwrite "/home/r/ppp.png" new-mat)
rlm@358 293
rlm@357 294 (view (ImagePlus. input))
rlm@361 295 (view (ImagePlus. output))))
rlm@357 296
rlm@364 297 (use 'clojure.java.shell)
rlm@363 298
rlm@363 299
rlm@364 300 (defn apply-gabor [kernel source dest]
rlm@364 301 (let [i (org.opencv.highgui.Highgui/imread source)
rlm@364 302 new-mat (Mat.)]
rlm@364 303
rlm@364 304 (println dest)
rlm@364 305 (if (not (.exists (File. dest)))
rlm@364 306 (do
rlm@364 307 (org.opencv.imgproc.Imgproc/filter2D i new-mat CvType/CV_32F kernel)
rlm@364 308 (org.opencv.highgui.Highgui/imwrite dest new-mat)
rlm@364 309 (println "mogrify" "-modulate" "1000%" dest)
rlm@364 310 (sh "mogrify" "-modulate" "1000%" dest)))))
rlm@363 311
rlm@363 312
rlm@364 313 (import java.io.File)
rlm@363 314
rlm@364 315 (defn images [path]
rlm@364 316 (sort (rest (file-seq (File. path)))))
rlm@364 317
rlm@364 318
rlm@364 319
rlm@364 320 (defn pics [file]
rlm@364 321 (images (str base file)))
rlm@364 322
rlm@364 323 (defn generate-gabor-images [kernel name]
rlm@364 324 (draw-kernel! kernel (str base name ".png"))
rlm@364 325
rlm@364 326 (.mkdir (File. (str base name)))
rlm@364 327
rlm@364 328 (let [main (map #(.getCanonicalPath %) (pics "main"))
rlm@364 329 targets (map #(str base name "/" (format "%07d.png" %))
rlm@364 330 (range 0 (count main)))]
rlm@364 331 (dorun (pmap (partial apply-gabor kernel) main targets))))
rlm@364 332
rlm@364 333
rlm@364 334 (def banks
rlm@364 335 [[(gabor-kernel 2.8 3.5) "bank-1-1"]
rlm@364 336 [(gabor-kernel 2.8 3.5 (/ Math/PI 2)) "bank-1-1-rot"]
rlm@364 337
rlm@364 338 ;; [(gabor-kernel 3.6 4.6) "bank-1-2"]
rlm@364 339 ;; [(gabor-kernel 4.5 5.6) "bank-2-1"]
rlm@364 340 ;; [(gabor-kernel 6.3 7.9) "bank-3-1"]
rlm@364 341 ;; [(gabor-kernel 7.3 9.1) "bank-3-2"]
rlm@364 342
rlm@364 343 [(gabor-kernel 12.3 15.4) "bank-6-1"]
rlm@364 344
rlm@364 345
rlm@364 346 ;; [(gabor-kernel 17 21.2) "bank-8-1"]
rlm@364 347 ;; [(gabor-kernel 18.2 22.8) "bank-8-2"]
rlm@364 348 ])
rlm@364 349
rlm@364 350
rlm@364 351 (defn make-all-images []
rlm@364 352 (dorun (map (partial apply generate-gabor-images) banks)))
rlm@364 353
rlm@364 354
rlm@364 355
rlm@364 356 (defn compile-left-right []
rlm@364 357 (.mkdir (File. (str base "left-right")))
rlm@364 358 (let [main (pics "main")
rlm@364 359 left (pics "bank-1-1")
rlm@364 360 right (pics "bank-1-1-rot")
rlm@364 361 left-kernel (repeat 20000 (File. (str base "bank-1-1.png")))
rlm@364 362 right-kernel (repeat 20000 (File. (str base "bank-1-1-rot.png")))
rlm@364 363 targets (map
rlm@364 364 #(File. (str base "left-right/" (format "%07d.png" %)))
rlm@364 365 (range 0 (count main)))]
rlm@364 366
rlm@364 367 (dorun
rlm@364 368 (pmap
rlm@364 369 (comp
rlm@364 370 (fn [[main left right left-kernel right-kernel target]]
rlm@364 371 (println target)
rlm@364 372 (if (not (.exists (File. target)))
rlm@364 373 (sh "convert"
rlm@364 374 "-size" "1940x515" "xc:white"
rlm@364 375 main "-geometry" "+0+0" "-composite"
rlm@364 376 left "-geometry" "+650+0" "-composite"
rlm@364 377 right "-geometry" "+1300+0" "-composite"
rlm@364 378 left-kernel "-geometry" "+960+485" "-composite"
rlm@364 379 right-kernel "-geometry" "+1610+485" "-composite"
rlm@364 380 target)))
rlm@364 381 (fn [& args] (map #(.getCanonicalPath %) args)))
rlm@364 382 main left right left-kernel right-kernel targets))))
rlm@364 383
rlm@364 384
rlm@364 385 (defn compile-big-small []
rlm@364 386 (.mkdir (File. (str base "big-small")))
rlm@364 387 (let [main (pics "main")
rlm@364 388 left (pics "bank-1-1")
rlm@364 389 right (pics "bank-6-1")
rlm@364 390 small-kernel (repeat 20000 (File. (str base "bank-1-1.png")))
rlm@364 391 big-kernel (repeat 20000 (File. (str base "bank-6-1.png")))
rlm@364 392 targets (map
rlm@364 393 #(File. (str base "big-small/" (format "%07d.png" %)))
rlm@364 394 (range 0 (count main)))]
rlm@364 395
rlm@364 396 (dorun
rlm@364 397 (pmap
rlm@364 398 (comp
rlm@364 399 (fn [[main left right small-kernel big-kernel target]]
rlm@364 400 (println target)
rlm@364 401 (if (not (.exists (File. target)))
rlm@364 402 (sh "convert"
rlm@364 403 "-size" "1940x610" "xc:white"
rlm@364 404 main "-geometry" "+0+0" "-composite"
rlm@364 405 left "-geometry" "+650+0" "-composite"
rlm@364 406 right "-geometry" "+1300+0" "-composite"
rlm@364 407 small-kernel "-geometry" "+960+485" "-composite"
rlm@364 408 big-kernel "-geometry" "+1560+485" "-composite"
rlm@364 409 target)))
rlm@364 410 (fn [& args] (map #(.getCanonicalPath %) args)))
rlm@364 411 main left right small-kernel big-kernel targets))))
rlm@364 412
rlm@364 413
rlm@364 414 (defn regen-everything []
rlm@364 415 (make-all-images)
rlm@364 416 (compile-left-right)
rlm@364 417 (compile-big-small))
rlm@364 418
rlm@364 419
rlm@364 420 #+end_src
rlm@364 421
rlm@364 422 #+name: make-left-right
rlm@364 423 #+begin_src sh
rlm@364 424 #!/bin/sh
rlm@364 425
rlm@364 426 ffmpeg -framerate 60 -i ./left-right/%07d.png -b:v 9000k\
rlm@365 427 -c:v mpeg4 -r 60 gabor-rotation.mp4
rlm@364 428
rlm@364 429 #+end_src
rlm@364 430
rlm@364 431
rlm@364 432 #+name: make-big-small
rlm@364 433 #+begin_src sh
rlm@364 434 #!/bin/sh
rlm@364 435
rlm@364 436 ffmpeg -framerate 60 -i ./big-small/%07d.png -b:v 9000k\
rlm@365 437 -c:v mpeg4 -r 60 gabor-big-small.mp4
rlm@363 438
rlm@356 439 #+end_src
rlm@356 440
rlm@365 441 #+begin_html
rlm@365 442 <div class="figure">
rlm@365 443 <video controls="controls" width="755">
rlm@365 444 <source src="../video/gabor-rotation.mp4" type="video/mp4"
rlm@365 445 preload="none" poster="../images/aurellem-1280x480.png" />
rlm@365 446 </video>
rlm@365 447 <br> <a href="http://youtu.be/r5Bn2aG7MO0"> YouTube </a>
rlm@365 448 <p>Two gabor filters with different values of theta are compared. The
rlm@365 449 horizontally aligned one does better in this example.</p>
rlm@365 450 </div>
rlm@365 451 #+end_html
rlm@365 452
rlm@365 453
rlm@365 454
rlm@365 455 #+begin_html
rlm@365 456 <div class="figure">
rlm@365 457 <video controls="controls" width="755">
rlm@365 458 <source src="../video/gabor-big-small.mp4" type="video/mp4"
rlm@365 459 preload="none" poster="../images/aurellem-1280x480.png" />
rlm@365 460 </video>
rlm@365 461 <br> <a href="http://youtu.be/r5Bn2aG7MO0"> YouTube </a>
rlm@365 462 <p>Here we compare a gabor filter from bank 1 with one from bank 6.</p>
rlm@365 463 </div>
rlm@365 464 #+end_html
rlm@365 465
rlm@365 466
rlm@356 467
rlm@356 468
rlm@356 469 * COMMENT Generate Source
rlm@356 470 #+begin_src clojure :tangle ../src/cortex/gabor.clj
rlm@356 471 <<gabor>>
rlm@364 472 <<gabor-tail>>
rlm@356 473 #+end_src
rlm@364 474
rlm@364 475 #+begin_src clojure :tangle ../render/gabor-1/make-rotation.sh
rlm@364 476 <<make-left-right>>
rlm@364 477 #+end_src
rlm@364 478
rlm@364 479 #+begin_src clojure :tangle ../render/gabor-1/make-big-small.sh
rlm@364 480 <<make-big-small>>
rlm@364 481 #+end_src