1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/src/clojure/contrib/mmap.clj	Sat Aug 21 06:25:44 2010 -0400
     1.3 @@ -0,0 +1,90 @@
     1.4 +;   Copyright (c) Chris Houser, April 2008. All rights reserved.
     1.5 +;   The use and distribution terms for this software are covered by the
     1.6 +;   Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
     1.7 +;   which can be found in the file epl-v10.html at the root of this distribution.
     1.8 +;   By using this software in any fashion, you are agreeing to be bound by
     1.9 +;   the terms of this license.
    1.10 +;   You must not remove this notice, or any other, from this software.
    1.11 +
    1.12 +; Functions for memory-mapping files, plus some functions that use a
    1.13 +; mmaped file for "normal" activies -- slurp, load-file, etc.
    1.14 +
    1.15 +(ns 
    1.16 +  ^{:author "Chris Houser",
    1.17 +     :doc "Functions for memory-mapping files, plus some functions that use a
    1.18 +mmaped file for \"normal\" activies -- slurp, load-file, etc."}
    1.19 +  clojure.contrib.mmap
    1.20 +    (:refer-clojure :exclude (slurp load-file))
    1.21 +    (:import (java.nio ByteBuffer CharBuffer)
    1.22 +             (java.io PushbackReader InputStream InputStreamReader
    1.23 +                      FileInputStream)))
    1.24 +
    1.25 +;(set! *warn-on-reflection* true)
    1.26 +
    1.27 +(def READ_ONLY ^{:private true}
    1.28 +  (java.nio.channels.FileChannel$MapMode/READ_ONLY))
    1.29 +
    1.30 +(defn mmap
    1.31 +  "Memory-map the file named f.  Returns a ByteBuffer."
    1.32 +  [f]
    1.33 +  (let [channel (.getChannel (FileInputStream. f))]
    1.34 +    (.map channel READ_ONLY 0 (.size channel))))
    1.35 +
    1.36 +(defn slurp
    1.37 +  "Reads the file named by f and returns it as a string."
    1.38 +  [^String f]
    1.39 +  (.. java.nio.charset.Charset (forName "UTF-8")
    1.40 +      (newDecoder) (decode (mmap f))))
    1.41 +
    1.42 +(defn buffer-stream
    1.43 +  "Returns an InputStream for a ByteBuffer, such as returned by mmap."
    1.44 +  [^ByteBuffer buf]
    1.45 +  (proxy [InputStream] []
    1.46 +    (available [] (.remaining buf))
    1.47 +    (read
    1.48 +      ([] (if (.hasRemaining buf) (.get buf) -1))
    1.49 +      ([dst offset len] (let [actlen (min (.remaining buf) len)]
    1.50 +                          (.get buf dst offset actlen)
    1.51 +                          (if (< actlen 1) -1 actlen))))))
    1.52 +
    1.53 +(defn load-file [f]
    1.54 +  "Like clojure.lang/load-file, but uses mmap internally."
    1.55 +  (with-open [rdr (-> f mmap buffer-stream InputStreamReader. PushbackReader.)]
    1.56 +    (load-reader rdr)))
    1.57 +
    1.58 +
    1.59 +(comment
    1.60 +
    1.61 +(alias 'mmap 'clojure.contrib.mmap)
    1.62 +(alias 'core 'clojure.core)
    1.63 +
    1.64 +;---
    1.65 +; zip_filter.clj is 95KB
    1.66 +(def tf "/home/chouser/build/clojure/src/clj/clojure/core.clj")
    1.67 +(println "\nload-file" tf)
    1.68 +(time (dotimes [_ 5] (core/load-file tf))) ; 5420.177813 msecs
    1.69 +(time (dotimes [_ 5] (mmap/load-file tf))) ; 7946.854434 msecs -- not so good
    1.70 +
    1.71 +;---
    1.72 +; kern.log.0 is 961KB
    1.73 +(def tf "/var/log/kern.log.0")
    1.74 +(println "\nslurp" tf)
    1.75 +(time (dotimes [_ 10] (.length (core/slurp tf)))) ; 435.767226 msecs
    1.76 +(time (dotimes [_ 10] (.length (mmap/slurp tf)))) ;  93.176858 msecs
    1.77 +
    1.78 +;---
    1.79 +; kern.log.0 is 961KB
    1.80 +(def tf "/var/log/kern.log.0")
    1.81 +(println "\nregex slurp large" tf)
    1.82 +(time (dotimes [_ 10] (count (re-seq #"EXT3.*" (core/slurp tf))))) ; 416
    1.83 +(time (dotimes [_ 10] (count (re-seq #"EXT3.*" (mmap/slurp tf))))) ; 101
    1.84 +
    1.85 +;---
    1.86 +; mmap.clj is about 3.1KB
    1.87 +(def tf "/home/chouser/proj/clojure-contrib/src/clojure/contrib/mmap.clj")
    1.88 +(println "\nregex slurp small" tf)
    1.89 +
    1.90 +(time (dotimes [_ 1000] (count (re-seq #"defn \S*" (core/slurp tf))))) ; 308
    1.91 +(time (dotimes [_ 1000] (count (re-seq #"defn \S*" (mmap/slurp tf))))) ; 198
    1.92 +
    1.93 +)