Mercurial > lasercutter

     1 ;   Copyright (c) Chris Houser, April 2008. All rights reserved.

     2 ;   The use and distribution terms for this software are covered by the

     3 ;   Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)

     4 ;   which can be found in the file epl-v10.html at the root of this distribution.

     5 ;   By using this software in any fashion, you are agreeing to be bound by

     6 ;   the terms of this license.

     7 ;   You must not remove this notice, or any other, from this software.

     8 

     9 ; Functions for memory-mapping files, plus some functions that use a

    10 ; mmaped file for "normal" activies -- slurp, load-file, etc.

    11 

    12 (ns 

    13   ^{:author "Chris Houser",

    14      :doc "Functions for memory-mapping files, plus some functions that use a

    15 mmaped file for \"normal\" activies -- slurp, load-file, etc."}

    16   clojure.contrib.mmap

    17     (:refer-clojure :exclude (slurp load-file))

    18     (:import (java.nio ByteBuffer CharBuffer)

    19              (java.io PushbackReader InputStream InputStreamReader

    20                       FileInputStream)))

    21 

    22 ;(set! *warn-on-reflection* true)

    23 

    24 (def READ_ONLY ^{:private true}

    25   (java.nio.channels.FileChannel$MapMode/READ_ONLY))

    26 

    27 (defn mmap

    28   "Memory-map the file named f.  Returns a ByteBuffer."

    29   [f]

    30   (let [channel (.getChannel (FileInputStream. f))]

    31     (.map channel READ_ONLY 0 (.size channel))))

    32 

    33 (defn slurp

    34   "Reads the file named by f and returns it as a string."

    35   [^String f]

    36   (.. java.nio.charset.Charset (forName "UTF-8")

    37       (newDecoder) (decode (mmap f))))

    38 

    39 (defn buffer-stream

    40   "Returns an InputStream for a ByteBuffer, such as returned by mmap."

    41   [^ByteBuffer buf]

    42   (proxy [InputStream] []

    43     (available [] (.remaining buf))

    44     (read

    45       ([] (if (.hasRemaining buf) (.get buf) -1))

    46       ([dst offset len] (let [actlen (min (.remaining buf) len)]

    47                           (.get buf dst offset actlen)

    48                           (if (< actlen 1) -1 actlen))))))

    49 

    50 (defn load-file [f]

    51   "Like clojure.lang/load-file, but uses mmap internally."

    52   (with-open [rdr (-> f mmap buffer-stream InputStreamReader. PushbackReader.)]

    53     (load-reader rdr)))

    54 

    55 

    56 (comment

    57 

    58 (alias 'mmap 'clojure.contrib.mmap)

    59 (alias 'core 'clojure.core)

    60 

    61 ;---

    62 ; zip_filter.clj is 95KB

    63 (def tf "/home/chouser/build/clojure/src/clj/clojure/core.clj")

    64 (println "\nload-file" tf)

    65 (time (dotimes [_ 5] (core/load-file tf))) ; 5420.177813 msecs

    66 (time (dotimes [_ 5] (mmap/load-file tf))) ; 7946.854434 msecs -- not so good

    67 

    68 ;---

    69 ; kern.log.0 is 961KB

    70 (def tf "/var/log/kern.log.0")

    71 (println "\nslurp" tf)

    72 (time (dotimes [_ 10] (.length (core/slurp tf)))) ; 435.767226 msecs

    73 (time (dotimes [_ 10] (.length (mmap/slurp tf)))) ;  93.176858 msecs

    74 

    75 ;---

    76 ; kern.log.0 is 961KB

    77 (def tf "/var/log/kern.log.0")

    78 (println "\nregex slurp large" tf)

    79 (time (dotimes [_ 10] (count (re-seq #"EXT3.*" (core/slurp tf))))) ; 416

    80 (time (dotimes [_ 10] (count (re-seq #"EXT3.*" (mmap/slurp tf))))) ; 101

    81 

    82 ;---

    83 ; mmap.clj is about 3.1KB

    84 (def tf "/home/chouser/proj/clojure-contrib/src/clojure/contrib/mmap.clj")

    85 (println "\nregex slurp small" tf)

    86 

    87 (time (dotimes [_ 1000] (count (re-seq #"defn \S*" (core/slurp tf))))) ; 308

    88 (time (dotimes [_ 1000] (count (re-seq #"defn \S*" (mmap/slurp tf))))) ; 198

    89 

    90 )
author	Robert McIntyre <rlm@mit.edu>
date	Sat, 21 Aug 2010 06:25:44 -0400
parents
children