Mercurial > lasercutter

comparison src/clojure/contrib/mmap.clj @ 10:ef7dbbd6452c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
added clojure source goodness
author Robert McIntyre <rlm@mit.edu>
date Sat, 21 Aug 2010 06:25:44 -0400
parents
children
comparison
equal deleted inserted replaced
9:35cf337adfcf 10:ef7dbbd6452c
1 ; Copyright (c) Chris Houser, April 2008. All rights reserved.
2 ; The use and distribution terms for this software are covered by the
3 ; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
4 ; which can be found in the file epl-v10.html at the root of this distribution.
5 ; By using this software in any fashion, you are agreeing to be bound by
6 ; the terms of this license.
7 ; You must not remove this notice, or any other, from this software.
8
9 ; Functions for memory-mapping files, plus some functions that use a
10 ; mmaped file for "normal" activies -- slurp, load-file, etc.
11
12 (ns
13 ^{:author "Chris Houser",
14 :doc "Functions for memory-mapping files, plus some functions that use a
15 mmaped file for \"normal\" activies -- slurp, load-file, etc."}
16 clojure.contrib.mmap
17 (:refer-clojure :exclude (slurp load-file))
18 (:import (java.nio ByteBuffer CharBuffer)
19 (java.io PushbackReader InputStream InputStreamReader
20 FileInputStream)))
21
22 ;(set! *warn-on-reflection* true)
23
24 (def READ_ONLY ^{:private true}
25 (java.nio.channels.FileChannel$MapMode/READ_ONLY))
26
27 (defn mmap
28 "Memory-map the file named f. Returns a ByteBuffer."
29 [f]
30 (let [channel (.getChannel (FileInputStream. f))]
31 (.map channel READ_ONLY 0 (.size channel))))
32
33 (defn slurp
34 "Reads the file named by f and returns it as a string."
35 [^String f]
36 (.. java.nio.charset.Charset (forName "UTF-8")
37 (newDecoder) (decode (mmap f))))
38
39 (defn buffer-stream
40 "Returns an InputStream for a ByteBuffer, such as returned by mmap."
41 [^ByteBuffer buf]
42 (proxy [InputStream] []
43 (available [] (.remaining buf))
44 (read
45 ([] (if (.hasRemaining buf) (.get buf) -1))
46 ([dst offset len] (let [actlen (min (.remaining buf) len)]
47 (.get buf dst offset actlen)
48 (if (< actlen 1) -1 actlen))))))
49
50 (defn load-file [f]
51 "Like clojure.lang/load-file, but uses mmap internally."
52 (with-open [rdr (-> f mmap buffer-stream InputStreamReader. PushbackReader.)]
53 (load-reader rdr)))
54
55
56 (comment
57
58 (alias 'mmap 'clojure.contrib.mmap)
59 (alias 'core 'clojure.core)
60
61 ;---
62 ; zip_filter.clj is 95KB
63 (def tf "/home/chouser/build/clojure/src/clj/clojure/core.clj")
64 (println "\nload-file" tf)
65 (time (dotimes [_ 5] (core/load-file tf))) ; 5420.177813 msecs
66 (time (dotimes [_ 5] (mmap/load-file tf))) ; 7946.854434 msecs -- not so good
67
68 ;---
69 ; kern.log.0 is 961KB
70 (def tf "/var/log/kern.log.0")
71 (println "\nslurp" tf)
72 (time (dotimes [_ 10] (.length (core/slurp tf)))) ; 435.767226 msecs
73 (time (dotimes [_ 10] (.length (mmap/slurp tf)))) ; 93.176858 msecs
74
75 ;---
76 ; kern.log.0 is 961KB
77 (def tf "/var/log/kern.log.0")
78 (println "\nregex slurp large" tf)
79 (time (dotimes [_ 10] (count (re-seq #"EXT3.*" (core/slurp tf))))) ; 416
80 (time (dotimes [_ 10] (count (re-seq #"EXT3.*" (mmap/slurp tf))))) ; 101
81
82 ;---
83 ; mmap.clj is about 3.1KB
84 (def tf "/home/chouser/proj/clojure-contrib/src/clojure/contrib/mmap.clj")
85 (println "\nregex slurp small" tf)
86
87 (time (dotimes [_ 1000] (count (re-seq #"defn \S*" (core/slurp tf))))) ; 308
88 (time (dotimes [_ 1000] (count (re-seq #"defn \S*" (mmap/slurp tf))))) ; 198
89
90 )