Mercurial > lasercutter

comparison src/clojure/contrib/http/agent.clj @ 10:ef7dbbd6452c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
added clojure source goodness
author Robert McIntyre <rlm@mit.edu>
date Sat, 21 Aug 2010 06:25:44 -0400
parents
children
comparison
equal deleted inserted replaced
9:35cf337adfcf 10:ef7dbbd6452c
1 ;;; http/agent.clj: agent-based asynchronous HTTP client
2
3 ;; by Stuart Sierra, http://stuartsierra.com/
4 ;; August 17, 2009
5
6 ;; Copyright (c) Stuart Sierra, 2009. All rights reserved. The use
7 ;; and distribution terms for this software are covered by the Eclipse
8 ;; Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
9 ;; which can be found in the file epl-v10.html at the root of this
10 ;; distribution. By using this software in any fashion, you are
11 ;; agreeing to be bound by the terms of this license. You must not
12 ;; remove this notice, or any other, from this software.
13
14 ;; DEPRECATED IN 1.2. Use direct Java bits, or take a look at
15 ;; http://github.com/technomancy/clojure-http-client
16
17 (ns ^{:deprecated "1.2"
18 :doc "Agent-based asynchronous HTTP client.
19
20 This is a HTTP client library based on Java's HttpURLConnection
21 class and Clojure's Agent system. It allows you to make multiple
22 HTTP requests in parallel.
23
24 Start an HTTP request with the 'http-agent' function, which
25 immediately returns a Clojure Agent. You will never deref this
26 agent; that is handled by the accessor functions. The agent will
27 execute the HTTP request on a separate thread.
28
29 If you pass a :handler function to http-agent, that function will be
30 called as soon as the HTTP response body is ready. The handler
31 function is called with one argument, the HTTP agent itself. The
32 handler can read the response body by calling the 'stream' function
33 on the agent.
34
35 The value returned by the handler function becomes part of the state
36 of the agent, and you can retrieve it with the 'result' function.
37 If you call 'result' before the HTTP request has finished, it will
38 block until the handler function returns.
39
40 If you don't provide a handler function, the default handler will
41 buffer the entire response body in memory, which you can retrieve
42 with the 'bytes', 'string', or 'stream' functions. Like 'result',
43 these functions will block until the HTTP request is completed.
44
45 If you want to check if an HTTP request is finished without
46 blocking, use the 'done?' function.
47
48 A single GET request could be as simple as:
49
50 (string (http-agent \"http://www.stuartsierra.com/\"))
51
52 A simple POST might look like:
53
54 (http-agent \"http...\" :method \"POST\" :body \"foo=1\")
55
56 And you could write the response directly to a file like this:
57
58 (require '[clojure.contrib.io :as d])
59
60 (http-agent \"http...\"
61 :handler (fn [agnt]
62 (with-open [w (d/writer \"/tmp/out\")]
63 (d/copy (stream agnt) w))))
64 "
65 :author "Stuart Sierra"
66 }
67
68 clojure.contrib.http.agent
69 (:refer-clojure :exclude [bytes])
70 (:require [clojure.contrib.http.connection :as c]
71 [clojure.contrib.io :as duck])
72 (:import (java.io InputStream ByteArrayOutputStream
73 ByteArrayInputStream)
74 (java.net HttpURLConnection)))
75
76
77 ;;; PRIVATE
78
79 (declare result stream)
80
81 (defn- setup-http-connection
82 "Sets the instance method, redirect behavior, and request headers of
83 the HttpURLConnection."
84 [^HttpURLConnection conn options]
85 (when-let [t (:connect-timeout options)]
86 (.setConnectTimeout conn t))
87 (when-let [t (:read-timeout options)]
88 (.setReadTimeout conn t))
89 (.setRequestMethod conn (:method options))
90 (.setInstanceFollowRedirects conn (:follow-redirects options))
91 (doseq [[name value] (:headers options)]
92 (.setRequestProperty conn name value)))
93
94 (defn- start-request
95 "Agent action that starts sending the HTTP request."
96 [state options]
97 (let [conn (::connection state)]
98 (setup-http-connection conn options)
99 (c/start-http-connection conn (:body options))
100 (assoc state ::state ::started)))
101
102 (defn- connection-success? [^HttpURLConnection conn]
103 "Returns true if the HttpURLConnection response code is in the 2xx
104 range."
105 (= 2 (quot (.getResponseCode conn) 100)))
106
107 (defn- open-response
108 "Agent action that opens the response body stream on the HTTP
109 request; this will block until the response stream is available." ;
110 [state options]
111 (let [^HttpURLConnection conn (::connection state)]
112 (assoc state
113 ::response-stream (if (connection-success? conn)
114 (.getInputStream conn)
115 (.getErrorStream conn))
116 ::state ::receiving)))
117
118 (defn- handle-response
119 "Agent action that calls the provided handler function, with no
120 arguments, and sets the ::result key of the agent to the handler's
121 return value."
122 [state handler options]
123 (let [conn (::connection state)]
124 (assoc state
125 ::result (handler)
126 ::state ::finished)))
127
128 (defn- disconnect
129 "Agent action that closes the response body stream and disconnects
130 the HttpURLConnection."
131 [state options]
132 (when (::response-stream state)
133 (.close ^InputStream (::response-stream state)))
134 (.disconnect ^HttpURLConnection (::connection state))
135 (assoc state
136 ::response-stream nil
137 ::state ::disconnected))
138
139 (defn- status-in-range?
140 "Returns true if the response status of the HTTP agent begins with
141 digit, an Integer."
142 [digit http-agnt]
143 (= digit (quot (.getResponseCode
144 ^HttpURLConnection (::connection @http-agnt))
145 100)))
146
147 (defn- ^ByteArrayOutputStream get-byte-buffer [http-agnt]
148 (let [buffer (result http-agnt)]
149 (if (instance? ByteArrayOutputStream buffer)
150 buffer
151 (throw (Exception. "Handler result was not a ByteArrayOutputStream")))))
152
153
154 (defn buffer-bytes
155 "The default HTTP agent result handler; it collects the response
156 body in a java.io.ByteArrayOutputStream, which can later be
157 retrieved with the 'stream', 'string', and 'bytes' functions."
158 [http-agnt]
159 (let [output (ByteArrayOutputStream.)]
160 (duck/copy (or (stream http-agnt) "") output)
161 output))
162
163
164 ;;; CONSTRUCTOR
165
166 (def *http-agent-defaults*
167 {:method "GET"
168 :headers {}
169 :body nil
170 :connect-timeout 0
171 :read-timeout 0
172 :follow-redirects true
173 :handler buffer-bytes})
174
175 (defn http-agent
176 "Creates (and immediately returns) an Agent representing an HTTP
177 request running in a new thread.
178
179 options are key/value pairs:
180
181 :method string
182
183 The HTTP method name. Default is \"GET\".
184
185 :headers h
186
187 HTTP headers, as a Map or a sequence of pairs like
188 ([key1,value1], [key2,value2]) Default is nil.
189
190 :body b
191
192 HTTP request entity body, one of nil, String, byte[], InputStream,
193 Reader, or File. Default is nil.
194
195 :connect-timeout int
196
197 Timeout value, in milliseconds, when opening a connection to the
198 URL. Default is zero, meaning no timeout.
199
200 :read-timeout int
201
202 Timeout value, in milliseconds, when reading data from the
203 connection. Default is zero, meaning no timeout.
204
205 :follow-redirects boolean
206
207 If true, HTTP 3xx redirects will be followed automatically. Default
208 is true.
209
210 :handler f
211
212 Function to be called when the HTTP response body is ready. If you
213 do not provide a handler function, the default is to buffer the
214 entire response body in memory.
215
216 The handler function will be called with the HTTP agent as its
217 argument, and can use the 'stream' function to read the response
218 body. The return value of this function will be stored in the state
219 of the agent and can be retrieved with the 'result' function. Any
220 exceptions thrown by this function will be added to the agent's
221 error queue (see agent-errors). The default function collects the
222 response stream in a memory buffer.
223 "
224 ([uri & options]
225 (let [opts (merge *http-agent-defaults* (apply array-map options))]
226 (let [a (agent {::connection (c/http-connection uri)
227 ::state ::created
228 ::uri uri
229 ::options opts})]
230 (send-off a start-request opts)
231 (send-off a open-response opts)
232 (send-off a handle-response (partial (:handler opts) a) opts)
233 (send-off a disconnect opts)))))
234
235
236 ;;; RESPONSE BODY ACCESSORS
237
238 (defn result
239 "Returns the value returned by the :handler function of the HTTP
240 agent; blocks until the HTTP request is completed. The default
241 handler function returns a ByteArrayOutputStream."
242 [http-agnt]
243 (await http-agnt)
244 (::result @http-agnt))
245
246 (defn stream
247 "Returns an InputStream of the HTTP response body. When called by
248 the handler function passed to http-agent, this is the raw
249 HttpURLConnection stream.
250
251 If the default handler function was used, this function returns a
252 ByteArrayInputStream on the buffered response body."
253 [http-agnt]
254 (let [a @http-agnt]
255 (if (= (::state a) ::receiving)
256 (::response-stream a)
257 (ByteArrayInputStream.
258 (.toByteArray (get-byte-buffer http-agnt))))))
259
260 (defn bytes
261 "Returns a Java byte array of the content returned by the server;
262 nil if the content is not yet available."
263 [http-agnt]
264 (.toByteArray (get-byte-buffer http-agnt)))
265
266 (defn string
267 "Returns the HTTP response body as a string, using the given
268 encoding.
269
270 If no encoding is given, uses the encoding specified in the server
271 headers, or clojure.contrib.io/*default-encoding* if it is
272 not specified."
273 ([http-agnt]
274 (await http-agnt) ;; have to wait for Content-Encoding
275 (string http-agnt (or (.getContentEncoding
276 ^HttpURLConnection (::connection @http-agnt))
277 duck/*default-encoding*)))
278 ([http-agnt ^String encoding]
279 (.toString (get-byte-buffer http-agnt) encoding)))
280
281
282 ;;; REQUEST ACCESSORS
283
284 (defn request-uri
285 "Returns the URI/URL requested by this HTTP agent, as a String."
286 [http-agnt]
287 (::uri @http-agnt))
288
289 (defn request-headers
290 "Returns the request headers specified for this HTTP agent."
291 [http-agnt]
292 (:headers (::options @http-agnt)))
293
294 (defn method
295 "Returns the HTTP method name used by this HTTP agent, as a String."
296 [http-agnt]
297 (:method (::options @http-agnt)))
298
299 (defn request-body
300 "Returns the HTTP request body given to this HTTP agent.
301
302 Note: if the request body was an InputStream or a Reader, it will no
303 longer be usable."
304 [http-agnt]
305 (:body (::options @http-agnt)))
306
307
308 ;;; RESPONSE ACCESSORS
309
310 (defn done?
311 "Returns true if the HTTP request/response has completed."
312 [http-agnt]
313 (if (#{::finished ::disconnected} (::state @http-agnt))
314 true false))
315
316 (defn status
317 "Returns the HTTP response status code (e.g. 200, 404) for this
318 request, as an Integer, or nil if the status has not yet been
319 received."
320 [http-agnt]
321 (when (done? http-agnt)
322 (.getResponseCode ^HttpURLConnection (::connection @http-agnt))))
323
324 (defn message
325 "Returns the HTTP response message (e.g. 'Not Found'), for this
326 request, or nil if the response has not yet been received."
327 [http-agnt]
328 (when (done? http-agnt)
329 (.getResponseMessage ^HttpURLConnection (::connection @http-agnt))))
330
331 (defn headers
332 "Returns a map of HTTP response headers. Header names are converted
333 to keywords in all lower-case Header values are strings. If a
334 header appears more than once, only the last value is returned."
335 [http-agnt]
336 (reduce (fn [m [^String k v]]
337 (assoc m (when k (keyword (.toLowerCase k))) (last v)))
338 {} (.getHeaderFields
339 ^HttpURLConnection (::connection @http-agnt))))
340
341 (defn headers-seq
342 "Returns the HTTP response headers in order as a sequence of
343 [String,String] pairs. The first 'header' name may be null for the
344 HTTP status line."
345 [http-agnt]
346 (let [^HttpURLConnection conn (::connection @http-agnt)
347 f (fn thisfn [^Integer i]
348 ;; Get value first because first key may be nil.
349 (when-let [value (.getHeaderField conn i)]
350 (cons [(.getHeaderFieldKey conn i) value]
351 (thisfn (inc i)))))]
352 (lazy-seq (f 0))))
353
354
355 ;;; RESPONSE STATUS CODE ACCESSORS
356
357 (defn success?
358 "Returns true if the HTTP response code was in the 200-299 range."
359 [http-agnt]
360 (status-in-range? 2 http-agnt))
361
362 (defn redirect?
363 "Returns true if the HTTP response code was in the 300-399 range.
364
365 Note: if the :follow-redirects option was true (the default),
366 redirects will be followed automatically and a the agent will never
367 return a 3xx response code."
368 [http-agnt]
369 (status-in-range? 3 http-agnt))
370
371 (defn client-error?
372 "Returns true if the HTTP response code was in the 400-499 range."
373 [http-agnt]
374 (status-in-range? 4 http-agnt))
375
376 (defn server-error?
377 "Returns true if the HTTP response code was in the 500-599 range."
378 [http-agnt]
379 (status-in-range? 5 http-agnt))
380
381 (defn error?
382 "Returns true if the HTTP response code was in the 400-499 range OR
383 the 500-599 range."
384 [http-agnt]
385 (or (client-error? http-agnt)
386 (server-error? http-agnt)))