Mercurial > lasercutter

     1 ;;; http/agent.clj: agent-based asynchronous HTTP client

     2 

     3 ;; by Stuart Sierra, http://stuartsierra.com/

     4 ;; August 17, 2009

     5 

     6 ;; Copyright (c) Stuart Sierra, 2009. All rights reserved.  The use

     7 ;; and distribution terms for this software are covered by the Eclipse

     8 ;; Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)

     9 ;; which can be found in the file epl-v10.html at the root of this

    10 ;; distribution.  By using this software in any fashion, you are

    11 ;; agreeing to be bound by the terms of this license.  You must not

    12 ;; remove this notice, or any other, from this software.

    13 

    14 ;; DEPRECATED IN 1.2. Use direct Java bits, or take a look at

    15 ;;                    http://github.com/technomancy/clojure-http-client

    16 

    17 (ns ^{:deprecated "1.2"

    18       :doc "Agent-based asynchronous HTTP client.

    19 

    20   This is a HTTP client library based on Java's HttpURLConnection

    21   class and Clojure's Agent system.  It allows you to make multiple

    22   HTTP requests in parallel.

    23 

    24   Start an HTTP request with the 'http-agent' function, which

    25   immediately returns a Clojure Agent.  You will never deref this

    26   agent; that is handled by the accessor functions.  The agent will

    27   execute the HTTP request on a separate thread.

    28 

    29   If you pass a :handler function to http-agent, that function will be

    30   called as soon as the HTTP response body is ready.  The handler

    31   function is called with one argument, the HTTP agent itself.  The

    32   handler can read the response body by calling the 'stream' function

    33   on the agent.

    34 

    35   The value returned by the handler function becomes part of the state

    36   of the agent, and you can retrieve it with the 'result' function.

    37   If you call 'result' before the HTTP request has finished, it will

    38   block until the handler function returns.

    39 

    40   If you don't provide a handler function, the default handler will

    41   buffer the entire response body in memory, which you can retrieve

    42   with the 'bytes', 'string', or 'stream' functions.  Like 'result',

    43   these functions will block until the HTTP request is completed.

    44 

    45   If you want to check if an HTTP request is finished without

    46   blocking, use the 'done?' function.

    47 

    48   A single GET request could be as simple as:

    49 

    50     (string (http-agent \"http://www.stuartsierra.com/\"))

    51 

    52   A simple POST might look like:

    53 

    54     (http-agent \"http...\" :method \"POST\" :body \"foo=1\")

    55 

    56   And you could write the response directly to a file like this:

    57 

    58     (require '[clojure.contrib.io :as d])

    59 

    60     (http-agent \"http...\"

    61                 :handler (fn [agnt] 

    62                            (with-open [w (d/writer \"/tmp/out\")] 

    63                              (d/copy (stream agnt) w))))

    64 "

    65        :author "Stuart Sierra"

    66        }

    67 

    68   clojure.contrib.http.agent

    69   (:refer-clojure :exclude [bytes])

    70   (:require [clojure.contrib.http.connection :as c]

    71             [clojure.contrib.io :as duck])

    72   (:import (java.io InputStream ByteArrayOutputStream

    73                     ByteArrayInputStream)

    74            (java.net HttpURLConnection)))

    75 

    76 

    77 ;;; PRIVATE

    78 

    79 (declare result stream)

    80 

    81 (defn- setup-http-connection

    82   "Sets the instance method, redirect behavior, and request headers of

    83   the HttpURLConnection."

    84   [^HttpURLConnection conn options]

    85   (when-let [t (:connect-timeout options)]

    86     (.setConnectTimeout conn t))

    87   (when-let [t (:read-timeout options)]

    88     (.setReadTimeout conn t))

    89   (.setRequestMethod conn (:method options))

    90   (.setInstanceFollowRedirects conn (:follow-redirects options))

    91   (doseq [[name value] (:headers options)]

    92     (.setRequestProperty conn name value)))

    93 

    94 (defn- start-request

    95   "Agent action that starts sending the HTTP request."

    96   [state options]

    97   (let [conn (::connection state)]

    98     (setup-http-connection conn options)

    99     (c/start-http-connection conn (:body options))

   100     (assoc state ::state ::started)))

   101 

   102 (defn- connection-success? [^HttpURLConnection conn]

   103   "Returns true if the HttpURLConnection response code is in the 2xx

   104   range."

   105   (= 2 (quot (.getResponseCode conn) 100)))

   106 

   107 (defn- open-response

   108   "Agent action that opens the response body stream on the HTTP

   109   request; this will block until the response stream is available." ;

   110   [state options]

   111   (let [^HttpURLConnection conn (::connection state)]

   112     (assoc state

   113       ::response-stream (if (connection-success? conn)

   114                           (.getInputStream conn)

   115                           (.getErrorStream conn))

   116       ::state ::receiving)))

   117 

   118 (defn- handle-response

   119   "Agent action that calls the provided handler function, with no

   120   arguments, and sets the ::result key of the agent to the handler's

   121   return value."

   122   [state handler options]

   123   (let [conn (::connection state)]

   124     (assoc state

   125       ::result (handler)

   126       ::state ::finished)))

   127 

   128 (defn- disconnect

   129   "Agent action that closes the response body stream and disconnects

   130   the HttpURLConnection."

   131   [state options]

   132   (when (::response-stream state)

   133     (.close ^InputStream (::response-stream state)))

   134   (.disconnect ^HttpURLConnection (::connection state))

   135   (assoc state

   136     ::response-stream nil

   137     ::state ::disconnected))

   138 

   139 (defn- status-in-range?

   140   "Returns true if the response status of the HTTP agent begins with

   141   digit, an Integer."

   142   [digit http-agnt]

   143   (= digit (quot (.getResponseCode

   144                               ^HttpURLConnection (::connection @http-agnt))

   145                              100)))

   146 

   147 (defn- ^ByteArrayOutputStream get-byte-buffer [http-agnt]

   148   (let [buffer (result http-agnt)]

   149     (if (instance? ByteArrayOutputStream buffer)

   150       buffer

   151       (throw (Exception. "Handler result was not a ByteArrayOutputStream")))))

   152 

   153 

   154 (defn buffer-bytes

   155   "The default HTTP agent result handler; it collects the response

   156   body in a java.io.ByteArrayOutputStream, which can later be

   157   retrieved with the 'stream', 'string', and 'bytes' functions."

   158   [http-agnt]

   159   (let [output (ByteArrayOutputStream.)]

   160     (duck/copy (or (stream http-agnt) "") output)

   161     output))

   162 

   163 

   164 ;;; CONSTRUCTOR

   165 

   166 (def *http-agent-defaults*

   167      {:method "GET"

   168       :headers {}

   169       :body nil

   170       :connect-timeout 0

   171       :read-timeout 0

   172       :follow-redirects true

   173       :handler buffer-bytes})

   174 

   175 (defn http-agent

   176   "Creates (and immediately returns) an Agent representing an HTTP

   177   request running in a new thread.

   178 

   179   options are key/value pairs:

   180 

   181   :method string

   182 

   183   The HTTP method name.  Default is \"GET\".

   184 

   185   :headers h

   186 

   187   HTTP headers, as a Map or a sequence of pairs like 

   188   ([key1,value1], [key2,value2])  Default is nil.

   189 

   190   :body b

   191   

   192   HTTP request entity body, one of nil, String, byte[], InputStream,

   193   Reader, or File.  Default is nil.

   194 

   195   :connect-timeout int

   196 

   197   Timeout value, in milliseconds, when opening a connection to the

   198   URL.  Default is zero, meaning no timeout.

   199 

   200   :read-timeout int

   201 

   202   Timeout value, in milliseconds, when reading data from the

   203   connection.  Default is zero, meaning no timeout.

   204 

   205   :follow-redirects boolean

   206 

   207   If true, HTTP 3xx redirects will be followed automatically.  Default

   208   is true.

   209 

   210   :handler f

   211 

   212   Function to be called when the HTTP response body is ready.  If you

   213   do not provide a handler function, the default is to buffer the

   214   entire response body in memory.

   215 

   216   The handler function will be called with the HTTP agent as its

   217   argument, and can use the 'stream' function to read the response

   218   body.  The return value of this function will be stored in the state

   219   of the agent and can be retrieved with the 'result' function.  Any

   220   exceptions thrown by this function will be added to the agent's

   221   error queue (see agent-errors).  The default function collects the

   222   response stream in a memory buffer.

   223   "

   224   ([uri & options]

   225      (let [opts (merge *http-agent-defaults* (apply array-map options))]

   226        (let [a (agent {::connection (c/http-connection uri)

   227                        ::state ::created

   228                        ::uri uri

   229                        ::options opts})]

   230          (send-off a start-request opts)

   231          (send-off a open-response opts)

   232          (send-off a handle-response (partial (:handler opts) a) opts)

   233          (send-off a disconnect opts)))))

   234 

   235 

   236 ;;; RESPONSE BODY ACCESSORS

   237 

   238 (defn result

   239   "Returns the value returned by the :handler function of the HTTP

   240   agent; blocks until the HTTP request is completed.  The default

   241   handler function returns a ByteArrayOutputStream."

   242   [http-agnt]

   243   (await http-agnt)

   244   (::result @http-agnt))

   245 

   246 (defn stream

   247   "Returns an InputStream of the HTTP response body.  When called by

   248   the handler function passed to http-agent, this is the raw

   249   HttpURLConnection stream.

   250 

   251   If the default handler function was used, this function returns a

   252   ByteArrayInputStream on the buffered response body."

   253   [http-agnt]

   254   (let [a @http-agnt]

   255     (if (= (::state a) ::receiving)

   256       (::response-stream a)

   257       (ByteArrayInputStream.

   258        (.toByteArray (get-byte-buffer http-agnt))))))

   259 

   260 (defn bytes

   261   "Returns a Java byte array of the content returned by the server;

   262   nil if the content is not yet available."

   263   [http-agnt]

   264   (.toByteArray (get-byte-buffer http-agnt)))

   265 

   266 (defn string

   267   "Returns the HTTP response body as a string, using the given

   268   encoding.

   269 

   270   If no encoding is given, uses the encoding specified in the server

   271   headers, or clojure.contrib.io/*default-encoding* if it is

   272   not specified."

   273   ([http-agnt]

   274      (await http-agnt) ;; have to wait for Content-Encoding

   275      (string http-agnt (or (.getContentEncoding

   276                             ^HttpURLConnection (::connection @http-agnt))

   277                            duck/*default-encoding*)))

   278   ([http-agnt ^String encoding]

   279      (.toString (get-byte-buffer http-agnt) encoding)))

   280 

   281 

   282 ;;; REQUEST ACCESSORS

   283 

   284 (defn request-uri

   285   "Returns the URI/URL requested by this HTTP agent, as a String."

   286   [http-agnt]

   287   (::uri @http-agnt))

   288 

   289 (defn request-headers

   290   "Returns the request headers specified for this HTTP agent."

   291   [http-agnt]

   292   (:headers (::options @http-agnt)))

   293 

   294 (defn method

   295   "Returns the HTTP method name used by this HTTP agent, as a String."

   296   [http-agnt]

   297   (:method (::options @http-agnt)))

   298 

   299 (defn request-body

   300   "Returns the HTTP request body given to this HTTP agent.  

   301 

   302   Note: if the request body was an InputStream or a Reader, it will no

   303   longer be usable."

   304   [http-agnt]

   305   (:body (::options @http-agnt)))

   306 

   307 

   308 ;;; RESPONSE ACCESSORS

   309 

   310 (defn done?

   311   "Returns true if the HTTP request/response has completed."

   312   [http-agnt]

   313   (if (#{::finished ::disconnected} (::state @http-agnt))

   314     true false))

   315 

   316 (defn status

   317   "Returns the HTTP response status code (e.g. 200, 404) for this

   318   request, as an Integer, or nil if the status has not yet been

   319   received."

   320   [http-agnt]

   321   (when (done? http-agnt)

   322     (.getResponseCode ^HttpURLConnection (::connection @http-agnt))))

   323 

   324 (defn message

   325   "Returns the HTTP response message (e.g. 'Not Found'), for this

   326   request, or nil if the response has not yet been received."

   327   [http-agnt]

   328   (when (done? http-agnt)

   329     (.getResponseMessage ^HttpURLConnection (::connection @http-agnt))))

   330 

   331 (defn headers

   332   "Returns a map of HTTP response headers.  Header names are converted

   333   to keywords in all lower-case Header values are strings.  If a

   334   header appears more than once, only the last value is returned."

   335   [http-agnt]

   336   (reduce (fn [m [^String k v]]

   337             (assoc m (when k (keyword (.toLowerCase k))) (last v)))

   338           {} (.getHeaderFields

   339               ^HttpURLConnection (::connection @http-agnt))))

   340 

   341 (defn headers-seq

   342   "Returns the HTTP response headers in order as a sequence of

   343   [String,String] pairs.  The first 'header' name may be null for the

   344   HTTP status line."

   345   [http-agnt]

   346   (let [^HttpURLConnection conn (::connection @http-agnt)

   347         f (fn thisfn [^Integer i]

   348             ;; Get value first because first key may be nil.

   349             (when-let [value (.getHeaderField conn i)]

   350               (cons [(.getHeaderFieldKey conn i) value]

   351                     (thisfn (inc i)))))]

   352     (lazy-seq (f 0))))

   353 

   354 

   355 ;;; RESPONSE STATUS CODE ACCESSORS

   356 

   357 (defn success?

   358   "Returns true if the HTTP response code was in the 200-299 range."

   359   [http-agnt]

   360   (status-in-range? 2 http-agnt))

   361 

   362 (defn redirect?

   363   "Returns true if the HTTP response code was in the 300-399 range.

   364 

   365   Note: if the :follow-redirects option was true (the default),

   366   redirects will be followed automatically and a the agent will never

   367   return a 3xx response code."

   368   [http-agnt]

   369   (status-in-range? 3 http-agnt))

   370 

   371 (defn client-error?

   372   "Returns true if the HTTP response code was in the 400-499 range."

   373   [http-agnt]

   374   (status-in-range? 4 http-agnt))

   375 

   376 (defn server-error?

   377   "Returns true if the HTTP response code was in the 500-599 range."

   378   [http-agnt]

   379   (status-in-range? 5 http-agnt))

   380 

   381 (defn error?

   382   "Returns true if the HTTP response code was in the 400-499 range OR

   383   the 500-599 range."

   384   [http-agnt]

   385   (or (client-error? http-agnt)

   386       (server-error? http-agnt)))
author	Robert McIntyre <rlm@mit.edu>
date	Sat, 21 Aug 2010 06:25:44 -0400
parents
children