# HG changeset patch # User Robert McIntyre # Date 1330606057 25200 # Node ID fca75c0e8f40062b3276d94f481b0f16db12886b # Parent 12d1367cf1aa76c1f530fe06000be056ab7c1f08 added stories.clj diff -r 12d1367cf1aa -r fca75c0e8f40 src/fanfiction/stories.clj --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/fanfiction/stories.clj Thu Mar 01 05:47:37 2012 -0700 @@ -0,0 +1,196 @@ +(ns fanfiction.stories) + +(use 'clojure.java.io) + +(import org.htmlcleaner.HtmlCleaner) +(import org.htmlcleaner.TagNode) +(import java.io.File) +(import java.net.URL) +(use 'clojure.contrib.def) + +(def mlp + (URL. "http://www.fanfiction.net/cartoon/My_Little_Pony")) +;; It Takes a Village +;; http://www.fanfiction.net/s/7490980/1/It_Takes_a_Village +;; Reviews: 1058 +;; Chapters: 17 +;; Progress +;; http://www.fanfiction.net/s/6982210/1/Progress +;; Reviews: 612 +;; Chapters: 27 +;; On a Cross and Arrow +;; http://www.fanfiction.net/s/7211729/1/On_a_Cross_and_Arrow +;; Reviews: 324 +;; Chapters: 12 +;; Growing Pains +;; http://www.fanfiction.net/s/7252679/1/Growing_Pains +;; Reviews: 223 +;; Chapters: 23 +;; My Little Pony: FiM The Romancing Quest +;; http://www.fanfiction.net/s/7171004/1/My_Little_Pony_FiM_The_Romancing_Quest +;; Reviews: 174 +;; Chapters: 13 +;; Don't Let the Sun Catch You Crying +;; http://www.fanfiction.net/s/7032385/1/Dont_Let_the_Sun_Catch_You_Crying +;; Reviews: 165 +;; Chapters: 6 + + +(def chobits (URL. "http://www.fanfiction.net/anime/Chobits/")) +;; TTTRRRRRRAAAAAAASSSSSSHHHHH!!!!!!! + +(def ccs (URL. "http://www.fanfiction.net/anime/Card_Captor_Sakura/")) +;; Jagged Amber +;; http://www.fanfiction.net/s/1348047/1/Jagged_Amber +;; Reviews: 4767 +;; Chapters: 25 +;; Misty Dreams +;; http://www.fanfiction.net/s/1250008/1/Misty_Dreams +;; Reviews: 4460 +;; Chapters: 27 +;; Butterflies +;; http://www.fanfiction.net/s/1667519/1/Butterflies +;; Reviews: 3029 +;; Chapters: 36 +;; Mischievous Love +;; http://www.fanfiction.net/s/1020198/1/Mischievous_Love +;; Reviews: 3013 +;; Chapters: 25 +;; Black Wings +;; http://www.fanfiction.net/s/2737015/1/Black_Wings +;; Reviews: 2822 +;; Chapters: 23 +;; Cherry, My Love +;; http://www.fanfiction.net/s/968234/1/Cherry_My_Love +;; Reviews: 2612 +;; Chapters: 26 +;; Deeper +;; http://www.fanfiction.net/s/4027405/1/Deeper +;; Reviews: 2260 +;; Chapters: 26 +;; Through A Looking Glass +;; http://www.fanfiction.net/s/2729427/1/Through_A_Looking_Glass +;; Reviews: 2009 +;; Chapters: 9 +;; Butterflies: In Spring +;; http://www.fanfiction.net/s/2284162/1/Butterflies_In_Spring +;; Reviews: 1815 +;; Chapters: 17 +;; The New Trials of Card Captor Sakura and Friends +;; http://www.fanfiction.net/s/197453/1/The_New_Trials_of_Card_Captor_Sakura_and_Friends +;; Reviews: 1794 +;; Chapters: 111 +;; Ice Queen +;; http://www.fanfiction.net/s/1927065/1/Ice_Queen +;; Reviews: 1725 +;; Chapters: 38 +;; Crystal Tears +;; http://www.fanfiction.net/s/751067/1/Crystal_Tears +;; Reviews: 1623 +;; Chapters: 26 + + +(def bible (URL. "http://www.fanfiction.net/book/Bible/")) +;; TTTRRRRRRAAAAAAASSSSSSHHHHH!!!!!!! + +(def dark (URL. "http://www.fanfiction.net/book/His_Dark_Materials/")) +;; TTTRRRRRRAAAAAAASSSSSSHHHHH!!!!!!! + +(def junjo (URL. "http://www.fanfiction.net/anime/Junjo_Romantica/")) +;;The Talk +;; http://www.fanfiction.net/s/4351277/1/The_Talk +;; Reviews: 250 +;; Chapters: 1 + +(def rye (URL. "http://www.fanfiction.net/book/Catcher_in_the_Rye/")) +;; TTTRRRRRRAAAAAAASSSSSSHHHHH!!!!!!! + + +(defn tags-by-name + [#^TagNode node #^String element] + (seq (.getElementListByName node element true))) + +(defn-memo parse + "parse a web page using HtmlCleaner" + [#^URL url] + (println "parsing" (.getPath url)) + (if (not (nil? url)) + (.clean (HtmlCleaner.) (input-stream url)))) + +(defn attributes + "get a hash map of the attributes of an element" + [#^TagNode node] + (into {} (.getAttributes node))) + +(def fanfiction-base "http://www.fanfiction.net") + +(defn next-fanfiction-url + "get the next url to visit from the current page we are visiting" + [#^URL url] + (let + [;; extract all links + links (tags-by-name (parse url) "a") + ;; extract the "Next" link + next-node (first (filter #(re-matches #".*Next.*" (.getText %)) links))] + (if (not (nil? next-node)) + + (URL. (str fanfiction-base + ((attributes next-node) "href")))))) + +(defn get-stories [#^URL url] + (filter + #(= "z-list" ((attributes %) "class")) + (tags-by-name (parse url) "div"))) + + +(defn story-url [#^TagNode node] + (str fanfiction-base + ((attributes + (first (tags-by-name node "a"))) "href"))) + +(defn story-desc [#^TagNode node] + (.getText (second (tags-by-name node "div")))) + +(defn story-reviews [#^TagNode node] + (Integer/parseInt + (get + (re-matches #".*Reviews: (\d+).*" + (story-desc node)) 1 "0"))) + +(defn story-chapters [#^TagNode node] + (Integer/parseInt + (get + (re-matches #".*Chapters: (\d+).*" + (story-desc node)) 1 "1"))) + +(defn story-title [#^TagNode node] + (str (.getText (first (tags-by-name node "a"))))) + +(defn all-stories [n #^URL start-url] + (loop [stories [] + index 0 + current-url start-url] + (let [stories* (concat stories (get-stories current-url)) + next (next-fanfiction-url current-url)] + (if (and (not= index n) + (not (nil? next))) + (do + (recur stories* (inc index) next)) + stories*)))) + +(defn best-stories [n search-limit #^URL start-url] + (dorun + (map + #(println (story-title %) "\n" + (story-url %) "\n" + " Reviews: " (story-reviews %) "\n" + " Chapters:" (story-chapters %)) + + (take n (reverse + (sort-by + #(/ (story-reviews %) (inc (story-chapters %))) + ;;story-reviews + (all-stories search-limit start-url))))))) + + + \ No newline at end of file