Skip to content

Commit

Permalink
changes needed in order to parse a feed to flipthis-webhook
Browse files Browse the repository at this point in the history
  • Loading branch information
@xicubed
xicubed committed Sep 17, 2013
1 parent 307f4f7 commit abab6b7
Show file tree
Hide file tree
Showing 4 changed files with 91 additions and 45 deletions.
5 changes: 4 additions & 1 deletion project.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,7 @@
:license {:name "Eclipse Public License"
:url "http://www.eclipse.org/legal/epl-v10.html"}
:dependencies [[org.clojure/clojure "1.5.1"]
[org.clojure/data.zip "0.1.1"]])
[org.clojure/data.zip "0.1.1"]
[clj-http "0.7.6"]
[clj-time "0.6.0"]
[overtone/at-at "1.2.0"]])
35 changes: 18 additions & 17 deletions src/ariane/atom.clj
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
(ns ariane.atom
"Namespace for parsing Atom feeds."
(:require [clojure.zip :as zip]
[clojure.data.zip.xml :as zip-xml]))
[clojure.data.zip.xml :as zipxml]
))

(defn- add-map-entry
"Add key/value entry to map only if value is not nil."
Expand All @@ -14,37 +15,37 @@
[links]
(vec (for [link links]
(let [link-spec {:href
(first (zip-xml/xml-> link (zip-xml/attr :href)))}]
(if-let [rel (first (zip-xml/xml-> link (zip-xml/attr :rel)))]
(first (zipxml/xml-> link (zipxml/attr :href)))}]
(if-let [rel (first (zipxml/xml-> link (zipxml/attr :rel)))]
(assoc link-spec :rel rel)
link-spec)))))

(defn- infos
[root]
(-> {}
(add-map-entry [:title (first (zip-xml/xml-> root :title zip-xml/text))])
(add-map-entry [:description (first (zip-xml/xml-> root :summary zip-xml/text))])
(add-map-entry [:updated (first (zip-xml/xml-> root :updated zip-xml/text))])
(add-map-entry [:links (links (zip-xml/xml-> root :link))])))
(add-map-entry [:title (first (zipxml/xml-> root :title zipxml/text))])
(add-map-entry [:description (first (zipxml/xml-> root :summary zipxml/text))])
(add-map-entry [:updated (first (zipxml/xml-> root :updated zipxml/text))])
(add-map-entry [:links (links (zipxml/xml-> root :link))])))

(defn- authors
[authors]
(vec (for [author authors]
{:name (first (zip-xml/xml-> author :name zip-xml/text))
:uri (first (zip-xml/xml-> author :uri zip-xml/text))})))
{:name (first (zipxml/xml-> author :name zipxml/text))
:uri (first (zipxml/xml-> author :uri zipxml/text))})))

(defn- entries
[root]
(for [entry (zip-xml/xml-> root :entry)]
(for [entry (zipxml/xml-> root :entry)]
(-> {}
(add-map-entry [:id (first (zip-xml/xml-> entry :id zip-xml/text))])
(add-map-entry [:title (first (zip-xml/xml-> entry :title zip-xml/text))])
(add-map-entry [:links (links (zip-xml/xml-> entry :link))])
(add-map-entry [:updated (first (zip-xml/xml-> entry :updated zip-xml/text))])
(add-map-entry [:authors (authors (zip-xml/xml-> entry :author))])
(add-map-entry [:id (first (zipxml/xml-> entry :id zipxml/text))])
(add-map-entry [:title (first (zipxml/xml-> entry :title zipxml/text))])
(add-map-entry [:links (links (zipxml/xml-> entry :link))])
(add-map-entry [:updated (first (zipxml/xml-> entry :updated zipxml/text))])
(add-map-entry [:authors (authors (zipxml/xml-> entry :author))])
(add-map-entry [:description
{:type (first (zip-xml/xml-> entry :content (zip-xml/attr :type)))
:content (first (zip-xml/xml-> entry :content zip-xml/text))}]))))
{:type (first (zipxml/xml-> entry :content (zipxml/attr :type)))
:content (first (zipxml/xml-> entry :content zipxml/text))}]))))

(defn parse-atom
"Parse Atom feed generated by ariane.core/parse. "
Expand Down
12 changes: 10 additions & 2 deletions src/ariane/core.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@
"Generic namespace for feed type autodetection and parsing."
(:require [clojure.xml :as xml]
[ariane.rss :as rss]
[ariane.atom :as atom]))
[ariane.atom :as atom]
[clj-time.core :as ct]
[clj-time.format :as ctf]))



(defn- rss-feed?
[feed]
Expand All @@ -19,4 +23,8 @@
(cond
(rss-feed? feed) (rss/parse-rss feed)
(atom-feed? feed) (atom/parse-atom feed)
:else (println "Unsupported format"))))
:else (println "Unsupported format")))
)



84 changes: 59 additions & 25 deletions src/ariane/rss.clj
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,29 @@
(ns ariane.rss
"Namespace for parsing RSS feeds."
(:require [clojure.zip :as zip]
[clojure.data.zip.xml :as zip-xml]))
[clojure.data.zip.xml :as zipxml]
[clj-http.client :as client]
[clj-time.core :as ct]
[clj-time.format :as ctf]
[overtone.at-at :as ot]))

(def lastrun (atom (slurp "lastrun.txt") ))

(defn updatedate
[x]
(spit "lastrun.txt" x)
)

(defn getlastrun
[]
(slurp "lastrun.txt")
)

; (def fixedlast (slurp "lastrun.txt"))
(def now (atom ""))

; for date calcs
(def custom-formatter (ctf/formatter "EEE, dd MMM yyyy HH:mm:ss Z"))

(defn- add-map-entry
"Add key/value entry to map only if value is not nil."
Expand All @@ -14,42 +36,54 @@
(defn- links
[links]
(vec (for [link links]
(let [link-spec {:href
(first (zip-xml/xml-> link zip-xml/text))}]
(if-let [rel (first (zip-xml/xml-> link (zip-xml/attr :rel)))]

(let [link-spec {:href
(first (zipxml/xml-> link zipxml/text))}]
(if-let [rel (first (zipxml/xml-> link (zipxml/attr :rel)))]
(assoc link-spec :rel rel)
link-spec)))))
link-spec)
)


)) )

(defn- infos
[root]
(-> {}
(add-map-entry [:title (first (zip-xml/xml-> root :title zip-xml/text))])
(add-map-entry [:description (first (zip-xml/xml-> root :description zip-xml/text))])
(add-map-entry [:updated (first (zip-xml/xml-> root :lastBuildDate zip-xml/text))])
(add-map-entry [:links (links (zip-xml/xml-> root :link))])))

(reset! now (first (zipxml/xml-> root :lastBuildDate zipxml/text)) )
(spit "lastrun.txt" (first (zipxml/xml-> root :lastBuildDate zipxml/text)) )

)

(defn- authors
[authors]
(vec (for [author authors]
{:name (first (zip-xml/xml-> author :name zip-xml/text))
:uri (first (zip-xml/xml-> author :uri zip-xml/text))})))
{:name (first (zipxml/xml-> author :name zipxml/text))
:uri (first (zipxml/xml-> author :uri zipxml/text))})))

(defn- entries
(defn- entries
[root]
(for [entry (zip-xml/xml-> root :item)]
(-> {}
(add-map-entry [:id (first (zip-xml/xml-> entry :guid zip-xml/text))])
(add-map-entry [:title (first (zip-xml/xml-> entry :title zip-xml/text))])
(add-map-entry [:links (links (zip-xml/xml-> entry :link))])
(add-map-entry [:updated (first (zip-xml/xml-> entry :pubDate zip-xml/text))])
(add-map-entry [:authors (authors (zip-xml/xml-> entry :author))])
(add-map-entry [:description
{:content (first (zip-xml/xml-> entry :description zip-xml/text))}]))))
(for [entry (zipxml/xml-> root :item)]
(if (ct/before? (ctf/parse custom-formatter (str @lastrun)) (ctf/parse custom-formatter (first (zipxml/xml-> entry :pubDate zipxml/text))))
(client/get "http://127.0.0.1:5000" {:query-params {"url" (first (zipxml/xml-> entry :link zipxml/text) ), "title" (first (zipxml/xml-> entry :title zipxml/text))}} )
(reset! lastrun @now)
)
)
)

(defn update
[]
;(swap! lastrun (fn [lastrun] (lr)))
(reset! lastrun @now)
nil
)

(defn parse-rss
"Parse Atom feed generated by ariane.core/parse. "
[feed]
(let [root (zip/xml-zip feed)
channel (first (zip-xml/xml-> root :channel))]
{:infos (infos channel)
:entries (entries channel)}))
channel (first (zipxml/xml-> root :channel))]
{:infos (infos channel)
:entries (entries channel)
})
)

0 comments on commit abab6b7

Please sign in to comment.