DRSS

rss.d (7167B)

---

 module drss.rss;
 import std.net.curl;
 import std.stdio;
 import std.container.slist;
 import std.datetime;
 import std.algorithm;
 import std.string;
 import std.range;
 static import std.conv;
 import kxml.xml;
 
 /**
  * A basic Entry-type.
  * 
  * If it is sufficient, you don't have to define your own...
  */
 struct Entry{
 	///Content-Type of the entry.
 	enum Type{
 		HTML,
 		Plain
 	}
 	static immutable string[] TypeTable=[Type.HTML:"html", Type.Plain:"text"];
 	///The Content-type of this entry.
 	Type type;
 	///The ID of the entry
 	string id;
 	///The raw content to be displayed either as HTML or Plain-text (Depending on type)
 	string content;
 	///The publishdate of the entry
 	SysTime date;
 	///Compares according to the publishdate
 	int opCmp(inout ref Entry b) const{
 		return date.opCmp(b.date);
 	}
 	///ditto
 	int opCmp(inout Entry b) const{
 		return date.opCmp(b.date);
 	}
 	
 	///Generates a atom-xmlnode for the entry
 	XmlNode toXML(){
 		XmlNode e=new XmlNode("entry");
 		e.addChild(new XmlNode("id").addCData(id));
 		e.addChild(new XmlNode("published").addCData(date.toISOExtString()));
 		e.addChild(new XmlNode("content").setAttribute("type",TypeTable[type]).addCData(content));
 		return e;
 	}
 }
 
 /**
  * The main RSS-Class.
  * 
  * Its template-parameter is to specify the entry-type, which is of 
  * Type drss.rss.Entry by default.
  * 
  */
 abstract class DRSS(T=Entry){
 	private string lastUpdated;
 	SList!(T) entries;
 	uint new_entries=0;
 	/**
 	 * Determines wether the date set by parse is reliable or not.
 	 * 
 	 * This has an impact in adding it to the entry-list entries.
 	 * 
 	 */
 	enum DateReliable{
 		///Always iterate through the entries and check if the ID exists
 		NO,
 		///Only iterate through the entries and check if the ID exists, if the new entry is newer than the newest old one
 		SEMI,	//If entry is newer, check the id additionally.
 		///Never iterate through the entries to check if the ID exists
 		YES
 	};
 	///
 	DateReliable date_reliability=DateReliable.SEMI;
 	///At what point shall the entries be discarded.
 	uint max_entries=100;
 	
 	protected uint entry_count=0;
 	protected HTTP http;
 	protected ubyte[] buffer;
 	
 	/**
 	 * Parse a raw document.
 	 * 
 	 * This function should extract the Posts from document and add
 	 * them using addEntry
 	 * ---
 	 * override void parse(string document){
 	 * 	foreach(Entry e; splitDocumentIntoEntries(document)){
 	 * 		this.addEntry(e);
 	 * 	}
 	 * }
 	 * ---
 	 * 
 	 */
 	abstract void parse(string document);
 
 	/**
 	 * Initialize the Feed by url
 	 * Params:
 	 * 	url = The url to fetch the raw-data from
 	 */
 	this(string url){
 		auto h=HTTP();
 		h.url=url;
 		this(h);
 	}
 	/**
 	 * Initialize the feed by a custom HTTP-request
 	 * 
 	 * This allows for things like:
 	 * ---
 	 * this(){
 	 * 	auto h=HTTP();
 	 * 	h.url="http://herp.derp/rss.xml";
 	 * 	h.setUserAgent("MyCustomUserAgent 1.0");
 	 * 	super(h);
 	 * }
 	 * ---
 	 * Params:
 	 * 	http = The http-request to execute when updateing
 	 */
 	this(HTTP http){
 		this.http=http;
 		this.http.onReceive = &addContent;
 		this.http.onReceiveHeader = &parseHeader;
 	}
 	
 	/**
 	 * Check if the entry newent is new.
 	 * Operation depends on the value of date_reliability.
 	 * $(SEE_ALSO enum DateReliable)
 	 * 
 	 * Params:
 	 * 	newent = the entry to check
 	 * Returns:
 	 * 	false if the entry is already in entries, true otherwise
 	 */
 	private bool checkIfNewEntry(in ref T newent){
 		if(entries.empty){
 			return true;
 		}
 		switch(date_reliability){
 			case DateReliable.NO:
 				return !canFind!("a.id==b.id")(entries[], newent);
 				break;
 			case DateReliable.YES: 
 				return (newent>entries.front);
 				break;
 			case DateReliable.SEMI:
 				if(newent>entries.front){
 					return !canFind!("a.id==b.id")(entries[], newent);
 				}
 				break;
 			default:
 				throw new Exception("Unknonwn reliability"); 
 		}
 		return false;
 	}
 	
 	/**
 	 * Add an entry to the list.
 	 * 
 	 * Params:
 	 * 	newent = The entry to add
 	 * Returns:
 	 * 	true if the entry was added, false otherwise.
 	 * 
 	 */
 	bool addEntry(T newent){
 		if(checkIfNewEntry(newent)){
 			entries.insertFront(newent);
 			new_entries++;
 			entry_count++;
 			return true;
 		}
 		else{
 			return false;
 		}
 	}
 	
 	/**
 	 * Updates the whole stream.
 	 * 
 	 * $(OL
 	 * 	$(LI Calls `fetch();`, if document is null)
 	 * 	$(LI Passes the result to `update();`)
 	 * 	$(LI Discards the old Entries with `removeOldEntries();`
 	 * )
 	 * 
 	 * Params:
 	 * 	document = The document to parse or null, to fetch it.
 	 * Returns:
 	 * 	true if the feed has new entries.
 	 * 
 	 */
 	final bool update(ubyte[] document=null){
 		if(document is null){
 			if(!fetch()){
 				return false;
 			}
 			document=buffer;
 		}
 		new_entries=0;
 		parse(cast(string)document);
 		if(entry_count>max_entries){
 			removeOldEntries();
 		}
 		return (new_entries>0);
 	}
 	
 	///Return the total count of entries
 	@property size_t length(){
 		return entry_count;
 	}
 	
 	/**
 	 * Removes every entry older than the max_entries newest.
 	 */
 	private void removeOldEntries()
 	out{
 		size_t real_length=entries[].walkLength;
 		assert(real_length==max_entries);
 		assert(real_length==entry_count);
 	}
 	body{
 		auto r=entries[];
 		auto i=max_entries;
 		while(i-->0 && !r.empty){
 			r.popFront();
 		}
 		if(!r.empty){
 			entries.linearRemove(r);
 			entry_count=max_entries;
 		}
 	}
 	
 	///Returns a range with the new entries since the last call to update()
 	@property auto news(){
 		return entries[].take(new_entries);
 	}
 
 	///Sets the new entries since the last call to update to 0.
 	public void discardNews(){
 		new_entries=0;
 	}
 	
 	private void parseHeader(in char[] key, in char[] value){
 		if(key=="date"){
 			lastUpdated=value.dup;
 		}
 	}
 	
 	private auto addContent(ubyte[] data){
 		buffer~=data;
 		return data.length;
 	}
 	/**
 	 * Fetches the document and saves it internally.
 	 * 
 	 * Returns:
 	 * 	true if upstream reports to have new entries, false otherwise
 	 */
 	public bool fetch(){
 		debug(RSS) writeln("Updateing");
 		if(lastUpdated.length!=0){
 			debug(RSS) writeln("Adding if-not-modified-since");
 			http.addRequestHeader("If-Modified-Since",lastUpdated);
 		}
 		debug(RSS) writeln("Performing HTTP request");
 		http.perform();
 		if(http.statusLine.code==200){
 			debug(RSS) writeln("Have news, parsing and returning xml");
 			return true;
 		}
 		else if(http.statusLine.code==304){
 			debug(RSS) writeln("Have no news, returning null");
 			return false;
 		}
 		else{
 			throw new Exception("Error in HTTP-Request: statuscode "~std.conv.to!string(http.statusLine.code));
 		}
 	}
 }
 
 ///
 unittest{
 	class TeRSSt : DRSS!(){
 		private uint counter;
 		this(){
 			date_reliability=DateReliable.YES;
 			max_entries=10;
 			super(HTTP());
 		}
 		override void parse(string document){
 			import std.datetime;
 			import std.conv;
 			
 			string cnt=to!string(counter);
 			SysTime st=SysTime(DateTime(1986,01,counter+1));
 			assert(addEntry(Entry(Entry.Type.Plain, "ID"~cnt, "This is id number "~cnt, st)));
 			counter++;
 		}
 		override bool fetch(){
 			return true;
 		}
 	}
 	TeRSSt t=new TeRSSt();
 	foreach(i; 0..30){
 		assert(t.update());
 		assert(t.news.walkLength==1);
 		assert(t.news.front.id=="ID"~to!string(i));
 		assert(t.length==min(t.max_entries,i+1));
 	}
 }