Mercurial > eagle-eye
view pyikriam/lazy/www/work/fetch.py @ 175:9f248c8460ce
Simplize xpath patterns.
- Use 'descendant' aix name simplize patterns.
author | Thinker K.F. Li <thinker@branda.to> |
---|---|
date | Sun, 02 Nov 2008 09:59:51 +0800 |
parents | a4c364888197 |
children | 60c4b4b78a01 |
line wrap: on
line source
import urllib2 import urllib import cookielib import os def install_opener(cookiefile): COOKIEFILE = cookiefile cj = cookielib.LWPCookieJar() if os.path.isfile(COOKIEFILE): cj.load(COOKIEFILE) else: cj.save(cookiefile) opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-TW; rv:1.8.1.12pre) Gecko/20071220 BonEcho/2.0.0.12pre')] urllib2.install_opener(opener) class Fetcher: opener = None working_product = None """ A Semi Production Decoration for content fetching. handles content restriving. >>> o = Fetcher( SemiProduct(source="http://localhost:8080") ) >>> o.get().working_product.content 'It works!!\\n' """ def __init__(self, working_product): self.working_product = working_product def get(self, data = {}): """ send datas via http get method. """ res = urllib2.urlopen(self.working_product.source, urllib.urlencode(data)) return res.read() def post(self, data = {} ): """ send datas via http post method. >>> o = Fetcher( SemiProduct(source="http://localhost:8080") ) >>> o.post({'a':'b'}).working_product.content 'It works!!\\n' """ res = urllib2.urlopen(self.working_product.source, urllib.urlencode(data)) return res.read() def refer(self, refer_url): """ refer getter/setter. >>> o = Fetcher( SemiProduct(source="http://localhost:8080") ) >>> o.refer('http://www.example.com') """ raise NotImplementedError def retry(self, count = 0, intval = 0, timeout = 0): """ retry to fetch the content. >>> o = Fetcher( SemiProduct(source="http://localhost:8080") ) >>> o.retry(4) """ raise NotImplementedError class Retry: """ A Fetcher Decoration for retry goal. """ def __init__(self, fetcher): raise NotImplementedError if __name__ == '__main__': import doctest doctest.testmod()