|
62
|
1 import urllib2
|
|
|
2 import urllib
|
|
|
3 import cookielib
|
|
|
4 import os
|
|
|
5
|
|
|
6 def install_opener(cookiefile):
|
|
|
7 COOKIEFILE = cookiefile
|
|
|
8 cj = cookielib.LWPCookieJar()
|
|
|
9 if os.path.isfile(COOKIEFILE):
|
|
|
10 cj.load(COOKIEFILE)
|
|
|
11 else:
|
|
|
12 cj.save(cookiefile)
|
|
|
13 opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
|
|
|
14 opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-TW; rv:1.8.1.12pre) Gecko/20071220 BonEcho/2.0.0.12pre')]
|
|
|
15 urllib2.install_opener(opener)
|
|
|
16
|
|
|
17 class Fetcher:
|
|
|
18
|
|
|
19 opener = None
|
|
|
20
|
|
|
21 working_product = None
|
|
|
22
|
|
|
23 """
|
|
|
24 A Semi Production Decoration for content fetching.
|
|
|
25
|
|
|
26 handles content restriving.
|
|
|
27
|
|
|
28 >>> o = Fetcher( SemiProduct(source="http://localhost:8080") )
|
|
|
29 >>> o.get().working_product.content
|
|
|
30 'It works!!\\n'
|
|
|
31 """
|
|
|
32 def __init__(self, working_product):
|
|
|
33 self.working_product = working_product
|
|
|
34
|
|
|
35 def get(self, data = {}):
|
|
|
36 """
|
|
|
37 send datas via http get method.
|
|
|
38 """
|
|
|
39 res = urllib2.urlopen(self.working_product.source, urllib.urlencode(data))
|
|
|
40 return res.read()
|
|
|
41
|
|
|
42 def post(self, data = {} ):
|
|
|
43 """
|
|
|
44 send datas via http post method.
|
|
|
45
|
|
|
46 >>> o = Fetcher( SemiProduct(source="http://localhost:8080") )
|
|
|
47 >>> o.post({'a':'b'}).working_product.content
|
|
|
48 'It works!!\\n'
|
|
|
49 """
|
|
|
50 res = urllib2.urlopen(self.working_product.source, urllib.urlencode(data))
|
|
|
51 return res.read()
|
|
|
52
|
|
|
53 def refer(self, refer_url):
|
|
|
54 """
|
|
|
55 refer getter/setter.
|
|
|
56
|
|
|
57 >>> o = Fetcher( SemiProduct(source="http://localhost:8080") )
|
|
|
58 >>> o.refer('http://www.example.com')
|
|
|
59 """
|
|
|
60 raise NotImplementedError
|
|
|
61
|
|
|
62 def retry(self, count = 0, intval = 0, timeout = 0):
|
|
|
63 """
|
|
|
64 retry to fetch the content.
|
|
|
65
|
|
|
66 >>> o = Fetcher( SemiProduct(source="http://localhost:8080") )
|
|
|
67 >>> o.retry(4)
|
|
|
68 """
|
|
|
69 raise NotImplementedError
|
|
|
70
|
|
|
71 class Retry:
|
|
|
72
|
|
|
73 """
|
|
|
74 A Fetcher Decoration for retry goal.
|
|
|
75
|
|
|
76
|
|
|
77 """
|
|
|
78 def __init__(self, fetcher):
|
|
|
79 raise NotImplementedError
|
|
|
80
|
|
|
81 if __name__ == '__main__':
|
|
|
82 import doctest
|
|
|
83 doctest.testmod() |
