# HG changeset patch # User "Hisn Yi, Chen " # Date 1228062307 -28800 # Node ID 60c4b4b78a01e34f9d288c0da44894f26c361386 # Parent fd6c3660cd3815dde57dbc6e433331801e632241 code clean diff -r fd6c3660cd38 -r 60c4b4b78a01 pyikriam/__init__.py --- a/pyikriam/__init__.py Fri Nov 14 02:05:48 2008 +0800 +++ b/pyikriam/__init__.py Mon Dec 01 00:25:07 2008 +0800 @@ -1,3 +1,94 @@ -from ikariam import Ikariam +from lazy.www import c +from lconf import LoadConfigfile +import cookielib +import os +import urllib2 +import urllib +class Ikariam: + + cities = {} + + def __init__(self, DEBUG=False): + self.COOKIEFILE = '/tmp/ikariam.lwp' + self.confdata=LoadConfigfile().cd + #self.baseurl='http://'+self.confdata['server'] + if DEBUG: + self.debug = True + self.baseurl = self.debug_url() + else: + self.baseurl = 'http://s4.ikariam.tw' + self.cj = cookielib.LWPCookieJar() + if os.path.isfile(self.COOKIEFILE): + self.cj.load(self.COOKIEFILE) + + opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj)) + opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-TW; rv:1.8.1.12pre) Gecko/20071220 BonEcho/2.0.0.12pre')] + urllib2.install_opener(opener) + + self.login() + + def debug_url(self): + """ + get the path of testing datas. + >>> + + @return: file path with scheme 'file://' + """ + return 'file://'+ os.path.abspath(os.path.curdir) + '/tests' -__all__ = ('Ikariam',) + def login(self): + print "login to %s...." % self.confdata['server'] + params = {"universe":self.confdata['server'], \ + "name":self.confdata['user'], \ + "password":self.confdata['pass']} + ret = c(self.baseurl+'/index.php?action=loginAvatar&function=login').get(params).get_content() + self.cj.save(self.COOKIEFILE) + + def logout(self): + print "logut from %s...." % self.confdata['server'] + c(self.baseurl+'/index.php?action=loginAvatar&function=logout') + os.remove(self.COOKIEFILE) + + def find_number_in(self, html, xpath): + return parse_to_int(self.find_data_in(html, xpath)) + + def find_data_in(self, work, xpath): + if work is None or xpath is None: return + return work.find(xpath) + + def city(self, id): + return self.cities.get(id, IkariamCity(id=id, core=self) ) + +class IkariamCity: + + def __init__(self, id, core ): + self.core = core + self.id = id + self.params = {'view':'city','id':id} + self.resources = {'gold':0,'wood':0,'wine':0,'marble':0,'crystal':0,'sulfur':0} + + # define xpath queries + self.xpath = { + 'gold':"/html/body[@id='city']/div[@id='container']/div[@id='container2']/div[@id='globalResources']/ul/li[2]/a/span[@id='value_gold']/text()", + 'wood':"/html/body[@id='city']/div[@id='container']/div[@id='container2']/div[@id='cityResources']/ul/li[3]/span[@id='value_wood']/text()", + 'wine':"/////div[@id='cityResources']/ul/li[4]/span[@id='value_wine']/text()", + 'marble':"/////div[@id='cityResources']/ul/li[4]/span[@id='value_wine']/text()", + 'crystal':"/////div[@id='cityResources']/ul/li[4]/span[@id='value_wine']/text()", + 'sulfur':"/////div[@id='cityResources']/ul/li[4]/span[@id='value_wine']/text()" + } + + def sync(self): + print "pull datas of the city %s" % self.id + + if self.core.debug: + work = c(self.core.baseurl+'/viewcity.html').get() + else: + work = c(self.core.baseurl).get(self.params) + + #print work.find(self.xpath['gold']).get_content() + print work.find(self.xpath['wood']).get_content() + print work.find(self.xpath['wood']).get_content() + +def parse_to_int( str ): pass + # print str[0] + # if str: return int( str.strip().replace(',','')) \ No newline at end of file diff -r fd6c3660cd38 -r 60c4b4b78a01 pyikriam/buildings.py --- a/pyikriam/buildings.py Fri Nov 14 02:05:48 2008 +0800 +++ b/pyikriam/buildings.py Mon Dec 01 00:25:07 2008 +0800 @@ -1,4 +1,4 @@ -from lazy.www import c +from lazy.www import c, Resource from lxml import etree from StringIO import StringIO from sync_utils import sync_tagclass, sync_tagvalue @@ -192,6 +192,25 @@ pass pass +class safehouse(building): + def __init__(self, city_id, idx, baseurl): + super(shipyard, self).__init__('safehouse', city_id, idx, baseurl) + self.data_patterns = { + 'vacancy_spys_total':"/::input[@id='spyCount']/@value", + 'trained_spys_total':"/descendant::div[@id='mainview']/div[4]/div[1]/p/span/span[2]/text()"} + pass + + def __sync__(self, page_dom): + workflow = create_workflow('find', page_dom) + attrs = workflow.findall(self.data_patterns).get_content() + for attr_name, attr_value in attrs: + setattr(self, attr_name, attr_value) + pass + pass + + def _mk_spy(self): + import spys + class empty_pos(position): res_patterns = { 'wood': 'wood', diff -r fd6c3660cd38 -r 60c4b4b78a01 pyikriam/ikariam.py --- a/pyikriam/ikariam.py Fri Nov 14 02:05:48 2008 +0800 +++ b/pyikriam/ikariam.py Mon Dec 01 00:25:07 2008 +0800 @@ -115,3 +115,7 @@ return build_type pass + + + + \ No newline at end of file diff -r fd6c3660cd38 -r 60c4b4b78a01 pyikriam/lazy/www/__init__.py --- a/pyikriam/lazy/www/__init__.py Fri Nov 14 02:05:48 2008 +0800 +++ b/pyikriam/lazy/www/__init__.py Mon Dec 01 00:25:07 2008 +0800 @@ -15,7 +15,7 @@ return {'msg':s} try: - c("http://example.tw/").get().find("#id > div") \ + c("http://example.tw/").get().find("////ul/text()") \ .build_param( func ).post_to("http://example2.com") \ .save_as('hellow.html') except: @@ -38,10 +38,21 @@ print "unknow error." """ from lazy.www.work import WorkFlow -from lazy.www.work.fetch import Fetcher, install_opener +from lazy.www.work.fetch import Fetcher +from lazy.www.work.storage import FileStorager from lazy.www.core import SemiProduct +import os +import sys +import re -def c(url): +def parse_scheme(scheme): + try: + return re.findall("(\w+):\/\/(.*\/?)",scheme)[0] + except: + sys.stdout.write("the scheme is not supported.") + sys.exit() + +def c(scheme): """ connect to a web apge @@ -51,14 +62,17 @@ >>> c('http://localhost:8080').get().find('//text()') 'It works!!\\n' """ - s= SemiProduct(source=url) - w = WorkFlow(Fetcher(s)) - return w + target_type, path = parse_scheme(scheme) -def lz_install(**kwds): - if('opener' == kwds.get('name')): - install_opener(kwds.get('cookiefile')) + #@todo: SemiProduct Factory. + if 'file' == target_type: + s= SemiProduct(source=path) + worker = FileStorager(s) + else: + s= SemiProduct(source=scheme) + worker = Fetcher(s) + return WorkFlow(worker) if __name__ == '__main__': import doctest - doctest.testmod() \ No newline at end of file + doctest.testmod() diff -r fd6c3660cd38 -r 60c4b4b78a01 pyikriam/lazy/www/core/__init__.py --- a/pyikriam/lazy/www/core/__init__.py Fri Nov 14 02:05:48 2008 +0800 +++ b/pyikriam/lazy/www/core/__init__.py Mon Dec 01 00:25:07 2008 +0800 @@ -9,4 +9,38 @@ self.source = kwds.get('source','') def __str__(self): - return self.content \ No newline at end of file + return self.content + +class Resource(object): + + datas = {} + + data_patterns = {} + + def __init__(self, **kwds): + self.datas = kwds.get('datas') + + def __get__(self, key): + try: + return self.datas[key] + except KeyError: + return self.key + + def __set__(self, key, value): + try: + self.datas[key] + except KeyError: + self.key = value + + def sync(self): + if not self.data_patterns: raise AttributeError("data patterns not defined.") + + express = {} + for attr in self.attr_xpath.keys(): + express[attr] = self.root_xpath+self.attr_xpath[attr] + + def fn(e): + for x in e: e[x] = e[x][0] + return e + + self.datas = self.core.open(self.param).findall(express).process(fn).get_content() \ No newline at end of file diff -r fd6c3660cd38 -r 60c4b4b78a01 pyikriam/lazy/www/work/__init__.py --- a/pyikriam/lazy/www/work/__init__.py Fri Nov 14 02:05:48 2008 +0800 +++ b/pyikriam/lazy/www/work/__init__.py Mon Dec 01 00:25:07 2008 +0800 @@ -21,14 +21,15 @@ return self.working_product.content def change_worker(self, new_worker): - self.serial_number += 1 + self.serial_number += 1 + self.last_work = self self.worker = new_worker def is_fetcher(self, obj): - if obj is not None: return True + if obj.__class__.__name__ == 'Fetcher': return True def get(self, data = {} ): - if not self.is_fetcher(self.worker) : + if self.worker.__class__.__name__ != 'FileStorager' and not self.is_fetcher(self.worker) : self.change_worker( Fetcher(self.working_product) ) self.working_product.content = self.worker.get(data) @@ -42,11 +43,31 @@ return self def is_finder(self, obj): - if obj is not None: return True + if obj.__class__.__name__ == 'Finder': return True + + def findall(self, expresses): + if not self.is_finder(self.worker): + self.change_worker( Finder(self.working_product) ) + + ret = {} + for e in expresses.keys(): + try: + ret[e] = self.worker.find(expresses[e]) + except: + pass + + self.working_product.content = ret + + return self def find(self, express): #if not self.is_finder(self.worker): - self.worker = Finder(self.working_product) + self.change_worker( Finder(self.working_product) ) + self.last_working_product = self.working_product self.working_product.content = self.worker.find(express) + return self + + def process(self, fn): + self.working_product.content = fn(self.working_product.content) return self \ No newline at end of file diff -r fd6c3660cd38 -r 60c4b4b78a01 pyikriam/lazy/www/work/fetch.py --- a/pyikriam/lazy/www/work/fetch.py Fri Nov 14 02:05:48 2008 +0800 +++ b/pyikriam/lazy/www/work/fetch.py Mon Dec 01 00:25:07 2008 +0800 @@ -3,17 +3,6 @@ import cookielib import os -def install_opener(cookiefile): - COOKIEFILE = cookiefile - cj = cookielib.LWPCookieJar() - if os.path.isfile(COOKIEFILE): - cj.load(COOKIEFILE) - else: - cj.save(cookiefile) - opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) - opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-TW; rv:1.8.1.12pre) Gecko/20071220 BonEcho/2.0.0.12pre')] - urllib2.install_opener(opener) - class Fetcher: opener = None @@ -31,15 +20,14 @@ """ def __init__(self, working_product): self.working_product = working_product - - def get(self, data = {}): - """ - send datas via http get method. - """ - res = urllib2.urlopen(self.working_product.source, urllib.urlencode(data)) - return res.read() - def post(self, data = {} ): + def get(self, **kwds): + return self.open(kwds) + + def post(self, **kwds): + return self.open(kwds) + + def open(self, data = {} ): """ send datas via http post method. diff -r fd6c3660cd38 -r 60c4b4b78a01 pyikriam/lazy/www/work/find.py --- a/pyikriam/lazy/www/work/find.py Fri Nov 14 02:05:48 2008 +0800 +++ b/pyikriam/lazy/www/work/find.py Mon Dec 01 00:25:07 2008 +0800 @@ -10,14 +10,18 @@ self.working_prodcut = working_product self.encoding = 'utf8' - parser = etree.HTMLParser(encoding=self.encoding) - self.dom_tree = etree.parse(StringIO(self.working_prodcut.content), parser) def find(self, express , callback = None): + + if self.dom_tree is None: self.set_dom_tree(self.working_prodcut.content) + xpath = self.dom_tree.xpath(express) - if callback is None: - ret = xpath - else: - ret = self.callback(xpath) - return ret + if callback: return self.callback(xpath) + return xpath + + def set_dom_tree(self, content): + stream = StringIO(content) + + parser = etree.HTMLParser(encoding=self.encoding) + self.dom_tree = etree.parse(stream, parser) \ No newline at end of file