--- a/Ikariam.pm	Thu Oct 23 15:51:20 2008 +0800
+++ b/Ikariam.pm	Fri Oct 24 21:46:33 2008 +0800
@@ -13,6 +13,79 @@
     namespace => 'Ikariam',
 );
 
+package Ikariam::Extractor;
+use strict;
+use Data::Dumper;
+use XML::LibXML;
+use IO::Uncompress::Gunzip qw(gunzip $GunzipError) ;
+use Carp;
+use utf8;
+
+sub new {
+    Carp::croak("Options should be key/value pairs, not hash reference") if ref($_[1]) eq 'HASH'; 
+
+    my($class, %conf) = @_;
+
+    my $self = bless {
+        doc => undef,
+        gzip => 1,
+    }, $class;
+
+    $self->{gzip} = $conf{'gzip'} if(defined($conf{'gzip'}));
+    $self->parse($conf{'content'}) if(defined($conf{'content'}));
+
+    return $self;
+}
+
+# parse($Content);
+sub parse {
+    my ($self, $content) = @_;
+    my $string;
+    my $parser = XML::LibXML->new('1.0','utf-8');
+
+    if($self->{gzip} == 1) {
+        gunzip \$content => \$string
+            or die "gunzip failed: $GunzipError\n";
+    } else {
+        $string = $content;
+    }
+
+    $self->{doc} = $parser->parse_html_string ($string, { suppress_errors => 1 });
+    return;
+}
+
+# find($XPathQuery);
+sub find {
+    my $self = shift;
+    my $query = shift;
+    my $out = [];
+
+    my $result = $self->{doc}->find($query);
+
+    return undef unless defined($result);
+
+    if ( $result->isa( 'XML::LibXML::NodeList' ) ) {
+        return undef if($result->size() == 0);
+        foreach ( @$result ) {
+            # $_ is XML::LibXML::Element, XML::LibXML::Node
+            my $literal = $_->to_literal() , "\n";
+            utf8::encode($literal);
+            # warn $_->toString(1) , "\n";
+            return $literal unless wantarray;
+            push( @$out, $literal);
+        }
+    } else {
+        Carp::croak("Unsupported data type"); 
+    }
+    # TODO
+    # XML::LibXML::Literal
+    # XML::LibXML::Boolean
+
+    return @$out;
+}
+
+1;
+
 package Ikariam;
 use strict;
 use Data::Dumper;
@@ -22,7 +95,9 @@
 use HTTP::Cookies;
 use WWW::Mechanize;
 use HTML::TagParser;
+use XML::LibXML qw(:encoding);
 use IO::Uncompress::Gunzip qw(gunzip $GunzipError) ;
+use utf8;
 
 sub new
 {
@@ -347,186 +422,187 @@
     my $cityId = shift;
 
     my $res = $self->{mech}->get(sprintf('http://%s/index.php?view=researchOverview&id=%s', $self->{server}, $cityId));
+    
+    my @urls = Ikariam::Extractor->new(content => $res->content)->find('//ul[@class="explored"]//a/@href');
+    my $out = {};
+    foreach(@urls) {
+        if(/view=researchDetail&id=\d+&position=\d+&researchId=(\d+)$/) {
+            # we ignore the chinese name of technology researched.
+            @$out{$1} = 1;
+        }
+    }
+    return $out;
+}
 
-    my $content;
-    gunzip \$res->content => \$content 
-        or die "gunzip failed: $GunzipError\n";
-    my $html = HTML::TagParser->new($content);
+sub checkCity {
+    my $self = shift;
+    my $cityId = shift;
+
+    # search for goods
+    my $res = $self->{mech}->get(sprintf("http://%s/index.php?view=city&id=%d", $self->{server}, $cityId));
+    my $extractor = new Ikariam::Extractor(content => $res->content);
+
+    $self->{'cities'}->{$cityId}->{resources}->{gold} = $extractor->find('//span[@id="value_gold"]/text()');
+    $self->{'cities'}->{$cityId}->{resources}->{gold} =~ s/,//g;
+
+    $self->{'cities'}->{$cityId}->{name} = $extractor->find('//span[@class="city"]/text()');
+
+    $self->{'cities'}->{$cityId}->{construction} = defined($extractor->find('//*[@class="constructionSite"]')) ? 1 : 0;
 
-    my @elems = $html->getElementsByAttribute('class', 'explored');
+    # maxCapacity
+    my $page = $extractor->{doc}->toString(1);
+    if($page =~ /maxCapacity : {\s+wood: (\d+),\s+wine: (\d+),\s+marble: (\d+),\s+crystal: (\d+),\s+sulfur: (\d+)\s+}/s) {
+        $self->{'cities'}->{$cityId}->{maxCapacity}->{wood} = $1;
+        $self->{'cities'}->{$cityId}->{maxCapacity}->{wine} = $2;
+        $self->{'cities'}->{$cityId}->{maxCapacity}->{marble} = $3;
+        $self->{'cities'}->{$cityId}->{maxCapacity}->{crystal} = $4;
+        $self->{'cities'}->{$cityId}->{maxCapacity}->{sulfur} = $5;
+    }
+
+    foreach my $good (qw/wood wine marble crystal sulfur/) {
+        $self->{'cities'}->{$cityId}->{resources}->{$good} = $extractor->find(sprintf('//span[@id="value_%s"]', $good));
+        $self->{'cities'}->{$cityId}->{resources}->{$good} =~ s/,//g;
+    }
 
-    my $out = {};
-    foreach my $elem (@elems) {
-        my @items = getElementsByTagName($elem, "a");
-        foreach my $item (@items) {
-            if($item->getAttribute('href') =~ /view=researchDetail&id=\d+&position=\d+&researchId=(\d+)$/) {
-                @$out{$1} = $item->innerText();
+    foreach my $i (0..14) {
+        my @buildings = $extractor->find(sprintf('//*[@id="position%s"]/@class', $i));
+        foreach my $building (@buildings) {
+            if (!($building =~ /buildingGround/) && !($building =~ /townhall/)) {
+                $self->{'cities'}->{$cityId}->{locations}[$i] = $building;
+
+                my $span;
+                if($building eq 'townHall') {
+                    # this is stupid, the new vession give two cityTown information.
+                    $span = ($extractor->find(sprintf('//*[@id="position%s"]//span[@class="textLabel"]/text()', $i)))[1];
+                } else {
+                    $span = $extractor->find(sprintf('//*[@id="position%s"]//span[@class="textLabel"]/text()', $i));
+                }
+                my (undef, undef, $level) = split(/ /, $span);
+                $self->{'cities'}->{$cityId}->{buildings}->{$building} = $level;
             }
         }
     }
-    return $out;
+
+    $self->{'cities'}->{$cityId}->{transporters}->{avail} = $extractor->find('//span[@id="value_transAvail"]/text()');
+    $self->{'cities'}->{$cityId}->{transporters}->{sum} = $extractor->find('//span[@id="value_transSum"]/text()');
+    $self->{'cities'}->{$cityId}->{transporters}->{sum} =~ s/[\(|\)]//g;
+    $self->{'cities'}->{$cityId}->{maxActionPoints} = $extractor->find('//span[@id="value_maxActionPoints"]');;
+}
+
+
+sub checkMilitaryAdvisorMilitaryMovements  {
+    my $self = shift;
+    # TODO
+    # http://s2.ikariam.tw/index.php?view=militaryAdvisorMilitaryMovements
+    my $res = $self->{mech}->get(sprintf('http://%s/index.php?view=militaryAdvisorMilitaryMovements', $self->{server}));
+}
+
+sub checkMilitaryAdvisorCombatReports {
+    my $self = shift;
+    my $content;
+    my $res = $self->{mech}->get(sprintf('http://%s/index.php?view=militaryAdvisorCombatReports', $self->{server}));
+    gunzip \$res->content => \$content 
+        or die "gunzip failed: $GunzipError\n";
+
+    open(OUT,">foo.html");
+    print OUT $content;
+    close(OUT);
+ 
+    # TODO
+    # $self->{'cities'}->{$cityId}->{force}->{attacks} = $1 if($content =~ /敵人攻擊: (\d+)/);
+    # $self->{'cities'}->{$cityId}->{force}->{wars} = $1 if($content =~ /我方軍隊行程: (\d+)/);
+
+    # list down reports.
+    # /index.php?view=militaryAdvisorReportView&combatId=1887662
+    # //form[@id='finishedReports']//a
+}
+
+sub checkTownHall {
+    my $self = shift;
+    my $cityId = shift;
+
+    my $res = $self->{mech}->get(sprintf('http://%s/index.php?view=townHall&id=%d', $self->{server}, $cityId));
+    my $extractor = new Ikariam::Extractor(content => $res->content);
+
+    # //div[@id='SatisfactionOverview']//div[@class='value']
+    # //div[@id='SatisfactionOverview']//div[@class='text']
+    $self->{"cities"}->{$cityId}->{happiness} = $extractor->find('//div[@id="SatisfactionOverview"]//div[@class="value"]');
+
+    # 取 SatisfactionOverview 中的其他值
+    # 一個城鎮的市民滿意度結合了多方面的因素
+    # check happiness
+    # Happiness = Basic bonuses (196 + Capital Bonus + Holiday Bonus(25)) + 
+    #             Wine (Tavern Base(12*level) + 
+    #             Tavern Bonus(80*step)) + 
+    #             Culture (Museum Base(20*level) + 
+    #             Cultural Goods Bonus(50*Cultural Goods)) - 
+    #             Population (population) - 
+    #             Corruption (Corruption rate * population)
+    #
+    # Growth Rate = Happiness * 0.02
+
+    # Space, 房屋數
+    $self->{'cities'}->{$cityId}->{"space"} = {};
+    $self->{'cities'}->{$cityId}->{"space"}->{'total'} = $extractor->find('//span[@class="value total"]/text()');
+    $self->{'cities'}->{$cityId}->{"space"}->{'occupied'} = $extractor->find('//span[@class="value occupied"]/text()');
+
+    my @values = $extractor->find('//div[@id="CityOverview"]//span[@class="value"]');
+    $self->{'cities'}->{$cityId}->{"growth"} = $values[0];
+    $self->{'cities'}->{$cityId}->{"incomegold"} = $values[1];
+
+    $self->{'cities'}->{$cityId}->{corruption} = $extractor->find('//li[@class="corruption"]//span[@title="目前腐敗程度"]');
+    $self->{'cities'}->{$cityId}->{corruption} =~ s/%//g;
+
+    my @citizens_type = qw/citizens woodworkers specialworkers scientists/;
+    @values = $extractor->find('//div[@id="PopulationGraph"]//span[@class="count"]');
+    foreach my $i (0..$#citizens_type)
+    {
+        $self->{'cities'}->{$cityId}->{'citizens'}->{$citizens_type[$i]} = $values[$i];
+        $self->{'cities'}->{$cityId}->{'citizens'}->{total} += $values[$i];
+    }
+}
+
+
+sub checkArmies
+{
+    my $self = shift;
+    my $cityId = shift;
+    my %force_types;
+
+    $force_types{'army'} = [ qw/undef undef Slinger Swordsman Phalanx Ram Archer Catapult Gunsman Mortar SteamGiant Gyrocopter Bombardier Doctor Cook/ ];
+    $force_types{'fleet'} = [ qw/undef undef Ram-Ship BallistaShip Flamethrower CatapultShip MortarShip PaddleWheelRam DivingBoat/ ];
+
+    foreach my $x (qw/army fleet/) {
+        $self->{'cities'}->{$cityId}->{$x} = {};
+        my $res = $self->{mech}->get(sprintf('http://%s/index.php?view=cityMilitary-%s&id=%d', $self->{server}, $x, $cityId));
+        my @numbers = Ikariam::Extractor->new(content => $res->content)->find('//table//tr/td');
+        foreach my $j (0..$#{$force_types{$x}}) {
+            if ($numbers[$j] == '-') {
+                $self->{'cities'}->{$cityId}->{$x}->{$force_types{$x}[$j]} = 0;
+            } else {
+                $self->{'cities'}->{$cityId}->{$x}->{$force_types{$x}[$j]} = $numbers[$j];
+            }
+        }
+    }
 }
 
 sub check
 {
     my $self = shift;
 
+    # MilitaryAdvisor
+    $self->checkMilitaryAdvisorMilitaryMovements();
+    $self->checkMilitaryAdvisorCombatReports();
+
     # looking for cities
     foreach my $cityId (keys(%{$self->{'cities'}})) 
     {
-        # search for goods
-        my $res = $self->{mech}->post(sprintf('http://%s/index.php', $self->{server}), [
-            action => 'header',
-            cityId => $cityId,
-            function => 'changeCurrentCity',
-            id => $cityId,
-            oldView => 'city',
-        ]);
-
-        my $content;
-        gunzip \$res->content => \$content 
-            or die "gunzip failed: $GunzipError\n";
-
-        my $html = HTML::TagParser->new($content);
-        my @elems;
-
-        my ($elem) = $html->getElementsByAttribute("id", "value_gold");
-        $self->{'cities'}->{$cityId}->{resources}->{gold} = $elem->innerText();
-        $self->{'cities'}->{$cityId}->{resources}->{gold} =~ s/,//g;
-
-        # XXX
-        my ($elem) = $html->getElementsByAttribute("class", "city");
-        $self->{'cities'}->{$cityId}->{name} = $elem->innerText();
-
-        my ($elem) = $html->getElementsByAttribute("class", 'constructionSite');
-        $self->{'cities'}->{$cityId}->{construction} = 0;
-        $self->{'cities'}->{$cityId}->{construction} = 1 if(defined($elem));
-
-        # check goods
-        foreach my $good (qw/wood wine marble crystal sulfur/) {
-            my ($elem) = $html->getElementsByAttribute("id", "value_" . $good);
-            $self->{'cities'}->{$cityId}->{resources}->{$good} = $elem->innerText();
-            $self->{'cities'}->{$cityId}->{resources}->{$good} =~ s/,//g;
-        }
-
-        # search locations
-        foreach my $i (0..14) {
-            my ($elem) = $html->getElementsByAttribute("id", "position" . $i);
-            my $building = $elem->getAttribute('class');
-            if (!($building =~ /buildingGround/)) {
-                $self->{'cities'}->{$cityId}->{locations}[$i] = $building;
-                my $span = getElementsByAttribute($elem, "class", "textLabel");
-                my (undef, undef, $level) = split(/ /, $span->innerText());
-                $self->{'cities'}->{$cityId}->{buildings}->{$building} = $level;
-            }
-        }
-
-        # transporters
-        my ($elem) = $html->getElementsByAttribute("class", 'transAvail');
-        $self->{'cities'}->{$cityId}->{transporters}->{avail} = $elem->innerText();
-        my ($elem) = $html->getElementsByAttribute("class", 'transSum');
-        $self->{'cities'}->{$cityId}->{transporters}->{sum} = $elem->innerText();
-        $self->{'cities'}->{$cityId}->{transporters}->{sum} =~ s/\(//;
-        $self->{'cities'}->{$cityId}->{transporters}->{sum} =~ s/\)//;
-
-        $res = $self->{mech}->get(sprintf('http://%s/index.php?view=militaryAdvisorMilitaryMovements', $self->{server}, $cityId));
-        gunzip \$res->content => \$content 
-            or die "gunzip failed: $GunzipError\n";
-
-        $self->{'cities'}->{$cityId}->{force}->{attacks} = $1 if($content =~ /敵人攻擊: (\d+)/);
-        $self->{'cities'}->{$cityId}->{force}->{wars} = $1 if($content =~ /我方軍隊行程: (\d+)/);
-        # if($content =~ /更新戰鬥報告: (\d+)/);
-        # if($content =~ /新的戰鬥報告: (\d+)/);
-
-        # sub checkTownHall {
-        $res = $self->{mech}->get(sprintf('http://%s/index.php?view=townHall&id=%d', $self->{server}, $cityId));
-        gunzip \$res->content => \$content 
-            or die "gunzip failed: $GunzipError\n";
-
-        # check happiness
-        # Happiness = Basic bonuses (196 + Capital Bonus + Holiday Bonus(25)) + 
-        #             Wine (Tavern Base(12*level) + 
-        #             Tavern Bonus(80*step)) + 
-        #             Culture (Museum Base(20*level) + 
-        #             Cultural Goods Bonus(50*Cultural Goods)) - 
-        #             Population (population) - 
-        #             Corruption (Corruption rate * population)
-        #
-        # Growth Rate = Happiness * 0.02
-        $html = HTML::TagParser->new($content);
-
-        my @happiness = ("ecstatic", "happy", "neutral", "sad", "outraged");
-        foreach my $j (0..$#happiness) {
-            my ($elem) = $html->getElementsByAttribute("class", sprintf("happiness happiness_%s", $happiness[$j]));
-            if(defined($elem)) {
-                $self->{'cities'}->{$cityId}->{happiness} = $j;
-                $self->{'cities'}->{$cityId}->{happiness_text} = $happiness[$j];
-            }
-        }
-
-        # Space, 房屋數
-        $self->{'cities'}->{$cityId}->{"space"} = {};
-        foreach my $j (qw/occupied total/) {
-            my ($elem) = $html->getElementsByAttribute("class", sprintf("value %s", $j));
-            if(defined($elem)) {
-                $self->{'cities'}->{$cityId}->{"space"}->{$j} = $elem->innerText();
-            }
-        }
-
-        # Actions
-        # <span id="value_maxActionPoints">1</span>
-
-        # <li class="incomegold incomegold_negative">
-        # <li class="incomegold incomegold_positive">
-        # <span class="value">-178</span>
-
-        my ($elem) = $html->getElementsByAttribute("title", "目前腐敗程度");
-        if(defined($elem)) {
-            $self->{'cities'}->{$cityId}->{corruption} = $elem->innerText();
-            $self->{'cities'}->{$cityId}->{corruption} =~ s/%//g;
-        }
-
-        # countCiizens
-        my @citizens_type = qw/citizens woodworkers specialworkers scientists/;
-        @elems = $html->getElementsByAttribute('class', 'count');
-        $self->{'cities'}->{$cityId}->{'citizens'} = {};
-        $self->{'cities'}->{$cityId}->{'citizens'}->{total} = 0;
-
-        foreach my $i (0..$#citizens_type)
-        {
-            $self->{'cities'}->{$cityId}->{'citizens'}->{$citizens_type[$i]} = $elems[$i]->innerText();
-            $self->{'cities'}->{$cityId}->{'citizens'}->{total} += $elems[$i]->innerText();;
-        }
-
-        # } 
-        
+        $self->checkCity($cityId);
+        $self->checkTownHall($cityId);
+        $self->checkArmies($cityId);
         $self->{'cities'}->{$cityId}->{'research'} = $self->checkResearch($cityId);
-
-        # sub checkArmies {
-        my %force_types;
-        $force_types{'army'} = [ qw/undef undef Slinger Swordsman Phalanx Ram Archer Catapult Gunsman Mortar SteamGiant Gyrocopter Bombardier Doctor Cook/ ];
-        $force_types{'fleet'} = [ qw/undef undef Ram-Ship BallistaShip Flamethrower CatapultShip MortarShip PaddleWheelRam DivingBoat/ ];
-        foreach my $x (qw/army fleet/)
-        {
-            $self->{'cities'}->{$cityId}->{$x} = {};
-            # search army
-            $res = $self->{mech}->get(sprintf('http://%s/index.php?view=cityMilitary-%s&id=%d', $self->{server}, $x, $cityId));
-            gunzip \$res->content => \$content 
-                or die "gunzip failed: $GunzipError\n";
-
-            $html = HTML::TagParser->new($content);
-            @elems = $html->getElementsByTagName('td');
-            foreach my $j (0..$#{$force_types{$x}}) {
-                next if($force_types{$x}[$j] eq 'undef');
-                if($elems[$j]->innerText() == '-') {
-                    $self->{'cities'}->{$cityId}->{$x}->{$force_types{$x}[$j]} = 0;
-                } else {
-                    $self->{'cities'}->{$cityId}->{$x}->{$force_types{$x}[$j]} = $elems[$j]->innerText();
-                }
-            }
-        }
     }
-    # print Dumper($self->{'cities'});
+    print Dumper($self->{'cities'});
     return $self->{'cities'};
 }
 
@@ -544,6 +620,7 @@
         name => $self->{user},
         password => $self->{pass},
         ]);
+
     my $c;
     my $status = gunzip \$res->content => \$c 
         or die "gunzip failed: $GunzipError\n";
@@ -552,14 +629,14 @@
     {
         die ("password error\n");
     } else {
-        my $html = HTML::TagParser->new($c);
-        my @elems;
-        
-        # XXX
-        @elems = $html->getElementsByAttribute("class", "avatarCities coords");
-        foreach my $elem (@elems) {
-            # my cities
-            $self->{'cities'}->{$elem->getAttribute('value')} = {};
+        my $result = XML::LibXML->new('1.0','utf-8')
+                                 ->parse_html_string ($c, { suppress_errors => 1 })
+                                 ->find( '//option[@class="avatarCities coords"]' );
+
+        if ( $result->isa( 'XML::LibXML::NodeList' ) ) {
+            foreach ( @$result ) {
+                $self->{'cities'}->{$_->getAttribute('value')} = {};
+            }
         }
     }
 }