import urllib, htmlentitydefs, re, os

## stolen code from http://effbot.org/librarybook/htmlentitydefs.htm
pattern = re.compile("&(\w+?);")

def descape_entity(m, defs=htmlentitydefs.entitydefs):
    # callback: translate one entity to its ISO Latin value
    try:
        return defs[m.group(1)]
    except KeyError:
        return m.group(0) # use as is

def descape(string):
    return pattern.sub(descape_entity, string)
## end stolen code


def fetch_seen(num):
    url = "http://baka-tsuki.net/project/index.php?title=Clannad:SEEN"+num
    page = urllib.urlopen(url)
    text = page.read()
    start = text.find("<pre>") + 5
    end = text.find("</pre>", start)
    print start, end
    text = text[start:end]
    text = descape(text)
    text = text.replace("\\b", "").replace("\\u", "")
    f = open("seen_new/Seen"+num+".utf", "w")
    f.write(text)
    f.close()

#os.chdir("C:/overflow/clannad_patch")
for x in os.listdir("seen_src"):
    if ".utf" in x:# and x not in os.listdir("seen.new"):
        try:
            fetch_seen(x[4:8])
        except Exception, e:
            print e