import urllib2
import urlparse
import oaipmh
import nuxconvert
import os

def getDocumentUrl(url):
    """Lift url to document from dspace generate page.
    """
    f = urllib2.urlopen(url)
    html = f.read()
    url = f.geturl() # get url of actual location, may have redirected
    f.close()
    j = html.find('">View/Open')
    if j == -1:
        return None
    i = html.rfind('"', 0, j)
    if i == -1:
        return None
    i += 1
    found = html[i:j]
    found_tuple = urlparse.urlparse(found)
    if found_tuple[0] == '' and found_tuple[1] == '':
        url_tuple = urlparse.urlparse(url)
        new_tuple = url_tuple[:2] + found_tuple[2:]
        return urlparse.urlunparse(new_tuple)
    else:
        return found

def scrapeUrl(url):
    """Scrape doc from url and return textual representation.
    """
    doc_url = getDocumentUrl(url)
    if doc_url is None:
        print "error:", url
        return
    doc = urllib2.urlopen(doc_url)
    data = doc.read()
    doc.close()
    converter = nuxconvert.getConverter(data)
    return converter.getText()

def scrapeOAI(oai_url, path):
    """Scrape all documents indicated by OAI repository. Note,
    getIdentifiers must spit out handle urls.. Dump text in path.
    """
    server = oaipmh.ServerProxy(oai_url)
    for header in server.listIdentifiers(metadataPrefix='oai_dc'):
        hdl = header.identifier()[4:] # strip off 'hdl:' part
        text = scrapeUrl('http://hdl.handle.net/' + hdl)
	if text is None:
	    continue 
        filename = makeFilename(hdl)
        f = open(os.path.join(path, filename), 'wb')
        f.write(text)
        f.close()
    
def makeFilename(hdl):
    return hdl.replace('/', '-')
    
def main():
    scrapeOAI('http://hpds1.mit.edu/oai/',
              '/tmp/dump')

if __name__ == '__main__':
    main()
    
