from OFS import SimpleItem, Folder
import Globals
import Acquisition
from Persistence import Persistent
from Globals import PersistentMapping
from AccessControl import ClassSecurityInfo, Permissions
from Products.ZCatalog.CatalogPathAwareness import CatalogPathAware
from Products.PageTemplates.PageTemplateFile import PageTemplateFile
from DateTime import DateTime
import time, os
from helpers import add_and_edit, createRandomId
from Products.ZCTextIndex.ZCTextIndex import ZCTextIndex
from schemaRegistry import schemaRegistry
import oaipmh
from fakeserver import FakeServerProxy

class Dummy:
    pass

class El:
    def __init__(self, **kw):
        self.__dict__.update(kw)
        
class OAIService(Folder.Folder):

    security = ClassSecurityInfo()

    meta_type = 'OAIPMH Service'

    manage_options = (
        {'label':'Edit', 'action':'manage_main'},
        ) + SimpleItem.SimpleItem.manage_options
    
    def __init__(self, id, repository_uri, fake_server_directory=None):
        self.id = id
        self._datestamp = None
        self._sets = []
        self._repository_uri = repository_uri
        self._max_records = None
        self._fulltext_directory = None
        self._fake_server_directory = fake_server_directory

    # MANIPULATORS

    manage_editForm = PageTemplateFile(
        "www/oaiServiceEdit", globals(),
        __name__='manage_editForm')
    manage_main = manage_editForm
    
    security.declareProtected('View management screens', 'manage_edit')
    def manage_edit(self, datestamp, repository_uri, max_records,
                    fulltext_directory, metadata_formats=None, ignore_bad_characters=None, REQUEST=None):
        """Edit settings.
        """
        if datestamp.strip():
            self._datestamp = DateTime(datestamp)
        else:
            self._datestamp = None
        self._repository_uri = repository_uri
        if max_records.strip():
            self._max_records = int(max_records)
        else:
            self._max_records = None
        self._setFulltextDirectory(fulltext_directory)
        if metadata_formats:
            for metadata_prefix in metadata_formats:
                self.createStorage(metadata_prefix)
                # XXX temporarily hardcoded
                schema = self.getSchema(metadata_prefix)
                if schema is not None:
                    self._registerSchema(metadata_prefix, schema)
        if REQUEST is not None:
            return self.manage_main(manage_tabs_message="Changed Settings")

    def _registerSchema(self, id, schema):
        dc = oaipmh.MetadataSchema(id, schema.getNamespaces())
        for field in schema.getFields():
            dc.addFieldDescription(field.id, 'textList', field.getXpath())
        oaipmh.addMetadataSchema(dc)
            
    def _setFulltextDirectory(self, dir):
        """Set the fulltext directory for OAI data.
        """
        # check whether we have a dir and whether it says it is actually
        # an OAI dir by having a 'OAI' file in there, as extra security
        # measure
        if os.path.isdir(dir) and os.path.isfile(os.path.join(dir, 'OAI')):
            self._fulltext_directory = dir
        else:
            self._fulltext_directory = None

    security.declareProtected('View management screens', 'manage_update')
    def manage_update(self):
        """Trigger update.
        """
        self.update(self._datestamp, DateTime())
        return self.manage_main(manage_tabs_message="Harvested records")

    security.declareProtected('View management screens', 'update')
    def update(self, from_datestamp=None, until_datestamp=None):
        """Update by harvesting.
        """
        for metadata_prefix in self.getMetadataFormats():
            self.updateMetadataFormat(metadata_prefix, from_datestamp, until_datestamp)
        self._datestamp = until_datestamp

    security.declareProtected('View management screens', 'updateMetadataFormat')
    def updateMetadataFormat(self, prefix, datestamp, update_datestamp):
        server = self.getServer()
        identify = server.identify()
        granularity = identify.granularity()
        if granularity == 'YYYY-MM-DD':
            use_low_granularity = 1
        else:
            use_low_granularity = 0
        kw = {}
        if datestamp:
            from_ = datestamp.HTML4()
            if use_low_granularity:
                from_ = from_[:10]
            kw['from_'] = from_
        until = update_datestamp.HTML4()
        if use_low_granularity:
            until = until[:10]
        kw['until'] = until
        if self._max_records is not None:
            kw['max'] = self._max_records
        kw['metadataPrefix'] = prefix
        records = server.listRecords(**kw)
        for record in records:
            self.updateRecord(prefix, record, self.getSchema(prefix))
        # store set information
        self._sets = [set for set in server.listSets()]
            
    security.declareProtected('View management screens', 'updateRecord')
    def updateRecord(self, prefix, record, schema):
        # update/create/delete records
        header, metadata, about = record
        identifier = header.identifier()
        local_record = self.getRecordByIdentifier(prefix, identifier)
        if header.isDeleted():
            if local_record is not None:
                self.deleteRecord(local_record)
            return
        if metadata is not None:
            map = metadata.getMap()
            schema.preprocess(map)
            if local_record is None:
                local_record = self.createRecord(
                    prefix, identifier, header.setSpec(), DateTime(
                        header.datestamp()), map)
                return
            local_record._update_metadata(
                DateTime(header.datestamp()),
                map)

    security.declareProtected('View management screens', 'createRecord')
    def createRecord(self, metadata_prefix, identifier, setSpec, datestamp, map):
        """Create a record in the storage.
        Id is generated by using timestamp.
        """
        storage = self.getStorage(metadata_prefix)
        while 1:
            id = createRandomId(9)
            if not hasattr(storage.aq_base, id):
                break
        record = self._recordFactory(id, identifier, setSpec, datestamp, map)
        storage._setObject(id, record)    
        return storage._getOb(id)

    security.declareProtected('View management screens', 'deleteRecord')
    def deleteRecord(self, record):
        record.aq_parent.manage_delObjects([record.id])
    
    def _recordFactory(self, id, identifier, setSpec, datestamp, map):
        """Can be overridden by subclass.
        """
        return Record(id, identifier, setSpec, datestamp, map)

    def _getRecordName(self):
        """Can be overridden by subclass.
        """
        return 'OAIPMH Record'
    
    security.declareProtected('View management screens', 'setRepositoryURI')
    def setRepositoryURI(self, repository_uri):
        self._repository_uri = repository_uri

    security.declareProtected('View management screens', 'manage_cleanup')
    def manage_cleanup(self):
        """Trigger cleanup.
        """
        self.cleanup()
        return self.manage_main(manage_tabs_message="Cleaned up records")
      
    security.declareProtected('View management screens', 'cleanup')
    def cleanup(self):
        """Clean all records from the storage.
        """
        for metadata_prefix in self.getMetadataFormats():
            storage = self.getStorage(metadata_prefix)
            record_name = self._getRecordName()
            ids = [object.id for object in storage.objectValues(
                ) if object.meta_type == record_name]
            storage.manage_delObjects(ids)

    security.declareProtected('View management screens', 'manage_createStorage')
    def manage_createStorage(self, metadata_prefix):
        """Trigger storage creation.
        """
        self.createStorage(metadata_prefix)
        return self.manage_main(
            manage_tabs_message="Created and initialized storage for %s" % (
                metadata_prefix))
                                    
    security.declareProtected('View management screens', 'createStorage')
    def createStorage(self, metadata_prefix):
        """Add a storage to the service for a particular
        metadataformat.
        """
        storage = self.getStorage(metadata_prefix)
        if storage is None:
            # XXX adds in parent, should become add in self when OAIService
            # becomes folderish.
            self.manage_addProduct[
                'BTreeFolder2'].manage_addBTreeFolder(metadata_prefix,'')

    security.declareProtected('View management screens', 'manage_createCatalogs')
    def manage_createCatalogs(self):
        """Trigger catalog creation.
        """
        for metadata_prefix in self.getMetadataFormats():
            self.createCatalog(metadata_prefix)
        return self.manage_main(
            manage_tabs_message="Created and initialized catalogs")
                                    
    security.declareProtected('View management screens', 'createCatalog')
    def createCatalog(self, metadata_prefix):
        """Can be overridden in subclasses to create different indexes.
        """
        catalog = self.getCatalog(metadata_prefix)
        if catalog is None:
            self.getStorage(metadata_prefix).manage_addProduct[
                'ZCatalog'].manage_addZCatalog('service_oai_catalog', '')
            catalog = self.getCatalog(metadata_prefix)
            # XXX ugh, hardcoded dependency on names in ZCTextIndex
            catalog.manage_addProduct['ZCTextIndex'].manage_addLexicon(
                'oai_lexicon', 
                elements=[
                El(group='Case Normalizer', name='Case Normalizer'),
                El(group='Stop Words', name=" Don't remove stop words"),
                El(group='Word Splitter', name="Unicode Whitespace splitter"),
                ]
                )
        # XXX check whether indexes exist?
        catalog.manage_addIndex('datestamp', 'DateIndex')
        catalog.manage_addIndex('identifier', 'FieldIndex')
        catalog.manage_addIndex('meta_type', 'FieldIndex')
        catalog.manage_addIndex('setSpec', 'KeywordIndex')     
        catalog.manage_addColumn('id')
        
        schema = self.getSchema(metadata_prefix)
        if schema is not None:
            schema.installIndexes(catalog)
        if schema.getFulltextFields() or self._fulltext_directory:
            dummy = Dummy()
            dummy.index_type = 'Okapi BM25 Rank'
            dummy.lexicon_id = 'oai_lexicon'
            catalog.manage_addProduct['ZCTextIndex'].manage_addZCTextIndex(
                'fulltext', extra=dummy)
        
    # ACCESSORS

    security.declarePublic('getStorage')
    def getStorage(self, id):
        """Get a storage by id.
        """
        try:
            return self.aq_inner._getOb(id)
        except AttributeError:
            pass
        return None
        
    security.declarePublic('getCatalog')
    def getCatalog(self, metadata_prefix):
        """Get the catalog.
        """
        storage = self.getStorage(metadata_prefix)
        if storage == None:
            return None
        if not hasattr(storage.aq_base, 'service_oai_catalog'):
            return None
        return storage.service_oai_catalog
    
    security.declarePublic('getServer')
    def getServer(self):
        """Get the OAI server.
        """
        if self._fake_server_directory is not None:
            return FakeServerProxy(self._fake_server_directory)
        server = oaipmh.ServerProxy(self._repository_uri)
        server.ignoreBadCharacters(1)
        return server
    
    security.declarePublic('getSets')
    def getSets(self):
        """Get list of all sets.
        """
        return self._sets

    security.declarePublic('getServerMetadataFormats')
    def getServerMetadataFormats(self):
        """Get a list of supported metadataformats.
        """
        return [metadata_prefix for metadata_prefix, schema, ns in self.getServer().listMetadataFormats()]

    security.declarePublic('getMetadataFormats')
    def getMetadataFormats(self):
        """Get a list of supported metadataformats.
        """
        return self.aq_inner.objectIds('BTreeFolder2')
    
    security.declarePublic('getSchema')
    def getSchema(self, metadataformat):
        return schemaRegistry.get(metadataformat)
    
    def getSubSets(self, parent_id):
        """Get all set keys and descriptions below parent_id as a dictionary.

        If parent_id is None, get top level sets.
        """
        sets = self.getSets()
        result = {}
        if parent_id is None:
            for setSpec, name, about in sets:
                if setSpec.find(':') == -1:
                    result[setSpec] = name
            return result
        for setSpec, name, about in sets:
            if setSpec.startswith(parent_id + ':'):
                result[setSpec] = name
        return result

    security.declarePublic('getSetName')
    def getSetName(self, id):
        """Get name of given set id.
        """
        for setSpec, name, about in self.getSets():
            if setSpec == id:
                return name
        return None

    security.declarePublic('getRepositoryURI')
    def getRepositoryURI(self):
        """Get url of the OAI repository
        """
        return self._repository_uri

    security.declarePublic('getRecordByIdentifier')
    def getRecordByIdentifier(self, metadata_prefix, identifier):
        """Get record based on identifier.
        """
        result = self.getCatalog(metadata_prefix).searchResults(
            meta_type=self._getRecordName(), identifier=identifier)
        if result:
            return result[0].getObject()
        else:
            return None

    security.declarePublic('getDatestamp')
    def getDatestamp(self):
        """Get datestamp of last update.
        """
        return self._datestamp

    security.declarePublic('getISODatestamp')
    def getISODatestamp(self):
        """Get ISO datestamp of last update.
        """
        if self._datestamp is not None:
            return self._datestamp.HTML4()
        else:
            return None
    
    security.declarePublic('getMaxRecords')
    def getMaxRecords(self):
        """Get max records to update in one go.
        """
        return self._max_records

    security.declarePublic('getIdentifiers')
    def getIdentifiers(self):
        """Get all identifiers (may be handle urls) in storage.
        """
        result = self.getCatalog().searchResults(
            meta_type=self._getRecordName())
        return '\n'.join([item.getObject().identifier() for item in result])

    security.declarePublic('getFulltextDirectory')
    def getFulltextDirectory(self):
        """Return fulltext directory on filesystem.
        """
        return getattr(self.aq_base, '_fulltext_directory', None)

    security.declarePublic('getTTWFulltextDirectory')
    def getTTWFulltextDirectory(self):
        """Return fulltext directory on filesystem to a http request. Can't
        return None, because that's a http-error, so we return .
        """
        fTDir = getattr(self.aq_base, '_fulltext_directory', None)
        if fTDir is not None:
            return fTDir
        else:
            return "no directory specified"

    security.declarePublic('getTTWFulltextDirectory')
    def getTTWISODatestamp(self):
        """Return ISO Date to a http request. Can't
        return None, because that's a http-error, so we return .
        """
        date = getattr(self.aq_base, '_datestamp', None)
        if date is not None:
            return date.HTML4()
        else:
            return "no date specified"
        
Globals.InitializeClass(OAIService)

manage_addOAIServiceForm = PageTemplateFile(
    "www/oaiServiceAdd", globals(),
    __name__='manage_addOAIServiceForm')

def manage_addOAIService(self, id, repository_uri, fake_server_directory=None, REQUEST=None):
    """Add OAI Service.
    """
    object = OAIService(id, repository_uri, fake_server_directory)
    self._setObject(id, object)
    add_and_edit(self, id, REQUEST)
    return ''

class Record(CatalogPathAware, SimpleItem.SimpleItem, Acquisition.Implicit):
    """An OAI Record.

    A resouce of metadata as gathered by the metadata harvester.
    """
    #__implements__ = IRecord

    security = ClassSecurityInfo()

    meta_type = 'OAIPMH Record'
    default_catalog = 'service_oai_catalog'

    manage_options = (
        {'label':'Overview', 'action':'manage_metadataForm'},
        {'label':'Fulltext', 'action':'manage_fulltextForm'},
        ) + SimpleItem.Item.manage_options
    
    def __init__(self, id, identifier, setSpec, datestamp, map):
        self.id = id
        self.title = identifier
        self._identifier = identifier
        self._setSpec = tuple(setSpec)
        self._datestamp = datestamp
        if map is not None:
            for key, value in map.items():
                if key[:9] == "metadata_":
                    setattr(self, key, value)
            
    # MANIPULATORS

    def _update_metadata(self, datestamp, map):
        
        self._datestamp = datestamp
        for key, value in map.items():
            if key[:9] == "metadata_":
                setattr(self, key, value)
        self.reindex_object()

    def initContentType(self):
        """Initialize content type.
        XXX hack to support page template.
        """
        self.REQUEST.RESPONSE.setHeader(
            'Content-Type',
            'text/html;charset=utf-8')
        
    # ACCESSORS
    security.declareProtected('View management screens', 'manage_metadataForm')
    manage_metadataForm = PageTemplateFile(
        "www/oaiRecordMetadata", globals(),
        __name__='manage_metadataForm')

    security.declareProtected('View management screens', 'manage_fulltextForm')
    manage_fulltextForm = PageTemplateFile(
        "www/oaiRecordFulltext", globals(),
        __name__="manage_fulltextForm")
    
    security.declarePublic('identifier')
    def identifier(self):
        """Get identifier.
        """
        return self._identifier

    security.declarePublic('setSpec')
    def setSpec(self):
        """Get setSpec tuple.
        """
        return self._setSpec

    security.declarePublic('datestamp')
    def datestamp(self):
        """Get datestamp of last updated metadata.
        """
        return self._datestamp
    
    security.declarePublic('getFieldNames')
    def getFieldNames(self):
        names = ['datestamp', 'setSpec', 'identifier']
        schema = self.getSchema()
        if schema is not None:
            for field in schema.getFieldsAndSortables():
                names.append(field.id)
        names.sort()
        return names
    
        security.declarePublic('getField')

    security.declarePublic('getFieldText')
    def getFieldText(self, name):
         """Get field as flattened text.
         """
         if name == "datestamp":
             return self.datestamp()
         if name == "setSpec":
             return self.setSpec()
         if name == "identifier":
             return self.identifier()
         data = None
         if hasattr(self.aq_base, name):
             data = getattr(self.aq_base, name)
             if type(data) == type([]):
                 data = ' '.join(data)
         return data
        
    security.declarePublic('getDocumentUrls')
    def getDocumentUrls(self):
        """Get the urls pointing to the actual documents from filesystem.
        """

        path = self._getFulltextPath()
        if path is None:
            return []
        f = open(path, 'r')
        uris = []
        while 1:
            line = unicode(f.readline(), 'latin1', 'ignore')
            if line.find('|FULLTEXT|') != -1:
                return uris
            uris.append(line.strip())
        else:
            # couldn't find any FULLTEXT reference, so nothing there
            return []

    security.declarePublic('getDocumentFulltext')
    def getDocumentFulltext(self):
        """Get list of fulltext data for each document from filesystem.
        """
        path = self._getFulltextPath()
        if path is None:
            return []
        f = open(path, 'r')
        data = unicode(f.read(), 'latin1', 'ignore')
        f.close()
        i = data.find('|FULLTEXT|')
        if i == -1:
            return []
        # get part containing actual document data
        document_data = data[i + len('|FULLTEXT|'):]
        # now get individual documents and add them (first entry is always
        # empty string so skip that)
        return document_data.split('|DOCUMENT|')[1:]
    
    security.declarePublic('getDocumentInfo')
    def getDocumentInfo(self):
        """Get a list of url, fulltext dicts.
        """
        urls = self.getDocumentUrls()
        fulltexts = self.getDocumentFulltext()
        result = []
        for url, fulltext in zip(urls, fulltexts):
            d = {
                'url': url,
                'fulltext': fulltext
                }
            result.append(d)
        return result

    security.declarePublic('fulltext')
    def fulltext(self):
        """Fulltext body.
        """
        result = []
        schema = self.getSchema()
        if schema is not None:
            for field in schema.getFulltextFields():
                value = getattr(self, field)
                if value:
                    if type(value) == type([]):
                        result.extend(value)
                    else:
                        result.append(value)
        result.extend(self.getDocumentFulltext())
        return result
    
    def _getFulltextPath(self):
        """Path to filesystem file containing information.
        Return None if no such path can be found.
        """
        filename = self.identifier()[4:].replace('/', '-')
        directory =  self.aq_inner.aq_parent.aq_parent.getFulltextDirectory()
        if directory is None:
            return None
        path = os.path.join(directory, filename)
        if not os.path.isfile(path):
            return None
        return path

    def metadata_prefix(self):
        return self.aq_inner.aq_parent.id

    def getSchema(self):
        return self.aq_inner.aq_parent.aq_parent.getSchema(self.metadata_prefix())
