# -*- coding: utf-8 -*-
import datetime
import re
import sys

from .._globals import IDENTITY, GLOBAL_LOCKER
from .._compat import PY2, integer_types, basestring
from ..connection import ConnectionPool
from ..objects import Field, Query, Expression
from ..helpers.classes import SQLALL
from ..helpers.methods import use_common_filters
from .base import NoSQLAdapter

long = integer_types[-1]


class IMAPAdapter(NoSQLAdapter):

    """ IMAP server adapter

    This class is intended as an interface with
    email IMAP servers to perform simple queries in the
    web2py DAL query syntax, so email read, search and
    other related IMAP mail services (as those implemented
    by brands like Google(r), and Yahoo!(r)
    can be managed from web2py applications.

    The code uses examples by Yuji Tomita on this post:
    http://yuji.wordpress.com/2011/06/22/python-imaplib-imap-example-with-gmail/#comment-1137
    and is based in docs for Python imaplib, python email
    and email IETF's (i.e. RFC2060 and RFC3501)

    This adapter was tested with a small set of operations with Gmail(r). Other
    services requests could raise command syntax and response data issues.

    It creates its table and field names "statically",
    meaning that the developer should leave the table and field
    definitions to the DAL instance by calling the adapter's
    .define_tables() method. The tables are defined with the
    IMAP server mailbox list information.

    .define_tables() returns a dictionary mapping dal tablenames
    to the server mailbox names with the following structure:

    {<tablename>: str <server mailbox name>}

    Here is a list of supported fields:

    ===========   ============== ===========
    Field         Type           Description
    ===========   ============== ===========
    uid           string
    answered      boolean        Flag
    created       date
    content       list:string    A list of dict text or html parts
    to            string
    cc            string
    bcc           string
    size          integer        the amount of octets of the message*
    deleted       boolean        Flag
    draft         boolean        Flag
    flagged       boolean        Flag
    sender        string
    recent        boolean        Flag
    seen          boolean        Flag
    subject       string
    mime          string         The mime header declaration
    email         string         The complete RFC822 message (*)
    attachments   list           Each non text part as dict
    encoding      string         The main detected encoding
    ===========   ============== ===========

    (*) At the application side it is measured as the length of the RFC822
    message string

    WARNING: As row id's are mapped to email sequence numbers,
    make sure your imap client web2py app does not delete messages
    during select or update actions, to prevent
    updating or deleting different messages.
    Sequence numbers change whenever the mailbox is updated.
    To avoid this sequence numbers issues, it is recommended the use
    of uid fields in query references (although the update and delete
    in separate actions rule still applies).
    ::

        # This is the code recommended to start imap support
        # at the app's model:

        imapdb = DAL("imap://user:password@server:port", pool_size=1) # port 993 for ssl
        imapdb.define_tables()

    Here is an (incomplete) list of possible imap commands::

        # Count today's unseen messages
        # smaller than 6000 octets from the
        # inbox mailbox

        q = imapdb.INBOX.seen == False
        q &= imapdb.INBOX.created == datetime.date.today()
        q &= imapdb.INBOX.size < 6000
        unread = imapdb(q).count()

        # Fetch last query messages
        rows = imapdb(q).select()

        # it is also possible to filter query select results with limitby and
        # sequences of mailbox fields

        set.select(<fields sequence>, limitby=(<int>, <int>))

        # Mark last query messages as seen
        messages = [row.uid for row in rows]
        seen = imapdb(imapdb.INBOX.uid.belongs(messages)).update(seen=True)

        # Delete messages in the imap database that have mails from mr. Gumby

        deleted = 0
        for mailbox in imapdb.tables
            deleted += imapdb(imapdb[mailbox].sender.contains("gumby")).delete()

        # It is possible also to mark messages for deletion instead of ereasing them
        # directly with set.update(deleted=True)


        # This object give access
        # to the adapter auto mailbox
        # mapped names (which native
        # mailbox has what table name)

        imapdb.mailboxes <dict> # tablename, server native name pairs

        # To retrieve a table native mailbox name use:
        imapdb.<table>.mailbox

        ### New features v2.4.1:

        # Declare mailboxes statically with tablename, name pairs
        # This avoids the extra server names retrieval

        imapdb.define_tables({"inbox": "INBOX"})

        # Selects without content/attachments/email columns will only
        # fetch header and flags

        imapdb(q).select(imapdb.INBOX.sender, imapdb.INBOX.subject)

    """
    drivers = ('imaplib',)
    types = {
            'string': str,
            'text': str,
            'date': datetime.date,
            'datetime': datetime.datetime,
            'id': long,
            'boolean': bool,
            'integer': int,
            'bigint': long,
            'blob': str,
            'list:string': str
        }

    dbengine = 'imap'

    REGEX_URI = re.compile('^(?P<user>[^:]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?$')

    def __init__(self,
                 db,
                 uri,
                 pool_size=0,
                 folder=None,
                 db_codec ='UTF-8',
                 credential_decoder=IDENTITY,
                 driver_args={},
                 adapter_args={},
                 do_connect=True,
                 after_connection=None):

        super(IMAPAdapter, self).__init__(
            db=db,
            uri=uri,
            pool_size=pool_size,
            folder=folder,
            db_codec=db_codec,
            credential_decoder=credential_decoder,
            driver_args=driver_args,
            adapter_args=adapter_args,
            do_connect=do_connect,
            after_connection=after_connection)

        # db uri: user@example.com:password@imap.server.com:123
        # TODO: max size adapter argument for preventing large mail transfers

        if do_connect: self.find_driver(adapter_args)
        self.credential_decoder = credential_decoder
        self.driver_args = driver_args
        self.adapter_args = adapter_args
        self.mailbox_size = None
        self.static_names = None
        self.charset = sys.getfilesystemencoding()
        # imap class
        self.imap4 = None
        uri = uri.split("://")[1]

        """ MESSAGE is an identifier for sequence number"""

        self.flags = {'deleted': '\\Deleted', 'draft': '\\Draft',
                      'flagged': '\\Flagged', 'recent': '\\Recent',
                      'seen': '\\Seen', 'answered': '\\Answered'}
        self.search_fields = {
            'id': 'MESSAGE', 'created': 'DATE',
            'uid': 'UID', 'sender': 'FROM',
            'to': 'TO', 'cc': 'CC',
            'bcc': 'BCC', 'content': 'TEXT',
            'size': 'SIZE', 'deleted': '\\Deleted',
            'draft': '\\Draft', 'flagged': '\\Flagged',
            'recent': '\\Recent', 'seen': '\\Seen',
            'subject': 'SUBJECT', 'answered': '\\Answered',
            'mime': None, 'email': None,
            'attachments': None
            }

        db['_lastsql'] = ''

        m = self.REGEX_URI.match(uri)
        user = m.group('user')
        password = m.group('password')
        host = m.group('host')
        port = int(m.group('port'))
        over_ssl = False
        if port==993:
            over_ssl = True

        driver_args.update(host=host,port=port, password=password, user=user)
        def connector(driver_args=driver_args):
            # it is assumed sucessful authentication alLways
            # TODO: support direct connection and login tests
            if over_ssl:
                self.imap4 = self.driver.IMAP4_SSL
            else:
                self.imap4 = self.driver.IMAP4
            connection = self.imap4(driver_args["host"], driver_args["port"])
            data = connection.login(driver_args["user"], driver_args["password"])

            # static mailbox list
            connection.mailbox_names = None

            # dummy dbapi functions
            connection.cursor = lambda : self.fake_cursor
            connection.close = lambda : None
            connection.commit = lambda : None

            return connection

        self.db.define_tables = self.define_tables
        self.connector = connector
        if do_connect: self.reconnect()

    def reconnect(self, f=None):
        """
        IMAP4 Pool connection method

        imap connection lacks of self cursor command.
        A custom command should be provided as a replacement
        for connection pooling to prevent uncaught remote session
        closing

        """
        if getattr(self, 'connection', None) is not None:
            return
        if f is None:
            f = self.connector

        if not self.pool_size:
            self.connection = f()
            self.cursor = self.connection.cursor()
        else:
            POOLS = ConnectionPool.POOLS
            uri = self.uri
            while True:
                GLOBAL_LOCKER.acquire()
                if not uri in POOLS:
                    POOLS[uri] = []
                if POOLS[uri]:
                    self.connection = POOLS[uri].pop()
                    GLOBAL_LOCKER.release()
                    self.cursor = self.connection.cursor()
                    if self.cursor and self.check_active_connection:
                        try:
                            # check if connection is alive or close it
                            result, data = self.connection.list()
                        except:
                            # Possible connection reset error
                            # TODO: read exception class
                            self.connection = f()
                    break
                else:
                    GLOBAL_LOCKER.release()
                    self.connection = f()
                    self.cursor = self.connection.cursor()
                    break
        self.after_connection_hook()

    def get_last_message(self, tablename):
        last_message = None
        # request mailbox list to the server if needed.
        if not isinstance(self.connection.mailbox_names, dict):
            self.get_mailboxes()
        try:
            result = self.connection.select(
                self.connection.mailbox_names[tablename])
            last_message = int(result[1][0])
            # Last message must be a positive integer
            if last_message == 0:
                last_message = 1
        except (IndexError, ValueError, TypeError, KeyError):
            e = sys.exc_info()[1]
            self.db.logger.debug("Error retrieving the last mailbox" +
                         " sequence number. %s" % str(e))
        return last_message

    def get_uid_bounds(self, tablename):
        if not isinstance(self.connection.mailbox_names, dict):
            self.get_mailboxes()
        # fetch first and last messages
        # return (first, last) messages uid's
        last_message = self.get_last_message(tablename)
        result, data = self.connection.uid("search", None, "(ALL)")
        uid_list = data[0].strip().split()
        if len(uid_list) <= 0:
            return None
        else:
            return (uid_list[0], uid_list[-1])

    def convert_date(self, date, add=None, imf=False):
        if add is None:
            add = datetime.timedelta()
        """ Convert a date object to a string
        with d-Mon-Y style for IMAP or the inverse
        case

        add <timedelta> adds to the date object
        """
        months = [None, "JAN","FEB","MAR","APR","MAY","JUN",
                  "JUL", "AUG","SEP","OCT","NOV","DEC"]
        if isinstance(date, basestring):
            # Prevent unexpected date response format
            try:
                if "," in date:
                    dayname, datestring = date.split(",")
                else:
                    dayname, datestring = None, date
                date_list = datestring.strip().split()
                year = int(date_list[2])
                month = months.index(date_list[1].upper())
                day = int(date_list[0])
                hms = list(map(int, date_list[3].split(":")))
                return datetime.datetime(year, month, day,
                    hms[0], hms[1], hms[2]) + add
            except (ValueError, AttributeError, IndexError) as e:
                self.db.logger.error("Could not parse date text: %s. %s" %
                             (date, e))
                return None
        elif isinstance(date, (datetime.date, datetime.datetime)):
            if imf: date_format = "%a, %d %b %Y %H:%M:%S %z"
            else: date_format = "%d-%b-%Y"
            return (date + add).strftime(date_format)
        else:
            return None

    @staticmethod
    def header_represent(f, r):
        from email.header import decode_header
        text, encoding = decode_header(f)[0]
        if encoding:
            text = text.decode(encoding).encode('utf-8')
        return text

    def encode_text(self, text, charset, errors="replace"):
        """ convert text for mail to unicode"""
        if text is None:
            text = ""
        if PY2:
            if isinstance(text, str):
                if charset is None:
                    text = unicode(text, "utf-8", errors)
                else:
                    text = unicode(text, charset, errors)
            else:
                raise Exception("Unsupported mail text type %s" % type(text))
            return text.encode("utf-8")
        else:
            if isinstance(text, bytes):
                return text.decode("utf-8")
            return text

    def get_charset(self, message):
        charset = message.get_content_charset()
        return charset

    def get_mailboxes(self):
        """ Query the mail database for mailbox names """
        if self.static_names:
            # statically defined mailbox names
            self.connection.mailbox_names = self.static_names
            return self.static_names.keys()

        mailboxes_list = self.connection.list()
        self.connection.mailbox_names = dict()
        mailboxes = list()
        x = 0
        for item in mailboxes_list[1]:
            x = x + 1
            item = item.strip()
            if not "NOSELECT" in item.upper():
                sub_items = item.split("\"")
                sub_items = [sub_item for sub_item in sub_items \
                if len(sub_item.strip()) > 0]
                # mailbox = sub_items[len(sub_items) -1]
                mailbox = sub_items[-1].strip()
                # remove unwanted characters and store original names
                # Don't allow leading non alphabetic characters
                mailbox_name = re.sub('^[_0-9]*', '', re.sub('[^_\w]','',re.sub('[/ ]','_',mailbox)))
                mailboxes.append(mailbox_name)
                self.connection.mailbox_names[mailbox_name] = mailbox

        return mailboxes

    def get_query_mailbox(self, query):
        nofield = True
        tablename = None
        attr = query
        while nofield:
            if hasattr(attr, "first"):
                attr = attr.first
                if isinstance(attr, Field):
                    return attr.tablename
                elif isinstance(attr, Query):
                    pass
                else:
                    return None
            else:
                return None
        return tablename

    def is_flag(self, flag):
        if self.search_fields.get(flag, None) in self.flags.values():
            return True
        else:
            return False

    def define_tables(self, mailbox_names=None):
        """
        Auto create common IMAP fileds

        This function creates fields definitions "statically"
        meaning that custom fields as in other adapters should
        not be supported and definitions handled on a service/mode
        basis (local syntax for Gmail(r), Ymail(r)

        Returns a dictionary with tablename, server native mailbox name
        pairs.
        """
        if mailbox_names:
            # optional statically declared mailboxes
            self.static_names = mailbox_names
        else:
            self.static_names = None
        if not isinstance(self.connection.mailbox_names, dict):
            self.get_mailboxes()

        names = self.connection.mailbox_names.keys()

        for name in names:
            self.db.define_table("%s" % name,
                Field("uid", writable=False),
                Field("created", "datetime", writable=False),
                Field("content", "text", writable=False),
                Field("to", writable=False),
                Field("cc", writable=False),
                Field("bcc", writable=False),
                Field("sender", writable=False),
                Field("size", "integer", writable=False),
                Field("subject", writable=False),
                Field("mime", writable=False),
                Field("email", "text", writable=False, readable=False),
                Field("attachments", "text", writable=False, readable=False),
                Field("encoding", writable=False),
                Field("answered", "boolean"),
                Field("deleted", "boolean"),
                Field("draft", "boolean"),
                Field("flagged", "boolean"),
                Field("recent", "boolean", writable=False),
                Field("seen", "boolean")
                )

            # Set a special _mailbox attribute for storing
            # native mailbox names
            self.db[name].mailbox = \
                self.connection.mailbox_names[name]

            # decode quoted printable
            self.db[name].to.represent = self.db[name].cc.represent = \
            self.db[name].bcc.represent = self.db[name].sender.represent = \
            self.db[name].subject.represent = self.header_represent

        # Set the db instance mailbox collections
        self.db.mailboxes = self.connection.mailbox_names
        return self.db.mailboxes

    def create_table(self, *args, **kwargs):
        # not implemented
        # but required by DAL
        pass

    def select(self, query, fields, attributes):
        """  Searches and Fetches records and return web2py rows
        """
        # move this statement elsewhere (upper-level)
        if use_common_filters(query):
            query = self.common_filter(query, [self.get_query_mailbox(query),])

        import email
        # get records from imap server with search + fetch
        # convert results to a dictionary
        tablename = None
        fetch_results = list()

        if isinstance(query, Query):
            tablename = self.get_table(query)
            mailbox = self.connection.mailbox_names.get(tablename, None)
            if mailbox is None:
                raise ValueError("Mailbox name not found: %s" % mailbox)
            else:
                # select with readonly
                result, selected = self.connection.select(mailbox, True)
                if result != "OK":
                    raise Exception("IMAP error: %s" % selected)
                self.mailbox_size = int(selected[0])
                search_query = "(%s)" % str(query).strip()
                search_result = self.connection.uid("search", None, search_query)
                # Normal IMAP response OK is assumed (change this)
                if search_result[0] == "OK":
                    # For "light" remote server responses just get the first
                    # ten records (change for non-experimental implementation)
                    # However, light responses are not guaranteed with this
                    # approach, just fewer messages.
                    limitby = attributes.get('limitby', None)
                    messages_set = search_result[1][0].split()
                    # descending order
                    messages_set.reverse()
                    if limitby is not None:
                        # TODO: orderby, asc/desc, limitby from complete message set
                        messages_set = messages_set[int(limitby[0]):int(limitby[1])]

                    # keep the requests small for header/flags
                    if any([(field.name in ["content", "size",
                                            "attachments", "email"]) for
                           field in fields]):
                        imap_fields = "(RFC822 FLAGS)"
                    else:
                        imap_fields = "(RFC822.HEADER FLAGS)"

                    if len(messages_set) > 0:
                        # create fetch results object list
                        # fetch each remote message and store it in memmory
                        # (change to multi-fetch command syntax for faster
                        # transactions)
                        for uid in messages_set:
                            # fetch the RFC822 message body
                            typ, data = self.connection.uid("fetch", uid, imap_fields)
                            if typ == "OK":
                                fr = {"message": int(data[0][0].split()[0]),
                                      "uid": long(uid),
                                      "email": email.message_from_string(data[0][1]),
                                      "raw_message": data[0][1]}
                                fr["multipart"] = fr["email"].is_multipart()
                                # fetch flags for the message
                                if PY2:
                                    fr["flags"] = self.driver.ParseFlags(data[1])
                                else:
                                    fr["flags"] = self.driver.ParseFlags(
                                        bytes(data[1], "utf-8"))
                                fetch_results.append(fr)
                            else:
                                # error retrieving the message body
                                raise Exception("IMAP error retrieving the body: %s" % data)
                else:
                    raise Exception("IMAP search error: %s" % search_result[1])
        elif isinstance(query, (Expression, basestring)):
            raise NotImplementedError()
        else:
            raise TypeError("Unexpected query type")

        imapqry_dict = {}
        imapfields_dict = {}

        if len(fields) == 1 and isinstance(fields[0], SQLALL):
            allfields = True
        elif len(fields) == 0:
            allfields = True
        else:
            allfields = False
        if allfields:
            colnames = ["%s.%s" % (tablename, field) for field in self.search_fields.keys()]
        else:
            colnames = ["%s.%s" % (tablename, field.name) for field in fields]

        for k in colnames:
            imapfields_dict[k] = k

        imapqry_list = list()
        imapqry_array = list()
        for fr in fetch_results:
            attachments = []
            content = []
            size = 0
            n = int(fr["message"])
            item_dict = dict()
            message = fr["email"]
            uid = fr["uid"]
            charset = self.get_charset(message)
            flags = fr["flags"]
            raw_message = fr["raw_message"]
            # Return messages data mapping static fields
            # and fetched results. Mapping should be made
            # outside the select function (with auxiliary
            # instance methods)

            # pending: search flags states trough the email message
            # instances for correct output

            # preserve subject encoding (ASCII/quoted printable)

            if "%s.id" % tablename in colnames:
                item_dict["%s.id" % tablename] = n
            if "%s.created" % tablename in colnames:
                item_dict["%s.created" % tablename] = self.convert_date(message["Date"])
            if "%s.uid" % tablename in colnames:
                item_dict["%s.uid" % tablename] = uid
            if "%s.sender" % tablename in colnames:
                # If there is no encoding found in the message header
                # force utf-8 replacing characters (change this to
                # module's defaults). Applies to .sender, .to, .cc and .bcc fields
                item_dict["%s.sender" % tablename] = message["From"]
            if "%s.to" % tablename in colnames:
                item_dict["%s.to" % tablename] = message["To"]
            if "%s.cc" % tablename in colnames:
                if "Cc" in message.keys():
                    item_dict["%s.cc" % tablename] = message["Cc"]
                else:
                    item_dict["%s.cc" % tablename] = ""
            if "%s.bcc" % tablename in colnames:
                if "Bcc" in message.keys():
                    item_dict["%s.bcc" % tablename] = message["Bcc"]
                else:
                    item_dict["%s.bcc" % tablename] = ""
            if "%s.deleted" % tablename in colnames:
                item_dict["%s.deleted" % tablename] = "\\Deleted" in flags
            if "%s.draft" % tablename in colnames:
                item_dict["%s.draft" % tablename] = "\\Draft" in flags
            if "%s.flagged" % tablename in colnames:
                item_dict["%s.flagged" % tablename] = "\\Flagged" in flags
            if "%s.recent" % tablename in colnames:
                item_dict["%s.recent" % tablename] = "\\Recent" in flags
            if "%s.seen" % tablename in colnames:
                item_dict["%s.seen" % tablename] = "\\Seen" in flags
            if "%s.subject" % tablename in colnames:
                item_dict["%s.subject" % tablename] = message["Subject"]
            if "%s.answered" % tablename in colnames:
                item_dict["%s.answered" % tablename] = "\\Answered" in flags
            if "%s.mime" % tablename in colnames:
                item_dict["%s.mime" % tablename] = message.get_content_type()
            if "%s.encoding" % tablename in colnames:
                item_dict["%s.encoding" % tablename] = charset

            # Here goes the whole RFC822 body as an email instance
            # for controller side custom processing
            # The message is stored as a raw string
            # >> email.message_from_string(raw string)
            # returns a Message object for enhanced object processing
            if "%s.email" % tablename in colnames:
                # WARNING: no encoding performed (raw message)
                item_dict["%s.email" % tablename] = raw_message

            # Size measure as suggested in a Velocity Reviews post
            # by Tim Williams: "how to get size of email attachment"
            # Note: len() and server RFC822.SIZE reports doesn't match
            # To retrieve the server size for representation would add a new
            # fetch transaction to the process
            for part in message.walk():
                maintype = part.get_content_maintype()
                if ("%s.attachments" % tablename in colnames) or \
                   ("%s.content" % tablename in colnames):
                    payload = part.get_payload(decode=True)
                    if payload:
                        filename = part.get_filename()
                        values = {"mime": part.get_content_type()}
                        if ((filename or not "text" in maintype) and
                            ("%s.attachments" % tablename in colnames)):
                            values.update({"payload": payload,
                                "filename": filename,
                                "encoding": part.get_content_charset(),
                                "disposition": part["Content-Disposition"]})
                            attachments.append(values)
                        elif (("text" in maintype) and
                              ("%s.content" % tablename in colnames)):
                            values.update({"text": self.encode_text(payload,
                                               self.get_charset(part))})
                            content.append(values)

                if "%s.size" % tablename in colnames:
                    if part is not None:
                        size += len(str(part))
            item_dict["%s.content" % tablename] = content
            item_dict["%s.attachments" % tablename] = attachments
            item_dict["%s.size" % tablename] = size
            imapqry_list.append(item_dict)

        # extra object mapping for the sake of rows object
        # creation (sends an array or lists)
        for item_dict in imapqry_list:
            imapqry_array_item = list()
            for fieldname in colnames:
                imapqry_array_item.append(item_dict[fieldname])
            imapqry_array.append(imapqry_array_item)

        # parse result and return a rows object
        colnames = colnames
        processor = attributes.get('processor',self.parse)
        return processor(imapqry_array, fields, colnames)

    def insert(self, table, fields):
        def add_payload(message, obj):
            payload = Message()
            encoding = obj.get("encoding", "utf-8")
            if encoding and (encoding.upper() in
                             ("BASE64", "7BIT", "8BIT", "BINARY")):
                payload.add_header("Content-Transfer-Encoding", encoding)
            else:
                payload.set_charset(encoding)
            mime = obj.get("mime", None)
            if mime:
                payload.set_type(mime)
            if "text" in obj:
                payload.set_payload(obj["text"])
            elif "payload" in obj:
                payload.set_payload(obj["payload"])
            if "filename" in obj and obj["filename"]:
                payload.add_header("Content-Disposition",
                    "attachment", filename=obj["filename"])
            message.attach(payload)

        mailbox = table.mailbox
        d = dict(((k.name, v) for k, v in fields))
        date_time = d.get("created") or datetime.datetime.now()
        struct_time = date_time.timetuple()
        if len(d) > 0:
            message = d.get("email", None)
            attachments = d.get("attachments", [])
            content = d.get("content", [])
            flags = " ".join(["\\%s" % flag.capitalize() for flag in
                     ("answered", "deleted", "draft", "flagged",
                      "recent", "seen") if d.get(flag, False)])
            if not message:
                from email.message import Message
                mime = d.get("mime", None)
                charset = d.get("encoding", None)
                message = Message()
                message["from"] = d.get("sender", "")
                message["subject"] = d.get("subject", "")
                message["date"] = self.convert_date(date_time, imf=True)

                if mime:
                    message.set_type(mime)
                if charset:
                    message.set_charset(charset)
                for item in ("to", "cc", "bcc"):
                    value = d.get(item, "")
                    if isinstance(value, basestring):
                        message[item] = value
                    else:
                        message[item] = ";".join([i for i in
                            value])
                if (not message.is_multipart() and
                   (not message.get_content_type().startswith(
                        "multipart"))):
                    if isinstance(content, basestring):
                        message.set_payload(content)
                    elif len(content) > 0:
                        message.set_payload(content[0]["text"])
                else:
                    [add_payload(message, c) for c in content]
                    [add_payload(message, a) for a in attachments]
                message = message.as_string()

            result, data = self.connection.append(mailbox, flags, struct_time, message)
            if result == "OK":
                uid = int(re.findall("\d+", str(data))[-1])
                return self.db(table.uid==uid).select(table.id).first().id
            else:
                raise Exception("IMAP message append failed: %s" % data)
        else:
            raise NotImplementedError("IMAP empty insert is not implemented")

    def update(self, tablename, query, fields):
        # TODO: the adapter should implement an .expand method
        commands = list()
        rowcount = 0
        if use_common_filters(query):
            query = self.common_filter(query, [tablename,])
        mark = []
        unmark = []
        if query:
            for item in fields:
                field = item[0]
                name = field.name
                value = item[1]
                if self.is_flag(name):
                    flag = self.search_fields[name]
                    if (value is not None) and (flag != "\\Recent"):
                        if value:
                            mark.append(flag)
                        else:
                            unmark.append(flag)
            result, data = self.connection.select(
                self.connection.mailbox_names[tablename])
            string_query = "(%s)" % query
            result, data = self.connection.search(None, string_query)
            store_list = [item.strip() for item in data[0].split()
                          if item.strip().isdigit()]
            # build commands for marked flags
            for number in store_list:
                result = None
                if len(mark) > 0:
                    commands.append((number, "+FLAGS", "(%s)" % " ".join(mark)))
                if len(unmark) > 0:
                    commands.append((number, "-FLAGS", "(%s)" % " ".join(unmark)))

        for command in commands:
            result, data = self.connection.store(*command)
            if result == "OK":
                rowcount += 1
            else:
                raise Exception("IMAP storing error: %s" % data)
        return rowcount

    def count(self,query,distinct=None):
        counter = 0
        tablename = self.get_query_mailbox(query)
        if query and tablename is not None:
            if use_common_filters(query):
                query = self.common_filter(query, [tablename,])
            result, data = self.connection.select(self.connection.mailbox_names[tablename])
            string_query = "(%s)" % query
            result, data = self.connection.search(None, string_query)
            store_list = [item.strip() for item in data[0].split() if item.strip().isdigit()]
            counter = len(store_list)
        return counter

    def delete(self, tablename, query):
        counter = 0
        if query:
            if use_common_filters(query):
                query = self.common_filter(query, [tablename,])
            result, data = self.connection.select(self.connection.mailbox_names[tablename])
            string_query = "(%s)" % query
            result, data = self.connection.search(None, string_query)
            store_list = [item.strip() for item in data[0].split() if item.strip().isdigit()]
            for number in store_list:
                result, data = self.connection.store(number, "+FLAGS", "(\\Deleted)")
                if result == "OK":
                    counter += 1
                else:
                    raise Exception("IMAP store error: %s" % data)
            if counter > 0:
                result, data = self.connection.expunge()
        return counter

    def BELONGS(self, first, second):
        result = None
        name = self.search_fields[first.name]
        if name == "MESSAGE":
            values = [str(val) for val in second if str(val).isdigit()]
            result = "%s" % ",".join(values).strip()

        elif name == "UID":
            values = [str(val) for val in second if str(val).isdigit()]
            result = "UID %s" % ",".join(values).strip()

        else:
            raise Exception("Operation not supported")
        # result = "(%s %s)" % (self.expand(first), self.expand(second))
        return result

    def CONTAINS(self, first, second, case_sensitive=False):
        # silently ignore, only case sensitive
        result = None
        name = self.search_fields[first.name]

        if name in ("FROM", "TO", "SUBJECT", "TEXT"):
            result = "%s \"%s\"" % (name, self.expand(second))
        else:
            if first.name in ("cc", "bcc"):
                result = "%s \"%s\"" % (first.name.upper(), self.expand(second))
            elif first.name == "mime":
                result = "HEADER Content-Type \"%s\"" % self.expand(second)
            else:
                raise Exception("Operation not supported")
        return result

    def GT(self, first, second):
        result = None
        name = self.search_fields[first.name]
        if name == "MESSAGE":
            last_message = self.get_last_message(first.tablename)
            result = "%d:%d" % (int(self.expand(second)) + 1, last_message)
        elif name == "UID":
            # GT and LT may not return
            # expected sets depending on
            # the uid format implemented
            try:
                pedestal, threshold = self.get_uid_bounds(first.tablename)
            except TypeError:
                e = sys.exc_info()[1]
                self.db.logger.debug("Error requesting uid bounds: %s", str(e))
                return ""
            try:
                lower_limit = int(self.expand(second)) + 1
            except (ValueError, TypeError):
                e = sys.exc_info()[1]
                raise Exception("Operation not supported (non integer UID)")
            result = "UID %s:%s" % (lower_limit, threshold)
        elif name == "DATE":
            result = "SINCE %s" % self.convert_date(second, add=datetime.timedelta(1))
        elif name == "SIZE":
            result = "LARGER %s" % self.expand(second)
        else:
            raise Exception("Operation not supported")
        return result

    def GE(self, first, second):
        result = None
        name = self.search_fields[first.name]
        if name == "MESSAGE":
            last_message = self.get_last_message(first.tablename)
            result = "%s:%s" % (self.expand(second), last_message)
        elif name == "UID":
            # GT and LT may not return
            # expected sets depending on
            # the uid format implemented
            try:
                pedestal, threshold = self.get_uid_bounds(first.tablename)
            except TypeError:
                e = sys.exc_info()[1]
                self.db.logger.debug("Error requesting uid bounds: %s", str(e))
                return ""
            lower_limit = self.expand(second)
            result = "UID %s:%s" % (lower_limit, threshold)
        elif name == "DATE":
            result = "SINCE %s" % self.convert_date(second)
        else:
            raise Exception("Operation not supported")
        return result

    def LT(self, first, second):
        result = None
        name = self.search_fields[first.name]
        if name == "MESSAGE":
            result = "%s:%s" % (1, int(self.expand(second)) - 1)
        elif name == "UID":
            try:
                pedestal, threshold = self.get_uid_bounds(first.tablename)
            except TypeError:
                e = sys.exc_info()[1]
                self.db.logger.debug("Error requesting uid bounds: %s", str(e))
                return ""
            try:
                upper_limit = int(self.expand(second)) - 1
            except (ValueError, TypeError):
                e = sys.exc_info()[1]
                raise Exception("Operation not supported (non integer UID)")
            result = "UID %s:%s" % (pedestal, upper_limit)
        elif name == "DATE":
            result = "BEFORE %s" % self.convert_date(second)
        elif name == "SIZE":
            result = "SMALLER %s" % self.expand(second)
        else:
            raise Exception("Operation not supported")
        return result

    def LE(self, first, second):
        result = None
        name = self.search_fields[first.name]
        if name == "MESSAGE":
            result = "%s:%s" % (1, self.expand(second))
        elif name == "UID":
            try:
                pedestal, threshold = self.get_uid_bounds(first.tablename)
            except TypeError:
                e = sys.exc_info()[1]
                self.db.logger.debug("Error requesting uid bounds: %s", str(e))
                return ""
            upper_limit = int(self.expand(second))
            result = "UID %s:%s" % (pedestal, upper_limit)
        elif name == "DATE":
            result = "BEFORE %s" % self.convert_date(second, add=datetime.timedelta(1))
        else:
            raise Exception("Operation not supported")
        return result

    def NE(self, first, second=None):
        if (second is None) and isinstance(first, Field):
            # All records special table query
            if first.type == "id":
                return self.GE(first, 1)
        result = self.NOT(self.EQ(first, second))
        result =  result.replace("NOT NOT", "").strip()
        return result

    def EQ(self,first,second):
        name = self.search_fields[first.name]
        result = None
        if name is not None:
            if name == "MESSAGE":
                # query by message sequence number
                result = "%s" % self.expand(second)
            elif name == "UID":
                result = "UID %s" % self.expand(second)
            elif name == "DATE":
                result = "ON %s" % self.convert_date(second)

            elif name in self.flags.values():
                if second:
                    result = "%s" % (name.upper()[1:])
                else:
                    result = "NOT %s" % (name.upper()[1:])
            else:
                raise Exception("Operation not supported")
        else:
            raise Exception("Operation not supported")
        return result

    def AND(self, first, second):
        result = "%s %s" % (self.expand(first), self.expand(second))
        return result

    def OR(self, first, second):
        result = "OR %s %s" % (self.expand(first), self.expand(second))
        return "%s" % result.replace("OR OR", "OR")

    def NOT(self, first):
        result = "NOT %s" % self.expand(first)
        return result
