#include <../source/TAL/objbase.h>
#include <../source/PAL/dataman.h>
#include <ctype.h>

typedef struct  scanner_struct_s    /* simple lexical analyser */
{
    libv_object_t base;
    char_p      token_buffer_start; /* beginnig of buffer to parse */
    char_p      token_buffer_end;   /* pointer to 1st byte after buffer */
    char_p      token_ptr;          /* current position in buffer */
    u32_t       current_line;       /* number passed 0x0A codes */
    /* if false, tokNext() returns NULL if no more tokens on current line */
    vboolean    crossline;
    u8_stack_t  token;              /* buffer for last parsed token */
    u8_stack_t  unget_token;        /* waiting token */
    char        *delim;             /* list of delimiters */
    char        *last;              /* last parsed token | NULL if no more */
    vp_stack_t  posstack;           /* saved positions */
    file_t      mapped;             /* used if input file can be mmap()'ed */
}   scanner_struct_t,   *scanner_struct_p;

#ifdef  CHECKED_BUILD
#define SCANNER_MAGIC   0x53525052
V_LOCAL char            *scanner_object="scanner";
#endif  /* CHECKED_BUILD */

#define GET_SCANNER(handle,caller) \
        (CHECK_AND_GET_OBJECT(scanner_struct_t,handle,caller, \
            scanner_object,SCANNER_MAGIC))

V_LOCAL u32_t V_CALL
tokDestroyScanner(scanner_t self)
{
    scanner_struct_p sc=(scanner_struct_p)self;
    if(--sc->base.ref_count)
        return sc->base.ref_count;
    u8sClear(&sc->token);
    u8sClear(&sc->unget_token);
    vpsClear(&sc->posstack);
    if(sc->mapped != LIBV_INVALID_HANDLE)
        fileClose(sc->mapped);
    memFreeObject(self);
    return 0;
}

V_LOCAL scanner_struct_p
tokAllocScanner(char *delim,u32_t extra)
{
    scanner_struct_p s;
    /* reserve 255+1 bytes for delimiters */
    ALLOC_OBJECT(s,scanner_struct_t,256+extra,SCANNER_MAGIC);
    s->base.destroy=tokDestroyScanner;
    s->delim = (char*)(s+1);
    s->current_line = 1;
    s->crossline = true;
    s->last = NULL;
    if(delim)
        strCopy(s->delim,delim,256);
    else
        s->delim[0]=0;
    u8sInit(&s->token);
    u8sInit(&s->unget_token);
    vpsInit(&s->posstack);
    if(extra)
    {
        /* Set pointers to storage */
        s->token_buffer_start=s->token_ptr=s->delim+256;
        s->token_buffer_end=s->token_buffer_start+extra;
    }
    else
        s->token_buffer_start=s->token_ptr=s->token_buffer_end=NULL;
    s->mapped = LIBV_INVALID_HANDLE;
    return s;
}

void    V_GLOBAL
tokDestroy(scanner_t handle)
{
    libvCloseHandle(handle,"tokDestroy",scanner_object,SCANNER_MAGIC);
}

scanner_t V_GLOBAL
tokParseBuffer(char *buffer,u32_t buf_len,char *delim)
{
    scanner_struct_p s;
    if(!buffer)
    {
#ifdef  CHECKED_BUILD
        conlPrintf(LOG_ERROR,
            "tokParseBuffer('[NULL]',%d,'%s'): empty buffer.\n",
            buf_len,dbs(delim));
#endif
        return NULL;
    }
    s=tokAllocScanner(delim,0);
    s->token_ptr=s->token_buffer_start=buffer;
    s->token_buffer_end=buffer+buf_len;
    return (scanner_t)s;
}
    
scanner_t V_GLOBAL
tokParseString(char *str,char *delim)
{
    return tokParseBuffer(str,strlen(dbs(str)),delim);
}

/*
 * Determine type of file: ASCII, LE/BE UNICODE, or UTF-8.
 * 0 - ASCII
 * 1 - UTF-8
 * 2 - LE UNICODE
 * 3 - BE UNICODE
 */

#ifdef  FAKE_UNICODE

u8_t utf_to_cyrillic[] =
{
  /* ÀÁÂÃÄÅ¨ÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÜÛÚÝÞß */
  0xD0, 0x90, 0xD0, 0x91, 0xD0, 0x92,
  0xD0, 0x93, 0xD0, 0x94, 0xD0, 0x95,
  0xD0, 0x81, 0xD0, 0x96, 0xD0, 0x97, 
  0xD0, 0x98, 0xD0, 0x99, 0xD0, 0x9A, 
  0xD0, 0x9B, 0xD0, 0x9C, 0xD0, 0x9D, 
  0xD0, 0x9E, 0xD0, 0x9F, 0xD0, 0xA0, 
  0xD0, 0xA1, 0xD0, 0xA2, 0xD0, 0xA3, 
  0xD0, 0xA4, 0xD0, 0xA5, 0xD0, 0xA6, 
  0xD0, 0xA7, 0xD0, 0xA8, 0xD0, 0xA9, 
  0xD0, 0xAC, 0xD0, 0xAB, 0xD0, 0xAA, 
  0xD0, 0xAD, 0xD0, 0xAE, 0xD0, 0xAF,
  /* àáâãäå¸æçèéêëìíîïðñòóôõö÷øùüûúýþÿ */
  0xD0, 0xB0, 0xD0, 0xB1, 0xD0, 0xB2,
  0xD0, 0xB3, 0xD0, 0xB4, 0xD0, 0xB5,
  0xD1, 0x91, 0xD0, 0xB6, 0xD0, 0xB7,
  0xD0, 0xB8, 0xD0, 0xB9, 0xD0, 0xBA,
  0xD0, 0xBB, 0xD0, 0xBC, 0xD0, 0xBD,
  0xD0, 0xBE, 0xD0, 0xBF, 0xD1, 0x80,
  0xD1, 0x81, 0xD1, 0x82, 0xD1, 0x83,
  0xD1, 0x84, 0xD1, 0x85, 0xD1, 0x86,
  0xD1, 0x87, 0xD1, 0x88, 0xD1, 0x89,
  0xD1, 0x8C, 0xD1, 0x8B, 0xD1, 0x8A,
  0xD1, 0x8D, 0xD1, 0x8E, 0xD1, 0x8F
}; 

V_LOCAL vboolean get_utf8_char(vhandle_t in_stream,u8_t *char_ret)
{
    u8_t ch, ch2;
    if(fileRead(in_stream,&ch,1) != 1)
        return false;
    if(ch < 128)
    {
        *char_ret = ch;
        return true;
    }
    if(fileRead(in_stream,&ch2,1) != 1)
        return false;
    /* HACK */
    if(ch != 0xD0 && ch != 0xD1)
    {
        *char_ret = ch2;
        return true;
    }
    else
        for(i=0;i<132;i+=2)
            if(utf_to_cyrillic[i+1]==ch2 && utf_to_cyrillic[i]==ch)
                break;
    i <<= 1;
    if(i < 33)
    {
        /* Uppercase cyrillic letters; FIXME */
        *char_ret = "ÀÁÂÃÄÅ¨ÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÜÛÚÝÞß"[i];
    }
    else
        if(i < 66)
        {
            /* Lowercase cyrillic letters; FIXME */
            i -= 33;
            *char_ret = "àáâãäå¸æçèéêëìíîïðñòóôõö÷øùüûúýþÿ"[i];
        }
        else
            *char_ret = ch2;
    return true;
}

/*
 * tokGetUnicodeType(): call to check either input text is UNICODE or not.
 *
 * Returns:
 *  0 - ANSI text or binary file;
 *  1 - UTF8 text;
 *  2 - UNICODE text, little-endian;
 *  3 - UNICODE text, big-endian.
 * Side effects: if unicode text detected, file pointer moved to the
 * start of data.
 */
V_LOCAL u32_t tokGetUnicodeType(file_t in_stream)
{
    char head[3];
    fileRead(in_stream,head,2);
    if(head[0] == 0xFF && head[1] == 0xFE)
        return 2; /* LE UNICODE */
    if(head[0] == 0xFE && head[1] == 0xFF)
        return 3; /* BE UNICODE */
    fileRead(in_stream,head+2,1);
    if(head[0] == 0xEF && head[1] == 0xBB && head[2] == 0xBF)
        return 1; /* UTF8 */
    fileSeek(in_stream,-3,SEEK_CUR);
    return 0;
}

V_LOCAL u32_t tokReadUnicodeFile(file_t in_stream,u8_stack_t dest,u32_t utype)
{
    /*
     * TODO
     */
    conlPuts(LOG_ERROR,"tokReadUnicodeFile('%s','%s'): not implemented.\n");
    u8sClear(&dest);
    return NULL;
}

#endif  /* FAKE_UNICODE */

V_LOCAL u32_t tokReadAsciiFile(file_t in_stream,u8_stack_t dest)
{
#define READ_PAGE_SIZE      (4096*64)
    u32_t read_in;
    u8_t *ptr = memAlloc(READ_PAGE_SIZE);
    V_ASSERT(ptr);
    for(;;)
    {
        read_in = fileRead(in_stream,ptr,READ_PAGE_SIZE);
        if(read_in)
            u8sPushBuffer(&dest,ptr,read_in);
        else
            break;
    }
    memFree(ptr);
    return u8sCount(&dest);
}
        
scanner_t V_GLOBAL
tokParseFile(char *filename,char *delim)
{
    void *base;
    scanner_struct_p s;
    file_t infile;
    u32_t ftype;
    u8_stack_t tmp_store;
#ifdef  FAKE_UNICODE
    u32_t utype;
#endif  /* FAKE_UNICODE */
    u8sInit(&tmp_store);
    infile=fileOpenRead(filename);
    if(!infile)
    {
#ifdef  CHECKED_BUILD
        conlPrintf(LOG_ERROR,
            "tokParseFile('%s','%s'): can't open file.\n",
            dbs(filename),dbs(delim));
#endif
        return LIBV_INVALID_HANDLE;
    }
    if(!fileControl(infile,pctl_get_type,&ftype))
    {
#ifdef  CHECKED_BUILD
        conlPrintf(LOG_WARNING,
            "tokParseFile('%s','%s'): can't stat() file.\n",
            filename,dbs(delim));
#endif
        ftype = 0;
    }
#ifdef  FAKE_UNICODE
    if(ftype && PCTL_BLOCK_DEVICE)
        utype = tokGetUnicodeType(infile);
    else
        utype = 0;
    if(utype == 0 && (ftype & PCTL_SUPPORT_MMAP) &&
        (fileControl(infile,pctl_map_ispace,&base)))
#else   /* !FAKE_UNICODE */
    if((ftype & PCTL_SUPPORT_MMAP) &&
        (fileControl(infile,pctl_map_ispace,&base)))
#endif  /* FAKE_UNICODE */
    {
        s=tokAllocScanner(delim,0);
        s->mapped=infile;
        s->token_ptr=s->token_buffer_start=(char_p)base;
        s->token_buffer_end=s->token_buffer_start+fileSize(infile);
        return s;
    }
#ifdef  FAKE_UNICODE
    if(utype == 0)
        tokReadAsciiFile(infile,tmp_store);
    else
        tokReadUnicodeFile(infile,tmp_store,utype);
#else   /* !FAKE_UNICODE */
    tokReadAsciiFile(infile,tmp_store);
#endif  /* !FAKE_UNICODE */
    fileClose(infile);
    if(u8sCount(&tmp_store) == 0)
    {
#ifdef  CHECKED_BUILD
        conlPrintf(LOG_ERROR,
            "tokParseFile('%s','%s'): can't read from file.\n",
            dbs(filename),dbs(delim));
#endif
        return LIBV_INVALID_HANDLE;
    }
    s=tokAllocScanner(delim,u8sCount(&tmp_store));
    memCopy(s->token_buffer_start,u8sGetStorage(&tmp_store),u8sCount(&tmp_store));
    u8sClear(&tmp_store);
    return s;
}

void    V_GLOBAL
tokRestart(scanner_t handle,char *new_buffer,char *new_delim)
{
    scanner_struct_p sc=GET_SCANNER(handle,"tokRestart");
    if(!sc)
        return;    
    if(new_buffer)
    {
        sc->token_buffer_start=new_buffer;
        sc->token_buffer_end=new_buffer+strlen(new_buffer);
    }
    if(new_delim)
        strCopy(sc->delim,new_delim,256);
    sc->token_ptr=sc->token_buffer_start;
    sc->last = NULL;
    vpsResize(&sc->posstack,0);
    return;
}

vboolean    V_GLOBAL
tokPush(scanner_t handle)
{
    scanner_struct_p sc=GET_SCANNER(handle,"tokPush");
    if(!sc)
        return false;
    vpsPush(&sc->posstack,sc->token_ptr);
    return true;
}

vboolean    V_GLOBAL
tokPop(scanner_t handle)
{
    scanner_struct_p sc=GET_SCANNER(handle,"tokPop");
    if(!sc)
        return false;
    return vpsPop(&sc->posstack,(void**)&sc->token_ptr);
}

#define IS_SPACE(c)             ((c)<=32)
#define IS_EOL(c)               ((c)==0x0D || (c)==0x0A)
#define IS_QUOTE(c)             ((c)=='"')
#define IS_LINECONT(c)          ((c)=='\\')
#define HAVE_CHARS(sc) { \
        (sc)->last = (char*)u8sGetStorage(&(sc)->token); \
        return (sc)->last; \
    }
#define NO_MORE_CHARS(sc) { \
        u8sResize(&(sc)->token,0); \
        (sc)->last = NULL; \
        return NULL; \
    }

V_LOCAL vboolean tokNextChar(scanner_struct_p sc,char_p char_ret)
{
    while(sc->token_ptr < sc->token_buffer_end)
    {
        *char_ret = *sc->token_ptr;
        if(*char_ret == 0x0A)
            sc->current_line++;
        /* EOL not allowed if 'crossline' not set */
        if(!sc->crossline && IS_EOL(*char_ret))
            break;
        if(IS_LINECONT(*char_ret))
        {
            /* Discard all chars between '\' and EOL */
            while(sc->token_ptr < sc->token_buffer_end &&
                    !IS_EOL(*sc->token_ptr))
                sc->token_ptr++;
            /* Discard EOL */
            while(sc->token_ptr < sc->token_buffer_end &&
                    IS_EOL(*sc->token_ptr))
            {
                if(*sc->token_ptr == 0x0A)
                    sc->current_line++;
                sc->token_ptr++;
            }
            continue;
        }
        sc->token_ptr++;
        return true;
    }
    return false;
}

V_LOCAL char_p
tokReady(scanner_struct_p sc)
{
    u32_t unget_len = u8sCount(&sc->unget_token);
    if(unget_len)
    {
        u8sResize(&sc->token,0);
        u8sPushBuffer(&sc->token,u8sGetStorage(&sc->unget_token),unget_len);
        u8sResize(&sc->unget_token,0);
        HAVE_CHARS(sc);
    }
    return NULL;
}

char_p  V_GLOBAL
tokNext(scanner_t handle)
{
    char next, *unget;
    scanner_struct_p sc;
    sc=GET_SCANNER(handle,"tokNext");
    if(!sc)
        return NULL;
    /* Check if token already waiting */
    unget = tokReady(sc);
    if(unget)
        return unget;
    /* Skip whitespace */
    do
    {
        if(!tokNextChar(sc,&next))
            NO_MORE_CHARS(sc);
    } while(IS_SPACE(next));
	/* Now we have one non-whitespace char in 'next' */
    u8sResize(&sc->token,0);    /* clear dest buffer */
    if(IS_QUOTE(next))
    {
        /* Extract quoted token; disallow EOL and EOF */
        vboolean old_crossline = sc->crossline;
        sc->crossline = false;
        for(;;)
        {
            if(!tokNextChar(sc,&next))
            {
#ifdef  CHECKED_BUILD
                conlPrintf(LOG_ERROR,"tokNext(): parse error on line %u:"
                    " EOL/EOF inside quoted token.\n",sc->current_line);
#endif  /* CHECKED_BUILD */
                sc->crossline = old_crossline;
				/* error in input: stop parsing */
                NO_MORE_CHARS(sc);
            }
            if(IS_QUOTE(next))
            {
                sc->crossline = old_crossline;
                break;
            }
            u8sPush(&sc->token,next);
        }
    }
    else
    {
        if(strchr(sc->delim,next))
            /* return delimiter */
            u8sPush(&sc->token,(u8_t)next);
        else
        {
            /* extract next token enclosed in delimiters */
            do
            {
                u8sPush(&sc->token,(u8_t)next);
                if(!tokNextChar(sc,&next))
                    break;
                /* we must NOT miss delimiter */
                if(strchr(sc->delim,next))
                {
                    u8sResize(&sc->unget_token,0);
                    u8sPush(&sc->unget_token,(u8_t)next);
                    u8sPush(&sc->unget_token,0);
                    break;
                }
            } while(!IS_SPACE(next));
        }
    }
   	u8sPush(&sc->token,0);
   	HAVE_CHARS(sc);
}

/*
 * Same as tokNext(), but handles whitespace & quotes as normal chars,
 * unless they are not in sc->delimiters.
 */
char_p  V_GLOBAL
tokStrict(scanner_t handle)
{
    char next, *unget;
    scanner_struct_p sc;
    sc=GET_SCANNER(handle,"tokStrict");
    if(!sc)
        return NULL;
    /* Check if token already waiting */
    unget = tokReady(sc);
    if(unget)
        return unget;
    u8sResize(&sc->token,0);    /* clear dest buffer */
	/* Read one char */
    if(!tokNextChar(sc,&next))
        NO_MORE_CHARS(sc);
	/* Now we have one non-whitespace char in 'next' */
    if(strchr(sc->delim,next))
        /* return delimiter */
        u8sPush(&sc->token,(u8_t)next);
    else
    {
        /* extract next token enclosed in delimiters */
        for(;;)
        {
            u8sPush(&sc->token,(u8_t)next);
            if(!tokNextChar(sc,&next))
                break;
            /* we must NOT miss delimiter */
            if(strchr(sc->delim,next))
            {
                u8sResize(&sc->unget_token,0);
                u8sPush(&sc->unget_token,(u8_t)next);
                u8sPush(&sc->unget_token,0);
                break;
            }
        }
    }
   	u8sPush(&sc->token,0);
   	HAVE_CHARS(sc);
}

/*
 * Extracts next string (all chars from current pos to EOL)
 */
char_p  V_GLOBAL
tokString(scanner_t handle)
{
    char next;
    scanner_struct_p sc;
    sc=GET_SCANNER(handle,"tokString");
    if(!sc)
        return NULL;
    /* Skip EOLs */
    do
    {
        if(!tokNextChar(sc,&next))
            NO_MORE_CHARS(sc);
    } while(IS_EOL(next));
	/* Now we have one non-whitespace char in 'next' */
    u8sResize(&sc->token,0);
    do
    {
        u8sPush(&sc->token,(u8_t)next);
        if(!tokNextChar(sc,&next))
            break;
    } while(!IS_EOL(next));
   	u8sPush(&sc->token,0);
   	HAVE_CHARS(sc);
}

/*
 *  Returns last extracted token | NULL when no more chars.
 */
char_p  V_GLOBAL
tokLast(scanner_t handle)
{
    scanner_struct_p sc;
    sc=GET_SCANNER(handle,"tokLast");
    if(!sc)
        return NULL;
    return sc->last;
}

u32_t   V_GLOBAL
tokCurrentLine(scanner_t handle)
{
    scanner_struct_p sc;
    sc=GET_SCANNER(handle,"tokCurrentLine");
    if(!sc)
        return 0;
    return sc->current_line;
}
