#include <../source/TAL/objbase.h>
#include <../source/PAL/dataman.h>
#include <ctype.h>

typedef struct	scanner_state_s
{
	char		comment_char;		/* 0 if no comment char defined */
	char_p		token_ptr;			/* current position in buffer */
	u32_t		current_line;       /* number of passed 0x0A codes */
	u32_t		last_line;			/* line of last parsed token */
	vboolean	crossline;
	vboolean	inside_quotes;
}	scanner_state_t, *scanner_state_p;

typedef struct  scanner_struct_s    /* simple lexical analyser */
{
	libv_object_t base;
	char_p      token_buffer_start; /* beginnig of buffer to parse */
	char_p      token_buffer_end;   /* pointer to 1st byte after buffer */
	/* if false, tokNext() returns NULL if no more tokens on current line */
	scanner_state_t	state;
	u8_stack_t  token;              /* buffer for last parsed token */
	u32_t		unget_char;			/* 0xFFFFFFFF means 'no unget char' */
	u8_stack_t  unget_token;        /* waiting token */
	char        *delim;             /* list of delimiters */
	char        *last;              /* last parsed token | NULL if no more */
	bstack_t    state_stack;        /* saved state(s) */
	vhandle_t   mapped;             /* used if input file can be mmap()'ed */
}   scanner_struct_t,   *scanner_struct_p;

#ifdef  CHECKED_BUILD
#define SCANNER_MAGIC   0x53525052
V_LOCAL char            *scanner_object="scanner";
#endif  /* CHECKED_BUILD */

#define GET_SCANNER(handle,caller) \
		(CHECK_AND_GET_OBJECT(scanner_struct_t,handle,caller, \
			scanner_object,SCANNER_MAGIC))

#define IS_SPACE(c)             ((unsigned char)(c) <= 32)
#define IS_EOL(c)               ((c)==0x0D || (c)==0x0A)
#define IS_QUOTE(c)             ((c)=='"')
#define IS_LINECONT(sc,c)       ((c)=='\\' && !(sc)->state.inside_quotes)
#define IS_COMMENT_CHAR(sc,c)   ((sc)->state.comment_char && (c) == (sc)->state.comment_char)
#define	IS_DELIM(sc,ch)			(strchr((sc)->delim,(ch)) ? true : false)
#define	UNGET_CHAR(sc,ch)		(sc)->unget_char = (u32_t)(ch)
#define	GET_UNGOT_CHAR(sc)		((char)((sc)->unget_char & 0xFF))
#define	UNGET_CLEAR(sc)			(sc)->unget_char = 0xFFFFFFFF
#define	HAVE_UNGOT_CHAR(sc)		((sc)->unget_char != 0xFFFFFFFF)

#define HAVE_CHARS(sc) { \
		(sc)->last = (char*)u8sGetStorage(&(sc)->token); \
		return (sc)->last; \
	}

#define NO_MORE_CHARS(sc) { \
		u8sResize(&(sc)->token,0); \
		(sc)->last = NULL; \
		return NULL; \
	}


V_LOCAL u32_t V_CALL
tokDestroyScanner(scanner_t self)
{
	scanner_struct_p sc=(scanner_struct_p)self;
	if(--sc->base.ref_count)
		return sc->base.ref_count;
	u8sClear(&sc->token);
	u8sClear(&sc->unget_token);
	stackClear(&sc->state_stack);
	if(sc->mapped != LIBV_INVALID_HANDLE)
		fileClose(sc->mapped);
	memFreeObject(self);
	return 0;
}

V_LOCAL scanner_struct_p
tokAllocScanner(char *delim,u32_t extra)
{
	scanner_struct_p s;
	/* reserve 255+1 bytes for delimiters */
	ALLOC_OBJECT(s,scanner_struct_t,256+extra,SCANNER_MAGIC);
	s->base.destroy=tokDestroyScanner;
	s->delim = (char*)(s+1);
	s->state.current_line = 1;
	s->state.last_line = 1;
	s->state.crossline = true;
	s->state.inside_quotes = false;
	s->state.comment_char = 0;
	s->last = NULL;
	if(delim)
		strCopy(s->delim,delim,256);
	else
		s->delim[0]=0;
	u8sInit(&s->token);
	u8sInit(&s->unget_token);
	UNGET_CLEAR(s);
	stackInit(&s->state_stack,sizeof(scanner_state_t));
	if(extra)
	{
		/* Set pointers to storage */
		s->token_buffer_start=s->state.token_ptr=s->delim+256;
		s->token_buffer_end=s->token_buffer_start+extra;
	}
	else
		s->token_buffer_start=s->state.token_ptr=s->token_buffer_end=NULL;
	s->mapped = LIBV_INVALID_HANDLE;
	return s;
}

void    V_GLOBAL
tokDestroy(scanner_t handle)
{
	libvCloseHandle(handle,"tokDestroy",scanner_object,SCANNER_MAGIC);
}

scanner_t V_GLOBAL
tokParseBuffer(char *buffer,u32_t buf_len,char *delim)
{
	scanner_struct_p s;
	if(!buffer)
	{
#ifdef  CHECKED_BUILD
		conlPrintf(LOG_ERROR,
			"tokParseBuffer('[NULL]',%d,'%s'): empty buffer.\n",
			buf_len,dbs(delim));
#endif
		return NULL;
	}
	s=tokAllocScanner(delim,0);
	s->state.token_ptr=s->token_buffer_start=buffer;
	s->token_buffer_end=buffer+buf_len;
	return (scanner_t)s;
}
	
scanner_t V_GLOBAL
tokParseString(char *str,char *delim)
{
	return tokParseBuffer(str,strlen(dbs(str)),delim);
}

#ifdef  FAKE_UNICODE

/*
 * TODO: convert to use libc tables
 */

/*
 * Determine type of file: ASCII, LE/BE UNICODE, or UTF-8.
 * 0 - ASCII
 * 1 - UTF-8
 * 2 - LE UNICODE
 * 3 - BE UNICODE
 */


u8_t utf_to_cyrillic[] =
{
  /* ÀÁÂÃÄÅ¨ÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÜÛÚÝÞß */
  0xD0, 0x90, 0xD0, 0x91, 0xD0, 0x92,
  0xD0, 0x93, 0xD0, 0x94, 0xD0, 0x95,
  0xD0, 0x81, 0xD0, 0x96, 0xD0, 0x97, 
  0xD0, 0x98, 0xD0, 0x99, 0xD0, 0x9A, 
  0xD0, 0x9B, 0xD0, 0x9C, 0xD0, 0x9D, 
  0xD0, 0x9E, 0xD0, 0x9F, 0xD0, 0xA0, 
  0xD0, 0xA1, 0xD0, 0xA2, 0xD0, 0xA3, 
  0xD0, 0xA4, 0xD0, 0xA5, 0xD0, 0xA6, 
  0xD0, 0xA7, 0xD0, 0xA8, 0xD0, 0xA9, 
  0xD0, 0xAC, 0xD0, 0xAB, 0xD0, 0xAA, 
  0xD0, 0xAD, 0xD0, 0xAE, 0xD0, 0xAF,
  /* àáâãäå¸æçèéêëìíîïðñòóôõö÷øùüûúýþÿ */
  0xD0, 0xB0, 0xD0, 0xB1, 0xD0, 0xB2,
  0xD0, 0xB3, 0xD0, 0xB4, 0xD0, 0xB5,
  0xD1, 0x91, 0xD0, 0xB6, 0xD0, 0xB7,
  0xD0, 0xB8, 0xD0, 0xB9, 0xD0, 0xBA,
  0xD0, 0xBB, 0xD0, 0xBC, 0xD0, 0xBD,
  0xD0, 0xBE, 0xD0, 0xBF, 0xD1, 0x80,
  0xD1, 0x81, 0xD1, 0x82, 0xD1, 0x83,
  0xD1, 0x84, 0xD1, 0x85, 0xD1, 0x86,
  0xD1, 0x87, 0xD1, 0x88, 0xD1, 0x89,
  0xD1, 0x8C, 0xD1, 0x8B, 0xD1, 0x8A,
  0xD1, 0x8D, 0xD1, 0x8E, 0xD1, 0x8F
}; 

V_LOCAL vboolean get_utf8_char(vhandle_t in_stream,u8_t *char_ret)
{
	u8_t ch, ch2;
	if(fileRead(in_stream,&ch,1) != 1)
		return false;
	if(ch < 128)
	{
		*char_ret = ch;
		return true;
	}
	if(fileRead(in_stream,&ch2,1) != 1)
		return false;
	/* HACK */
	if(ch != 0xD0 && ch != 0xD1)
	{
		*char_ret = ch2;
		return true;
	}
	else
		for(i=0;i<132;i+=2)
			if(utf_to_cyrillic[i+1]==ch2 && utf_to_cyrillic[i]==ch)
				break;
	i <<= 1;
	if(i < 33)
	{
		/* Uppercase cyrillic letters; FIXME */
		*char_ret = "ÀÁÂÃÄÅ¨ÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÜÛÚÝÞß"[i];
	}
	else
		if(i < 66)
		{
			/* Lowercase cyrillic letters; FIXME */
			i -= 33;
			*char_ret = "àáâãäå¸æçèéêëìíîïðñòóôõö÷øùüûúýþÿ"[i];
		}
		else
			*char_ret = ch2;
	return true;
}

/*
 * tokGetUnicodeType(): call to check either input text is UNICODE or not.
 *
 * Returns:
 *  0 - ANSI text or binary file;
 *  1 - UTF8 text;
 *  2 - UNICODE text, little-endian;
 *  3 - UNICODE text, big-endian.
 * Side effects: if unicode text detected, file pointer moved to the
 * start of data.
 */
V_LOCAL u32_t tokGetUnicodeType(file_t in_stream)
{
	char head[3];
	fileRead(in_stream,head,2);
	if(head[0] == 0xFF && head[1] == 0xFE)
		return 2; /* LE UNICODE */
	if(head[0] == 0xFE && head[1] == 0xFF)
		return 3; /* BE UNICODE */
	fileRead(in_stream,head+2,1);
	if(head[0] == 0xEF && head[1] == 0xBB && head[2] == 0xBF)
		return 1; /* UTF8 */
	fileSeek(in_stream,-3,SEEK_CUR);
	return 0;
}

V_LOCAL u32_t tokReadUnicodeFile(file_t in_stream,u8_stack_p dest,u32_t utype)
{
	/*
	 * TODO
	 */
	conlPuts(LOG_ERROR,"tokReadUnicodeFile('%s','%s'): not implemented.\n");
	u8sClear(dest);
	return 0;
}

#endif  /* FAKE_UNICODE */

V_LOCAL u32_t tokReadAsciiFile(file_t in_stream,u8_stack_p dest)
{
#define READ_PAGE_SIZE      (4096*64)
	u32_t read_in;
	u8_t *ptr = (u8_t*)memAlloc(READ_PAGE_SIZE);
	V_ASSERT(ptr);
	for(;;)
	{
		read_in = fileRead(in_stream,ptr,READ_PAGE_SIZE);
		if(read_in)
			u8sPushBuffer(dest,ptr,read_in);
		else
			break;
	}
	memFree(ptr);
	return u8sCount(dest);
}
		
scanner_t V_GLOBAL
tokParseFile(char *filename,char *delim)
{
	void *base;
	scanner_struct_p s;
	file_t infile;
	u32_t ftype;
	u8_stack_t tmp_store;
#ifdef  FAKE_UNICODE
	u32_t utype;
#endif  /* FAKE_UNICODE */
	u8sInit(&tmp_store);
	infile=fileOpenRead(filename);
	if(!infile)
	{
#ifdef  CHECKED_BUILD
		conlPrintf(LOG_ERROR,
			"tokParseFile('%s','%s'): can't open file.\n",
			dbs(filename),dbs(delim));
#endif
		return LIBV_INVALID_HANDLE;
	}
	if(!fileControl(infile,pctl_get_type,&ftype))
	{
#ifdef  CHECKED_BUILD
		conlPrintf(LOG_WARNING,
			"tokParseFile('%s','%s'): can't stat() file.\n",
			filename,dbs(delim));
#endif
		ftype = 0;
	}
	/*
	 * Check wheither file can be mmap()'ed
	 */
#ifdef  FAKE_UNICODE
	if(ftype && PCTL_BLOCK_DEVICE)
		utype = tokGetUnicodeType(infile);
	else
		utype = 0;
	if(utype == 0 && (ftype & PCTL_SUPPORT_MMAP) &&
		(fileControl(infile,pctl_map_ispace,&base)))
#else   /* !FAKE_UNICODE */
	if((ftype & PCTL_SUPPORT_MMAP) &&
		(fileControl(infile,pctl_map_ispace,&base)))
#endif  /* FAKE_UNICODE */
	{
		s=tokAllocScanner(delim,0);
		s->mapped=infile;
		s->state.token_ptr=s->token_buffer_start=(char_p)base;
		s->token_buffer_end=s->token_buffer_start+fileSize(infile);
		return s;
	}
	/*
	 * Can't be mmap()'ed; read file into temporary buffer
	 */
#ifdef  FAKE_UNICODE
	if(utype == 0)
		tokReadAsciiFile(infile,tmp_store);
	else
		tokReadUnicodeFile(infile,tmp_store,utype);
#else   /* !FAKE_UNICODE */
	tokReadAsciiFile(infile,&tmp_store);
#endif  /* !FAKE_UNICODE */
	fileClose(infile);
	if(!u8sCount(&tmp_store))
	{
#ifdef  CHECKED_BUILD
		conlPrintf(LOG_ERROR,
			"tokParseFile('%s','%s'): empty file.\n",
			dbs(filename),dbs(delim));
#endif
		return LIBV_INVALID_HANDLE;
	}
	s=tokAllocScanner(delim,u8sCount(&tmp_store));
	memCopy(s->token_buffer_start,u8sGetStorage(&tmp_store),u8sCount(&tmp_store));
	u8sClear(&tmp_store);
	return s;
}

void    V_GLOBAL
tokRestartEx(scanner_t handle,char *new_buffer,char *new_delim)
{
	scanner_struct_p sc=GET_SCANNER(handle,"tokRestartEx");
	if(sc==LIBV_INVALID_HANDLE)
		return;
	if(new_buffer)
	{
		vboolean another_buffer;
		another_buffer = sc->token_buffer_start != new_buffer ||
			strcmp(sc->token_buffer_start,new_buffer);
		/* Can't use old state stack with another buffer */
		if(another_buffer && stackCount(&sc->state_stack))
		{
#ifdef	CHECKED_BUILD
			conlPrintf(LOG_WARNING,"tokRestartEx(): new buffer, "
				"wiped %u elements in state stack.\n",stackCount(&sc->state_stack));
#endif	/* CHECKED_BUILD */
			stackResize(&sc->state_stack,0);
		}
		sc->token_buffer_start=new_buffer;
		sc->token_buffer_end=new_buffer+strlen(new_buffer);
	}
	if(new_delim)
		strCopy(sc->delim,new_delim,256);
	sc->state.token_ptr=sc->token_buffer_start;
	sc->state.current_line = 1;
	sc->state.crossline = true;
	sc->state.inside_quotes = false;
	sc->last = NULL;
	UNGET_CLEAR(sc);
	return;
}

void    V_GLOBAL
tokRestart(scanner_t handle)
{
	tokRestartEx(handle,NULL,NULL);
}

vboolean    V_GLOBAL
tokPush(scanner_t handle)
{
	scanner_struct_p sc=GET_SCANNER(handle,"tokPush");
	if(sc==LIBV_INVALID_HANDLE)
		return false;
	stackPush(&sc->state_stack,&sc->state);
	return true;
}

vboolean    V_GLOBAL
tokPop(scanner_t handle)
{
	scanner_struct_p sc=GET_SCANNER(handle,"tokPop");
	if(sc==LIBV_INVALID_HANDLE)
		return false;
	return stackPop(&sc->state_stack,&sc->state);
}

void    V_GLOBAL
tokClearStateStack(scanner_t handle)
{
	scanner_struct_p sc=GET_SCANNER(handle,"tokClearStateStack");
	if(sc==LIBV_INVALID_HANDLE)
		return;
	stackResize(&sc->state_stack,0);
}

V_LOCAL vboolean
tokNextChar(scanner_struct_p sc,char_p char_ret)
{
	if(HAVE_UNGOT_CHAR(sc))
	{
		*char_ret = GET_UNGOT_CHAR(sc);
		UNGET_CLEAR(sc);
		return true;
	}
	while(sc->state.token_ptr < sc->token_buffer_end)
	{
		*char_ret = *sc->state.token_ptr;
		if(*char_ret == 0x0A) /* it's Ok: _ignore_ 0x0Ds */
			sc->state.current_line++;
		/* EOL not allowed if 'state.crossline' not set */
		if(!sc->state.crossline && IS_EOL(*char_ret))
			break;
		if(IS_COMMENT_CHAR(sc,*char_ret))
		{
			/* discard end of line */
			while(sc->state.token_ptr < sc->token_buffer_end &&
				!IS_EOL(*sc->state.token_ptr))
				sc->state.token_ptr++;
			continue;
		}
		if(IS_LINECONT(sc,*char_ret))
		{
			/* Discard all chars between '\' and EOL */
			while(sc->state.token_ptr < sc->token_buffer_end &&
					!IS_EOL(*sc->state.token_ptr))
				sc->state.token_ptr++;
			/* Discard EOL */
			while(sc->state.token_ptr < sc->token_buffer_end &&
					IS_EOL(*sc->state.token_ptr))
			{
				if(*sc->state.token_ptr == 0x0A)
					sc->state.current_line++;
				sc->state.token_ptr++;
			}
			continue;
		}
		sc->state.token_ptr++;
		return true;
	}
	return false;
}

V_LOCAL char_p
tokReady(scanner_struct_p sc)
{
	u32_t unget_len = u8sCount(&sc->unget_token);
	if(unget_len)
	{
		u8sResize(&sc->token,0);
		u8sPushBuffer(&sc->token,u8sGetStorage(&sc->unget_token),unget_len);
		u8sResize(&sc->unget_token,0);
		HAVE_CHARS(sc);
	}
	return NULL;
}

char_p  V_GLOBAL
tokNext(scanner_t handle)
{
	char next, *unget;
	scanner_struct_p sc;
	sc=GET_SCANNER(handle,"tokNext");
	if(sc==LIBV_INVALID_HANDLE)
		return NULL;
	/* Check if token already waiting */
	unget = tokReady(sc);
	if(unget)
		return unget;
	/* Skip whitespace */
	do
	{
		if(!tokNextChar(sc,&next))
			NO_MORE_CHARS(sc);
	} while(IS_SPACE(next));
	/* Now we have one non-whitespace char in 'next' */
	sc->state.last_line = sc->state.current_line;
	u8sResize(&sc->token,0);    /* clear dest buffer */
	if(IS_QUOTE(next))
	{
		/* Extract quoted token; disallow EOL and EOF */
		vboolean old_crossline = sc->state.crossline;
		sc->state.inside_quotes = true;
		sc->state.crossline = false;
		for(;;)
		{
			if(!tokNextChar(sc,&next))
			{
#ifdef  CHECKED_BUILD
				conlPrintf(LOG_ERROR,"tokNext(): parse error on line %u:"
					" EOL/EOF inside quoted token.\n",sc->state.current_line-1);
#endif  /* CHECKED_BUILD */
				sc->state.crossline = old_crossline;
				sc->state.inside_quotes = false;
				/* error in input: stop parsing */
				NO_MORE_CHARS(sc);
			}
			if(IS_QUOTE(next))
			{
				sc->state.crossline = old_crossline;
				sc->state.inside_quotes = false;
				break;
			}
			u8sPush(&sc->token,next);
		}
	}
	else
	{
		if(IS_DELIM(sc,next))
		{
			/* return delimiter */
			u8sPush(&sc->token,(u8_t)next);
		}
		else
		{
			/* extract next token enclosed in delimiters */
			do
			{
				u8sPush(&sc->token,(u8_t)next);
				if(!tokNextChar(sc,&next))
					break;
				if(IS_QUOTE(next))
				{
					UNGET_CHAR(sc,next);
					break;
				}
				if(IS_DELIM(sc,next))
				{
					u8sResize(&sc->unget_token,0);
					u8sPush(&sc->unget_token,(u8_t)next);
					u8sPush(&sc->unget_token,0);
					break;
				}
			} while(!IS_SPACE(next));
		}
	}
	u8sPush(&sc->token,0);
	HAVE_CHARS(sc);
}

/*
 * Same as tokNext(), but handles whitespace & quotes as normal chars,
 * unless they are not in sc->delimiters.
 */
char_p  V_GLOBAL
tokStrict(scanner_t handle)
{
	char next, *unget;
	scanner_struct_p sc;
	sc=GET_SCANNER(handle,"tokStrict");
	if(sc==LIBV_INVALID_HANDLE)
		return NULL;
	/* Check if token already waiting */
	unget = tokReady(sc);
	if(unget)
		return unget;
	sc->state.last_line = sc->state.current_line;
	u8sResize(&sc->token,0);    /* clear dest buffer */
	/* Read one char */
	sc->state.inside_quotes = true;
	if(!tokNextChar(sc,&next))
	{
		sc->state.inside_quotes = false;
		NO_MORE_CHARS(sc);
	}
	/* Now we have one non-whitespace char in 'next' */
	if(IS_DELIM(sc,next))
	{
		/* return delimiter */
		u8sPush(&sc->token,(u8_t)next);
	}
	else
	{
		/* extract next token enclosed in delimiters */
		for(;;)
		{
			u8sPush(&sc->token,(u8_t)next);
			if(!tokNextChar(sc,&next))
				break;
			/* we must NOT miss delimiter */
			if(IS_DELIM(sc,next))
			{
				u8sResize(&sc->unget_token,0);
				u8sPush(&sc->unget_token,(u8_t)next);
				u8sPush(&sc->unget_token,0);
				break;
			}
		}
	}
	u8sPush(&sc->token,0);
	sc->state.inside_quotes = false;
	HAVE_CHARS(sc);
}

/*
 * Extracts next string (all chars from current pos to EOL)
 */
char_p  V_GLOBAL
tokString(scanner_t handle)
{
	char next;
	scanner_struct_p sc;
	sc=GET_SCANNER(handle,"tokString");
	if(sc==LIBV_INVALID_HANDLE)
		return NULL;
	/* Skip EOLs */
	do
	{
		if(!tokNextChar(sc,&next))
			NO_MORE_CHARS(sc);
	} while(IS_EOL(next));
	/* Now we have one non-whitespace char in 'next' */
	sc->state.last_line = sc->state.current_line;
	u8sResize(&sc->token,0);
	do
	{
		u8sPush(&sc->token,(u8_t)next);
		if(!tokNextChar(sc,&next))
			break;
	} while(!IS_EOL(next));
	u8sPush(&sc->token,0);
	HAVE_CHARS(sc);
}

/*
 *  Returns last extracted token | NULL when no more chars.
 */
char_p  V_GLOBAL
tokLast(scanner_t handle)
{
	scanner_struct_p sc;
	sc=GET_SCANNER(handle,"tokLast");
	if(sc==LIBV_INVALID_HANDLE)
		return NULL;
	return sc->last;
}

u32_t   V_GLOBAL
tokCurrentLine(scanner_t handle)
{
	scanner_struct_p sc;
	sc=GET_SCANNER(handle,"tokCurrentLine");
	if(sc==LIBV_INVALID_HANDLE)
		return 0;
	return sc->state.current_line;
}

u32_t   V_GLOBAL
tokLastLine(scanner_t handle)
{
	scanner_struct_p sc;
	sc=GET_SCANNER(handle,"tokCurrentLine");
	if(sc==LIBV_INVALID_HANDLE)
		return 0;
	return sc->state.last_line;
}

char	V_GLOBAL
tokCommentChar(scanner_t handle,char comment_char)
{
	char old_comment_char;
	scanner_struct_p sc;
	sc=GET_SCANNER(handle,"tokCommentChar");
	if(sc==LIBV_INVALID_HANDLE)
		return 0;
	old_comment_char = sc->state.comment_char;
	sc->state.comment_char = comment_char;
	return old_comment_char;
}

/*
 * Helper routines
 */
vboolean V_GLOBAL
tokNextIsEx(scanner_t scanner,char *expected_token,
	int (*str_comp)(const char *s1,const char *s2))
{
	char *tok = tokNext(scanner);
	if(tok && !str_comp(tok,dbs(expected_token)))
		return true;
	return false;
}

vboolean V_GLOBAL
tokLastWasEx(scanner_t scanner,char *expected_token,
	int (*str_comp)(const char *s1,const char *s2))
{
	char *tok = tokLast(scanner);
	if(tok && !str_comp(tok,dbs(expected_token)))
		return true;
	return false;
}

vboolean V_GLOBAL
tokNextIs(scanner_t scanner,char *expected_token)
{
	char *tok = tokNext(scanner);
	if(tok && !strcmp(tok,dbs(expected_token)))
		return true;
	return false;
}

vboolean V_GLOBAL
tokLastWas(scanner_t scanner,char *expected_token)
{
	char *tok = tokLast(scanner);
	if(tok && !strcmp(tok,dbs(expected_token)))
		return true;
	return false;
}

vboolean V_GLOBAL
tokSkipTo(vhandle_t scanner,char *token)
{
	while(tokNext(scanner))
		if(tokLastWas(scanner,token))
			return true;
	return false;
}

vboolean V_GLOBAL
tokSkipToEx(vhandle_t scanner,char *token,
	int (*str_comp)(const char *s1,const char *s2))
{
	while(tokNext(scanner))
		if(tokLastWasEx(scanner,token,str_comp))
			return true;
	return false;
}
	
vboolean V_GLOBAL
tokSkipBlock(scanner_t scanner,char skip_from)
{
	char *tok;
	char skip_to;
	int count = 1;
	switch(skip_from)
	{
		case '{': skip_to = '}'; break;
		case '}': skip_to = '{'; break;
		case '(': skip_to = ')'; break;
		case ')': skip_to = '('; break;
		case '[': skip_to = ']'; break;
		case ']': skip_to = '['; break;
		default:
		{
#ifdef  CHECKED_BUILD
			conlPrintf(LOG_ERROR,"tokSkipBlock('%c'): skip_from "
				"not in '{}()[]'.\n",skip_from);
#endif	/* CHECKED_BUILD */
			return false;
		}
	}
	while(count)
	{
		tok = tokNext(scanner);
		if(!tok)
		{
#ifdef  CHECKED_BUILD
			conlPuts(LOG_ERROR,"tokSkipBlock('%c'): unexpected EOF.\n");
#endif	/* CHECKED_BUILD */
			return false;
		}
		if(tok[0]==skip_from && !tok[1])
			count--;
		else
			if(tok[0]==skip_to && !tok[1])
				count++;
	}
	return true;
}

vboolean V_GLOBAL
tokNextInteger(scanner_t scanner,s32_t *ret_val)
{
	char *tok = tokNext(scanner);
	return strToInt(tok,ret_val);
}

vboolean V_GLOBAL
tokLastInteger(scanner_t scanner,s32_t *ret_val)
{
	char *tok = tokLast(scanner);
	return strToInt(tok,ret_val);
}

vboolean V_GLOBAL
tokNextFloat(scanner_t scanner,vec_t *ret_val)
{
	char *tok = tokNext(scanner);
	return strToFloat(tok,ret_val);
}

vboolean V_GLOBAL
tokLastFloat(scanner_t scanner,vec_t *ret_val)
{
	char *tok = tokLast(scanner);
	return strToFloat(tok,ret_val);
}

u32_t V_GLOBAL
tokCopyNextToken(vhandle_t scanner,char *buf,u32_t buf_len)
{
	tokNext(scanner);
	return tokCopyLastToken(scanner,buf,buf_len);
}

u32_t V_GLOBAL
tokCopyLastToken(vhandle_t scanner,char *buf,u32_t buf_len)
{
	u32_t token_len;
	char *token;
	if(!buf || !buf_len)
	{
#ifdef	CHECKED_BUILD
		conlPuts(LOG_ERROR,"tokCopyToken(): incorrect parameter(s).\n");
#endif	/* CHECKED_BUILD */
		return 0;
	}
	token = tokLast(scanner);
	if(token)
	{
		token_len = strlen(token);
		if(token_len >= buf_len)
			token_len = buf_len - 1;
		memCopy(buf,token,token_len);
		buf[token_len]=0;
		return token_len;
	}
#ifdef	CHECKED_BUILD
	conlPuts(LOG_ERROR,"tokCopyToken(): no more chars.\n");
#endif	/* CHECKED_BUILD */
	return 0;
}
