#include <../source/TAL/objbase.h>
#include <../source/PAL/dataman.h>
#include <ctype.h>

typedef struct	scanner_pos_s
{
	char_p		token_ptr;			/* current position in buffer */
	u32_t		current_line;	   /* number of passed 0x0A codes */
	u32_t		last_line;			/* line of last parsed token */
	vboolean	inside_quotes;		/* == true if open - quote encountered */
	vboolean	got_escape;			/* == true if prev char was an escape */
	vboolean	crossline_before_quote;
	u32_t		unget_char;
	char_p		unget_token;		/* used in push / pop operations */
	char_p		last_token;
}	scanner_pos_t, *scanner_pos_p;

#define	LEX_INITIAL_POS(pos) { \
		(pos)->token_ptr = NULL; \
		(pos)->current_line = 1; \
		(pos)->last_line = 1; \
		(pos)->inside_quotes = false; \
		(pos)->got_escape = false; \
		(pos)->unget_char = MAXLONG; \
		(pos)->unget_token = NULL; \
	}

typedef struct	scanner_state_s
{
	char		comment_char;
	char		quote_char;
	char		escape_char;
	s32_t		fp_domain;
	u8_t		flags[LEX_MAX_STATE_FLAG];
}	scanner_state_t, *scanner_state_p;


#define	LEX_DEFAULT_STATE(state) { \
		(state)->comment_char = 0; \
		(state)->quote_char = '"'; \
		(state)->escape_char = '\\'; \
		(state)->fp_domain = LEX_FP_DOMAIN_ANY; \
		(state)->flags[LEX_DEFAULT_DELIMITERS] = 1; \
		(state)->flags[LEX_LINE_MODE] = 0; \
		(state)->flags[LEX_LINE_ADVANCING] = 1; \
		(state)->flags[LEX_ONE_CHAR_COMMENTS] = 0; \
		(state)->flags[LEX_QUOTES_RETURNING] = 0; \
		(state)->flags[LEX_QUOTES_PROCESSING] = 1; \
		(state)->flags[LEX_ESCAPES_PROCESSING] = 1; \
		(state)->flags[LEX_ALIAS_TRANSLATION] = 0; \
		(state)->flags[LEX_FP_DOMAIN_CHECK] = 0; \
	}

/* fixme: reimplement delimiting facilities as array[256] of _flags_ */

#define	MAX_DELIM	256

typedef struct  scanner_struct_s	/* simple lexical analyser */
{
	libv_object_t base;
	char		source_name[_MAX_PATH];
	char_p	  token_buffer_start; /* beginnig of buffer to parse */
	char_p	  token_buffer_end;   /* pointer to 1st byte after buffer */
	scanner_pos_t	pos;
	scanner_state_t	state;
	u8_stack_t  token;			  /* buffer for last parsed token */
	u8_stack_t  unget_token;		/* waiting token */
	char		*delim;				/* list of delimiters */
	char		*last_token;		/* last parsed token | NULL if no more */
	bstack_t	pos_stack;			/* saved positions(s) */
	bstack_t	state_stack;		/* saved state(s) */
	vhandle_t	alias_hash;			/* aliases */
	char_p		string_store;		/* if non-NULL, mut memFree() it */
}   scanner_struct_t,   *scanner_struct_p;

#ifdef  CHECKED_BUILD
#define SCANNER_MAGIC   0x53525052
V_LOCAL char			*scanner_object="scanner";
#endif  /* CHECKED_BUILD */

#define GET_SCANNER(handle) \
		(CHECK_AND_GET_OBJECT(scanner_struct_t,handle,caller_name, \
			scanner_object,SCANNER_MAGIC))

#define IS_WHITESPACE(c)		((unsigned char)(c) <= 32) /* explainme: should i use u8_t? */

#define IS_EOL(c)			   (((c) == 0x0D) || ((c) == 0x0A))

#define IS_QUOTE_CHAR(sc,c) \
	((sc)->state.flags[LEX_QUOTES_PROCESSING] && \
		((c) == (sc)->state.quote_char) && \
		!(sc)->pos.got_escape)

#define IS_ESCAPE_CHAR(sc,c) \
	((sc)->state.flags[LEX_ESCAPES_PROCESSING] && \
		((c) == (sc)->state.escape_char))

#define IS_COMMENT_CHAR(sc,c)   \
	((sc)->state.flags[LEX_ONE_CHAR_COMMENTS] && \
		((c) == (sc)->state.comment_char) && \
		!(sc)->pos.inside_quotes)

#define	IS_DELIM(sc,ch)			(strchr((sc)->delim,(ch)) ? true : false)
#define	SET_UNGET_CHAR(sc,ch)	(sc)->pos.unget_char = (u32_t)(ch)
#define	GET_UNGOT_CHAR(sc)		((char)((sc)->pos.unget_char & 0xFF))
#define	UNGET_CLEAR(sc)			(sc)->pos.unget_char = MAXLONG
#define	HAVE_UNGOT_CHAR(sc)		((sc)->pos.unget_char != MAXLONG)

#define HAVE_CHARS(sc) { \
		(sc)->last_token = (char*)u8sGetStorage(&(sc)->token); \
		return; \
	}

#define HAVE_ONE_CHAR(sc,ch) { \
		u8sPush(&(sc)->token,(u8_t)(ch)); \
		u8sPush(&(sc)->token,0); \
		HAVE_CHARS(self); \
	}

#define NO_MORE_CHARS(sc) { \
		u8sResize(&(sc)->token,0); \
		(sc)->last_token = NULL; \
		return; \
	}

/*
 * lex_empty_pos_stack()
 */
V_LOCAL void
lex_empty_pos_stack(scanner_struct_p self)
{
	scanner_pos_t pos;
	while(stackPop(&self->pos_stack,&pos))
		if(pos.unget_token)
			memFree(pos.unget_token);
}

/*
 * lex_destroy_scanner()
 */
V_LOCAL u32_t V_CALL
lex_destroy_scanner(vhandle_t handle)
{
	scanner_struct_p self = (scanner_struct_p)handle;
	UNREF_EXIT(self);
	u8sClear(&self->token);
	u8sClear(&self->unget_token);
	lex_empty_pos_stack(self);
	stackClear(&self->pos_stack);
	stackClear(&self->state_stack);
	if(self->alias_hash != LIBV_INVALID_HANDLE)
		hashDestroy(self->alias_hash);
	if(self->string_store)
		memFree(self->string_store);
	memFreeObject(self);
	return 0;
}

/*
 * lex_alloc_scanner()
 */
V_LOCAL scanner_struct_p
lex_alloc_scanner(char *delim,u32_t buffer_len)
{
	scanner_struct_p self;
	/* reserve MAX_DELIM bytes for delimiters */
	ALLOC_OBJECT(self,scanner_struct_t,MAX_DELIM,SCANNER_MAGIC);
	self->base.destroy = lex_destroy_scanner;
	self->delim = (char*)(self + 1);
	/* init state */
	LEX_INITIAL_POS(&(self->pos));
	LEX_DEFAULT_STATE(&self->state);
	self->last_token = NULL;
	if(delim)
		strCopy(self->delim,delim,MAX_DELIM);
	else
		self->delim[0] = 0;
	u8sInit(&self->token);
	u8sInit(&self->unget_token);
	UNGET_CLEAR(self);
	stackInit(&self->state_stack,sizeof(scanner_state_t));
	stackInit(&self->pos_stack,sizeof(scanner_pos_t));
	self->alias_hash = LIBV_INVALID_HANDLE;
	if(buffer_len)
	{
		self->string_store = (char_p)memAlloc(buffer_len);
		/* Set pointers to storage */
		self->token_buffer_start = self->pos.token_ptr = self->string_store;
		self->token_buffer_end = self->token_buffer_start + buffer_len;
	}
	else
		self->token_buffer_start = self->pos.token_ptr
			= self->token_buffer_end = NULL;
	return self;
}

/* FIXME: should i use below global names_of_pnames? */

/*
 * lexEnable()
 */
vboolean	V_GLOBAL
lexEnable(vhandle_t handle,u32_t pname)
{
	CALLER_NAME(lexEnable)
	DEBUG_MESSAGE(unknown_pname,"unknown state flag")
	scanner_struct_p self;
	if(pname >= LEX_MAX_STATE_FLAG)
	{
		ERROR_GENERIC(unknown_pname);
		return false;
	}
	self = GET_SCANNER(handle);
	if(self == LIBV_INVALID_HANDLE)
		return false;
	self->state.flags[pname] = 1;
	return true;
}

/*
 * lexDisable()
 */
vboolean	V_GLOBAL
lexDisable(vhandle_t handle,u32_t pname)
{
	CALLER_NAME(lexDisable)
	DEBUG_MESSAGE(unknown_pname,"unknown state flag")
	scanner_struct_p self;
	if(pname >= LEX_MAX_STATE_FLAG)
	{
		ERROR_GENERIC(unknown_pname);
		return false;
	}
	self = GET_SCANNER(handle);
	if(self == LIBV_INVALID_HANDLE)
		return false;
	self->state.flags[pname] = 0;
	return true;
}

/*
 * lexIsEnabled()
 */
vboolean	V_GLOBAL
lexIsEnabled(vhandle_t handle,u32_t pname)
{
	CALLER_NAME(lexIsEnabled)
	DEBUG_MESSAGE(unknown_pname,"unknown state flag")
	scanner_struct_p self;
	if(pname >= LEX_MAX_STATE_FLAG)
	{
		ERROR_GENERIC(unknown_pname);
		return false;
	}
	self = GET_SCANNER(handle);
	if(self == LIBV_INVALID_HANDLE)
		return false;
	return self->state.flags[pname] ? true : false;
}

/*
 * lexDestroy()
 */
void	V_GLOBAL
lexDestroy(vhandle_t handle)
{
	CALLER_NAME(lexDestroy)
	libvCloseHandle(handle,caller_name,scanner_object,SCANNER_MAGIC);
}

/*
 * lexParseBuffer()
 */
vhandle_t V_GLOBAL
lexParseBuffer(char *buffer,u32_t buf_len,char *delim)
{
	CALLER_NAME(lexParseBuffer)
	scanner_struct_p self;
	if(!buffer || !buf_len)
	{
		ERROR_GENERIC(error_incorrect_parameters);
		return LIBV_INVALID_HANDLE;
	}
	self = lex_alloc_scanner(delim,0);
	self->pos.token_ptr = self->token_buffer_start = buffer;
	self->token_buffer_end = buffer + buf_len;
	strCopy(self->source_name,"[static buffer]",_MAX_PATH);
	return (vhandle_t)self;
}

/*
 * lexParseString()
 */
vhandle_t V_GLOBAL
lexParseString(char *str,char *delim)
{
	CALLER_NAME(lexParseString)
	if(!str || !*str)
	{
		ERROR_GENERIC(error_incorrect_parameters);
		return LIBV_INVALID_HANDLE;
	}
	return lexParseBuffer(str,strlen(strSafe(str)),delim);
}

/*
 * lex_read_file()
 */
V_LOCAL u32_t
lex_read_file(vhandle_t in_stream,u8_stack_p dest)
{
	/* *TODO*: conversion from unicode || and/or unicode parsing facilities */
#define READ_PAGE_SIZE	  (4096*64)
	u32_t read_in;
	u8_t *ptr = (u8_t*)memAlloc(READ_PAGE_SIZE);
	V_ASSERT(ptr);
	for(;;)
	{
		read_in = fileRead(in_stream,ptr,READ_PAGE_SIZE);
		if(read_in)
			u8sPushBuffer(dest,ptr,read_in);
		else
			break;
	}
	memFree(ptr);
	return u8sCount(dest);
}
		
/*
 * lexParseStream()
 */
vhandle_t V_GLOBAL
lexParseStream(vhandle_t infile,char *delim)
{
	CALLER_NAME(lexParseStream)
	u32_t len;
	scanner_struct_p self;
	u8_stack_t tmp_store;
	if(infile == LIBV_INVALID_HANDLE)
	{
		ERROR_GENERIC(error_incorrect_parameters);
		return LIBV_INVALID_HANDLE;
	}
	u8sInit(&tmp_store);
	len = lex_read_file(infile,&tmp_store);
	self = lex_alloc_scanner(delim,len);
	if(len)
		memCopy(self->token_buffer_start,u8sGetStorage(&tmp_store),len);
	u8sClear(&tmp_store);
	if(!fileControl(infile,pctl_get_name,self->source_name))
		strCopy(self->source_name,"[unnamed stream]",_MAX_PATH);
	return (vhandle_t)self;
}

/*
 * lexParseFile()
 */
vhandle_t V_GLOBAL
lexParseFile(char *filename,char *delim)
{
	CALLER_NAME(lexParseFile)
	DEBUG_MESSAGE(unable_to_open,"unable to open: ")
	vhandle_t infile, scanner;
	infile = fileOpenRead(filename);
	if(!infile)
	{
		ERROR_GENERIC_2ARG(unable_to_open,strSafe(filename));
		return LIBV_INVALID_HANDLE;
	}
	scanner = lexParseStream(infile,delim);
	fileClose(infile);
	return (vhandle_t)scanner;
}

/*
 * lexPush()
 */
vboolean	V_GLOBAL
lexPush(vhandle_t handle,u32_t bitfields)
{
	CALLER_NAME(lexPush)
	scanner_struct_p self = GET_SCANNER(handle);
	if(self == LIBV_INVALID_HANDLE)
		return false;
	if(bitfields & LEX_POSITION_BIT)
	{
		u32_t unget_len = u8sCount(&self->unget_token);
		if(unget_len)
		{
			char *unget = strDup((char*)u8sGetStorage(&self->unget_token));
			self->pos.unget_token = unget;
		}
		else
			self->pos.unget_token = NULL;
		if(self->last_token)
			self->pos.last_token = strDup(self->last_token);
		else
			self->pos.last_token = NULL;
		stackPush(&self->pos_stack,&self->pos);
	}
	if(bitfields & LEX_STATE_BIT)
		stackPush(&self->state_stack,&self->state);
	return true;
}

/*
 * lexPop()
 */
vboolean	V_GLOBAL
lexPop(vhandle_t handle,u32_t bitfields)
{
	CALLER_NAME(lexPop)
	DEBUG_MESSAGE(stack_underflow,"stack underflow")
	DEBUG_MESSAGE(position,"position ")
	DEBUG_MESSAGE(state,"state ")
	vboolean result = true;
	scanner_struct_p self = GET_SCANNER(handle);
	if(self == LIBV_INVALID_HANDLE)
		return false;
	if(bitfields & LEX_POSITION_BIT)
	{
		if(!stackPop(&self->pos_stack,&self->pos))
		{
			ERROR_GENERIC_2ARG(position,stack_underflow);
			result = false;
		}
		if(self->pos.unget_token)
		{
			u32_t len = strlen(self->pos.unget_token) + 1;
			u8sResize(&self->unget_token,0);
			u8sPushBuffer(&self->unget_token,(u8_p)self->pos.unget_token,len);
			memFree(self->pos.unget_token);
			self->pos.unget_token = NULL;
		}
		if(self->pos.last_token)
		{
			u32_t len = strlen(self->pos.last_token) + 1;
			u8sResize(&self->token,0);
			u8sPushBuffer(&self->token,(u8_p)self->pos.last_token,len);
			self->last_token = (char*)u8sGetStorage(&self->token);
			memFree(self->pos.last_token);
			self->pos.last_token = NULL;
		}
		else
			self->last_token = NULL;
	}
	if(bitfields & LEX_STATE_BIT)
	{
		if(!stackPop(&self->state_stack,&self->state))
		{
			ERROR_GENERIC_2ARG(state,stack_underflow);
			result = false;
		}
	}
	return result;
}

/*
 * lexClearStacks()
 */
void	V_GLOBAL
lexClearStacks(vhandle_t handle,u32_t bitfields)
{
	CALLER_NAME(lexClearStateStack)
	scanner_struct_p self = GET_SCANNER(handle);
	if(self == LIBV_INVALID_HANDLE)
		return;
	stackResize(&self->state_stack,0);
	if(bitfields & LEX_POSITION_BIT)
		lex_empty_pos_stack(self);
	if(bitfields & LEX_STATE_BIT)
		stackResize(&self->state_stack,0);
}

/*
 * lex_handle_escape()
 */
V_LOCAL vboolean
lex_handle_escape(scanner_struct_p self,char *char_ret,
	vboolean *to_continue)
{
	CALLER_NAME(lex_handle_escape)
	DEBUG_MESSAGE(unexpected_eof,"unexpected EOF")
	DEBUG_MESSAGE(wrong_hex_number,"wrong hexadecimal number")
	DEBUG_MESSAGE(wrong_oct_number,"wrong octal number")
	u32_t val;
	unsigned char code[4] = {0,0,0};
	*to_continue = false;
	/* skip it and get next character */
	if(self->pos.token_ptr >= self->token_buffer_end)
	{
		SYN_UNEXPECTED_EOF((vhandle_t)self);
		return false;
	}
	self->pos.token_ptr++;
	switch(*self->pos.token_ptr)
	{
		case 0x0D:
			if(self->pos.token_ptr < self->token_buffer_end)
				self->pos.token_ptr++;
			/* fall though */
		case 0x0A:
			self->pos.current_line++;
			self->pos.token_ptr++;
			*to_continue = true;
			return true;
		case 'a':
			*char_ret = 0x07; /* bell */
			break;
		case 'b':
			*char_ret = 0x08; /* backspace */
			break;
		case 'f':
			*char_ret = 0x0C; /* form feed */
			break;
		case 'n':
			*char_ret = 0x0A; /* line feed */
			break;
		case 'r':
			*char_ret = 0x0D; /* carriage return */
			break;
		case 't':
			*char_ret = 0x09; /* tab */
			break;
		case 'v':
			*char_ret = 0x0B; /* vertical tab */
			break;
		case 'x':
			if((self->pos.token_ptr + 3) > self->token_buffer_end)
			{
				ERROR_GENERIC(unexpected_eof);
				return false;
			}
			code[0] = self->pos.token_ptr[1];
			code[1] = self->pos.token_ptr[2];
			if(!strToHex((char*)code,&val))
			{
				ERROR_GENERIC(wrong_hex_number);
				return false;
			}
			*char_ret = (char)val;
			self->pos.token_ptr += 2;
			break;
		case '0':
		case '1':
		case '2':
		case '3':
		case '4':
		case '5':
		case '6':
		case '7':
			if((self->pos.token_ptr+3) > self->token_buffer_end)
			{
				ERROR_GENERIC(unexpected_eof);
				return false;
			}
			code[0] = (unsigned char)(self->pos.token_ptr[0] - '0');
			code[1] = self->pos.token_ptr[1];
			code[2] = self->pos.token_ptr[2];
			if(code[1] < '0' || code[1] > '7' ||
				code[2] < '0' || code[2] > '7')
			{
				ERROR_GENERIC(wrong_oct_number);
				return false;
			}
			val = 64 * (u32_t)(code[0]) + 8 * (u32_t)(code[1] - '0')
				+ (u32_t)(code[2] - '0');
			*char_ret = (char)val;
			self->pos.token_ptr += 2;
			break;
		default:
			*char_ret = *self->pos.token_ptr;
			break;
	}
	self->pos.token_ptr++;
	return true;
}


/*
 * lex_next_char()
 */
V_LOCAL vboolean
lex_next_char(scanner_struct_p self,char_p char_ret)
{
	if(HAVE_UNGOT_CHAR(self))
	{
		*char_ret = GET_UNGOT_CHAR(self);
		UNGET_CLEAR(self);
		goto process_char;
	}
	while(self->pos.token_ptr < self->token_buffer_end)
	{
		*char_ret = *self->pos.token_ptr;
process_char:;
		if(IS_ESCAPE_CHAR(self,*char_ret))
		{
			vboolean to_continue;
			self->pos.got_escape = true;
			if(!lex_handle_escape(self,char_ret,&to_continue))
				break;
			if(to_continue)
				continue;
			return true;
		}
		self->pos.got_escape = false;
		/* EOL not allowed if LEX_LINE_ADVANCING not set */
		if(!self->state.flags[LEX_LINE_ADVANCING] && IS_EOL(*char_ret))
			break;
		if(*char_ret == 0x0A) /* it's Ok: _ignore_ 0x0Ds completely */
			self->pos.current_line++;
		if(IS_COMMENT_CHAR(self,*char_ret))
		{
			/* discard end of line */
			while(self->pos.token_ptr < self->token_buffer_end &&
				!IS_EOL(*self->pos.token_ptr))
				self->pos.token_ptr++;
			continue;
		}
		self->pos.token_ptr++;
		return true;
	}
	return false;
}

/*
 * lex_token_ready()
 * Checks if next token is already parsed and if it is,
 * returns it. Otherwise, resurns NULL.
 */
V_LOCAL vboolean
lex_token_ready(scanner_struct_p sc)
{
	u32_t unget_len = u8sCount(&sc->unget_token);
	if(unget_len)
	{
		u8sResize(&sc->token,0);
		u8sPushBuffer(&sc->token,u8sGetStorage(&sc->unget_token),unget_len);
		u8sResize(&sc->unget_token,0);
		sc->last_token = (char*)u8sGetStorage(&sc->token);
		return true;
	}
	return false;
}

/*
 * lex_skip_whitespace()
 * Returns true if at least one character of input still available.
 */
V_LOCAL vboolean
lex_skip_whitespace(scanner_struct_p self,char_p char_ret)
{
	char next;
	do
	{
		if(!lex_next_char(self,&next))
			return false;
	} while(IS_WHITESPACE(next));
	*char_ret = next;
	return true;
}

/*
 * lex_extract_string()
 */
V_LOCAL void
lex_extract_string(scanner_struct_p self)
{
	CALLER_NAME(lex_extract_string)
	char next;
	for(;;)
	{
		if(!lex_next_char(self,&next))
		{
			/* error in input: stop parsing */
			SYN_UNEXPECTED_EOL(self);
			NO_MORE_CHARS(self);
		}
		if(IS_QUOTE_CHAR(self,next))
			break;
		u8sPush(&self->token,next);
	}
	if(u8sCount(&self->token))
	{
		/* non-empty string */
		u8sPush(&self->token,0);
		/* save the quote */
		u8sResize(&self->unget_token,0);
		u8sPush(&self->unget_token,(u8_t)next);
		u8sPush(&self->unget_token,0);
		HAVE_CHARS(self);
	}
	/* empty string */
	HAVE_ONE_CHAR(self,next);
}

/*
 * lex_extract_delimited_token()
 * Ignores quotes and whitespace.
 */
V_LOCAL void
lex_extract_delimited_token(scanner_struct_p self)
{
	char next;
	/* Read one char */
	if(!lex_next_char(self,&next))
		NO_MORE_CHARS(self);
	/* We have al least one char */
	if(IS_DELIM(self,next))
		/* return delimiter */
		HAVE_ONE_CHAR(self,next);
	/* extract next token enclosed in delimiters */
	for(;;)
	{
		u8sPush(&self->token,(u8_t)next);
		if(!lex_next_char(self,&next))
			break;
		if(IS_DELIM(self,next))
		{
			/* store delimiter in 'unget_token' */
			u8sResize(&self->unget_token,0);
			u8sPush(&self->unget_token,(u8_t)next);
			u8sPush(&self->unget_token,0);
			break;
		}
	}
	u8sPush(&self->token,0);
	HAVE_CHARS(self);
}

/*
 * lex_extract_line()
 * Extracts next line of input (all chars from current pos to EOL).
 */
V_LOCAL	void
lex_extract_line(scanner_struct_p self)
{
	char next;
	/* Skip EOLs */
	do
	{
		if(!lex_next_char(self,&next))
			NO_MORE_CHARS(self);
	} while(IS_EOL(next));
	/* We have at least one non-EOL char */
	self->pos.last_line = self->pos.current_line;
	do
	{
		u8sPush(&self->token,(u8_t)next);
		if(!lex_next_char(self,&next))
			break;
	} while(!IS_EOL(next));
	u8sPush(&self->token,0);
	HAVE_CHARS(self);
}

V_LOCAL void
lex_get_next_token(scanner_struct_p self,vboolean *is_quote)
{
	char next;
	vboolean have_chars;
	*is_quote = false;
	/* Check if token already waiting */
	if(lex_token_ready(self))
	{
		*is_quote = (IS_QUOTE_CHAR(self,self->last_token[0]) && 
			!(self->last_token[1]));
		return;
	}
	u8sResize(&self->token,0); /* clear dest buffer */
	if(self->pos.inside_quotes)
	{
		lex_extract_string(self);
		self->state.flags[LEX_LINE_ADVANCING] = 
			(u8_t)((self->pos.crossline_before_quote) ? 1 : 0);
		self->pos.inside_quotes = false;
		return;
	}
	if(!self->state.flags[LEX_LINE_ADVANCING] &&
		(self->pos.current_line > self->pos.last_line))
		/* can't advance to the next line */
		NO_MORE_CHARS(self);
	self->pos.last_line = self->pos.current_line;
	/* Check mode flags */
	if(!self->state.flags[LEX_DEFAULT_DELIMITERS])
	{
		lex_extract_delimited_token(self);
		return;
	}
	if(self->state.flags[LEX_LINE_MODE])
	{
		lex_extract_line(self);
		return;
	}
	/* Skip whitespace */
	have_chars = lex_skip_whitespace(self,&next);
	self->pos.last_line = self->pos.current_line;
	if(!have_chars)
		NO_MORE_CHARS(self);
	/* Now we have one non-whitespace char in 'next' */
	if(IS_QUOTE_CHAR(self,next))
	{
		*is_quote = true;
		/* Extract quoted token; disallow EOL */
		self->pos.crossline_before_quote = 
			self->state.flags[LEX_LINE_ADVANCING] ? true : false;
		self->pos.inside_quotes = true;
		self->state.flags[LEX_LINE_ADVANCING] = 0;
		HAVE_ONE_CHAR(self,self->state.quote_char);
	}
	/* extract next non-quoted token */
	if(IS_DELIM(self,next))
		/* 'next' is a delimiter; return it */
		HAVE_ONE_CHAR(self,next);
	/* extract next token enclosed in delimiters */
	do
	{
		u8sPush(&self->token,(u8_t)next);
		if(!lex_next_char(self,&next))
			break;
		if(IS_QUOTE_CHAR(self,next))
		{
			*is_quote = true;
			SET_UNGET_CHAR(self,next);
			break;
		}
		if(IS_DELIM(self,next))
		{
			/* store delimiter in 'unget_token' */
			u8sResize(&self->unget_token,0);
			u8sPush(&self->unget_token,(u8_t)next);
			u8sPush(&self->unget_token,0);
			break;
		}
	} while(!IS_WHITESPACE(next));
	u8sPush(&self->token,0);
	HAVE_CHARS(self);
}

/*
 * lex_next_r()
 */
V_LOCAL char_p
lex_next_r(scanner_struct_p self)
{
	vboolean is_quote;
	lex_get_next_token(self,&is_quote);
	if(!self->last_token)
		return NULL;
	if(self->state.flags[LEX_ALIAS_TRANSLATION] &&
		(self->alias_hash != LIBV_INVALID_HANDLE))
	{
		char *map;
		map = hashStringForKey(self->alias_hash,self->last_token,NULL);
		while(map && strcmp(map,self->last_token))
		{
			u8sResize(&self->token,0);
			u8sPushBuffer(&self->token,(u8_p)map,strlen(map) + 1);
			self->last_token = (char*)u8sGetStorage(&self->token);
			map = hashStringForKey(self->alias_hash,self->last_token,NULL);
		}
	}
	if(is_quote && !self->state.flags[LEX_QUOTES_RETURNING] &&
		!self->state.flags[LEX_LINE_MODE] && 
		self->state.flags[LEX_DEFAULT_DELIMITERS])
			return lex_next_r(self);
	return self->last_token;
}

/*
 * lexNext()
 */
char_p  V_GLOBAL
lexNext(vhandle_t handle)
{
	CALLER_NAME(lexNext)
	scanner_struct_p self = GET_SCANNER(handle);
	if(self == LIBV_INVALID_HANDLE)
		return NULL;
	return lex_next_r(self);
}

/*
 * lexLast()
 * Returns last extracted token or NULL if no more chars available.
 */
char_p  V_GLOBAL
lexLast(vhandle_t handle)
{
	CALLER_NAME(lexLast)
	scanner_struct_p self;
	self = GET_SCANNER(handle);
	if(self == LIBV_INVALID_HANDLE)
		return NULL;
	return self->last_token;
}

/*
 * lex_restart()
 */ 
V_LOCAL void
lex_restart(scanner_struct_p self)
{
	u8sResize(&self->unget_token,0);
	u8sResize(&self->token,0);
	LEX_INITIAL_POS(&self->pos);
	UNGET_CLEAR(self);
	self->pos.token_ptr = self->token_buffer_start;
	self->last_token = NULL;
}

/*
 * lex_change_source()
 */
V_LOCAL void
lex_change_source(scanner_struct_p self,char_p new_buffer)
{
	CALLER_NAME(lex_change_source)
	vboolean another_buffer;
	strCopy(self->source_name,"[static buffer]",_MAX_PATH);
	another_buffer = self->token_buffer_start != new_buffer ||
		strcmp(self->token_buffer_start,new_buffer);
	/* Can't use old position stack with another buffer */
	if(another_buffer && stackCount(&self->pos_stack))
	{
#ifdef	CHECKED_BUILD
		conlPrintf(LOG_WARNING,"%s(): new buffer, "
			"discarded %u elements in position stack.\n",
			caller_name,stackCount(&self->pos_stack));
#endif	/* CHECKED_BUILD */
		lex_empty_pos_stack(self);
	}
	if(self->string_store)
	{
		memFree(self->string_store);
		self->string_store = NULL;
	}
	self->token_buffer_start = new_buffer;
	self->token_buffer_end = new_buffer + strlen(new_buffer);
	lex_restart(self);
}

/*
 * lexGetInteger()
 */
s32_t	V_GLOBAL
lexGetInteger(vhandle_t handle,u32_t pname)
{
	CALLER_NAME(lexGetInteger)
	DEBUG_MESSAGE(unknown_pname,"unknown 'pname'")
	scanner_struct_p self;
	self = GET_SCANNER(handle);
	if(self == LIBV_INVALID_HANDLE)
		return 0;
	switch(pname)
	{
		case LEX_CURRENT_LINE:
			return self->pos.current_line;
		case LEX_LAST_LINE:
			return self->pos.last_line;
		case LEX_COMMENT_CHAR:
			return self->state.comment_char;
		case LEX_QUOTE_CHAR:
			return self->state.quote_char;
		case LEX_ESCAPE_CHAR:
			return self->state.escape_char;
		case LEX_FP_DOMAIN:
			return self->state.fp_domain;
		default:
			ERROR_GENERIC(unknown_pname);
			break;
	}
	return 0;
}

vboolean V_GLOBAL
lexSetInteger(vhandle_t handle,u32_t pname,s32_t param)
{
	CALLER_NAME(lexSetInteger)
	DEBUG_MESSAGE(unknown_pname,"unknown 'pname'")
	DEBUG_MESSAGE(wrong_param,"wrong 'param'")
	DEBUG_MESSAGE(read_only,"attempt to write read-only 'pname'")
	char dummy = 0;
	scanner_struct_p self;
	self = GET_SCANNER(handle);
	if(self == LIBV_INVALID_HANDLE)
		return false;
	switch(pname)
	{
		case LEX_COMMENT_CHAR:
			self->state.comment_char = (char)param;
			return true;
		case LEX_QUOTE_CHAR:
			self->state.quote_char = (char)param;
			return true;
		case LEX_ESCAPE_CHAR:
			self->state.escape_char = (char)param;
			return true;
		case LEX_CURRENT_LINE:
			if(!param)
			{
				ERROR_GENERIC(wrong_param);
				break;
			}
			lex_restart(self);
			if(param == 1)
				return true;
			while(self->pos.current_line != (u32_t)param)
				if(!lex_next_char(self,&dummy))
				{
					ERROR_GENERIC(wrong_param);
					break;
				}
			SET_UNGET_CHAR(self,dummy);
			return true;
		case LEX_LAST_LINE:
			ERROR_GENERIC(read_only);
			break;
		case LEX_FP_DOMAIN:
			switch(param)
			{
				case LEX_FP_DOMAIN_ANY:
				case LEX_FP_DOMAIN_POSITIVE:
					self->state.fp_domain = param;
					break;
				default:
					ERROR_GENERIC(wrong_param);
					break;
			}
			break;
		default:
			ERROR_GENERIC(unknown_pname);
			break;
	}
	return false;
}

char_p	V_GLOBAL
lexGetString(vhandle_t handle,u32_t pname)
{
	CALLER_NAME(lexGetString)
	DEBUG_MESSAGE(unknown_pname,"unknown 'pname'")
	DEBUG_MESSAGE(write_only,"attempt to read write-only 'pname'")
	scanner_struct_p self;
	self = GET_SCANNER(handle);
	if(self == LIBV_INVALID_HANDLE)
		return NULL;
	switch(pname)
	{
		case LEX_DELIMITER_STRING:
			return self->delim;
		case LEX_SOURCE_NAME:
			return self->source_name;
		case LEX_SOURCE_STRING:
			ERROR_GENERIC(write_only);
			break;
		default:
			ERROR_GENERIC(unknown_pname);
			break;
	}
	return NULL;
}

vboolean V_GLOBAL
lexSetString(vhandle_t handle,u32_t pname,char_p param)
{
	CALLER_NAME(lexSetString)
	DEBUG_MESSAGE(unknown_pname,"unknown 'pname'")
	DEBUG_MESSAGE(read_only,"attempt to write read-only 'pname'")
	scanner_struct_p self;
	if(!param)
	{
		ERROR_GENERIC(error_incorrect_parameters);
		return false;
	}
	self = GET_SCANNER(handle);
	if(self == LIBV_INVALID_HANDLE)
		return false;
	switch(pname)
	{
		case LEX_DELIMITER_STRING:
			strCopy(self->delim,param,MAX_DELIM);
			return true;
		case LEX_SOURCE_STRING:
			lex_change_source(self,param);
			return true;
		case LEX_SOURCE_NAME:
			ERROR_GENERIC(read_only);
			break;
		default:
			ERROR_GENERIC(unknown_pname);
			break;
	}
	return false;
}

vboolean	V_GLOBAL
lexGetBoolean(vhandle_t handle,u32_t pname)
{
	CALLER_NAME(lexGetBoolean)
	DEBUG_MESSAGE(unknown_pname,"unknown 'pname'")
	scanner_struct_p self;
	self = GET_SCANNER(handle);
	if(self == LIBV_INVALID_HANDLE)
		return false;
	switch(pname)
	{
		case LEX_INSIDE_QUOTES:
			return self->pos.inside_quotes;
		default:
			ERROR_GENERIC(unknown_pname);
			break;
	}
	return false;
}

#include <../source/PAL/argsused.h>
vboolean	V_GLOBAL
lexSetBoolean(vhandle_t handle,u32_t pname,vboolean param)
{
	CALLER_NAME(lexGetBoolean)
	DEBUG_MESSAGE(unknown_pname,"unknown 'pname'")
	DEBUG_MESSAGE(read_only,"attempt to write read-only 'pname'")
	scanner_struct_p self;
	self = GET_SCANNER(handle);
	if(self == LIBV_INVALID_HANDLE)
		return false;
	switch(pname)
	{
		case LEX_INSIDE_QUOTES:
			ERROR_GENERIC(read_only);
			break;
		default:
			ERROR_GENERIC(unknown_pname);
			break;
	}
	return false;
}

vboolean	V_GLOBAL
lexDefineAlias(vhandle_t handle,char *value,char *replacement)
{
	CALLER_NAME(lexDefineAlias)
	scanner_struct_p self;
	self = GET_SCANNER(handle);
	if(self == LIBV_INVALID_HANDLE)
		return false;
	if(self->alias_hash == LIBV_INVALID_HANDLE)
		self->alias_hash = hashCreate();
	if(!value)
	{
		ERROR_GENERIC(error_incorrect_parameters);
		return false;
	}
	if(replacement)
		return hashSetString(self->alias_hash,value,replacement);
	return hashRemoveKey(self->alias_hash,value);
}

#define	LEX_WRONG_DOMAIN(self, expected) \
	conlPrintf(LOG_ERROR, \
		"Error parsing \"%s\", line %u:" \
		" wrong FP domain (expected %s, got %s)\n", \
		self->source_name, self->pos.last_line, expected, self->last_token);

static vboolean
lex_fp_domain_check(scanner_struct_p self, vec_t val)
{
	switch(self->state.fp_domain)
	{
		case LEX_FP_DOMAIN_POSITIVE:
			if(val > 0)
				break;
			LEX_WRONG_DOMAIN(self, "positive");
			return false;
		case LEX_FP_DOMAIN_ANY:
		default:
			break;
	}
	return true;
}

vboolean V_GLOBAL
lexNextFloat(vhandle_t handle,vec_t *ret_val)
{
	CALLER_NAME(lexNextFloat)
	vec_t retval;
	char *token;
	scanner_struct_p self = GET_SCANNER(handle);
	if(self == LIBV_INVALID_HANDLE)
		return false;
	token = lex_next_r(self);
	if(!strToFloat(token, &retval))
		return false;
	if(self->state.flags[LEX_FP_DOMAIN_CHECK])
		if(!lex_fp_domain_check(self, retval))
			return false;
	if(ret_val)
		*ret_val = retval;
	return true;
}

vboolean V_GLOBAL
lexLastFloat(vhandle_t handle,vec_t *ret_val)
{
	CALLER_NAME(lexLastFloat)
	vec_t retval;
	scanner_struct_p self = GET_SCANNER(handle);
	if(self == LIBV_INVALID_HANDLE)
		return false;
	if(!strToFloat(self->last_token, &retval))
		return false;
	if(self->state.flags[LEX_FP_DOMAIN_CHECK])
		if(!lex_fp_domain_check(self, retval))
			return false;
	if(ret_val)
		*ret_val = retval;
	return true;
}

/* -------------------------------------------------------------------- *\
						'User-space' helper routines
\* -------------------------------------------------------------------- */

vboolean V_GLOBAL
lexNextIsEx(vhandle_t scanner,char *expected_token,
	int (LIBC_CALL *str_comp)(const char *s1,const char *s2))
{
	char *tok = lexNext(scanner);
	if(tok && !str_comp(tok,strSafe(expected_token)))
		return true;
	return false;
}

vboolean V_GLOBAL
lexLastWasEx(vhandle_t scanner,char *expected_token,
	int (LIBC_CALL *str_comp)(const char *s1,const char *s2))
{
	char *tok = lexLast(scanner);
	if(tok && !str_comp(tok,strSafe(expected_token)))
		return true;
	return false;
}

vboolean V_GLOBAL
lexNextIs(vhandle_t scanner,char *expected_token)
{
	char *tok = lexNext(scanner);
	if(tok && !strcmp(tok,strSafe(expected_token)))
		return true;
	return false;
}

vboolean V_GLOBAL
lexLastWas(vhandle_t scanner,char *expected_token)
{
	char *tok = lexLast(scanner);
	if(tok && !strcmp(tok,strSafe(expected_token)))
		return true;
	return false;
}

vboolean V_GLOBAL
lexSkipTo(vhandle_t scanner,char *token)
{
	while(lexNext(scanner))
		if(lexLastWas(scanner,token))
			return true;
	return false;
}

vboolean V_GLOBAL
lexSkipToEx(vhandle_t scanner,char *token,
	int (LIBC_CALL *str_comp)(const char *s1,const char *s2))
{
	while(lexNext(scanner))
		if(lexLastWasEx(scanner,token,str_comp))
			return true;
	return false;
}
	
vboolean V_GLOBAL
lexSkipBlock(vhandle_t scanner,char skip_from)
{
	CALLER_NAME(lexSkipBlock)
	DEBUG_MESSAGE(wrong_skip_from,"wrong 'skip_from'")
	char *tok;
	char skip_to;
	int count = 1;
	switch(skip_from)
	{
		case '{': skip_to = '}'; break;
		case '}': skip_to = '{'; break;
		case '(': skip_to = ')'; break;
		case ')': skip_to = '('; break;
		case '[': skip_to = ']'; break;
		case ']': skip_to = '['; break;
		default:
		{
			ERROR_GENERIC(wrong_skip_from);
			return false;
		}
	}
	while(count)
	{
		tok = lexNext(scanner);
		if(!tok)
		{
			SYN_UNEXPECTED_EOF(scanner);
			return false;
		}
		if(tok[0]==skip_from && !tok[1])
			count--;
		else
			if(tok[0]==skip_to && !tok[1])
				count++;
	}
	return true;
}

vboolean V_GLOBAL
lexNextInteger(vhandle_t scanner,s32_t *ret_val)
{
	char *tok = lexNext(scanner);
	return strToInt(tok,ret_val);
}

vboolean V_GLOBAL
lexLastInteger(vhandle_t scanner,s32_t *ret_val)
{
	char *tok = lexLast(scanner);
	return strToInt(tok,ret_val);
}

#if 0

/*
 * 13 nov 2011: lexNextFloat() and lexLastFloat() moved to the core.
 */
vboolean V_GLOBAL
lexNextFloat(vhandle_t scanner,vec_t *ret_val)
{
	char *tok = lexNext(scanner);
	return strToFloat(tok,ret_val);
}

vboolean V_GLOBAL
lexLastFloat(vhandle_t scanner,vec_t *ret_val)
{
	char *tok = lexLast(scanner);
	return strToFloat(tok,ret_val);
}

#endif

u32_t V_GLOBAL
lexCopyLastToken(vhandle_t scanner,char *buf,u32_t buf_len)
{
	CALLER_NAME(lexCopyLastToken)
	u32_t token_len;
	char *token;
	if(!buf || !buf_len)
	{
		ERROR_GENERIC(error_incorrect_parameters);
		return 0;
	}
	token = lexLast(scanner);
	if(!token)
	{
		SYN_UNEXPECTED_EOF(scanner);
		return 0;
	}
	token_len = strlen(token);
	if(token_len >= buf_len)
		token_len = buf_len - 1;
	memCopy(buf,token,token_len);
	buf[token_len] = 0;
	return token_len;
}

u32_t V_GLOBAL
lexCopyNextToken(vhandle_t scanner,char *buf,u32_t buf_len)
{
	lexNext(scanner);
	return lexCopyLastToken(scanner,buf,buf_len);
}

vboolean V_GLOBAL
lexLastMatchesEx(vhandle_t scanner,char *table[],
	u32_t table_size,u32_p idx_ret,
	int (LIBC_CALL *str_comp)(const char *s1,const char *s2))
{
	CALLER_NAME(lexLastMatchesEx)
	char_p last;
	u32_t i;
	if(!table_size)
		return false;
	if(!str_comp || !table)
	{
		ERROR_GENERIC(error_incorrect_parameters);
		return false;
	}
	last = lexLast(scanner);
	if(!last)
	{
		SYN_UNEXPECTED_EOF(scanner);
		return false;
	}
	for(i=0;i < table_size;i++)
	{
		if(str_comp(last,table[i]))
			continue;
		if(idx_ret)
			*idx_ret = i;
		return true;
	}
	return false;
}

#if	defined(__WATCOMC__) || defined(__BORLANDC__)

V_LOCAL int LIBC_CALL strcmp_default(const char *s1,const char *s2)
{
	return strcmp(s1,s2);
}

#else

#define	strcmp_default	strcmp

#endif

vboolean V_GLOBAL
lexNextMatches(vhandle_t scanner,char *table[],
	u32_t table_size,u32_p idx_ret)
{
	lexNext(scanner);
	return lexLastMatchesEx(scanner,table,table_size,idx_ret,strcmp_default);
}

vboolean V_GLOBAL
lexLastMatches(vhandle_t scanner,char *table[],
	u32_t table_size,u32_p idx_ret)
{
	return lexLastMatchesEx(scanner,table,table_size,idx_ret,strcmp_default);
}

vboolean V_GLOBAL
lexNextMatchesEx(vhandle_t scanner,char *table[],
	u32_t table_size,u32_p idx_ret,
	int (LIBC_CALL *str_comp)(const char *s1,const char *s2))
{
	lexNext(scanner);
	return lexLastMatchesEx(scanner,table,table_size,idx_ret,str_comp);
}

