A token describing a syntactic unit of the Bond scripting language (e.g. keywords, identifiers, literal constants and operators). More...

#include <bond/compiler/token.h>

Public Types
enum	TokenType { KEY_ALIGNOF , KEY_BOOL , KEY_BREAK , KEY_CASE , KEY_CAST , KEY_CHAR , KEY_CONST , KEY_CONTINUE , KEY_DEFAULT , KEY_DO , KEY_DOUBLE , KEY_ELSE , KEY_ENUM , KEY_FLOAT , KEY_FOR , KEY_IF , KEY_INCLUDE , KEY_INT , KEY_LONG , KEY_NAMESPACE , KEY_NATIVE , KEY_RETURN , KEY_SHORT , KEY_SIZEOF , KEY_STRUCT , KEY_SWITCH , KEY_THIS , KEY_UCHAR , KEY_UINT , KEY_ULONG , KEY_USHORT , KEY_VOID , KEY_WHILE , IDENTIFIER , OP_PLUS , OP_MINUS , OP_STAR , OP_DIV , OP_MOD , OP_INC , OP_DEC , OP_LT , OP_LTE , OP_GT , OP_GTE , OP_LEFT , OP_RIGHT , OP_EQUAL , OP_NOT_EQUAL , OP_NOT , OP_AND , OP_OR , OP_AMP , OP_BIT_OR , OP_BIT_XOR , OP_BIT_NOT , OP_TERNARY , OP_ARROW , ASSIGN , ASSIGN_LEFT , ASSIGN_RIGHT , ASSIGN_PLUS , ASSIGN_MINUS , ASSIGN_MULT , ASSIGN_DIV , ASSIGN_MOD , ASSIGN_AND , ASSIGN_OR , ASSIGN_XOR , OPAREN , CPAREN , OBRACE , CBRACE , OBRACKET , CBRACKET , PERIOD , SEMICOLON , COLON , SCOPE , COMMA , CONST_BOOL , CONST_CHAR , CONST_INT , CONST_UINT , CONST_LONG , CONST_ULONG , CONST_FLOAT , CONST_DOUBLE , CONST_STRING , CONST_NULL , INVALID , END , NUM_TOKEN_TYPES }
	An enumeration describing the various types of Tokens. More...

enum	Annotation { OCTAL = 1 , HEX = 2 }
	An enumeration describing a characteristic of the token as it appears in the Bond source. Currently only used to qualify integer literals as being expressed as hex or octal. More...

Public Member Functions
	Token ()
	Default constructs an empty Token object. More...

	Token (const char *text, TokenType tokenType)
	Constructs a Token object with the given text and type. More...

	Token (const StreamPos &startPos, const StreamPos &endPos, const Value &value, const char text, const char fileName, TokenType tokenType, short annotations=0)
	Constructs a fully described token. More...

const StringView &	GetText () const
	Returns the text that forms the token as it appears in the Bond source. More...

const char *	GetRawText () const
	Like GetText, but returns the text as a C-style string. The text may be truncated if it contains a null character. More...

void	SetText (const char *text)
	Assigns the text that forms the token. The text is assumed to be null terminated. More...

void	SetText (const char *text, size_t length)
	Assigns the text that forms the token. The text may contain null characters.. More...

const char *	GetFileName () const
	Returns the name of the Bond source file from which the Token was generated. More...

void	SetFileName (const char *fileName)
	Assigns the name of the Bond source file from which the Token was generated. More...

const StreamPos &	GetStartPos () const
	Returns the starting position of the Token in the Bond source file. More...

void	SetStartPos (const StreamPos &pos)
	Assigns the starting position of the Token in the Bond source file. More...

size_t	GetEndIndex () const
	Returns the offset into the Bond source of the end of the token. To save space, the entire position information is not available. More...

void	SetEndIndex (size_t index)
	Assigns the offset into the Bond source of the end of the token. More...

TokenType	GetTokenType () const
	Returns the type of the token. More...

void	SetTokenType (const TokenType &type)
	Assigns the type of the Token. More...

void	AddAnnotation (const Annotation &annotation)
	Adds an annotation to the token. More...

bool	HasAnnotation (const Annotation &annotation) const
	Returns whether the Token as a specified annotation. More...

const Value &	GetValue () const
	Returns the value of the token. Valid only for constant literal tokens. More...

bool	GetBoolValue () const
	Returns the boolean value of the Token. Valid only for Tokens of type `CONST_BOOL`. More...

void	SetBoolValue (bool value)
	Assigns the boolean value of the Token. Valid only for Tokens of type `CONST_BOOL`. More...

int32_t	GetIntValue () const
	Returns the integral value of the Token. Valid only for Tokens of type `CONST_INT`. More...

void	SetIntValue (int32_t value)
	Assigns the integral value of the Token. Valid only for Tokens of type `CONST_INT`. More...

uint32_t	GetUIntValue () const
	Returns the integral value of the Token. Valid only for Tokens of type `CONST_UINT`. More...

void	SetUIntValue (uint32_t value)
	Assigns the integral value of the Token. Valid only for Tokens of type `CONST_UINT`. More...

int64_t	GetLongValue () const
	Returns the integral value of the Token. Valid only for Tokens of type `CONST_LONG`. More...

void	SetLongValue (int64_t value)
	Assigns the integral value of the Token. Valid only for Tokens of type `CONST_LONG`. More...

uint64_t	GetULongValue () const
	Returns the integral value of the Token. Valid only for Tokens of type `CONST_ULONG`. More...

void	SetULongValue (uint64_t value)
	Assigns the integral value of the Token. Valid only for Tokens of type `CONST_ULONG`. More...

float	GetFloatValue () const
	Returns the floating point value of the Token. Valid only for Tokens of type `CONST_FLOAT`. More...

void	SetFloatValue (float value)
	Assigns the floating point value of the Token. Valid only for Tokens of type `CONST_FLOAT`. More...

double	GetDoubleValue () const
	Returns the floating point value of the Token. Valid only for Tokens of type `CONST_DOUBLE`. More...

void	SetDoubleValue (double value)
	Assigns the floating point value of the Token. Valid only for Tokens of type `CONST_DOUBLE`. More...

const StringView	GetStringValue () const
	Returns the string value of the Token. Valid only for Tokens of type `CONST_STRING`. More...

size_t	GetStringLength () const
	A short-hand for GetStringValue().length(). More...

void	SetStringValue (const StringView &str)
	Assigns the string value of the Token. Valid only for Tokens of type `CONST_STRING`. More...

const char *	GetTokenName () const
	Returns a descriptive name for the token's type. More...

Static Public Member Functions
static const char *	GetTokenName (TokenType type)
	Returns a descriptive name for the given TokenType. More...

Detailed Description

A token describing a syntactic unit of the Bond scripting language (e.g. keywords, identifiers, literal constants and operators).

A Token is typically created by a Lexer as it scans Bond source code. It contains the text from which the token is formed as it appears in the Bond source, the position at which it appears in the Bond source, a type and a value for tokens which are literal constants.

See also: Lexer

Member Enumeration Documentation

◆ Annotation

enum Bond::Token::Annotation

An enumeration describing a characteristic of the token as it appears in the Bond source. Currently only used to qualify integer literals as being expressed as hex or octal.

Enumerator
OCTAL
HEX

◆ TokenType

enum Bond::Token::TokenType

An enumeration describing the various types of Tokens.

Enumerator
KEY_ALIGNOF
KEY_BOOL
KEY_BREAK
KEY_CASE
KEY_CAST
KEY_CHAR
KEY_CONST
KEY_CONTINUE
KEY_DEFAULT
KEY_DO
KEY_DOUBLE
KEY_ELSE
KEY_ENUM
KEY_FLOAT
KEY_FOR
KEY_IF
KEY_INCLUDE
KEY_INT
KEY_LONG
KEY_NAMESPACE
KEY_NATIVE
KEY_RETURN
KEY_SHORT
KEY_SIZEOF
KEY_STRUCT
KEY_SWITCH
KEY_THIS
KEY_UCHAR
KEY_UINT
KEY_ULONG
KEY_USHORT
KEY_VOID
KEY_WHILE
IDENTIFIER
OP_PLUS
OP_MINUS
OP_STAR
OP_DIV
OP_MOD
OP_INC
OP_DEC
OP_LT
OP_LTE
OP_GT
OP_GTE
OP_LEFT
OP_RIGHT
OP_EQUAL
OP_NOT_EQUAL
OP_NOT
OP_AND
OP_OR
OP_AMP
OP_BIT_OR
OP_BIT_XOR
OP_BIT_NOT
OP_TERNARY
OP_ARROW
ASSIGN
ASSIGN_LEFT
ASSIGN_RIGHT
ASSIGN_PLUS
ASSIGN_MINUS
ASSIGN_MULT
ASSIGN_DIV
ASSIGN_MOD
ASSIGN_AND
ASSIGN_OR
ASSIGN_XOR
OPAREN
CPAREN
OBRACE
CBRACE
OBRACKET
CBRACKET
PERIOD
SEMICOLON
COLON
SCOPE
COMMA
CONST_BOOL
CONST_CHAR
CONST_INT
CONST_UINT
CONST_LONG
CONST_ULONG
CONST_FLOAT
CONST_DOUBLE
CONST_STRING
CONST_NULL
INVALID
END
NUM_TOKEN_TYPES

Constructor & Destructor Documentation

◆ Token() [1/3]

Bond::Token::Token ( )

inline

Default constructs an empty Token object.

◆ Token() [2/3]

Bond::Token::Token	(	const char *	text,
		TokenType	tokenType
	)

inline

Constructs a Token object with the given text and type.

Parameters

text	The text as the token appears in the Bond source.
tokenType	The type of token.

◆ Token() [3/3]

Bond::Token::Token	(	const StreamPos &	startPos,
		const StreamPos &	endPos,
		const Value &	value,
		const char *	text,
		const char *	fileName,
		TokenType	tokenType,
		short	annotations = `0`
	)

inline

Constructs a fully described token.

Parameters

startPos	The starting position of the token in the Bond source.
endPos	The ending position of the token in the Bond source.
text	The text as the token appears in the Bond source.
value	The value of a the token for constant literals.
fileName	The name of the Bond source file in which the token appears.
tokenType	The type of token.
annotations	Bit field of Annotation values.

Member Function Documentation

◆ AddAnnotation()

void Bond::Token::AddAnnotation ( const Annotation & annotation )

inline

Adds an annotation to the token.

◆ GetBoolValue()

bool Bond::Token::GetBoolValue ( ) const

inline

Returns the boolean value of the Token. Valid only for Tokens of type CONST_BOOL.

◆ GetDoubleValue()

double Bond::Token::GetDoubleValue ( ) const

inline

Returns the floating point value of the Token. Valid only for Tokens of type CONST_DOUBLE.

◆ GetEndIndex()

size_t Bond::Token::GetEndIndex ( ) const

inline

Returns the offset into the Bond source of the end of the token. To save space, the entire position information is not available.

◆ GetFileName()

const char * Bond::Token::GetFileName ( ) const

inline

Returns the name of the Bond source file from which the Token was generated.

◆ GetFloatValue()

float Bond::Token::GetFloatValue ( ) const

inline

Returns the floating point value of the Token. Valid only for Tokens of type CONST_FLOAT.

◆ GetIntValue()

int32_t Bond::Token::GetIntValue ( ) const

inline

Returns the integral value of the Token. Valid only for Tokens of type CONST_INT.

◆ GetLongValue()

int64_t Bond::Token::GetLongValue ( ) const

inline

Returns the integral value of the Token. Valid only for Tokens of type CONST_LONG.

◆ GetRawText()

const char * Bond::Token::GetRawText ( ) const

inline

Like GetText, but returns the text as a C-style string. The text may be truncated if it contains a null character.

◆ GetStartPos()

const StreamPos & Bond::Token::GetStartPos ( ) const

inline

Returns the starting position of the Token in the Bond source file.

◆ GetStringLength()

size_t Bond::Token::GetStringLength ( ) const

inline

A short-hand for GetStringValue().length().

◆ GetStringValue()

const StringView Bond::Token::GetStringValue ( ) const

inline

Returns the string value of the Token. Valid only for Tokens of type CONST_STRING.

◆ GetText()

const StringView & Bond::Token::GetText ( ) const

inline

Returns the text that forms the token as it appears in the Bond source.

◆ GetTokenName() [1/2]

const char * Bond::Token::GetTokenName ( ) const

Returns a descriptive name for the token's type.

◆ GetTokenName() [2/2]

static const char * Bond::Token::GetTokenName ( TokenType type )

static

Returns a descriptive name for the given TokenType.

◆ GetTokenType()

TokenType Bond::Token::GetTokenType ( ) const

inline

Returns the type of the token.

◆ GetUIntValue()

uint32_t Bond::Token::GetUIntValue ( ) const

inline

Returns the integral value of the Token. Valid only for Tokens of type CONST_UINT.

◆ GetULongValue()

uint64_t Bond::Token::GetULongValue ( ) const

inline

Returns the integral value of the Token. Valid only for Tokens of type CONST_ULONG.

◆ GetValue()

const Value & Bond::Token::GetValue ( ) const

inline

Returns the value of the token. Valid only for constant literal tokens.

◆ HasAnnotation()

bool Bond::Token::HasAnnotation ( const Annotation & annotation ) const

inline

Returns whether the Token as a specified annotation.

◆ SetBoolValue()

void Bond::Token::SetBoolValue ( bool value )

inline

Assigns the boolean value of the Token. Valid only for Tokens of type CONST_BOOL.

◆ SetDoubleValue()

void Bond::Token::SetDoubleValue ( double value )

inline

Assigns the floating point value of the Token. Valid only for Tokens of type CONST_DOUBLE.

◆ SetEndIndex()

void Bond::Token::SetEndIndex ( size_t index )

inline

Assigns the offset into the Bond source of the end of the token.

◆ SetFileName()

void Bond::Token::SetFileName ( const char * fileName )

inline

Assigns the name of the Bond source file from which the Token was generated.

◆ SetFloatValue()

void Bond::Token::SetFloatValue ( float value )

inline

Assigns the floating point value of the Token. Valid only for Tokens of type CONST_FLOAT.

◆ SetIntValue()

void Bond::Token::SetIntValue ( int32_t value )

inline

Assigns the integral value of the Token. Valid only for Tokens of type CONST_INT.

◆ SetLongValue()

void Bond::Token::SetLongValue ( int64_t value )

inline

Assigns the integral value of the Token. Valid only for Tokens of type CONST_LONG.

◆ SetStartPos()

void Bond::Token::SetStartPos ( const StreamPos & pos )

inline

Assigns the starting position of the Token in the Bond source file.

◆ SetStringValue()

void Bond::Token::SetStringValue ( const StringView & str )

Assigns the string value of the Token. Valid only for Tokens of type CONST_STRING.

◆ SetText() [1/2]

void Bond::Token::SetText ( const char * text )

inline

Assigns the text that forms the token. The text is assumed to be null terminated.

◆ SetText() [2/2]

void Bond::Token::SetText	(	const char *	text,
		size_t	length
	)

inline

Assigns the text that forms the token. The text may contain null characters..

Parameters

text	The pointer to the string of text.
length	The number of characters in the text.

◆ SetTokenType()

void Bond::Token::SetTokenType ( const TokenType & type )

inline

Assigns the type of the Token.

◆ SetUIntValue()

void Bond::Token::SetUIntValue ( uint32_t value )

inline

Assigns the integral value of the Token. Valid only for Tokens of type CONST_UINT.

◆ SetULongValue()

void Bond::Token::SetULongValue ( uint64_t value )

inline

Assigns the integral value of the Token. Valid only for Tokens of type CONST_ULONG.

The documentation for this class was generated from the following file:

include/bond/compiler/token.h

Public Types

Public Member Functions

Static Public Member Functions

Detailed Description

Member Enumeration Documentation

◆ Annotation

◆ TokenType

Constructor & Destructor Documentation

◆ Token() [1/3]

◆ Token() [2/3]

◆ Token() [3/3]

Member Function Documentation

◆ AddAnnotation()

◆ GetBoolValue()

◆ GetDoubleValue()

◆ GetEndIndex()

◆ GetFileName()

◆ GetFloatValue()

◆ GetIntValue()

◆ GetLongValue()

◆ GetRawText()

◆ GetStartPos()

◆ GetStringLength()

◆ GetStringValue()

◆ GetText()

◆ GetTokenName() [1/2]

◆ GetTokenName() [2/2]

◆ GetTokenType()

◆ GetUIntValue()

◆ GetULongValue()

◆ GetValue()

◆ HasAnnotation()

◆ SetBoolValue()

◆ SetDoubleValue()

◆ SetEndIndex()

◆ SetFileName()

◆ SetFloatValue()

◆ SetIntValue()

◆ SetLongValue()

◆ SetStartPos()

◆ SetStringValue()

◆ SetText() [1/2]

◆ SetText() [2/2]

◆ SetTokenType()

◆ SetUIntValue()

◆ SetULongValue()