libsnobol4 1.0.2
High-performance SNOBOL4-style pattern matching library
Loading...
Searching...
No Matches
lexer.h File Reference

Lexer for SNOBOL pattern syntax. More...

#include <stdbool.h>
#include "snobol/c23_compat.h"
#include <stddef.h>
#include <stdint.h>
Include dependency graph for lexer.h:
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes

struct  token_t
struct  snobol_lexer_state_t

Macros

#define SNOBOL_LEXER_ERROR_MAX   160

Typedefs

typedef struct snobol_lexer snobol_lexer_t

Enumerations

enum  token_type_t {
  TOKEN_EOF , TOKEN_LIT , TOKEN_IDENT , TOKEN_CHARCLASS ,
  TOKEN_INTEGER , TOKEN_PIPE , TOKEN_LPAREN , TOKEN_RPAREN ,
  TOKEN_STAR , TOKEN_PLUS , TOKEN_QUESTION , TOKEN_ANCHOR_START ,
  TOKEN_ANCHOR_END , TOKEN_AT , TOKEN_COLON , TOKEN_LBRACKET ,
  TOKEN_RBRACKET , TOKEN_DOT , TOKEN_EQUALS , TOKEN_COMMA ,
  TOKEN_ERROR
}

Functions

snobol_lexer_t * snobol_lexer_create (const char *source, size_t len)
token_t snobol_lexer_next (snobol_lexer_t *lexer)
token_t snobol_lexer_peek (snobol_lexer_t *lexer)
size_t snobol_lexer_get_pos (snobol_lexer_t *lexer)
size_t snobol_lexer_get_line (snobol_lexer_t *lexer)
bool snobol_lexer_has_error (const snobol_lexer_t *lexer)
const char * snobol_lexer_get_error (const snobol_lexer_t *lexer)
snobol_lexer_state_t snobol_lexer_save (snobol_lexer_t *lexer)
void snobol_lexer_restore (snobol_lexer_t *lexer, snobol_lexer_state_t state)
void snobol_lexer_destroy (snobol_lexer_t *lexer)
const char * snobol_token_name (token_type_t type)

Detailed Description

Lexer for SNOBOL pattern syntax.

Converts UTF-8 source text into a stream of tokens. Opaque handle pattern - implementation details hidden from callers.

Definition in file lexer.h.

Macro Definition Documentation

◆ SNOBOL_LEXER_ERROR_MAX

#define SNOBOL_LEXER_ERROR_MAX   160

Maximum length of a lexer error message.

Definition at line 124 of file lexer.h.

Typedef Documentation

◆ snobol_lexer_t

typedef struct snobol_lexer snobol_lexer_t

Opaque lexer state

Definition at line 73 of file lexer.h.

Enumeration Type Documentation

◆ token_type_t

Token types matching SNOBOL pattern syntax

Enumerator
TOKEN_EOF 
TOKEN_LIT 
TOKEN_IDENT 
TOKEN_CHARCLASS 
TOKEN_INTEGER 
TOKEN_PIPE 
TOKEN_LPAREN 
TOKEN_RPAREN 
TOKEN_STAR 
TOKEN_PLUS 
TOKEN_QUESTION 
TOKEN_ANCHOR_START 
TOKEN_ANCHOR_END 
TOKEN_AT 
TOKEN_COLON 
TOKEN_LBRACKET 
TOKEN_RBRACKET 
TOKEN_DOT 
TOKEN_EQUALS 
TOKEN_COMMA 
TOKEN_ERROR 

Definition at line 24 of file lexer.h.

Function Documentation

◆ snobol_lexer_create()

snobol_lexer_t * snobol_lexer_create ( const char * source,
size_t len )

Create a new lexer for the given source

Parameters
sourceUTF-8 source text (must remain valid until lexer is destroyed)
lenLength of source (or -1 for strlen)
Returns
New lexer (caller owns, must free with snobol_lexer_destroy)

◆ snobol_lexer_destroy()

void snobol_lexer_destroy ( snobol_lexer_t * lexer)

Destroy lexer and free resources

Parameters
lexerLexer to destroy (NULL is safe)

◆ snobol_lexer_get_error()

const char * snobol_lexer_get_error ( const snobol_lexer_t * lexer)

Get the lexer's error message (describes the offending character and its position). Valid only while snobol_lexer_has_error() is true.

Parameters
lexerLexer instance
Returns
Static message string (owned by the lexer, do not free)

◆ snobol_lexer_get_line()

size_t snobol_lexer_get_line ( snobol_lexer_t * lexer)

Get current line number (for error reporting)

Parameters
lexerLexer instance
Returns
1-based line number

◆ snobol_lexer_get_pos()

size_t snobol_lexer_get_pos ( snobol_lexer_t * lexer)

Get current position in source (for error reporting)

Parameters
lexerLexer instance
Returns
0-based byte offset in source

◆ snobol_lexer_has_error()

bool snobol_lexer_has_error ( const snobol_lexer_t * lexer)

Check whether the lexer encountered an unrecognized character. Once an error is set it is sticky: all subsequent tokens are TOKEN_ERROR.

Parameters
lexerLexer instance (NULL returns false)
Returns
true if a lexical error has been recorded

◆ snobol_lexer_next()

token_t snobol_lexer_next ( snobol_lexer_t * lexer)

Get the next token from the lexer

Parameters
lexerLexer instance
Returns
Next token (TOKEN_EOF when done)

◆ snobol_lexer_peek()

token_t snobol_lexer_peek ( snobol_lexer_t * lexer)

Peek at the next token without consuming it

Parameters
lexerLexer instance
Returns
Next token (TOKEN_EOF when done)

◆ snobol_lexer_restore()

void snobol_lexer_restore ( snobol_lexer_t * lexer,
snobol_lexer_state_t state )

Restore lexer state

Parameters
lexerLexer instance
stateState to restore

◆ snobol_lexer_save()

snobol_lexer_state_t snobol_lexer_save ( snobol_lexer_t * lexer)

Save lexer state for later restoration

Parameters
lexerLexer instance
Returns
Saved state

◆ snobol_token_name()

const char * snobol_token_name ( token_type_t type)

Get human-readable token type name for error messages

Parameters
typeToken type
Returns
Static string (do not free)