Lexer.cpp

// ============================================================
// LEXICAL ANALYSIS
// ==============================================================

// Purpose: LEXICAL ANALYSIS implementation
//          Scans source code left-to-right, consuming characters
//          and emitting Token objects.
// ============================================================

#include "Lexer.h"
#include <cctype>   
#include <stdexcept>

// Constructor – store source, reset position and line counter
Lexer::Lexer(const std::string& source)
    : source_(source), pos_(0), line_(1) {}

// current() function returns the current character without consuming it
char Lexer::current() const {
    if (pos_ >= (int)source_.size()) return '\0';  // Past end → null char
    return source_[pos_];
}

// peek() looks ahead by 'offset' characters without consuming
char Lexer::peek(int offset) const {
    int idx = pos_ + offset;                         // Target index
    if (idx >= (int)source_.size()) return '\0';     // Past end → null char
    return source_[idx];
}

// advance() consumes the current character and moves forward
char Lexer::advance() {
    char c = source_[pos_++];   // Read & move forward
    if (c == '\n') line_++;     // Track newlines for line numbers
    return c;
}

// skipWhitespaceAndComments() – ignore spaces, tabs, newlines, and comments
void Lexer::skipWhitespaceAndComments() {
    while (pos_ < (int)source_.size()) {
        char c = current();

        if (isspace(c)) {
            advance();   // Skip spaces, tabs, newlines
        }
        else if (c == '/' && peek() == '/') {
            // Single-line comment: skip until end of line
            while (current() != '\n' && current() != '\0')
                advance();
        }
        else {
            break;   // Not whitespace or comment – stop skipping
        }
    }
}

// readNumber() – scan a numeric literal (integer or float)
Token Lexer::readNumber() {
    int         startLine = line_;
    std::string lexeme;

    // Consume digits before decimal point
    while (isdigit(current())) {
        lexeme += advance();
    }

    // Consume optional decimal point and digits after it
    if (current() == '.' && isdigit(peek())) {
        lexeme += advance();   // consume '.'
        while (isdigit(current())) {
            lexeme += advance();
        }
    }

    return Token(TokenType::NUMBER, lexeme, startLine);
}

// readString() – scan a string literal, handling escape sequences
Token Lexer::readString() {
    int         startLine = line_;
    std::string lexeme;

    advance();   // Consume opening '"'

    while (current() != '"' && current() != '\0') {
        if (current() == '\\') {
            advance();   // Consume backslash
            char escaped = advance();
            // Handle common escape sequences
            switch (escaped) {
                case 'n':  lexeme += '\n'; break;
                case 't':  lexeme += '\t'; break;
                case '"':  lexeme += '"';  break;
                case '\\': lexeme += '\\'; break;
                default:   lexeme += escaped;
            }
        } else {
            lexeme += advance();   // Normal character
        }
    }

    if (current() == '\0')
        throw std::runtime_error("Unterminated string on line " + std::to_string(startLine));

    advance();   // Consume closing '"'
    return Token(TokenType::STRING_LIT, lexeme, startLine);
}

// readIdentOrKeyword() – scan an identifier or reserved keyword
Token Lexer::readIdentOrKeyword() {
    int         startLine = line_;
    std::string lexeme;

    // Identifiers start with a letter or '_', then alphanumeric / '_'
    while (isalnum(current()) || current() == '_') {
        lexeme += advance();
    }

    // Check if the identifier is a reserved keyword
    if (lexeme == "int")    return Token(TokenType::INT,    lexeme, startLine);
    if (lexeme == "float")  return Token(TokenType::FLOAT,  lexeme, startLine);
    if (lexeme == "if")     return Token(TokenType::IF,     lexeme, startLine);
    if (lexeme == "else")   return Token(TokenType::ELSE,   lexeme, startLine);
    if (lexeme == "while")  return Token(TokenType::WHILE,  lexeme, startLine);
    if (lexeme == "return") return Token(TokenType::RETURN, lexeme, startLine);
    if (lexeme == "print")  return Token(TokenType::PRINT,  lexeme, startLine);

    // Not a keyword → it is a user-defined identifier
    return Token(TokenType::IDENTIFIER, lexeme, startLine);
}

// readOperatorOrDelim() – scan an operator or delimiter, handling multi-char ops
Token Lexer::readOperatorOrDelim() {
    int  startLine = line_;
    char c = advance();   // Consume the first character

    switch (c) {
        case '+': return Token(TokenType::PLUS,      "+",  startLine);
        case '-': return Token(TokenType::MINUS,     "-",  startLine);
        case '*': return Token(TokenType::STAR,      "*",  startLine);
        case '/': return Token(TokenType::SLASH,     "/",  startLine);
        case ';': return Token(TokenType::SEMICOLON, ";",  startLine);
        case '(': return Token(TokenType::LPAREN,    "(",  startLine);
        case ')': return Token(TokenType::RPAREN,    ")",  startLine);
        case '{': return Token(TokenType::LBRACE,    "{",  startLine);
        case '}': return Token(TokenType::RBRACE,    "}",  startLine);
        case ',': return Token(TokenType::COMMA,     ",",  startLine);

        case '=':
            // '==' or '='
            if (current() == '=') { advance(); return Token(TokenType::EQ_EQ,    "==", startLine); }
            return Token(TokenType::EQUAL, "=", startLine);

        case '!':
            if (current() == '=') { advance(); return Token(TokenType::NOT_EQ,   "!=", startLine); }
            return Token(TokenType::UNKNOWN, "!", startLine);

        case '<':
            if (current() == '=') { advance(); return Token(TokenType::LESS_EQ,  "<=", startLine); }
            return Token(TokenType::LESS, "<", startLine);

        case '>':
            if (current() == '=') { advance(); return Token(TokenType::GREATER_EQ, ">=", startLine); }
            return Token(TokenType::GREATER, ">", startLine);

        default:
            // Unknown character
            return Token(TokenType::UNKNOWN, std::string(1, c), startLine);
    }
}

// tokenize() – main function to scan the entire source and produce a list of tokens
std::vector<Token> Lexer::tokenize() {
    std::vector<Token> tokens;   // Result list

    while (true) {
        skipWhitespaceAndComments();   // Ignore whitespace between tokens

        if (pos_ >= (int)source_.size()) {
            // End of source → emit EOF token and stop
            tokens.emplace_back(TokenType::END_OF_FILE, "", line_);
            break;
        }

        char c = current();

        if (isdigit(c)) {
            tokens.push_back(readNumber());
        }
        else if (c == '"') {
            tokens.push_back(readString());
        }
        else if (isalpha(c) || c == '_') {
            tokens.push_back(readIdentOrKeyword());
        }
        else {
            Token t = readOperatorOrDelim();
            if (t.type == TokenType::UNKNOWN) {
                // Warn but don't crash; let the parser handle it
                tokens.push_back(t);
            } else {
                tokens.push_back(t);
            }
        }
    }

    return tokens;
}