unit Token;

interface

uses
  ILexer;

type
  TSourceLocation = record
    Lexer: ILexer;
    Start: Integer;
    Stop: Integer;
    function Range(const initLex: TSourceLocation): TSourceLocation;
  end;

  TOptional<T> = TNullable<T>;

  TToken = record
    Value: string;
    TkType: Integer;
    Loc: TOptional<TSourceLocation>; // Optional TSourceLocation
    NoExpand: TOptional<Boolean>; // Optional Boolean
    TreatAsRelax: TOptional<Boolean>; // Optional Boolean
    function Range(const EndToken: TToken; const Text: string): TToken;
  end;

implementation

{ TSourceLocation }

function TSourceLocation.Range(const initLex: TSourceLocation): TSourceLocation;
begin
  if (initLex.Lexer <> Self.Lexer) or (initLex.Start < Self.Start) then  // The Lexer passed = current Lexer, and it's Start pos < current Start pos.
    Result := Default(TSourceLocation) // Return an empty TSourceLocation.
  else
    Result := TSourceLocation.Create(Self.Lexer, Self.Start, Other.Stop);
end;

{ TToken }

function TToken.Range(const EndToken: TToken; const Text: string): TToken;
begin
  Result.Value := Text;
  Result.Loc := Self.Loc.Range(EndToken.Loc); // Use the Range method for TOptional<TSourceLocation>
  Result.NoExpand := Self.NoExpand; // Preserve the NoExpand value
  Result.TreatAsRelax := Self.TreatAsRelax; // Preserve the TreatAsRelax value
end;

end.

// @flow
import SourceLocation from "./SourceLocation";

/**
 * Interface required to break circular dependency between Token, Lexer, and
 * ParseError.
 */
export interface LexerInterface {input: string, tokenRegex: RegExp}

/**
 * The resulting token returned from `lex`.
 *
 * It consists of the token text plus some position information.
 * The position information is essentially a range in an input string,
 * but instead of referencing the bare input string, we refer to the lexer.
 * That way it is possible to attach extra metadata to the input string,
 * like for example a file name or similar.
 *
 * The position information is optional, so it is OK to construct synthetic
 * tokens if appropriate. Not providing available position information may
 * lead to degraded error reporting, though.
 */
export class Token {
    text: string;
    loc: ?SourceLocation;
    noexpand: ?boolean; // don't expand the token
    treatAsRelax: ?boolean; // used in \noexpand

    constructor(
        text: string,           // the text of this token
        loc: ?SourceLocation,
    ) {
        this.text = text;
        this.loc = loc;
    }

    /**
     * Given a pair of tokens (this and endToken), compute a `Token` encompassing
     * the whole input range enclosed by these two.
     */
    range(
        endToken: Token,  // last token of the range, inclusive
        text: string,     // the text of the newly constructed token
    ): Token {
        return new Token(text, SourceLocation.range(this, endToken));
    }
}