#
#  File: lexer.k3
#  Tokenizes the incoming AGFL grammar
#
#  Copyright 2005 Radboud University of Nijmegen
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Library General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
#  CVS ID: "$Id: lexer.k3,v 1.10 2006/01/25 15:57:14 marcs Exp $"
#
MODULE lexer =	CURR_FNAME_BASE CURR_FNAME_EXT
		CURR_TOKEN CURR_LINE CURR_COLUMN
		NEXT_TOKEN NEXT_LINE NEXT_COLUMN
		MY_FILE MY_LINE MY_COLUMN MY_CHAR.

DEFINES PRED try open grammar file (>TEXT),
        PRED try open lexicon file (>TEXT),
        ACTION close current input file,
        TEST eof,

        FUNCTION get current position (POS>),
        FUNCTION get current module name (TEXT>),
	FUNCTION syntax error (>TEXT),

        TEST curr token is (>TOKEN),
	TEST next token is number (INT>),
        PRED is token (>TOKEN),
        ACTION should be token (>TOKEN),

        PRED is number (INT>),
        PRED is dot number (INT>),
        ACTION should be number (INT>),

        PRED is quoted string (ID>),
        PRED is word form (ID>),

	TEST is valid affix nonterminal (>TEXT),
        PRED is identifier (ID>),
        PRED is affix nonterminal (ID>),
	PRED is affix nonterminal instance (ID>, INT>),
        PRED is affix terminal (ID>),

        PRED is module name (TEXT>),
        PRED is special name (TEXT>),
        
        PRED are pragmats.

USES globals, id, io, rule, tokens, properties, textstorage.

#
# The current filename ID, extension, token, line and column
# as well as the next token line and column. We need an LL(2)
# lexer to distinguish between frequencies ( [ number ] and
# options ( [ members ] ).
#
CURR_FNAME_BASE = TEXT.
CURR_FNAME_EXT = TEXT.
CURR_TOKEN = TOKEN.
CURR_LINE = INT.
CURR_COLUMN = INT.
NEXT_TOKEN = TOKEN.
NEXT_LINE = INT.
NEXT_COLUMN = INT.

FUNCTION get current position (POS>):
  [ module CURR_FNAME_BASE extension CURR_FNAME_EXT line CURR_LINE at CURR_COLUMN -> POS ].

FUNCTION get current module name (TEXT>):
  [ CURR_FNAME_BASE -> TEXT ].

#
# Syntax error for lexing and parsing
# We should be able to do better than directly abort.
#
FUNCTION syntax error (>TEXT):
  get current position (POS), convert (POS) to (TEXT2),
    write error ("Syntax error at " + TEXT2 + ":" + TEXT),
    !.

#
# Lexical interface
# Context sensitive part
#

#
# Module names consist of letters, digits and ornaments without containing spaces
#
PRED is module name (TEXT>):
  is identifier part (TEXT).

#
# Affix nonterminals consist of capital letters, digits but may not end in digits
#
PRED is affix nonterminal (ID>):
  get current position (POS), ahead identifier part (TEXT),
    is valid affix nonterminal (TEXT), read token,
    get affix nonterminal id (POS, TEXT, ID).

PRED is affix nonterminal instance (ID>, INT>):
  get current position (POS), ahead identifier part (TEXT),
    split off instance (TEXT, TEXT2, TEXT3),
    is valid affix nonterminal (TEXT2), read token,
    get affix nonterminal id (POS, TEXT2, ID),
      ( [ TEXT3 -> "" ], [ -1 -> INT ];
	decbin (TEXT3, INT);
    	error ("Integer overflow when converting instance number", POS), [ 42 -> INT ]
      ).
	  
FUNCTION split off instance (>TEXT, TEXT2>, TEXT3>):
  length (TEXT, INT1),
    locate numeric suffix (TEXT, INT1, INT2),
      ( [ INT1 = INT2 ], [ TEXT -> TEXT2 ], [ "" -> TEXT3 ];
        extract (TEXT, 0, INT2, TEXT2), extract (TEXT, INT2, INT1, TEXT3)
      ).

FUNCTION locate numeric suffix (>TEXT, >INT, INT2>):
  between (TEXT, INT - 1, "0", "9"), [ INT - 1 -> INT ],
    locate numeric suffix (TEXT, INT, INT2);
  [ INT -> INT2 ].

# Misschien naar id.k3 overzetten?
ACTION get affix nonterminal id (>POS, >TEXT, ID>):
  [ TEXT -> "TEXT" ], get text affix (ID);
  [ TEXT -> "INT" ], get integer affix (ID);
  get id (TEXT, ID).

#
# Affix terminals consist of one identifier part
# containing only lower case letters
# 
PRED is affix terminal (ID>):
  is affix terminal (TEXT), get id (TEXT, ID).

PRED is affix terminal (TEXT>):
  ahead identifier part (TEXT),
    is valid affix terminal (TEXT), read token.

#
# Syntax nonterminals (the most liberal of all identifiers)
#
PRED is identifier (ID>):
  get current position (POS),
    is identifier (TEXT),
    get id (TEXT, ID).

PRED is identifier (TEXT>):
  is identifier part (TEXT1),
    (is rest identifier (TEXT2),
       [TEXT1 + " " + TEXT2 -> TEXT];
     [TEXT1 -> TEXT]).

PRED is rest identifier (TEXT>):
  is rest identifier token (TEXT1),
    (is rest identifier (TEXT2),
       [TEXT1 + " " + TEXT2 -> TEXT];
     [TEXT1 -> TEXT]).

PRED is rest identifier token (TEXT>):
  is token (grammar symbol),	[ "GRAMMAR" -> TEXT ];
  is token (defines symbol),	[ "DEFINES" -> TEXT ];
  is token (lexicon symbol),	[ "LEXICON" -> TEXT ];
  is token (includes symbol),	[ "INCLUDES" -> TEXT ];
  is token (root symbol),	[ "ROOT" -> TEXT ];
  is token (cond symbol),	[ "COND" -> TEXT ];
  is token (rule symbol),	[ "RULE" -> TEXT ];
  is token (option symbol),	[ "OPTION" -> TEXT ];
  is identifier part (TEXT).

#
# Recognition of special names
#
PRED is special name (TEXT>):
  [ CURR_TOKEN -> special name TEXT ], read token.

#
# Quoted strings as in affix expressions
#
PRED is quoted string (TEXT>):
  [ CURR_TOKEN -> quoted string TEXT ], read token.

PRED is quoted string (ID>):
  is quoted string (TEXT),
    get id (TEXT, ID).

ACTION should be quoted string (TEXT>):
  is quoted string (TEXT);
  syntax error ("quoted string expected"),
    ?. # never reached

#
# Word forms (i.e. syntax terminals) are quoted strings
# that are marked in copy word form to prepare them for use in the rts
#
PRED is word form (ID>):
  is quoted string (TEXT),
    copy word form (TEXT, MY_LINE, TEXT1),
    get id (TEXT1, ID).

#
# Recognition of {dot, signed} numbers
#
PRED is dot number (INT>):
  [ CURR_TOKEN -> dot number INT ], read token.

ACTION should be number (INT>):
  is number (INT);
  syntax error ("number expected"),
    ?. # never reached

PRED is number (INT>):
  [ CURR_TOKEN -> number INT ], read token.

#
#  Recognize pragmats
#
PRED are pragmats:
  is token (bar),
    should be pragmat list,
    should be token (bar).

ACTION should be pragmat list:
  is pragmat,
    may be rest pragmat list;
  syntax error ("directive expected").

ACTION may be rest pragmat list:
  is token (comma),
    should be pragmat list;
  +.

PRED is pragmat:
  get current position (POS),
    is pragmat text (TEXT),
    ( is valid pragmat (TEXT);
      error ("unknown pragmat \"" + TEXT + "\"", POS)
    ).

PRED is pragmat text (TEXT>):
  is identifier (TEXT).

PRED is valid pragmat (>TEXT):
  [ TEXT -> "translate off" ], set translate option off;
  [ TEXT -> "translate default" ], set translate option default;
  [ TEXT -> "translate" ],
    should be quoted string (TEXT),
    set translate option (TEXT);
  [ TEXT -> "details off" ], reset details option;
  [ TEXT -> "details on" ], set details option;
  [ TEXT -> "details" ], set details option;
  [ TEXT -> "line" ],
    should be number (INT), [ INT -> MY_LINE ], [ 1 -> MY_COLUMN ];
  [ TEXT -> "word separators" ],
    get current position (POS),
    error ("\"word separators\" is renamed to \"word terminators\"", POS),
    should be quoted string (TEXT),
    set word terminators option (TEXT);
  [ TEXT -> "word terminators" ],
    should be quoted string (TEXT),
    set word terminators option (TEXT);
  [ TEXT -> "invisible characters" ],
    should be quoted string (TEXT),
    set invisible characters option (TEXT);
  [ TEXT -> "segment mode" ], set segment mode option;
  [ TEXT -> "invisible" ],
    should be token (colon),
    should be visible nonterminal list.

ACTION should be visible nonterminal list:
  is visible nonterminal,
    ( is token(comma),
        should be visible nonterminal list;
      +
    );
  syntax error ("\"nonterminal/arity\" expected").

PRED is visible nonterminal:
  is identifier (ID),
    should be token (slash),
    should be number (INT),
    mark nonterminal invisible (ID, INT).

ACTION verbosity level option (INT>):
  is number (INT);
  [ 1 -> INT ].

#
# small tests on the current token
#
PRED is identifier part (TEXT>):
  ahead identifier part (TEXT), read token.

PRED is token (>TOKEN):
  curr token is (TOKEN), read token.

ACTION should be token (>TOKEN):
  is token (TOKEN);
  convert TOKEN to TEXT,
    syntax error ("\"" + TEXT + "\" expected"),
    ?. # never reached

TEST curr token is (>TOKEN):
  [ CURR_TOKEN = TOKEN ].

TEST next token is number (INT>):
  [ NEXT_TOKEN -> number INT ].

TEST eof:
  [ CURR_TOKEN -> end of file ].

TEST ahead identifier part (TEXT>):
  [ CURR_TOKEN -> identifier part TEXT ].

#
# Local administration for character reading
#
MY_FILE = FILE.
MY_LINE = INT.
MY_COLUMN = INT.
MY_CHAR = TEXT.

#
# Context free tokenization
#
ACTION read token:
  [ NEXT_TOKEN -> CURR_TOKEN ], [ NEXT_LINE -> CURR_LINE ], [ NEXT_COLUMN -> CURR_COLUMN ],
     may be layout, [ MY_LINE -> NEXT_LINE ], [ MY_COLUMN -> NEXT_COLUMN ],
       ( [ NEXT_TOKEN -> end of file ];
         no more characters, yield token (end of file);
         is letter (TEXT), scan rest identifier part (TEXT);
         is char ("$"), scan rest special name;
         is char ("\""), scan rest quoted string ("\"");
	 is char ("-"), scan rest signed number or identifier part ("-");
	 is char ("+"), scan rest signed number or identifier part ("+");
         is digit (TEXT), scan rest number (TEXT);
         is char (":"), scan rest colon;
         is char ("."), scan rest period;
         is char ("("), yield token (left parenthesis);
         is char ("{"), yield token (left curly bracket);
         is char ("["), yield token (left square bracket);
         is char (")"), yield token (right parenthesis);
	 is char (">"), yield token (right sharp bracket);
         is char ("}"), yield token (right curly bracket);
         is char ("]"), yield token (right square bracket);
         is char ("|"), yield token (bar);
         is char (","), yield token (comma);
         is char (";"), yield token (semicolon);
         is char ("!"), yield token (commit);
         is char ("/"), yield token (slash);
         is char ("&"), yield token (ampersand);
         get current position (POS),
	    error ("Illegal character '" + MY_CHAR + "'", POS),
	    read next character, read token
       ).

ACTION scan rest identifier part (>TEXT):
  is letment (TEXT1), scan rest identifier part (TEXT + TEXT1);
  [ TEXT -> "GRAMMAR" ],	yield token (grammar symbol);
  [ TEXT -> "DEFINES" ],	yield token (defines symbol);
  [ TEXT -> "LEXICON" ],	yield token (lexicon symbol);
  [ TEXT -> "INCLUDES" ],	yield token (includes symbol);
  [ TEXT -> "ROOT" ],		yield token (root symbol);
  [ TEXT -> "COND" ],		yield token (cond symbol);
  [ TEXT -> "RULE" ],		yield token (rule symbol);
  [ TEXT -> "OPTION" ],		yield token (option symbol);
  yield token (identifier part TEXT).

#
# Special names are constructed as a $ followed by a capital name
# As soon as we can treat them as identifiers we will remove the
# special token recognition
#
ACTION scan rest special name:
  should be capital name (TEXT),
    yield token (special name TEXT).

ACTION should be capital name (TEXT>):
  is capital name (TEXT);
  get current position (POS),
    error ("Capital name expected", POS), [ "" -> TEXT ].

PRED is capital name (TEXT>):
  is capital letter (TEXT1),
    ( is capital name (TEXT2), [ TEXT1 + TEXT2 -> TEXT ];
      [ TEXT1 -> TEXT ]
    ).

#
# Text denotations
#
ACTION scan rest quoted string (>TEXT):
  get current position (POS),
    ( ahead char ("\n"),
        error ("Unterminated string", POS), yield token (quoted string TEXT);
      is char ("\""), [ TEXT + "\"" -> TEXT ], yield token (quoted string TEXT);
      is char ("\\"),
	( ahead char ("\n"),
	    error ("Unterminated string", POS), yield token (quoted string TEXT);
	  pick current character (TEXT1), scan rest quoted string (TEXT + "\\" + TEXT1)
	);
      pick current character (TEXT1), scan rest quoted string (TEXT + TEXT1)
    ).

#
# Handling of numbers
#
ACTION scan rest period:
  is digit (TEXT), scan rest number (TEXT, INT),
    yield token (dot number INT);
  yield token (period).

ACTION scan rest number (>TEXT):
  scan rest number (TEXT, INT), yield token (number INT).

ACTION scan rest signed number or identifier part (>TEXT):
  is digit (TEXT2), scan rest number (TEXT2, INT),
    ( [ TEXT -> "-" ], yield token (number -INT);
      yield token (number INT)
    );
  is letter (TEXT2), scan rest identifier part (TEXT + TEXT2);
  get current position (POS),
    error ("Number or affix terminal part expected", POS),
    read token.

ACTION scan rest number (>TEXT, INT>):
  is digit (TEXT2), scan rest number (TEXT + TEXT2, INT);
  decbin (TEXT, INT);
  get current position (POS),
    error ("Integer overflow when converting", POS),
    [ 42 -> INT ].

#
# The yielding part
#
ACTION scan rest colon:
  is char (":"), yield token (double colon);
  yield token (colon).

ACTION yield token (>TOKEN):
  [ TOKEN -> NEXT_TOKEN ].

#
# Layout and comments
#
ACTION may be layout:
  is layout;
  +.

PRED is layout:
  is white space, may be layout;
  is char ("#"), read comment, may be layout.

ACTION read comment:
  is char ("\n");
  read next character, read comment.

#
# Checking of character classes
# Note that we allow digits in identifier parts
# The validity of affix nonterminals and affix terminals is checked elsewhere
#
PRED is letment (TEXT>):
  is capital letter (TEXT);
  is small letter (TEXT);
  is extended letter (TEXT);
  is digit (TEXT);
  is ornament (TEXT).

PRED is letter (TEXT>):
  is capital letter (TEXT);
  is small letter (TEXT);
  is extended letter (TEXT).

PRED is capital letter (TEXT>):
  is valid capital letter (MY_CHAR),
    [ MY_CHAR -> TEXT ], read next character.

PRED is small letter (TEXT>):
  is valid small letter (MY_CHAR),
    [ MY_CHAR -> TEXT ], read next character.

PRED is extended letter (TEXT>):
  is valid extended letter (MY_CHAR),
    [ MY_CHAR -> TEXT], read next character.

PRED is ornament (TEXT>):
  is valid ornament (MY_CHAR),
    [ MY_CHAR -> TEXT], read next character.

PRED is digit (TEXT>):
  is valid digit (MY_CHAR),
    [ MY_CHAR -> TEXT ], read next character.

PRED is white space:
  is char ("\n");
  is char ("\t");
  is char ("\f");
  is char ("\r");
  is char (" ").

PRED is char (>TEXT):
  [ MY_CHAR = TEXT ], read next character.

TEST no more characters:
  [ MY_CHAR -> "" ].

ACTION pick current character (TEXT>):
  [ MY_CHAR -> TEXT ], read next character.

#
# Special character recognition for affix nonterminals and affix terminals
# Affix nonterminals consist of upper case letters, extended letters and ornaments
#
TEST is valid affix nonterminal (>TEXT):
  prefix (TEXT, 1, TEXT1, TEXT2),
    is valid capital letter (TEXT1),
    is valid rest affix nonterminal (TEXT2).

TEST is valid rest affix nonterminal (>TEXT):
  prefix (TEXT, 1, TEXT1, TEXT2),
    ( is valid affix nonterminal letment (TEXT1),
        is valid rest affix nonterminal (TEXT2)
    );
  +.

TEST is valid affix nonterminal letment (>TEXT):
  between (TEXT, 0, "A", "Z");
  is valid extended letter (TEXT);
  is valid ornament (TEXT).

#
# Affix terminals consist of lower case letters, digits, extended letters and ornaments
# They must start with a lower case letter, + or -. For these latter two cases, they
# must be followed by a letter, which is enforced by read token.
#
TEST is valid affix terminal (>TEXT):
  prefix (TEXT, 1, TEXT1, TEXT2),
    is valid affix terminal first letment (TEXT1),
    is valid rest affix terminal (TEXT2).

TEST is valid rest affix terminal (>TEXT):
  prefix (TEXT, 1, TEXT1, TEXT2),
    ( is valid affix terminal letment (TEXT1),
        is valid rest affix terminal (TEXT2)
    );
  +.

TEST is valid affix terminal first letment (>TEXT):
  between (TEXT, 0, "a", "z");
  [ TEXT -> "-" ];
  [ TEXT -> "+" ].

TEST is valid affix terminal letment (>TEXT):
  is valid small letter (TEXT);
  is valid digit (TEXT);
  is valid extended letter (TEXT);
  is valid ornament (TEXT).

TEST is valid extended letter (>TEXT):
  asciicode (TEXT, 0, INT),
    lesseq (128, INT), lesseq (INT, 255).

TEST is valid ornament (>TEXT):
  [ TEXT -> "_" ];
  [ TEXT -> "@" ];
  [ TEXT -> "?" ];
  [ TEXT -> "-" ];
  [ TEXT -> "+" ].

TEST is valid capital letter (>TEXT):
  between (TEXT, 0, "A", "Z").

TEST is valid small letter (>TEXT):
  between (TEXT, 0, "a", "z").

TEST is valid digit (>TEXT):
  between (TEXT, 0, "0", "9").

#
# Actual character reading
# Note that when we read 0 characters we are at the end of file
# or a rather nasty error must have occurred
#
ACTION read next character:
  [ MY_CHAR -> "\n"], [ MY_LINE + 1 -> MY_LINE ],
    [ 1 -> MY_COLUMN ], actual read next character;
  [ MY_COLUMN + 1 -> MY_COLUMN ], actual read next character.
  
ACTION actual read next character:
  [ MY_CHAR -> "" ];
  read char (MY_FILE, MY_CHAR);
  [ "" -> MY_CHAR ].

TEST ahead char (>TEXT):
  [ MY_CHAR = TEXT ].

#
# Lexer initialization
#
PRED try open lexicon file (>TEXT):
  get lexicon file extension (TEXT1),
    try open input file (TEXT, TEXT1).

PRED try open grammar file (>TEXT):
  get grammar file extension (TEXT1),
    try open input file (TEXT, TEXT1).

PRED try open input file (>TEXT1, >TEXT2):
  open input file (TEXT1 + TEXT2, MY_FILE),
    [ TEXT1 -> CURR_FNAME_BASE ],
    [ TEXT2 -> CURR_FNAME_EXT ],
    [ 0 -> MY_LINE ],
    [ 0 -> MY_COLUMN ],
    [ "\n" -> MY_CHAR ],
    [ unknown token -> CURR_TOKEN ],
    [ unknown token -> NEXT_TOKEN ],
    read token,
    read token.

ACTION close current input file:
  close (MY_FILE).

EXTERNAL ACTION copy word form (>TEXT1, >INT, TEXT2>): 218
