/*
   File: lexicon_reading.c
   Reads a lexicon written by agfl-lexer

   This code is to be absorbed into the abase library

   Copyright 2006 Radboud University of Nijmegen
 
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.
 
   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Library General Public License for more details.
 
   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

   CVS ID: "$Id: lexicon_reading.c,v 1.14 2006/09/27 16:01:32 marcs Exp $"
*/

/* standard includes */
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include <sys/types.h>
#include <setjmp.h>
#include <string.h>

/* libabase includes */
#include <abase_memalloc.h>
#include <abase_dstring.h>
#include <abase_error.h>
#include <abase_repr.h>

/* local includes */
#include "lexicon.h"

/*-------------------------------------------------------------------------
// Structure for storing the lexicon affixes
//
// name: affix name
// bitset: the bitset associated with this affix
// is_nont: if false, then this affix is a terminal, otherwise it's a
//          nonterminal
// lhs[]: the indexes of the left-hand-sides (affix nonterminals)
//	  this affix belongs to.
//-----------------------------------------------------------------------*/
typedef int INT_AFFIX;
typedef char* TEXT_AFFIX;

typedef struct {
    char *name;
    Bitset bitset;
    int is_nont;
    size_t nr_lhsses;
    int *lhs;
} SET_AFFIX;

/* Structure for storing the nonterminal parameters */
enum LexParamTypes {
    LexParamLattice = 1,
    LexParamText = 2,
    LexParamInt = 3
};

typedef struct {
    char type;
    int index;
} PARAMETER;

/* Structure for storing the lexicon rules (nonterminals) */
typedef struct {
    char *name;
    int arity;
    PARAMETER *pars;
} NONTERMINAL;

/* Structure for storing the entries */
typedef struct {
    int nontnr;
    Penalty penalty;
    int *pars;
} ENTRY;

/* Generic structure for storing pairs */
typedef struct {
    int index;
    int pointer;
} PAIR;


/*------------------------------------------------------------------------------
// Lexicon structure
//----------------------------------------------------------------------------*/
struct lexicon_rec {
    Trie trie_buf;

    int nr_int_affixes;
    INT_AFFIX *int_affix;

    int nr_text_affixes;
    TEXT_AFFIX *text_affix;

    int nr_set_affixes;
    SET_AFFIX **set_affix;

    int nr_nonterminals;
    NONTERMINAL **nonterminal;

    int nr_entries;
    ENTRY **entry;

    int nr_entrylist_pairs;
    PAIR *entrylist_pair;
};

/*------------------------------------------------------------------------------
// Format of lexicon file (as delivered by agfl-lexer)
//
// 1a)	size_t (long/void *)	size of binary trie	(but not loaded as a size_t)
// 1b)	byte[]			trie in binary form	(brrr)
// 2a)	size_t			nr of integer affixes
// 2b)	long[]			integer affixes (read as bitsets i.e. longs)
// 3a)	size_t			nr of text affixes
// 3b)	(char *)[]		text affixes (each zero terminated)
// 4a)	size_t			nr of set affixes
// 4b)  set_affix[]		set affixes each consisting of
//				char *name, bitset value,
//				bool (read as bitset) is_nont,
//				#lhs's, off_t[] lhs_indexes (brrr)
// 5a)	size_t			nr of (syntax/lexicon) non terminals
// 5b)	nonterminal[]		all non terminals, each consisting of
//				char *name, short arity,
//				(byte type, (set_affix index)?)[] affix values
// 6a)	size_t			nr of entries
// 6b)	entry[]			all entries, each consisting of
//				nonterminal index, penalty,
				off_t[] affix_indexes
// 7a)	size_t			nr of entry list pairs
// 7b)	entry_list[]		all entrylist pairs, each consisting of
//				an entry index (as head) and a next
//				entrylist index (as tail; -1 = nil)
//
// In the code all numbers of items are read as size_t values
// and all indices are read as off_t values.
//----------------------------------------------------------------------------*/

/*------------------------------------------------------------------------------
// Lexicon file access routines
//----------------------------------------------------------------------------*/
static int read_byte (FILE *lex_file, char *ret_c)
{ int c = fgetc (lex_file);
  if (c == EOF) return (0);	/* fail if eof */

  *ret_c = (char) c;
  return (1);
}

static int read_bool (FILE *lex_file, int *ret_b)
{ char c;
  if (!read_byte (lex_file, &c)) return (0);

  *ret_b = (c == 'T');
  return (1);
}

static int read_string (FILE *lex_file, char **ret_s)
{ dstring tmp = abs_init_dstring (10);
  char c;
  do
     { if (!read_byte (lex_file, &c)) return (0);
       if (c) abs_append_dstring_c (tmp, c);
     }
  while (c);

  /* Note finish deallocates */
  *ret_s = abs_finish_dstring (tmp);
  return (1);
}

/* This code is only used for reading a nonterminal arity: to be replaced */
static int read_arity (FILE *lex_file, int *arity)
{ char bb1, bb2;
  int i, j;

  if (!read_byte (lex_file, &bb1)) return (0);
  if (!read_byte (lex_file, &bb2)) return (0);

  /* We only need the following ugly conversion until we abjure this format in the lexer */
  i = ((int) bb1) & 0xff;
  j = ((int) bb2) & 0xff;
  *arity = (i << 8) | j;
  return (1);
}

static int read_set (FILE* lex_file, Bitset *ret_set)
{ if (fread ((void *) ret_set, sizeof (Bitset), 1, lex_file) != 1) return (0);
  return (1);
}

/* Note: numbers are read as size_t's: brrr */
static int read_number (FILE *lex_file, int *ret_nr)
{ size_t res;

  if (fread (&res, sizeof (size_t), 1, lex_file) != 1) return (0);
  *ret_nr = (int) res;
  return (1);
}

/* Note: indexes are read as off_t's: brrr */
static int read_index (FILE *lex_file, int *ret_idx)
{ off_t res;

  if (fread (&res, sizeof (off_t), 1, lex_file) != 1) return (0);
  *ret_idx = (int) res;
  return (1);
}

static int read_penalty (FILE* lex_file, Penalty *ret_pen)
{ long old_res;		/* Just to get rid of foolish longs in Penalty typedef */

  if (fread (&old_res, sizeof(old_res), 1, lex_file) != 1) return (0);
  *ret_pen = (Penalty) old_res;
  return (1);
}

/*------------------------------------------------------------------------------
// Trie alloc/read/free routines:
//----------------------------------------------------------------------------*/
#define TS_LENGTH sizeof(long)
static int read_trie_size (FILE* lex_file, size_t *ret_size)
{ unsigned long ptr;

  if (fread((char *)&ptr, TS_LENGTH, 1, lex_file) != 1) return (0);

  DB_LEX(abs_printf("read_trie_size: size = %lu bytes\n", ptr));
  *ret_size = (size_t) ptr;
  return (1);
}

static int read_trie (FILE* lex_file, Lexicon lex)
{ size_t trie_size;

  if (!read_trie_size (lex_file, &trie_size)) return (0);
  lex -> trie_buf = abs_malloc (trie_size, "read_trie");
  
  if (fread (lex -> trie_buf, 1, trie_size, lex_file) != trie_size) return (0);
  DB_LEX(abs_printf("read_trie: %d bytes\n", trie_size));
  return (1);
}

static void free_trie (Lexicon lex)
{ abs_free (lex -> trie_buf, "trie");
}

Trie lxcn_get_lexicon_trie (Lexicon lex)
{ assert ((lex != NULL) && "lxcn_get_lexicon_trie: No lexicon.");
  return (lex -> trie_buf);
}

/*------------------------------------------------------------------------------
// INT affixes
//----------------------------------------------------------------------------*/
static int read_int_affixes (FILE* lex_file, Lexicon lex)
{ int nr, i; 

  if (!read_number (lex_file, &nr)) return (0);
  lex -> nr_int_affixes = nr;
  DB_LEX(abs_printf("reading %d INT affixes...\n", nr));

  if (nr)
    { lex -> int_affix = (INT_AFFIX *) abs_calloc (nr, sizeof(INT_AFFIX), "read_int_affixes");
      for (i = 0; i < nr; i++)
	{ Bitset set;
          if (!read_set (lex_file, &set)) return (0);
	  lex -> int_affix[i] = (int) set;
	};
    }
  else lex -> int_affix = NULL;
  return (1);
}

static void free_int_affixes (Lexicon lex)
{ /* Note abs_free does a nullity check */
  abs_free (lex -> int_affix, "free_int_affixes");
}

/*------------------------------------------------------------------------------
// TEXT affixes
//----------------------------------------------------------------------------*/
static int read_text_affixes (FILE* lex_file, Lexicon lex)
{ int nr, i;

  if (!read_number (lex_file, &nr)) return (0);
  lex -> nr_text_affixes = nr;
  DB_LEX(abs_printf("reading %d TEXT affixes...\n", nr));

  if (nr)
    { lex -> text_affix = (TEXT_AFFIX*) abs_calloc (nr, sizeof(TEXT_AFFIX), "lex->text_affix");
      for (i = 0; i < nr; i++)
	{ TEXT_AFFIX input;
	  if (!read_string (lex_file, &input)) return (0);
          lex -> text_affix[i] = input;
          DB_LEX(abs_printf("\tread affix: \"%s\"\n", input));
        };
    }
  else lex -> text_affix = NULL;
  return (1);
}

static void free_text_affixes (Lexicon lex)
{ if ((lex -> text_affix != NULL) && (lex -> nr_text_affixes > 0))
    { int i;
      for (i = 0; i < lex -> nr_text_affixes; i++)
        abs_free (lex -> text_affix[i], "free_text_affixes");

      abs_free (lex -> text_affix, "free_text_affixes: vector");
    };
}

/*------------------------------------------------------------------------------
// Affix names
//----------------------------------------------------------------------------*/
static int read_set_affixes (FILE* lex_file, Lexicon lex)
{ int nr, i;

  if (!read_number (lex_file, &nr)) return (0);
  lex -> nr_set_affixes = nr;
  DB_LEX(abs_printf("reading %d SET affixes...\n", nr));

  if (nr)
    { lex -> set_affix = (SET_AFFIX **) abs_calloc (nr, sizeof(SET_AFFIX*), "read_set_affixes");
      for (i = 0; i < nr; i++)
        { SET_AFFIX *set_affix = (SET_AFFIX*) abs_malloc (sizeof(SET_AFFIX), "read_set_affixes");
          int nr_lhsses, j;

	  if (!read_string (lex_file, &set_affix -> name)) return (0);
          DB_LEX(abs_printf("\tread affix %d: \"%s\"\n", i, set_affix -> name));
	  if (!read_set (lex_file, &set_affix -> bitset)) return (0);
	  if (!read_bool (lex_file, &set_affix -> is_nont)) return (0);
	  if (!read_number (lex_file, &nr_lhsses)) return (0);
          set_affix -> nr_lhsses = nr_lhsses;

	  if (nr_lhsses)
	    { set_affix -> lhs = (int*) abs_calloc (nr_lhsses, sizeof(int), "read_set_affixes");
              for (j = 0; j < nr_lhsses; j++)
		{ if (!read_index (lex_file, &set_affix -> lhs[j])) return (0);
                  DB_LEX(abs_printf("\t\twith LHS: %d\n", set_affix -> lhs[j]));
                }
            }
	  else set_affix -> lhs = NULL;
          lex -> set_affix[i] = set_affix;
        }
    }
  else lex -> set_affix = NULL;
  return (1);
}

static void free_set_affixes (Lexicon lex)
{ int nr = lex -> nr_set_affixes;
  
  if ((lex -> set_affix != NULL) && (lex -> nr_set_affixes > 0))
    { int i;
      for (i = 0; i < nr; i++)
        abs_free (lex -> set_affix[i], "free_set_affixes");
      abs_free (lex -> set_affix, "free_set_affixes");
    };
}

/*------------------------------------------------------------------------------
// Nonterminal names
//----------------------------------------------------------------------------*/
static int read_nonterminals (FILE* lex_file, Lexicon lex)
{ int nr, i;

  if (!read_number (lex_file, &nr)) return (0);
  lex -> nr_nonterminals = nr;
  DB_LEX(abs_printf("reading %d nonterminals\n", nr));

  if (nr)
    { lex -> nonterminal = (NONTERMINAL**) abs_calloc (nr, sizeof(NONTERMINAL*),
						       "read_nonterminals, lex -> nonterminal");
      memset (lex -> nonterminal, 0, nr * sizeof (NONTERMINAL *));

      for (i = 0; i < nr; i++)
    	{ NONTERMINAL* nonterminal;
          char* nont_name;

	  if (!read_string (lex_file, &nont_name)) return (0);
          if (nont_name[0] == '\0')
	    { /* empty nonterminal entry; nonterminal is not a lexicon nonterminal */
              DB_LEX(abs_printf("reading nonterminal %d: syntax nonterminal\n", i));
              abs_free (nont_name, "read_nonterminals");
              nonterminal = NULL;
            }
	  else
	    { /* We have a lexicon nonterminal entry */
    	      nonterminal = (NONTERMINAL*) abs_malloc (sizeof(NONTERMINAL), "read_nonterminals");
              nonterminal -> name = nont_name;
              DB_LEX(abs_printf("reading nonterminal %d: \"%s/\n", i, nonterminal -> name));
	      if (!read_arity (lex_file, &nonterminal -> arity)) return (0);
              DB_LEX(abs_printf("%d\"\n", nonterminal -> arity));
    
              if (nonterminal -> arity)
		{ int j;
		  nonterminal -> pars = (PARAMETER*) abs_calloc (nonterminal -> arity,
						 sizeof(PARAMETER), "read_nonterminals");
                  for (j = 0; j < nonterminal -> arity; j++)
		    { char type;
		      if (!read_byte (lex_file, &type)) return (0);
                      nonterminal -> pars[j].type = type;
                      switch (type)
		        { case LexParamLattice:
			    if (!read_index (lex_file, &(nonterminal -> pars[j].index)))
			       return (0);
                            break;
                          case LexParamText:
                          case LexParamInt:
                            break;
                          default:
                            DB_LEX(abs_printf("read_nonterminals: Unknown ptype 0x%x\n", type));
			    return (0);
                        }
                    };
		}
              else nonterminal -> pars = NULL;
            };
          lex -> nonterminal[i] = nonterminal;
        };
    }
  else lex -> nonterminal = NULL;
  return (1);
}

static void free_nonterminals (Lexicon lex)
{
    off_t i;
    size_t nr = lex->nr_nonterminals;

    /* TODO: AS, 30/4/05: not all is freed, not clear how or why */
    if ((lex->nonterminal != NULL) && (nr > 0)) {
        for (i = 0; i < nr; i++) {
            if (lex->nonterminal[i] != NULL) {
                abs_free ((lex->nonterminal[i])->name, "free_nonterminals: NONT");
                abs_free ((lex->nonterminal[i])->pars, "free_nonterminals: PARAMS");
            }
        }

        abs_free (lex->nonterminal, "free_nonterminals");
    }
}


/*------------------------------------------------------------------------------
// Lexicon entries handling
//----------------------------------------------------------------------------*/
static int read_entries (FILE* lex_file, Lexicon lex)
{ int nr, i;
  
  if (!read_number (lex_file, &nr)) return (0);
  lex -> nr_entries = nr;
  lex -> entry = (ENTRY**) abs_calloc (nr, sizeof(ENTRY*), "read_entries");
  DB_LEX(abs_printf("-- There are %d entries\n", nr));

  for (i = 0; i < nr; i++)
    { ENTRY *entry = (ENTRY *) abs_malloc (sizeof(ENTRY), "read_entries");
      int arity, j;

      /* Read nonterminal and penalty */
      if (!read_index (lex_file, &entry -> nontnr)) return (0);
      if (!read_penalty (lex_file, &entry -> penalty)) return (0);

      /* Obtain the arity from the nonterminal definition */
      arity = lex -> nonterminal[entry -> nontnr] -> arity;
      DB_LEX(abs_printf("Entry %d: %s", i, lex -> nonterminal[entry -> nontnr] -> name));
      if (arity)
        { entry -> pars = abs_calloc (arity, sizeof (int), "read_entries");
          DB_LEX(abs_printf("("));
          for (j = 0; j < arity; j++)
	    { /* read the index of the affix value */
	      if (!read_index (lex_file, &entry -> pars[j])) return (0);
#ifdef DEBUG_LEX
	      /* Note that we print the wrong names if we are not a set affix */
              abs_printf("\"%s\"", lex -> set_affix[entry -> pars[j]] -> name);
              if ((j + 1) != arity) abs_printf(", ");
#endif /* DEBUG_LEX */
            };
          DB_LEX(abs_printf(")"));
        }
      else entry -> pars = NULL;
      DB_LEX(abs_printf("\n"));
      lex -> entry[i] = entry;
    };
  return (1);
}

static void free_entries (Lexicon lex)
{ if ((lex -> entry != NULL) && (lex -> nr_entries > 0))
    { int i;
      for (i = 0; i < lex -> nr_entries; i++)
        abs_free (lex -> entry[i], "free_entries");
      abs_free (lex -> entry, "free_entries");
    };
}

/*------------------------------------------------------------------------------
// Entry list pairs
//----------------------------------------------------------------------------*/
static int read_entry_lists_pairs (FILE* lex_file, Lexicon lex)
{ int nr, i;

  if (!read_number (lex_file, &nr)) return (0);
  lex -> nr_entrylist_pairs = nr;
  lex -> entrylist_pair = (PAIR*) abs_calloc (nr, sizeof(PAIR), "read_entry_lists_pairs");
  DB_LEX(abs_printf("-- There are %d entry pairs\n", nr));

  for (i = 0; i < nr; i++)
    { if (!read_index (lex_file, &lex -> entrylist_pair[i].index)) return (0);
      if (!read_index (lex_file, &lex -> entrylist_pair[i].pointer)) return (0);
      DB_LEX(abs_printf("Entry pair %d: %d, %d\n", i, lex -> entrylist_pair[i].index, lex -> entrylist_pair[i].pointer));
    };
  return (1);
}

static void free_entry_list_pairs (Lexicon lex)
{ /* abs_free does a nullity check */
  abs_free (lex -> entrylist_pair, "free_entry_list_pairs");
}


/*--------------------------------------------------------------------------------
// Function:
//	int lexicon_affix_belongs_to_lhs (Lexicon lex, int aff_nr, int lhs_nr)
//
// Description:
//	Checks if affix aff_nr belongs to nonterminal lhs_nr.
//
// Return value:
//	true if check succeeds, false otherwise.
//------------------------------------------------------------------------------*/
int lexicon_affix_belongs_to_lhs (Lexicon lex, int aff_nr, int lhs_nr)
{ SET_AFFIX* aff_entry;
  int i;

  assert (lhs_nr <= lex -> nr_set_affixes);
  assert (aff_nr <= lex -> nr_set_affixes);

  aff_entry = lex -> set_affix[aff_nr];
  for (i = 0; i < aff_entry -> nr_lhsses; i++)
    if ((aff_entry -> lhs[i]) == lhs_nr)
      return (1);

  return (0);
}

/*------------------------------------------------------------------------------
// Function:
//	int lxcn_try_advance_to_next_entry_in_list(const Lexicon lex, int* lst)
//
// Description:
//	replaces the list index by the next entry's list index.
//
// Return value:
//	0 (false) if there are no more entries;
//	true otherwise.
//----------------------------------------------------------------------------*/
int lxcn_try_advance_to_next_entry_in_list (Lexicon lex, int* lst)
{ assert (lst);
  *lst = lex -> entrylist_pair[*lst].pointer;
  return ((*lst) != -1);
}

void lxcn_get_params_from_entry_in_list (Lexicon lex, int entry_list, int *nontnr, int *arity,
					 Penalty *penalty, TaggedValue **params)
{ ENTRY* cur_entry = lex -> entry[lex -> entrylist_pair[entry_list].index];
  NONTERMINAL* nonterminal;
  TaggedValue *cur_par;
  int i;

  *nontnr = cur_entry -> nontnr;
  nonterminal = lex -> nonterminal[*nontnr];
  *arity = nonterminal -> arity;
  *penalty = cur_entry -> penalty;

  if (!(*arity))
    { *params = NULL;
      return;
    }

  cur_par = (TaggedValue *) abs_calloc (*arity, sizeof(TaggedValue), "state");
  *params = cur_par;

  for (i = 0; i < (*arity); i++)
    { char type = nonterminal -> pars[i].type;
      int par_idx = cur_entry -> pars[i];
      switch (type)
        { case LexParamInt:
            cur_par -> kind = IntKind;
            cur_par -> value.int_par = lex -> int_affix[par_idx];
            break;
          case LexParamText:
            cur_par -> kind = TextKind;
            cur_par -> value.text_par = lex -> text_affix[par_idx];
            break;
          case LexParamLattice:
            cur_par -> kind = SetKind;
            cur_par -> value.set_par = lex -> set_affix[par_idx] -> bitset;
            break;
          default:
	    abs_bug ("lxcn_get_params_from_entry_in_list", "No such affix parameter type");
        };
      cur_par++;
    };
}


/*------------------------------------------------------------------------------
// Lexicon acces routines
//----------------------------------------------------------------------------*/
int lexicon_get_nr_nonterminals (Lexicon lex)
{ assert(lex);
  return (lex -> nr_nonterminals);
}

char* lexicon_get_nont_name (Lexicon lex, int nont_nr)
{ assert(lex);
  assert((0 <= nont_nr) && (nont_nr < lex -> nr_nonterminals));
  return (lex -> nonterminal[nont_nr] -> name);
}

/* Check: used to be > */
int lexicon_nont_exists (Lexicon lex, int nont_nr)
{ assert(lex);

  if ((nont_nr < 0) || (nont_nr >= lex -> nr_nonterminals)) return (0);
  return (lex -> nonterminal[nont_nr] != NULL);
}

int lexicon_get_nont_arity (Lexicon lex, int nont_nr)
{ assert(lex);
  assert((0 <= nont_nr) && (nont_nr < lex -> nr_nonterminals));
  return (lex -> nonterminal[nont_nr] -> arity);
}

/* Simple but expensive search, for now. Returns nontnr, or -1 if not found. */
int lxcn_find_lex_nonterminal_with_arity (Lexicon lex, char *nont_name, int arity)
{ int nr_nonterminals = lexicon_get_nr_nonterminals (lex);
  int retval;
  for (retval = 0; retval < nr_nonterminals; retval++)
    if (lexicon_nont_exists (lex, retval) &&
	(lex -> nonterminal[retval] -> arity == arity) &&
        !strcmp (lex -> nonterminal[retval] -> name, nont_name))
      return (retval);

  return (-1);  /* not found */
}

char* lexicon_get_nont_param_name (Lexicon lex, int nont_nr, int param_nr)
{ int idx;
  char type;

  assert (lex);
  assert ((0 <= nont_nr) && (nont_nr < lex -> nr_nonterminals));
  assert ((0 <= param_nr) && (param_nr < lex -> nonterminal[nont_nr] -> arity));

  idx = lex -> nonterminal[nont_nr] -> pars[param_nr].index;
  type = lex -> nonterminal[nont_nr] -> pars[param_nr].type;

  switch (type)
    { case LexParamText:    return ("TEXT");
      case LexParamInt:	    return ("INT");
      case LexParamLattice: return (lex -> set_affix[idx] -> name);
      default:
        /* Should be unreachable: */
        return ("Unknown type");
    };
}

int lexicon_get_nont_param_nr (Lexicon lex, int nont_nr, int param_nr)
{ assert (lex);
  assert ((0 <= nont_nr) && (nont_nr < lex -> nr_nonterminals));
  assert ((0 <= param_nr) && (param_nr < lex -> nonterminal[nont_nr] -> arity));

  return (lex -> nonterminal[nont_nr] -> pars[param_nr].index);
}

int lexicon_get_nr_setaffixes (Lexicon lex)
{ assert (lex);
  return (lex -> nr_set_affixes);
}

char* lexicon_get_setaffix_name (Lexicon lex, int aff_nr)
{ assert (lex);
  assert ((0 <= aff_nr) && (aff_nr < lex -> nr_set_affixes));
  return (lex -> set_affix[aff_nr] -> name);
}

Bitset lexicon_get_setaffix_bitset (Lexicon lex, int aff_nr)
{ assert (lex);
  assert ((0 <= aff_nr) && (aff_nr < lex -> nr_set_affixes));
  return (lex -> set_affix[aff_nr] -> bitset);
}

int lexicon_get_setaffix_nont_flag (Lexicon lex, int aff_nr)
{ assert (lex);
  assert ((0 <= aff_nr) && (aff_nr < lex -> nr_set_affixes));
  return (lex -> set_affix[aff_nr] -> is_nont);
}

int lexicon_get_setaffix_nr_lhsses(Lexicon lex, int aff_nr)
{ assert (lex);
  assert ((0 <= aff_nr) && (aff_nr < lex -> nr_set_affixes));
  return (lex -> set_affix[aff_nr] -> nr_lhsses);
}

char* lexicon_get_setaffix_lhs_name (Lexicon lex, int aff_nr, int lhs_nr)
{ int affix_idx;

  assert (lex);
  assert ((0 <= aff_nr) && (aff_nr < lex -> nr_set_affixes));
  assert ((0 <= lhs_nr) && (lhs_nr < lex -> set_affix[aff_nr] -> nr_lhsses));

  affix_idx = lex -> set_affix[aff_nr] -> lhs[lhs_nr];
  return (lexicon_get_setaffix_name (lex, affix_idx));
}


/*------------------------------------------------------------------------------
// Frees all lexicon tables
//----------------------------------------------------------------------------*/
void lexicon_free (Lexicon lex)
{ if (lex != NULL)
    { free_trie(lex);
      free_set_affixes(lex);
      free_text_affixes(lex);
      free_int_affixes(lex);
      free_nonterminals(lex);
      free_entries(lex);
      free_entry_list_pairs(lex);

      abs_free (lex, "lexicon_free");
    };
}


/*------------------------------------------------------------------------------
// lexicon initialization
// Note move all allocation stuff away
//----------------------------------------------------------------------------*/
Lexicon lexicon_new (FILE* lex_file)
{ Lexicon res = (Lexicon) abs_malloc (sizeof (struct lexicon_rec), "lexicon_new");

  /* set all to known value's (and don't trust the machine to do it) */
  memset (res, '\0', sizeof(struct lexicon_rec));

  /* but NULL != '\0', for that we need knowledge about the struct! */
  res -> trie_buf = NULL;
  res -> int_affix = NULL;
  res -> text_affix = NULL;
  res -> set_affix = NULL;
  res -> nonterminal = NULL;
  res -> entry = NULL;
  res -> entrylist_pair = NULL;

  if (read_trie (lex_file, res) &&
      read_int_affixes (lex_file, res) &&
      read_text_affixes (lex_file, res) &&
      read_set_affixes (lex_file, res) &&
      read_nonterminals (lex_file, res) &&
      read_entries (lex_file, res) &&
      read_entry_lists_pairs (lex_file, res))
    return (res);

  /* If a read error occurs, we return NULL, and free any memory allocated. */
  lexicon_free (res);
  return (NULL);
}

