
/*
 *    Copyright 1986, 1987 Pat Joseph Monardo. All rights reserved.
 *    Copying of this file is granted according to the provisions 
 *    specified in the file COPYING which must accompany this file.
 */


/*
 *		token.c
 */

#include "tex.h"
#include "cmds.h"
#include "heap.h"
#include "eq.h"
#include "hash.h"
#include "scan.h"
#include "io.h"
#include "char.h"
#include "box.h"
#include "cond.h"
#include "print.h"
#include "error.h"
#include "expand.h"
#include "align.h"
#include "tokenstack.h"
#include "token.h"

hword	cur_chr;
byte	cur_cmd;
ptr		cur_cs;
hword	cur_tok;

/*
 *	this is the TeX segment
 */

hword	tok[TOK_MAX-TOK_MIN+1];
hword	tok_link[TOK_MAX-TOK_MIN+1];
ptr		tok_head;
ptr		tok_low;
ptr		tok_end;
int		tok_used;

ptr		par_loc;
hword	par_token;

bool	force_eof;

get_token ()
{
	no_new_control_sequence = FALSE;
	get_next();
	no_new_control_sequence = TRUE;
	if (cur_cs == 0)
		cur_tok = cur_cmd * 0400 + cur_chr;
	else cur_tok = CS_TOKEN_FLAG + cur_cs;
}

#define	reduce_expanded_cc() \
	{if (buffer[k] == cur_chr && \
		cat == SUP_MARK && k < limit) { \
		cur_chr = buffer[k + 1]; \
		if (cur_chr < 0100) \
			buffer[k - 1] = cur_chr + 0100; \
		else buffer[k - 1] = cur_chr - 0100; \
		limit -= 2; \
		first -= 2; \
		while (k <= limit) { \
			buffer[k] = buffer[k + 2]; \
			incr(k);} \
		goto start_cs;}}
		
#define	any_state(CAT) \
	case MID_LINE + CAT: \
	case SKIP_BLANKS + CAT: \
	case NEW_LINE + CAT
			
#define	delims(CAT) \
	case MATH_SHIFT + CAT: \
	case TAB_MARK + CAT: \
	case MAC_PARAM + CAT: \
	case SUB_MARK + CAT: \
	case LETTER + CAT: \
	case OTHER_CHAR + CAT

#define	mid_line(CAT) \
	case MID_LINE + CAT

#define	new_line(CAT) \
	case NEW_LINE + CAT

#define	skip_blanks(CAT) \
	case SKIP_BLANKS + CAT

get_next ()
{
	int		k;
	hword	t;
	int		cat;

restart:
	cur_cs = 0;
	if (state != TOKEN_LIST) {
reread:
		if (loc <= limit) {
			cur_chr = buffer[loc];
			incr(loc);
reswitch:
			cur_cmd = cat_code(cur_chr);
			switch (state + cur_cmd)
			{
			any_state(IGNORE):
			skip_blanks(SPACER):
			new_line(SPACER):
				goto reread;
				break;

			any_state(ESCAPE):
				if (loc > limit)
					cur_cs = NULL_CS;
				else {
					start_cs: 
					k = loc;
					cur_chr = buffer[k];
					cat = cat_code(cur_chr);
					incr(k);
					if (cat == LETTER)
						state = SKIP_BLANKS;
					else if (cat == SPACER)
						state = SKIP_BLANKS;
					else state = MID_LINE;
					if (cat == LETTER && k <= limit) {
						do {
							cur_chr = buffer[k];
							incr(k);
							cat = cat_code(cur_chr);
						} while (cat == LETTER && k <= limit);
						reduce_expanded_cc();
						if (cat != LETTER)
							decr(k);
						if (k > loc + 1) {
							cur_cs = id_lookup(loc, k - loc);
							loc = k;
							goto found;
						}
					} else reduce_expanded_cc();
					cur_cs = SINGLE_BASE + buffer[loc];
					incr(loc);
				}
				found:
				cur_cmd = eq_type(cur_cs);
				cur_chr = equiv(cur_cs);
				if (cur_cmd >= OUTER_CALL)
					check_outer_validity();
				break;
						
			any_state(ACTIVE_CHAR):
				cur_cs = cur_chr + ACTIVE_BASE;
				cur_cmd = eq_type(cur_cs);
				cur_chr = equiv(cur_cs);
				state = MID_LINE;
				if (cur_cmd >= OUTER_CALL)
					check_outer_validity();
				break;
			
			any_state(SUP_MARK):
				if (cur_chr == buffer[loc] && loc < limit) {
					if (buffer[loc + 1] < 0100)
						cur_chr = buffer[loc + 1] + 0100;
					else cur_chr = buffer[loc + 1] - 0100;
					loc += 2;
					goto reswitch;
				}
				state = MID_LINE;
				break;

			any_state(INVALID_CHAR):
				print_err("Text line contains an invalid character");
				help_funny();
				deletions_allowed = FALSE;
				error();
				deletions_allowed = TRUE;
				goto restart;
				break;
			
			mid_line(SPACER):
				state = SKIP_BLANKS;
				cur_chr = ' ';
				break;
			
			mid_line(CAR_RET):
				loc = limit + 1;
				cur_cmd = SPACER;
				cur_chr = ' ';
				break;
			
			skip_blanks(CAR_RET):
			any_state(COMMENT):
				loc = limit + 1;
				goto reread;
				break;

			new_line(CAR_RET):
				loc = limit + 1;
				cur_cs = par_loc;
				cur_cmd = eq_type(cur_cs);
				cur_chr = equiv(cur_cs);
				if (cur_cmd >= OUTER_CALL)
					check_outer_validity();
				break;
			
			mid_line(LEFT_BRACE):
				incr(align_state);
				break;

			skip_blanks(LEFT_BRACE):
			new_line(LEFT_BRACE):
				state = MID_LINE;
				incr(align_state);
				break;
			
			mid_line(RIGHT_BRACE):
				decr(align_state);
				break;
			
			skip_blanks(RIGHT_BRACE):
			new_line(RIGHT_BRACE):
				state = MID_LINE;
				decr(align_state);
				break;
			
			delims(SKIP_BLANKS):
			delims(NEW_LINE):
				state = MID_LINE;
				break;

			default:
				break;
			}
		} else {
			state = NEW_LINE; 
			if (name > 17) {
				incr(line);
				first = start;
				if (!force_eof) {
					if (input_ln(cur_file, TRUE))
						firm_up_the_line();
					else force_eof = TRUE;
				}
				if (force_eof) {
					print_char(')');
					force_eof = FALSE;
					update_terminal();
					end_file_reading();
					check_outer_validity();
					goto restart;
				}
				if (end_line_char < 0 || end_line_char > 127)
					decr(limit);
				else buffer[limit] = end_line_char;
				first = limit + 1;
				loc = start;
			} else {
				if (!terminal_input) {
					cur_cmd = 0;
					cur_chr = 0;
					return;
				}
				if (input_ptr > 0) {
					end_file_reading();
					goto restart;
				}
				if (selector < LOG_ONLY)
					open_log_file();
				if (interaction > NONSTOP_MODE) {
					if (limit == start)
						print_nl("(Please type a command or say `\\end')");
					print_ln();
					first = start;
					prompt_input("*");
					limit = last;
					if (end_line_char < 0 || end_line_char > 127)
						decr(limit);
					else buffer[limit] = end_line_char;
					first = limit + 1;
					loc = start;
				} else
					fatal_error(
						"*** (job aborted, no legal \\end found)");
			}
			check_interrupt();
			goto reread;
		}
	} else {
		if (loc != NULL) {
			t = token(loc);
			loc = token_link(loc);
			if (t >= CS_TOKEN_FLAG) {
				cur_cs = t - CS_TOKEN_FLAG;
				cur_cmd = eq_type(cur_cs);
				cur_chr = equiv(cur_cs);
				if (cur_cmd >= OUTER_CALL) {
					if (cur_cmd == DONT_EXPAND) {
						cur_cs = token(loc) - CS_TOKEN_FLAG;
						loc = NULL;
						cur_cmd = eq_type(cur_cs);
						cur_chr = equiv(cur_cs);
						if (cur_cmd > MAX_COMMAND) {
							cur_cmd = RELAX;
							cur_chr = NO_EXPAND_FLAG;
						}
					} else check_outer_validity();
				}
			} else {
				cur_cmd = t / 0400;
				cur_chr = t % 0400;
				switch (cur_cmd)
				{
				case LEFT_BRACE:
					incr(align_state);
					break;

				case RIGHT_BRACE:
					decr(align_state);
					break;

				case OUT_PARAM:
					begin_token_list(
						param_stack[param_start + cur_chr - 1],
						PARAMETER);
					goto restart;
					break;

				default:
					break;
				}
			}
		} else {
			end_token_list();
			goto restart;
		}
	}
	if (cur_cmd <= CAR_RET &&
		cur_cmd >= TAB_MARK &&
		align_state == 0) {
		if (scanner_status == ALIGNING)
			fatal_error("(interwoven alignment preambles are not allowed)");
		cur_cmd = extra_info(cur_align);
		extra_info(cur_align) = cur_chr;
		if (cur_cmd == OMIT)
			begin_token_list(omit_template, (qword) V_TEMPLATE);
		else begin_token_list((ptr) v_part(cur_align), (qword) V_TEMPLATE);
		align_state = 1000000;
		goto restart;
	}
}

check_outer_validity ()
{
	ptr		p;
	ptr		q;

	if (scanner_status != NORMAL) {
		deletions_allowed = FALSE;
		if (cur_cs != 0) {
			if (state == TOKEN_LIST || name < 1 || name > 17) {
				p = new_token();
				token(p) = CS_TOKEN_FLAG + cur_cs;
				back_list(p);
			}
			cur_cmd = SPACER;
			cur_chr = ' ';
		}
		if (scanner_status > SKIPPING) {
			runaway();
			if (cur_cs == 0) 
				print_err("File ended");
			else {
				cur_cs = 0;
				print_err("Forbidden control sequence found");
			}
			print(" while scanning ");
			p = new_token();
			switch (scanner_status)
			{
			case DEFINING:
				print("definition");
				token(p) = RIGHT_BRACE_TOKEN + '}';
				break;

			case MATCHING:
				print("use");
				token(p) = par_token;
				long_state = OUTER_CALL;
				break;

			case ALIGNING:
				print("preamble");
				token(p) = RIGHT_BRACE_TOKEN + '}';
				q = p;
				p = new_token();
				token_link(p) = q;
				token(p) = CS_TOKEN_FLAG + FROZEN_CR;
				align_state = -1000000;
				break;

			case ABSORBING:
				print("text"); 
				token(p) = RIGHT_BRACE_TOKEN + '}';
				break;
			}
			ins_list(p);
			print(" of ");
			sprint_cs(warning_index);
			help_scanner();
			error();
		} else {
			print_err("Incomplete ");
			print_cmd_chr(IF_TEST, cur_if);
			print("; all text was ignored after line ");
			print_val(skip_line);
			help_skif();
			if (cur_cs != 0)
				cur_cs = 0;
			else
				help_line[0] = 
					"The file ended while I was skipping conditional text.";
			cur_tok = CS_TOKEN_FLAG + FROZEN_FI;
			ins_error();
		}
		deletions_allowed = TRUE;
	}
}

firm_up_the_line ()
{
	int		k;

	limit = last;
	if (pausing > 0 && interaction > NONSTOP_MODE) {
		wake_up_terminal();
		print_ln();
		if (start < limit) {
			for (k = start; k < limit; incr(k))
				print_char(buffer[k]);
		}
		first = limit;
		prompt_input("=>");
		if (last > first) {
			for (k = first; k < last; incr(k))
				buffer[k + start - first] = buffer[k];
			limit = start + last - first;
		}
	}
}

ptr
new_token ()
{
	ptr		p;

	p = tok_head;
	if (p != NULL)
		tok_head = token_link(tok_head);
	else if (tok_end < TOK_MAX) {
		incr(tok_end);
		p = tok_end;
	} else {
		decr(tok_low);
		p = tok_low;
		if (tok_low <= TOK_MIN) {
			runaway();
			overflow("token memory size", TOK_MAX - TOK_MIN + 1);
		}
	}
	token_link(p) = NULL;
#ifdef STAT
	incr(tok_used);
#endif
	return p;
}

init_tok_mem()
{
	int		i;

#ifdef INIT
	tok_head = NULL;
	tok_end = TOK_TOP;
	tok_low = tok_high;
	for (i = TOK_TOP; i > tok_low; decr(i)) {
		token(i) = 0;
		token_link(i) = NULL;
	}
	token(omit_template) = END_TEMPLATE_TOKEN;
	tok_used = tok_usage;
#endif

}

/*
 *	Help text
 */

help_scanner ()
{
	help4("I suspect you have forgotten a `}', causing me",
	"to read past where you wanted me to stop.",
	"Either type `I}' to try recovering,",
	"or type 'X' and fix your file.");
}

help_funny ()
{
	help2("A funny symbol that I can't read has just been input.",
	"Continue, and I'll forget that it ever happened.");
}

help_skif ()
{
	help3("A forbidden control sequence occurred in skipped text.",
	"This kind of error happens when you say `\\if...' and forget",
	"the matching `\\fi'. I've inserted a `\\fi'; this might work.");
}
