package ij.macro;
import ij.*;
import java.io.*;

/** This class converts an imageJ macro file file into a token stream. */
public class Tokenizer implements MacroConstants {

    private StreamTokenizer st;
    private int token;
    private String tokenString;
    private double tokenValue;
    private Program pgm;
    private int lineNumber;

    /** Uses a StreamTokenizer to convert an ImageJ macro file into a token stream. */
    public Program tokenize(String program) {
        if (program.contains("/*") && program.contains("*/"))
            program = addSpacesToEmptyLines(program);
        st = new StreamTokenizer(new StringReader(program));
        st.ordinaryChar('-');
        st.ordinaryChar('/');
        st.ordinaryChar('.');
        st.wordChars('_', '_');
        st.whitespaceChars(128, 255);
        st.slashStarComments(true);
        st.slashSlashComments(true);
        pgm = new Program();
        do {
            getToken();
            addToken();
        } while (token!=EOF);
        if (pgm.hasFunctions)
            addUserFunctions();
        //IJ.log(program.length()+" "+pgm.getSize()+" "+IJ.d2s((double)program.length()/pgm.getSize(),1)+" "+program.length()/10);
        return pgm;
    }

    final void getToken() {
        try {
            token = st.nextToken();
            lineNumber = st.lineno();
            String ret = null;
            int nextToken;
            switch (st.ttype) {
                case StreamTokenizer.TT_EOF:
                    ret = "EOF";
                    token = EOF;
                    break;
                case StreamTokenizer.TT_WORD:
                    ret = st.sval;
                    token = WORD;
                    break;
                case StreamTokenizer.TT_NUMBER:
                    ret = ""+st.nval;
                    tokenValue = st.nval;
                    if (tokenValue==0.0)
                        tokenValue = getHexConstant();
                    else if (tryScientificNotation())
                        ret += st.sval;
                    token = NUMBER;
                    break;
                case '"': case '\'':
                    ret = ""+st.sval;
                    token = STRING_CONSTANT;
                    break;
                case '+':
                    nextToken = st.nextToken();
                    if (nextToken=='+')
                        token = PLUS_PLUS;
                    else if (nextToken=='=')
                        token = PLUS_EQUAL;
                    else
                        st.pushBack();
                    break;
                case '-':
                    nextToken = st.nextToken();
                    if (nextToken=='-')
                        token = MINUS_MINUS;
                    else if (nextToken=='=')
                        token = MINUS_EQUAL;
                    else
                        st.pushBack();
                    break;
                case '*':
                    nextToken = st.nextToken();
                    if (nextToken=='=')
                        token = MUL_EQUAL;
                    else
                        st.pushBack();
                    break;
                case '/':
                    nextToken = st.nextToken();
                    if (nextToken=='=')
                        token = DIV_EQUAL;
                    else
                        st.pushBack();
                    break;
                case '=':
                    nextToken = st.nextToken();
                    if (nextToken=='=')
                        token = EQ;
                    else
                        st.pushBack();
                    break;
                case '!':
                    nextToken = st.nextToken();
                    if (nextToken=='=')
                        token = NEQ;
                    else
                        st.pushBack();
                    break;
                case '>':
                    nextToken = st.nextToken();
                    if (nextToken=='=')
                        token = GTE;
                    else if (nextToken=='>')
                        token = SHIFT_RIGHT;
                    else {
                        st.pushBack();
                        token = GT;
                    }
                    break;
                case '<':
                    nextToken = st.nextToken();
                    if (nextToken=='=')
                        token = LTE;
                    else if (nextToken=='<')
                        token = SHIFT_LEFT;
                    else {
                        st.pushBack();
                        token = LT;
                    }
                    break;
                case '&':
                    nextToken = st.nextToken();
                    if (nextToken=='&')
                        token = LOGICAL_AND;
                    else
                        st.pushBack();
                    break;
                case '|':
                    nextToken = st.nextToken();
                    if (nextToken=='|')
                        token = LOGICAL_OR;
                    else
                        st.pushBack();
                    break;
                default:
            }
            tokenString = ret;
        } catch (Exception e) {
            return;
        }
    }

    final void addToken() {
            int tok = token;
            switch (token) {
                case WORD:
                Symbol symbol = pgm.lookupWord(tokenString);
                if (symbol!=null) {
                    int type = symbol.getFunctionType();
                    if (type==0) {
                        tok = symbol.type;
                        switch (tok) {
                            case FUNCTION: pgm.hasFunctions=true; break;
                            case VAR: pgm.hasVars=true; break;
                            case MACRO: pgm.macroCount++; break;
                        }
                    } else
                        tok = type;
                    tok += pgm.symTabLoc<<TOK_SHIFT;
                } else {
                    pgm.addSymbol(new Symbol(token, tokenString));
                    tok += pgm.stLoc<<TOK_SHIFT;
                }
                break;
            case STRING_CONSTANT:
                pgm.addSymbol(new Symbol(token, tokenString));
                tok += pgm.stLoc<<TOK_SHIFT;
                break;
            case NUMBER:
                pgm.addSymbol(new Symbol(tokenValue));
                tok += pgm.stLoc<<TOK_SHIFT;
                break;
            default:
                break;
        }
        pgm.addToken(tok, lineNumber);
    }

    double getHexConstant() {
        try {
            token = st.nextToken();
        } catch (Exception e) {
            return 0.0;
        }
        if (st.ttype != StreamTokenizer.TT_WORD) {
            st.pushBack();
            return 0.0;
        }
        if (!st.sval.startsWith("x")) {
            st.pushBack();
            return 0.0;
        }
        String s = st.sval.substring(1, st.sval.length());
        double n = 0.0;
        try {
            n = Long.parseLong(s, 16);
        } catch (NumberFormatException e) {
            st.pushBack();
            n = 0.0;
        }
        return n;
    }

    boolean tryScientificNotation() {
        String sval = "" + tokenValue;
        try {
            int next = st.nextToken();
            String sval2 = st.sval;
            if (st.ttype == st.TT_WORD && (sval2.startsWith("e")||sval2.startsWith("E"))) {
                // Usually we would just append the sval, but "-" is special...
                if (sval2.equalsIgnoreCase("e")) {
                    //if (st.nextToken() != st.TT_WORD || !st.sval.equals("-"))
                    next = st.nextToken();
                    if (next == '-')
                        sval2 += "-";
                    else if (next != '+')
                        throw new Exception();
                    if (st.nextToken() != st.TT_NUMBER)
                        throw new Exception();
                    sval2 += st.nval;
                }
                if (sval2.endsWith(".0"))
                    sval2 = sval2.substring(0, sval2.length() - 2);
                tokenValue = Double.parseDouble(sval + sval2);
                return true;
            }
            st.pushBack();
        } catch(Exception e) {}
        return false;
    }

    /** Adds user-defined functions to the symbol table. */
    void addUserFunctions() {
        int[] code = pgm.getCode();
        int nextToken, address, address2;
        for (int i=0; i<code.length; i++) {
            token = code[i]&TOK_MASK;
            if (token==FUNCTION) {
                nextToken = code[i+1]&TOK_MASK;
                if (nextToken==WORD || (nextToken>=PREDEFINED_FUNCTION&&nextToken<=ARRAY_FUNCTION)) {
                    address = address2 = code[i+1]>>TOK_SHIFT;
                    if (nextToken!=WORD) { // override built in function
                        pgm.addSymbol(new Symbol(WORD, pgm.getSymbolTable()[address].str));
                        address2 = pgm.stLoc;
                        code[i+1] = WORD + (address2<<TOK_SHIFT);
                    }
                    Symbol sym = pgm.getSymbolTable()[address2];
                    sym.type = USER_FUNCTION;
                    sym.value = i+1;  //address of function
                    for (int j=0; j<code.length; j++) {
                        token = code[j]&TOK_MASK;
                        if ((token==WORD || (token>=PREDEFINED_FUNCTION&&token<=ARRAY_FUNCTION))
                        && (code[j]>>TOK_SHIFT)==address && (j==0||(code[j-1]&0xfff)!=FUNCTION)) {
                            code[j] = USER_FUNCTION;
                            code[j] += address2<<TOK_SHIFT;
                            //IJ.log((code[j]&TOK_MASK)+" "+(code[j]>>TOK_SHIFT)+" "+USER_FUNCTION+" "+address);
                        } else if (token==EOF)
                            break;
                    }
                    //IJ.log(i+"  "+pgm.decodeToken(nextToken, address));
                }                   
            } else if (token==EOF)
                break;
        }
    }
    
    private String addSpacesToEmptyLines(String pgm) {
        StringBuilder sb = new StringBuilder();
        int len = pgm.length();
        for (int jj=0; jj<len-1; jj++) {//insert a space if line is empty
            char c = pgm.charAt(jj);
            sb.append(c);
            if (jj<len-1 && c=='\n' && pgm.charAt(jj+1)=='\n')
                sb.append(" ");
        }
        sb.append(pgm.charAt(len-1)); //add last char
        return sb.toString(); //program no longer contain empty lines
    }

}