diff options
Diffstat (limited to 'code/Interpreter2/src/Interpreter/TokenScanner.java')
-rw-r--r-- | code/Interpreter2/src/Interpreter/TokenScanner.java | 179 |
1 files changed, 0 insertions, 179 deletions
diff --git a/code/Interpreter2/src/Interpreter/TokenScanner.java b/code/Interpreter2/src/Interpreter/TokenScanner.java deleted file mode 100644 index c9249a4..0000000 --- a/code/Interpreter2/src/Interpreter/TokenScanner.java +++ /dev/null @@ -1,179 +0,0 @@ -package Interpreter; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class TokenScanner { - private String sourceCode; - List<Token> tokens = new ArrayList<>(); - private int tokenStart=0; - private int currentLoc=0; - - //Extract tokens from the source code by reading character by character - List<Token> extractTokens(String sourceCode){ - this.sourceCode=sourceCode; - while (!checkEOF()){ - tokenStart=currentLoc; - readToken(); - } - tokens.add(new Token(TokenType.EOF, "", null)); - return tokens; - } - - //Extract a single token - private void readToken(){ - char checkChar = sourceCode.charAt(currentLoc); - switch(checkChar){ - - case ' ':break; - case '\n':break; - case '\r':break; - case '\t': - break; - - case '(': createTokenNull(TokenType.LEFT_PAREN); break; - case ')': createTokenNull(TokenType.RIGHT_PAREN); break; - case '+': createTokenNull(TokenType.PLUS); break; - case '-': createTokenNull(TokenType.MINUS); break; - case '*': createTokenNull(TokenType.STAR); break; - case '/': createTokenNull(TokenType.SLASH); break; - case ';': createTokenNull(TokenType.SEMI_COLON); break; - - //Some tokens are multiple characters long (==, <=) etc - //so need to check next char as well - case '=': - if (checkNextChar('=')){ - createTokenNull(TokenType.EQUALITY); - break; - } else { - createTokenNull(TokenType.EQUALS); - break; - } - case ':': - if (checkNextChar(':')){ - createTokenNull(TokenType.DEFINE); - break; - } else { - createTokenNull(TokenType.COLON); - break; - } - case '<': - if (checkNextChar('=')){ - createTokenNull(TokenType.LESS_EQUAL); - break; - } else { - createTokenNull(TokenType.LESS); - break; - } - case '>': - if (checkNextChar('=')){ - createTokenNull(TokenType.GREATER_EQUAL); - break; - } else { - createTokenNull(TokenType.GREATER); - break; - } - default: - - //Check for numer - if (checkIsDigit(checkChar)){ - while (checkIsDigit(lookAhead())){ - currentLoc++; - } - //Check if number contains a decimal point - if (lookAhead()=='.' && checkIsDigit(lookTwoAhead())){ - currentLoc++; - while (checkIsDigit(lookAhead())){ - currentLoc++; - } - } - createToken(TokenType.NUMBER, Double.parseDouble(sourceCode.substring(tokenStart, currentLoc+1))); - } - else if (checkIsAlpha(checkChar)){ - while (checkIsAlpha(lookAhead())){ - currentLoc++; - } - String text = sourceCode.substring(tokenStart, currentLoc+1); - TokenType type = keywords.get(text); - if(type == null){ - createToken(TokenType.IDENTIFIER, text); - } else{ - createToken(type, text); - } - - } else { - Language.displayError("Unexpected Character"); - } - } - currentLoc++; - - } - - //Test for end of file - private boolean checkEOF(){ - return currentLoc>=sourceCode.length(); - } - - //Create a token without a value - private void createTokenNull(TokenType type){ - createToken(type, null); - } - - //Create token - private void createToken(TokenType type, Object value){ - String tokenText = sourceCode.substring(tokenStart, currentLoc+1); - tokens.add(new Token(type, tokenText, value)); - } - - //Check if the next char matches a given char - private boolean checkNextChar(char matchChar){ - if (checkEOF()){ - return false; - } - if (sourceCode.charAt(currentLoc+1)==matchChar){ - currentLoc++; - return true; - } - return false; - } - - //Look at the next char in the source code - private char lookAhead(){ - if (currentLoc+1>=sourceCode.length()){ - return ' '; - - } else { - return sourceCode.charAt(currentLoc+1); - } - } - - //Look 2 chars ahead in the source code - private char lookTwoAhead(){ - if (currentLoc+2>=sourceCode.length()){ - return ' '; - - } else { - return sourceCode.charAt(currentLoc+2); - } - } - - //Check if a given char is a digit - private boolean checkIsDigit(char checkChar){ - return checkChar>='0' && checkChar<='9'; - } - - private boolean checkIsAlpha(char checkChar){ - return ('a'<=checkChar && checkChar<='z')|| - ('A'<=checkChar && checkChar<='Z'); - } - - private static final Map<String, TokenType> keywords; - - static { - keywords = new HashMap<>(); - keywords.put("var", TokenType.VAR); - keywords.put("print", TokenType.PRINT); - } -} |