Added basic lexical analysis

1 files changed, 132 insertions, 0 deletions

diff --git a/code/Interpreter/TokenScanner.java b/code/Interpreter/TokenScanner.java
new file mode 100644
index 0000000..87f1e4b
--- /dev/null
+++ b/code/Interpreter/TokenScanner.java
@@ -0,0 +1,132 @@
+package Interpreter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TokenScanner {
+    private String sourceCode;
+    List<Token> tokens = new ArrayList<>();
+    private int tokenStart=0;
+    private int currentLoc=0;
+
+    //Extract tokens from the source code by reading character by character
+    List<Token> extractTokens(String sourceCode){
+        this.sourceCode=sourceCode;
+        while (!checkEOF()){
+            tokenStart=currentLoc;
+            readToken();
+        }
+        return tokens;
+    }
+
+    //Extract a single token
+    private void readToken(){
+        char checkChar = sourceCode.charAt(currentLoc);
+        switch(checkChar){
+            case '(': createTokenNull(TokenType.LEFT_PAREN); break;
+            case ')': createTokenNull(TokenType.RIGHT_PAREN); break;
+            case '+': createTokenNull(TokenType.PLUS); break;
+            case '-': createTokenNull(TokenType.MINUS); break;
+            case '*': createTokenNull(TokenType.STAR); break;
+            case '/': createTokenNull(TokenType.SLASH); break;
+
+            //Some tokens are multiple characters long (==, <=) etc
+            //so need to check next char as well
+            case '=': 
+                if (checkNextChar('=')){
+                    createTokenNull(TokenType.EQUALITY);
+                    break;
+                } else {
+                    createTokenNull(TokenType.EQUALS);
+                    break;
+                }
+            case '<': 
+                if (checkNextChar('=')){
+                    createTokenNull(TokenType.LESS_EQUAL);
+                    break;
+                } else {
+                    createTokenNull(TokenType.LESS);
+                    break;
+                }
+            case '>': 
+                if (checkNextChar('=')){
+                    createTokenNull(TokenType.GREATER_EQUAL);
+                    break;
+                } else {
+                    createTokenNull(TokenType.GREATER);
+                    break;
+                }
+            default:
+
+                //Check for numer
+                if (checkIsDigit(checkChar)){
+                    while (checkIsDigit(lookAhead())){
+                        currentLoc++;
+                    }   
+                    //Check if number contains a decimal point
+                    if (lookAhead()=='.' && checkIsDigit(lookTwoAhead())){
+                        currentLoc++;
+                        while (checkIsDigit(lookAhead())){
+                            currentLoc++;
+                        }
+                    }
+                    createToken(TokenType.NUMBER, Double.parseDouble(sourceCode.substring(tokenStart, currentLoc+1)));
+                }
+        }
+        currentLoc++;
+
+    }
+
+    //Test for end of file
+    private boolean checkEOF(){
+        return currentLoc>=sourceCode.length();
+    }
+
+    //Create a token without a value
+    private void createTokenNull(TokenType type){
+        createToken(type, null);
+    }
+
+    //Create token
+    private void createToken(TokenType type, Object value){
+        String tokenText = sourceCode.substring(tokenStart, currentLoc+1);
+        tokens.add(new Token(type, tokenText, value));
+    }
+
+    //Check if the next char matches a given char
+    private boolean checkNextChar(char matchChar){
+        if (checkEOF()){
+            return false;
+        }
+        if (sourceCode.charAt(currentLoc+1)==matchChar){
+            currentLoc++;
+            return true;
+        } 
+        return false;
+    }
+
+    //Look at the next char in the source code
+    private char lookAhead(){
+        if (currentLoc+1>=sourceCode.length()){
+            return ' ';
+
+        } else {
+            return sourceCode.charAt(currentLoc+1);
+        }
+    }
+
+    //Look 2 chars ahead in the source code
+    private char lookTwoAhead(){
+        if (currentLoc+2>=sourceCode.length()){
+            return ' ';
+
+        } else {
+            return sourceCode.charAt(currentLoc+2);
+        }
+    }
+
+    //Check if a given char is a digit
+    private boolean checkIsDigit(char checkChar){
+        return checkChar>='0' && checkChar<='9';
+    }
+}