Skip to content

Commit

Permalink
Update TokenStream.java
Browse files Browse the repository at this point in the history
  • Loading branch information
@hideonfruit
hideonfruit authored Dec 15, 2024
1 parent 4a249b4 commit 0e99858
Showing 1 changed file with 59 additions and 38 deletions.
97 changes: 59 additions & 38 deletions src/main/java/com/scanner/project/TokenStream.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.regex.Pattern;

public class TokenStream {

Expand All @@ -32,7 +33,6 @@ public boolean isEoFile() {
public TokenStream(String fileName) {
try {
input = new BufferedReader(new FileReader(fileName));
//readChar();
} catch (FileNotFoundException e) {
System.out.println("File not found: " + fileName);
// System.exit(1); // Removed to allow ScannerDemo to continue
Expand All @@ -47,14 +47,9 @@ public Token nextToken() { // Main function of the scanner
t.setType("Other"); // For now it is Other
t.setValue("");


// First check for whitespaces and bypass them
skipWhiteSpace();

if (isEof) {
return null;
}

// Then check for a comment, and bypass it
// but remember that / may also be a division operator.
while (nextChar == '/') {
Expand All @@ -65,17 +60,13 @@ public Token nextToken() { // Main function of the scanner
// skip rest of line - it's a comment.
// TODO TO BE COMPLETED
// look for <cr>, <lf>, <ff>
while (nextChar != '\n' && nextChar != '\r' && !isEof) {
while(!isEndOfLine(nextChar) && !isEof){
nextChar = readChar();

}
skipWhiteSpace();

if (isEof) {
return null;
}

} else {
skipWhiteSpace();
if(isEof) return t;
}
else {
// A slash followed by anything else must be an operator.
t.setValue("/");
t.setType("Operator");
Expand All @@ -93,45 +84,49 @@ public Token nextToken() { // Main function of the scanner
case '<':
// <=
nextChar = readChar();
if (nextChar == '=') {
if( nextChar == '='){
t.setValue(t.getValue() + nextChar);
nextChar = readChar();
return t;
} else {
}
else{
t.setValue("<");
}
return t;

case '>':
// >=
nextChar = readChar();
if (nextChar == '=') {
if( nextChar == '='){
t.setValue(t.getValue() + nextChar);
nextChar = readChar();
return t;
} else {
}
else{
t.setValue(">");
}

return t;
case '=':
// ==
nextChar = readChar();
if (nextChar == '=') {
if( nextChar == '='){
t.setValue(t.getValue() + nextChar);
nextChar = readChar();
return t;
} else {
}
else{
t.setType("Other");
}
return t;
case '!':
// !=
nextChar = readChar();
if (nextChar == '=') {
if( nextChar == '='){
t.setValue(t.getValue() + nextChar);
nextChar = readChar();
return t;
} else {
}
else{
t.setValue("!");
}
return t;
Expand All @@ -144,32 +139,47 @@ public Token nextToken() { // Main function of the scanner
return t;
} else {
t.setType("Other");
nextChar = readChar();
}
return t;

case '&':
// Look for &&
// Look or &&
nextChar = readChar();
if (nextChar == '&') {
t.setValue(t.getValue() + nextChar);
nextChar = readChar();
return t;
} else {
t.setType("Other");
nextChar = readChar();
}
return t;

case ':':
// Look for :=
return t;
/*case '*':
nextChar = readChar();
if (nextChar == '=') {
if(nextChar == '*'){
t.setValue(t.getValue() + nextChar);
nextChar = readChar();
return t;
} else {
}
else{
t.setType("Other");
nextChar = readChar();
}
return t;
return t;*/
case ':':
nextChar = readChar();
if(nextChar == '='){
t.setValue(t.getValue() + nextChar);
nextChar = readChar();
return t;
}
else{
t.setType("Other");
nextChar = readChar();
}
return t;

default: // all other operators
nextChar = readChar();
Expand All @@ -190,14 +200,16 @@ public Token nextToken() { // Main function of the scanner
if (isLetter(nextChar)) {
// Set to an identifier
t.setType("Identifier");
while (isLetter(nextChar) || isDigit(nextChar)) {
while ((isLetter(nextChar) || isDigit(nextChar))) {
t.setValue(t.getValue() + nextChar);
nextChar = readChar();
if(isEndOfToken(nextChar)){
break;
}
}
// now see if this is a keyword
if (isKeyword(t.getValue())) {
t.setType("Keyword");
return t;
} else if (t.getValue().equals("True") || t.getValue().equals("False")) {
t.setType("Literal");
}
Expand Down Expand Up @@ -242,7 +254,6 @@ private char readChar() {
if (isEof)
return (char) 0;
System.out.flush();

try {
i = input.read();
} catch (IOException e) {
Expand All @@ -257,7 +268,9 @@ private char readChar() {

private boolean isKeyword(String s) {
// TODO TO BE COMPLETED
return (s.equals("bool") || s.equals("else") || s.equals("if") || s.equals("integer") || s.equals("main") || s.equals("while"));

if(s.equals("bool") || s.equals("else") || s.equals("if") || s.equals("integer") || s.equals("main") || s.equals("while")) return true;
return false;
}

private boolean isWhiteSpace(char c) {
Expand All @@ -281,22 +294,30 @@ private void skipWhiteSpace() {

private boolean isSeparator(char c) {
// TODO TO BE COMPLETED
return(c == '(' || c == ')' || c == '{' || c == '}' || c == ';' || c == ',');
if(c == '(' || c == ')' || c == '{' || c == '}' || c == ';' || c == ',') return true;
return false;
}

private boolean isOperator(char c) {
// Checks for characters that start operators
// TODO TO BE COMPLETED
return (c == '+' || c == '-' || c == '*' || c == '/' || c == '<' || c == '>' || c == '=' || c == '!' || c == '&' || c == '|' || c == ':');
if(c == '+' || c == '-' || c == '*' || c == '/' || c == '<' || c == '>' || c == '=' || c == '&' || c == '|' || c == '!' || c == ':') return true;
return false;
}



private boolean isLetter(char c) {
return (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z');
}

private boolean isDigit(char c) {
// TODO TO BE COMPLETED
return(c >= '0' && c <= '9');


//return (c == '0' || c == '1' || c == '2' || c == '3' || c == '4' || c == '5' || c == '6' || c == '7' || c == '8' || c == '9');
if(c >= '0' && c <= '9') return true;
return false;
}

public boolean isEndofFile() {
Expand Down

0 comments on commit 0e99858

Please sign in to comment.