Skip to content

Commit

Permalink
Allow for larger files
Browse files Browse the repository at this point in the history
  • Loading branch information
@HakonEriksen
HakonEriksen committed Jan 16, 2024
1 parent 20169a5 commit b0706e9
Show file tree
Hide file tree
Showing 11 changed files with 1,124 additions and 725 deletions.
27 changes: 27 additions & 0 deletions Step/src/org/bimserver/step/ByteBuffer.java
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,40 @@
package org.bimserver.step;

import java.io.IOException;
import java.io.InputStream;

class ByteBuffer {

private ByteBufferPage first = new ByteBufferPage(1 << 20);
private ByteBufferPage writePage = first;
private ByteBufferPage readPage = first;
private int writePageIndex = 0;
private int readPageIndex = 0;
private long length;

public ByteBuffer() {
}

public ByteBuffer(byte[] content) {
append(content, 0, content.length);
}

public ByteBuffer(String content) {
this(content.getBytes());
}

public ByteBuffer(InputStream in) throws IOException {
append(in);
}

public void append(InputStream in) throws IOException {
byte[] buffer = new byte[4096];
int length = 0;
while ((length = in.read(buffer, 0, buffer.length)) > 0) {
append(buffer, 0, length);
}
}

public void append(byte[] values, int offset, int length) {
int pageIndex = (int) (this.length >> 20);
ByteBufferPage page = writePage;
Expand Down
8 changes: 6 additions & 2 deletions Step/src/org/bimserver/step/StepAttribute.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,12 @@ public interface StepAttribute {
String getTokenValue();
boolean isUnset();
boolean isRedeclared();
boolean isBinary();
boolean isEnum();
boolean isList();
boolean isInstanceName();
boolean isInline();
boolean isInstanceName();
boolean isList();
boolean isInteger();
boolean isReal();
boolean isString();
}
176 changes: 103 additions & 73 deletions Step/src/org/bimserver/step/StepAttributeImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ public class StepAttributeImpl implements StepAttribute {
private final ByteBuffer dataBuffer;
private final TokenBuffer tokenBuffer;
private final int index;
private long token;
private StepToken token;

private static byte[] buffer = new byte[0];

Expand All @@ -23,81 +23,92 @@ public int tokenLength() {

@Override
public Object getValue() {
long offset = StepTokenizer.tokenPosition(token);
int length = StepTokenizer.tokenLength(token);
switch (StepTokenizer.tokenType(token)) {
case StepTokenizer.TOKEN_INTEGER:
{
if (buffer.length < length) {
buffer = new byte[length];
long offset = token.getPosition();
int length = token.getLength();
switch (token.getType()) {
case StepToken.TOKEN_INTEGER: {
if (buffer.length < length) {
buffer = new byte[length];
}
dataBuffer.bytesAt(buffer, offset, length);
String str = new String(buffer, 0, length);
try {
return Long.valueOf(str);
} catch (NumberFormatException e) {
}
throw new RuntimeException("Invalid integer " + new String(buffer));
}
dataBuffer.bytesAt(buffer, offset, length);
String str = new String(buffer, 0, length);
try {
return Long.valueOf(str);
} catch (NumberFormatException e) {
case StepToken.TOKEN_REAL: {
if (buffer.length < length) {
buffer = new byte[length];
}
dataBuffer.bytesAt(buffer, offset, length);
String str = new String(buffer, 0, length);
try {
return Double.valueOf(str);
} catch (NumberFormatException e) {
}
throw new RuntimeException("Invalid real " + new String(buffer));
}
throw new RuntimeException("Invalid integer " + new String(buffer));
}
case StepTokenizer.TOKEN_REAL:
{
if (buffer.length < length) {
buffer = new byte[length];
}
dataBuffer.bytesAt(buffer, offset, length);
String str = new String(buffer, 0, length);
try {
return Double.valueOf(str);
} catch (NumberFormatException e) {
}
throw new RuntimeException("Invalid real " + new String(buffer));
}
case StepTokenizer.TOKEN_INSTANCE_NAME:
{
if (buffer.length < length) {
buffer = new byte[length];
}
dataBuffer.bytesAt(buffer, offset, length);
String str = new String(buffer, 0, length);
try {
return Long.valueOf(str);
} catch (NumberFormatException e) {
case StepToken.TOKEN_INSTANCE_NAME: {
offset++;
length--;
if (buffer.length < length) {
buffer = new byte[length];
}
dataBuffer.bytesAt(buffer, offset, length);
String str = new String(buffer, 0, length);
try {
return Long.valueOf(str);
} catch (NumberFormatException e) {
}
throw new RuntimeException("Invalid instance id " + new String(buffer));
}
throw new RuntimeException("Invalid instance id " + new String(buffer));
}
case StepTokenizer.TOKEN_ENUM:
{
if (buffer.length < length) {
buffer = new byte[length];
case StepToken.TOKEN_ENUM: {
offset++;
length -= 2;
if (buffer.length < length) {
buffer = new byte[length];
}
dataBuffer.bytesAt(buffer, offset, length);
return new String(buffer, 0, length);
}
dataBuffer.bytesAt(buffer, offset, length);
return new String(buffer, 0, length);
}
case StepTokenizer.TOKEN_STRING:
{
if (buffer.length < length) {
buffer = new byte[length];
case StepToken.TOKEN_BINARY: {
offset++;
length -= 2;
if (buffer.length < length) {
buffer = new byte[length];
}
dataBuffer.bytesAt(buffer, offset, length);
return new String(buffer, 0, length);
}
dataBuffer.bytesAt(buffer, offset, length);
return StepStringDecoder.decode(buffer, 0, length);
}
case StepTokenizer.TOKEN_IDENTIFIER:
if (StepTokenizer.tokenType(tokenBuffer.tokenAt(index + 1)) == StepTokenizer.TOKEN_LPAREN) {
return new StepEntityInstanceImpl(dataBuffer, tokenBuffer, index);
} else {
throw new RuntimeException("Missing attribute list for inline instance");
case StepToken.TOKEN_STRING: {
offset++;
length -= 2;
if (buffer.length < length) {
buffer = new byte[length];
}
dataBuffer.bytesAt(buffer, offset, length);
return StepStringDecoder.decode(buffer, 0, length);
}
case StepTokenizer.TOKEN_UNSET:
case StepTokenizer.TOKEN_REDECLARED:
return null;
default:
throw new RuntimeException("Unknown token");
case StepToken.TOKEN_IDENTIFIER:
if (tokenBuffer.tokenAt(index + 1).getType() == StepToken.TOKEN_LPAREN) {
return new StepEntityInstanceImpl(dataBuffer, tokenBuffer, index);
} else {
throw new RuntimeException("Missing attribute list for inline instance");
}
case StepToken.TOKEN_UNSET:
case StepToken.TOKEN_REDECLARED:
return null;
default:
throw new RuntimeException("Unknown token");
}
}

@Override
public String getTokenValue() {
long offset = StepTokenizer.tokenPosition(token);
int length = StepTokenizer.tokenLength(token);
long offset = token.getPosition();
int length = token.getLength();
if (buffer.length < length) {
buffer = new byte[length];
}
Expand All @@ -107,17 +118,17 @@ public String getTokenValue() {

@Override
public boolean isUnset() {
return StepTokenizer.tokenType(token) == StepTokenizer.TOKEN_UNSET;
return token.getType() == StepToken.TOKEN_UNSET;
}

public void markAsUnset() {
token = StepTokenizer.token(StepTokenizer.tokenType(StepTokenizer.TOKEN_UNSET), StepTokenizer.tokenPosition(token), StepTokenizer.tokenLength(token));
token = new StepToken(StepToken.TOKEN_UNSET, token.getPosition(), token.getLength());
tokenBuffer.set(index, token);
}

@Override
public boolean isRedeclared() {
return StepTokenizer.tokenType(token) == StepTokenizer.TOKEN_REDECLARED;
return token.getType() == StepToken.TOKEN_REDECLARED;
}

@Override
Expand All @@ -127,18 +138,37 @@ public boolean isList() {

@Override
public boolean isInstanceName() {
return StepTokenizer.tokenType(token) == StepTokenizer.TOKEN_INSTANCE_NAME;
return token.getType() == StepToken.TOKEN_INSTANCE_NAME;
}

@Override
public boolean isEnum() {
return StepTokenizer.tokenType(token) == StepTokenizer.TOKEN_ENUM;
return token.getType() == StepToken.TOKEN_ENUM;
}

@Override
public boolean isInline() {
return StepTokenizer.tokenType(token) == StepTokenizer.TOKEN_IDENTIFIER &&
StepTokenizer.tokenType(tokenBuffer.tokenAt(index + 1)) == StepTokenizer.TOKEN_LPAREN;
return token.getType() == StepToken.TOKEN_IDENTIFIER &&
tokenBuffer.tokenAt(index + 1).getType() == StepToken.TOKEN_LPAREN;
}

@Override
public boolean isString() {
return token.getType() == StepToken.TOKEN_STRING;
}

@Override
public boolean isInteger() {
return token.getType() == StepToken.TOKEN_INTEGER;
}

@Override
public boolean isReal() {
return token.getType() == StepToken.TOKEN_REAL;
}

@Override
public boolean isBinary() {
return token.getType() == StepToken.TOKEN_BINARY;
}
}
60 changes: 30 additions & 30 deletions Step/src/org/bimserver/step/StepAttributeIterator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,40 +17,41 @@ public StepAttributeIterator(ByteBuffer dataBuffer,

@Override
public boolean hasNext() {
int tokenType = StepTokenizer.tokenType(tokenBuffer.tokenAt(index));
return tokenType != StepTokenizer.TOKEN_RPAREN;
int tokenType = tokenBuffer.tokenAt(index).getType();
return tokenType != StepToken.TOKEN_RPAREN;
}

@Override
public StepAttribute next() {
long token = tokenBuffer.tokenAt(index);
StepToken token = tokenBuffer.tokenAt(index);
StepAttribute attribute = null;
switch (StepTokenizer.tokenType(token)) {
case StepTokenizer.TOKEN_INTEGER:
case StepTokenizer.TOKEN_REAL:
case StepTokenizer.TOKEN_STRING:
case StepTokenizer.TOKEN_UNSET:
case StepTokenizer.TOKEN_REDECLARED:
case StepTokenizer.TOKEN_INSTANCE_NAME:
case StepTokenizer.TOKEN_ENUM:
attribute = new StepAttributeImpl(dataBuffer, tokenBuffer, index);
break;
case StepTokenizer.TOKEN_IDENTIFIER:
attribute = new StepAttributeImpl(dataBuffer, tokenBuffer, index);
if (StepTokenizer.tokenType(tokenBuffer.tokenAt(index + 1)) == StepTokenizer.TOKEN_LPAREN) {
int listLength = new StepAttributeListImpl(dataBuffer,
tokenBuffer, index + 1).tokenLength();
index += listLength;
}
break;
case StepTokenizer.TOKEN_LPAREN:
StepAttributeListImpl list = new StepAttributeListImpl(
dataBuffer, tokenBuffer, index);
index += list.tokenLength() - 1;
attribute = list;
break;
default:
throw new RuntimeException();
switch (token.getType()) {
case StepToken.TOKEN_INTEGER:
case StepToken.TOKEN_REAL:
case StepToken.TOKEN_STRING:
case StepToken.TOKEN_UNSET:
case StepToken.TOKEN_REDECLARED:
case StepToken.TOKEN_INSTANCE_NAME:
case StepToken.TOKEN_ENUM:
case StepToken.TOKEN_BINARY:
attribute = new StepAttributeImpl(dataBuffer, tokenBuffer, index);
break;
case StepToken.TOKEN_IDENTIFIER:
attribute = new StepAttributeImpl(dataBuffer, tokenBuffer, index);
if (tokenBuffer.tokenAt(index + 1).getType() == StepToken.TOKEN_LPAREN) {
int listLength = new StepAttributeListImpl(dataBuffer,
tokenBuffer, index + 1).tokenLength();
index += listLength;
}
break;
case StepToken.TOKEN_LPAREN:
StepAttributeListImpl list = new StepAttributeListImpl(
dataBuffer, tokenBuffer, index);
index += list.tokenLength() - 1;
attribute = list;
break;
default:
throw new RuntimeException();
}
index++;
return attribute;
Expand All @@ -60,5 +61,4 @@ public StepAttribute next() {
public void remove() {
}


}
Loading

0 comments on commit b0706e9

Please sign in to comment.