diff --git a/.gitignore b/.gitignore index 84e7a6bca..ffbdb448d 100644 --- a/.gitignore +++ b/.gitignore @@ -29,8 +29,14 @@ target/ .idea/ .eclipse/ *.iml +.vscode/ +.settings +.project +client/.classpath spark-importer.ipr spark-importer.iws .DS_Store + +examples/ diff --git a/client/src/main/java/com/vesoft/nebula/encoder/NebulaCodecImpl.java b/client/src/main/java/com/vesoft/nebula/encoder/NebulaCodecImpl.java index 605725eef..8e9e0345c 100644 --- a/client/src/main/java/com/vesoft/nebula/encoder/NebulaCodecImpl.java +++ b/client/src/main/java/com/vesoft/nebula/encoder/NebulaCodecImpl.java @@ -8,6 +8,7 @@ import com.vesoft.nebula.meta.ColumnDef; import com.vesoft.nebula.meta.ColumnTypeDef; import com.vesoft.nebula.meta.EdgeItem; +import com.vesoft.nebula.meta.GeoShape; import com.vesoft.nebula.meta.Schema; import com.vesoft.nebula.meta.TagItem; import java.nio.ByteBuffer; @@ -201,11 +202,13 @@ private SchemaProviderImpl genSchemaProvider(long ver, Schema schema) { boolean nullable = col.isSetNullable() && col.isNullable(); boolean hasDefault = col.isSetDefault_value(); int len = type.isSetType_length() ? type.getType_length() : 0; + GeoShape geoShape = type.isSetGeo_shape() ? type.getGeo_shape() : GeoShape.ANY; schemaProvider.addField(new String(col.getName()), type.type.getValue(), len, nullable, - hasDefault ? col.getDefault_value() : null); + hasDefault ? col.getDefault_value() : null, + geoShape.getValue()); } return schemaProvider; } diff --git a/client/src/main/java/com/vesoft/nebula/encoder/RowWriter.java b/client/src/main/java/com/vesoft/nebula/encoder/RowWriter.java index 492a1986d..774e0e1a5 100644 --- a/client/src/main/java/com/vesoft/nebula/encoder/RowWriter.java +++ b/client/src/main/java/com/vesoft/nebula/encoder/RowWriter.java @@ -7,6 +7,7 @@ import com.vesoft.nebula.Date; import com.vesoft.nebula.DateTime; +import com.vesoft.nebula.Geography; import com.vesoft.nebula.Time; public interface RowWriter { @@ -33,5 +34,7 @@ public interface RowWriter { void write(int index, DateTime v); + void write(int index, Geography v); + byte[] encodeStr(); } diff --git a/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java b/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java index 9a0fab6b7..65d8e1c6a 100644 --- a/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java +++ b/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java @@ -5,8 +5,13 @@ package com.vesoft.nebula.encoder; +import com.vesoft.nebula.Coordinate; import com.vesoft.nebula.Date; import com.vesoft.nebula.DateTime; +import com.vesoft.nebula.Geography; +import com.vesoft.nebula.LineString; +import com.vesoft.nebula.Point; +import com.vesoft.nebula.Polygon; import com.vesoft.nebula.Time; import com.vesoft.nebula.Value; import com.vesoft.nebula.meta.PropertyType; @@ -15,6 +20,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.apache.commons.codec.binary.Hex; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.io.ByteOrderValues; +import org.locationtech.jts.io.WKBWriter; public class RowWriterImpl implements RowWriter { private final SchemaProviderImpl schema; @@ -509,6 +518,7 @@ public void write(int index, byte[] v) { } int offset = headerLen + numNullBytes + field.offset(); switch (typeEnum) { + case GEOGRAPHY: case STRING: { strList.add(v); outOfSpaceStr = true; @@ -625,6 +635,31 @@ public void write(int index, DateTime v) { isSet.set(index, true); } + @Override + public void write(int index, Geography v) { + SchemaProvider.Field field = schema.field(index); + PropertyType typeEnum = PropertyType.findByValue(field.type()); + if (typeEnum == null) { + throw new RuntimeException("Incorrect field type " + field.type()); + } + if (typeEnum == PropertyType.GEOGRAPHY) { + if (field.geoShape() != 0 && field.geoShape() != v.getSetField()) { + throw new RuntimeException("Incorrect geo shape, expect " + + field.geoShape() + ", got " + + v.getSetField()); + } + } else { + throw new RuntimeException("Value: " + v + "'s type is unexpected"); + } + org.locationtech.jts.geom.Geometry jtsGeom = convertGeographyToJTSGeometry(v); + byte[] wkb = new org.locationtech.jts.io + .WKBWriter(2, this.byteOrder == ByteOrder.BIG_ENDIAN + ? ByteOrderValues.BIG_ENDIAN + : ByteOrderValues.LITTLE_ENDIAN) + .write(jtsGeom); + write(index, wkb); + } + @Override public byte[] encodeStr() { return buf.array(); @@ -666,6 +701,8 @@ public void setValue(int index, Object value) { write(index, (Date)value); } else if (value instanceof DateTime) { write(index, (DateTime)value); + } else if (value instanceof Geography) { + write(index, (Geography)value); } else { throw new RuntimeException("Unsupported value object `" + value.getClass() + "\""); } @@ -708,6 +745,9 @@ public void setValue(int index, Value value) { case Value.DTVAL: write(index, value.getDtVal()); break; + case Value.GGVAL: + write(index, value.getGgVal()); + break; default: throw new RuntimeException( "Unknown value: " + value.getFieldValue().getClass() @@ -790,6 +830,9 @@ public void checkUnsetFields() { // case Value.DTVAL: // write(i, defVal.getDtVal()); // break; + // case Value.GGVAL: + // write(i, defVal.getGgVal()); + // break; // default: // throw new RuntimeException("Unsupported default value type"); // } @@ -821,7 +864,8 @@ public ByteBuffer processOutOfSpace() { if (typeEnum == null) { throw new RuntimeException("Incorrect field type " + field.type()); } - if (typeEnum != PropertyType.STRING) { + if (typeEnum != PropertyType.STRING + && typeEnum != PropertyType.GEOGRAPHY) { continue; } int offset = headerLen + numNullBytes + field.offset(); @@ -867,4 +911,66 @@ private long getTimestamp() { long nanoTime = System.nanoTime(); return curTime + (nanoTime - nanoTime / 1000000 * 1000000) / 1000; } + + public org.locationtech.jts.geom.Geometry + convertGeographyToJTSGeometry(Geography geog) { + GeometryFactory geomFactory = new GeometryFactory(); + switch (geog.getSetField()) { + case Geography.PTVAL: { + Point point = geog.getPtVal(); + Coordinate coord = point.getCoord(); + return geomFactory.createPoint( + new org.locationtech.jts.geom.Coordinate(coord.x, coord.y)); + } + case Geography.LSVAL: { + LineString line = geog.getLsVal(); + List coordList = line.getCoordList(); + + List jtsCoordList = + new ArrayList<>(); + for (int i = 0; i < coordList.size(); ++i) { + jtsCoordList.add(new org.locationtech.jts.geom.Coordinate( + coordList.get(i).x, coordList.get(i).y)); + } + org.locationtech.jts.geom.Coordinate[] jtsCoordArray = + new org.locationtech.jts.geom.Coordinate[jtsCoordList.size()]; + return geomFactory.createLineString( + jtsCoordList.toArray(jtsCoordArray)); + } + case Geography.PGVAL: { + Polygon polygon = geog.getPgVal(); + List> coordListList = polygon.getCoordListList(); + if (coordListList.isEmpty()) { + throw new RuntimeException("Polygon must at least contain one loop"); + } + + List rings = new ArrayList<>(); + for (int i = 0; i < coordListList.size(); ++i) { + List coordList = coordListList.get(i); + List jtsCoordList = + new ArrayList<>(); + for (int j = 0; j < coordList.size(); ++j) { + jtsCoordList.add(new org.locationtech.jts.geom.Coordinate( + coordList.get(j).x, coordList.get(j).y)); + } + org.locationtech.jts.geom.Coordinate[] jtsCoordArray = + new org.locationtech.jts.geom.Coordinate[jtsCoordList.size()]; + rings.add(geomFactory.createLinearRing( + jtsCoordList.toArray(jtsCoordArray))); + } + org.locationtech.jts.geom.LinearRing shell = rings.get(0); + if (rings.size() == 1) { + return geomFactory.createPolygon(shell); + } else { + rings.remove(0); + org.locationtech.jts.geom.LinearRing[] holesArrary = + new org.locationtech.jts.geom.LinearRing[rings.size() - 1]; + return geomFactory.createPolygon(shell, rings.toArray(holesArrary)); + } + } + default: + throw new RuntimeException("Unknown geography: " + + geog.getFieldValue().getClass()); + } + } } diff --git a/client/src/main/java/com/vesoft/nebula/encoder/SchemaProvider.java b/client/src/main/java/com/vesoft/nebula/encoder/SchemaProvider.java index 9f6940134..89fc2e591 100644 --- a/client/src/main/java/com/vesoft/nebula/encoder/SchemaProvider.java +++ b/client/src/main/java/com/vesoft/nebula/encoder/SchemaProvider.java @@ -22,6 +22,8 @@ public interface Field { public int offset(); public int nullFlagPos(); + + public int geoShape(); } public long getVersion(); diff --git a/client/src/main/java/com/vesoft/nebula/encoder/SchemaProviderImpl.java b/client/src/main/java/com/vesoft/nebula/encoder/SchemaProviderImpl.java index 6aebb6298..114b868f5 100644 --- a/client/src/main/java/com/vesoft/nebula/encoder/SchemaProviderImpl.java +++ b/client/src/main/java/com/vesoft/nebula/encoder/SchemaProviderImpl.java @@ -26,6 +26,7 @@ static class SchemaField implements Field { private final int size; private final int offset; private final int nullFlagPos; + private final int geoShape; public SchemaField(String name, int type, @@ -34,7 +35,8 @@ public SchemaField(String name, byte[] defaultValue, int size, int offset, - int nullFlagPos) { + int nullFlagPos, + int geoShape) { this.name = name; this.type = type; this.nullable = nullable; @@ -43,6 +45,7 @@ public SchemaField(String name, this.size = size; this.offset = offset; this.nullFlagPos = nullFlagPos; + this.geoShape = geoShape; } @Override @@ -84,6 +87,11 @@ public int offset() { public int nullFlagPos() { return nullFlagPos; } + + @Override + public int geoShape() { + return geoShape; + } } public SchemaProviderImpl(long ver) { @@ -169,7 +177,8 @@ public void addField(String name, int type, int fixedStrLen, boolean nullable, - byte[] defaultValue) { + byte[] defaultValue, + int geoShape) { int size = fieldSize(type, fixedStrLen); int offset = 0; @@ -190,7 +199,8 @@ public void addField(String name, defaultValue, size, offset, - nullFlagPos)); + nullFlagPos, + geoShape)); fieldNameIndex.put(name, fields.size() - 1); } @@ -241,6 +251,8 @@ public int fieldSize(int type, int fixedStrLimit) { + Byte.BYTES // minute + Byte.BYTES // sec + Integer.BYTES; // microsec + case GEOGRAPHY: + return 8; // wkb offset + wkb length default: throw new RuntimeException("Incorrect field type " + type); } diff --git a/client/src/test/java/com/vesoft/nebula/encoder/MetaCacheImplTest.java b/client/src/test/java/com/vesoft/nebula/encoder/MetaCacheImplTest.java index 483c73560..aec661f4f 100644 --- a/client/src/test/java/com/vesoft/nebula/encoder/MetaCacheImplTest.java +++ b/client/src/test/java/com/vesoft/nebula/encoder/MetaCacheImplTest.java @@ -11,6 +11,7 @@ import com.vesoft.nebula.meta.ColumnDef; import com.vesoft.nebula.meta.ColumnTypeDef; import com.vesoft.nebula.meta.EdgeItem; +import com.vesoft.nebula.meta.GeoShape; import com.vesoft.nebula.meta.PropertyType; import com.vesoft.nebula.meta.Schema; import com.vesoft.nebula.meta.SpaceDesc; @@ -82,6 +83,19 @@ private Schema genNoDefaultVal() { new ColumnTypeDef(PropertyType.INT32)); columnDef.setNullable(true); columns.add(columnDef); + columnDef = new ColumnDef(("Col16").getBytes(), + new ColumnTypeDef(PropertyType.GEOGRAPHY, (short)0, GeoShape.POINT)); + columns.add(columnDef); + columnDef = new ColumnDef(("Col17").getBytes(), + new ColumnTypeDef(PropertyType.GEOGRAPHY, (short)0, GeoShape.LINESTRING)); + columns.add(columnDef); + columnDef = new ColumnDef(("Col18").getBytes(), + new ColumnTypeDef(PropertyType.GEOGRAPHY, (short)0, GeoShape.POLYGON)); + columns.add(columnDef); + columnDef = new ColumnDef(("Col19").getBytes(), + new ColumnTypeDef(PropertyType.GEOGRAPHY, (short)0, GeoShape.ANY)); + columnDef.setNullable(true); + columns.add(columnDef); return new Schema(columns, null); } diff --git a/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java b/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java index 31ed6fb08..0ed452b78 100644 --- a/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java +++ b/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java @@ -5,10 +5,15 @@ package test.java.com.vesoft.nebula.encoder; +import com.vesoft.nebula.Coordinate; import com.vesoft.nebula.Date; import com.vesoft.nebula.DateTime; +import com.vesoft.nebula.Geography; import com.vesoft.nebula.HostAddr; +import com.vesoft.nebula.LineString; import com.vesoft.nebula.NullType; +import com.vesoft.nebula.Point; +import com.vesoft.nebula.Polygon; import com.vesoft.nebula.Time; import com.vesoft.nebula.Value; import com.vesoft.nebula.encoder.MetaCacheImplTest; @@ -31,14 +36,23 @@ public class TestEncoder { private final MetaCacheImplTest cacheImplTest = new MetaCacheImplTest(); private final NebulaCodecImpl codec = new NebulaCodecImpl(); - final String allTypeValueExpectResult = "090cc001081000200000004000000000000000db0f494069" - + "57148b0abf05405d0000000c0000004e6562756c61204772617068bb334e5e000000" - + "00e40702140a1e2d00000000e40702140a1e2d00000000000000000000000000000000" - + "48656c6c6f20776f726c6421"; + final String allTypeValueExpectResult = "090ce001081000200000004000000000000000" + + "db0f49406957148b0abf05407d0000000c0000004e6562756c61204772617068bb334e5e" + + "00000000e40702140a1e2d00000000e40702140a1e2d0000000000000000000000000000" + + "000089000000150000009e00000039000000d70000009100000000000000000000004865" + + "6c6c6f20776f726c6421010100000000000000006066409a999999997956400102000000" + + "030000000000000000000000000000000000f03f000000000000f03f0000000000000040" + + "00000000000008400000000000001c4001030000000200000004000000cdcccccccc2c5b" + + "c0000000000080414000000000000059c00000000000404740cdccccccccac56c0333333" + + "3333734140cdcccccccc2c5bc000000000008041400400000066666666660659c0333333" + + "3333b344409a99999999b959c0cdcccccccccc424033333333333358c00000000000c042" + + "4066666666660659c03333333333b34440"; private List getCols() { return Arrays.asList("Col01","Col02", "Col03", "Col04", "Col05", "Col06", - "Col07","Col08", "Col09", "Col10", "Col11", "Col12", "Col13", "Col14"); + "Col07","Col08", "Col09", "Col10", "Col11", "Col12", "Col13", "Col14", + // Purposely skip the col15 + "Col16", "Col17", "Col18", "Col19"); } private List getValues() { @@ -61,8 +75,39 @@ private List getValues() { dateValue.setDVal(new Date((short)2020, (byte)2, (byte)20)); final Value nullVal = new Value(); nullVal.setNVal(NullType.__NULL__); + // geograph point + // POINT(179.0 89.9) + final Value geogPointVal = new Value(); + geogPointVal.setGgVal(Geography.ptVal(new Point(new Coordinate(179.0, 89.9)))); + // geography linestring + // LINESTRING(0 1, 1 2, 3 7) + final Value geogLineStringVal = new Value(); + List line = new ArrayList<>(); + line.add(new Coordinate(0, 1)); + line.add(new Coordinate(1, 2)); + line.add(new Coordinate(3, 7)); + geogLineStringVal.setGgVal(Geography.lsVal(new LineString(line))); + // geography polygon + // POLYGON((-108.7 35.0, -100.0 46.5, -90.7 34.9, -108.7 35.0), + // (-100.1 41.4, -102.9 37.6, -96.8 37.5, -100.1 41.4)) + final Value geogPolygonVal = new Value(); + List shell = new ArrayList<>(); + shell.add(new Coordinate(-108.7, 35.0)); + shell.add(new Coordinate(-100.0, 46.5)); + shell.add(new Coordinate(-90.7, 34.9)); + shell.add(new Coordinate(-108.7, 35.0)); + List hole = new ArrayList<>(); + hole.add(new Coordinate(-100.1, 41.4)); + hole.add(new Coordinate(-102.9, 37.6)); + hole.add(new Coordinate(-96.8, 37.5)); + hole.add(new Coordinate(-100.1, 41.4)); + List> rings = new ArrayList>(); + rings.add(shell); + rings.add(hole); + geogPolygonVal.setGgVal(Geography.pgVal(new Polygon(rings))); return Arrays.asList(true, 8, 16, 32, intVal, pi, e, strVal, fixVal, - timestampVal, dateValue, timeVal, datetimeValue, nullVal); + timestampVal, dateValue, timeVal, datetimeValue, nullVal, + geogPointVal, geogLineStringVal, geogPolygonVal, nullVal); } public int getSpaceVidLen(String spaceName) throws RuntimeException { diff --git a/pom.xml b/pom.xml index 5bccb3416..5076510d8 100644 --- a/pom.xml +++ b/pom.xml @@ -9,6 +9,7 @@ UTF-8 + 1.16.1 @@ -157,4 +158,13 @@ + + + + org.locationtech.jts + jts-core + 1.16.1 + + +