From 00bde55e572ee99bc9d25aff57250dd36395ddd5 Mon Sep 17 00:00:00 2001 From: jievince <38901892+jievince@users.noreply.github.com> Date: Wed, 3 Nov 2021 17:02:02 +0800 Subject: [PATCH 1/7] support geo encoder --- .gitignore | 6 ++ .../nebula/encoder/NebulaCodecImpl.java | 5 +- .../com/vesoft/nebula/encoder/RowWriter.java | 3 + .../vesoft/nebula/encoder/RowWriterImpl.java | 99 +++++++++++++++++++ .../vesoft/nebula/encoder/SchemaProvider.java | 2 + .../nebula/encoder/SchemaProviderImpl.java | 18 +++- .../nebula/encoder/MetaCacheImplTest.java | 9 ++ .../vesoft/nebula/encoder/TestEncoder.java | 54 +++++++++- pom.xml | 10 ++ 9 files changed, 199 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 84e7a6bca..ffbdb448d 100644 --- a/.gitignore +++ b/.gitignore @@ -29,8 +29,14 @@ target/ .idea/ .eclipse/ *.iml +.vscode/ +.settings +.project +client/.classpath spark-importer.ipr spark-importer.iws .DS_Store + +examples/ diff --git a/client/src/main/java/com/vesoft/nebula/encoder/NebulaCodecImpl.java b/client/src/main/java/com/vesoft/nebula/encoder/NebulaCodecImpl.java index 605725eef..8e9e0345c 100644 --- a/client/src/main/java/com/vesoft/nebula/encoder/NebulaCodecImpl.java +++ b/client/src/main/java/com/vesoft/nebula/encoder/NebulaCodecImpl.java @@ -8,6 +8,7 @@ import com.vesoft.nebula.meta.ColumnDef; import com.vesoft.nebula.meta.ColumnTypeDef; import com.vesoft.nebula.meta.EdgeItem; +import com.vesoft.nebula.meta.GeoShape; import com.vesoft.nebula.meta.Schema; import com.vesoft.nebula.meta.TagItem; import java.nio.ByteBuffer; @@ -201,11 +202,13 @@ private SchemaProviderImpl genSchemaProvider(long ver, Schema schema) { boolean nullable = col.isSetNullable() && col.isNullable(); boolean hasDefault = col.isSetDefault_value(); int len = type.isSetType_length() ? type.getType_length() : 0; + GeoShape geoShape = type.isSetGeo_shape() ? type.getGeo_shape() : GeoShape.ANY; schemaProvider.addField(new String(col.getName()), type.type.getValue(), len, nullable, - hasDefault ? col.getDefault_value() : null); + hasDefault ? col.getDefault_value() : null, + geoShape.getValue()); } return schemaProvider; } diff --git a/client/src/main/java/com/vesoft/nebula/encoder/RowWriter.java b/client/src/main/java/com/vesoft/nebula/encoder/RowWriter.java index 492a1986d..774e0e1a5 100644 --- a/client/src/main/java/com/vesoft/nebula/encoder/RowWriter.java +++ b/client/src/main/java/com/vesoft/nebula/encoder/RowWriter.java @@ -7,6 +7,7 @@ import com.vesoft.nebula.Date; import com.vesoft.nebula.DateTime; +import com.vesoft.nebula.Geography; import com.vesoft.nebula.Time; public interface RowWriter { @@ -33,5 +34,7 @@ public interface RowWriter { void write(int index, DateTime v); + void write(int index, Geography v); + byte[] encodeStr(); } diff --git a/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java b/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java index 9a0fab6b7..9f42d199b 100644 --- a/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java +++ b/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java @@ -5,8 +5,13 @@ package com.vesoft.nebula.encoder; +import com.vesoft.nebula.Coordinate; import com.vesoft.nebula.Date; import com.vesoft.nebula.DateTime; +import com.vesoft.nebula.Geography; +import com.vesoft.nebula.LineString; +import com.vesoft.nebula.Point; +import com.vesoft.nebula.Polygon; import com.vesoft.nebula.Time; import com.vesoft.nebula.Value; import com.vesoft.nebula.meta.PropertyType; @@ -15,6 +20,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.io.WKBWriter; public class RowWriterImpl implements RowWriter { private final SchemaProviderImpl schema; @@ -509,6 +516,7 @@ public void write(int index, byte[] v) { } int offset = headerLen + numNullBytes + field.offset(); switch (typeEnum) { + case GEOGRAPHY: case STRING: { strList.add(v); outOfSpaceStr = true; @@ -625,6 +633,27 @@ public void write(int index, DateTime v) { isSet.set(index, true); } + @Override + public void write(int index, Geography v) { + SchemaProvider.Field field = schema.field(index); + PropertyType typeEnum = PropertyType.findByValue(field.type()); + if (typeEnum == null) { + throw new RuntimeException("Incorrect field type " + field.type()); + } + if (typeEnum == PropertyType.GEOGRAPHY) { + if (field.geoShape() != 0 && field.geoShape() != v.getSetField()) { + throw new RuntimeException("Incorrect geo shape, expect " + + field.geoShape() + ", got " + + v.getSetField()); + } + } else { + throw new RuntimeException("Value: " + v + "'s type is unexpected"); + } + org.locationtech.jts.geom.Geometry jtsGeom = convertGeographyToJTSGeometry(v); + byte[] wkb = new org.locationtech.jts.io.WKBWriter().write(jtsGeom); + write(index, wkb); + } + @Override public byte[] encodeStr() { return buf.array(); @@ -666,6 +695,8 @@ public void setValue(int index, Object value) { write(index, (Date)value); } else if (value instanceof DateTime) { write(index, (DateTime)value); + } else if (value instanceof Geography) { + write(index, (Geography)value); } else { throw new RuntimeException("Unsupported value object `" + value.getClass() + "\""); } @@ -708,6 +739,9 @@ public void setValue(int index, Value value) { case Value.DTVAL: write(index, value.getDtVal()); break; + case Value.GGVAL: + write(index, value.getGgVal()); + break; default: throw new RuntimeException( "Unknown value: " + value.getFieldValue().getClass() @@ -790,6 +824,9 @@ public void checkUnsetFields() { // case Value.DTVAL: // write(i, defVal.getDtVal()); // break; + // case Value.GGVAL: + // write(i, defVal.getGgVal()); + // break; // default: // throw new RuntimeException("Unsupported default value type"); // } @@ -867,4 +904,66 @@ private long getTimestamp() { long nanoTime = System.nanoTime(); return curTime + (nanoTime - nanoTime / 1000000 * 1000000) / 1000; } + + public org.locationtech.jts.geom.Geometry + convertGeographyToJTSGeometry(Geography geog) { + GeometryFactory geomFactory = new GeometryFactory(); + switch (geog.getSetField()) { + case Geography.PTVAL: { + Point point = geog.getPtVal(); + Coordinate coord = point.getCoord(); + return geomFactory.createPoint( + new org.locationtech.jts.geom.Coordinate(coord.x, coord.y)); + } + case Geography.LSVAL: { + LineString line = geog.getLsVal(); + List coordList = line.getCoordList(); + + List jtsCoordList = + new ArrayList<>(); + for (int i = 0; i < coordList.size(); ++i) { + jtsCoordList.add(new org.locationtech.jts.geom.Coordinate( + coordList.get(i).x, coordList.get(i).y)); + } + org.locationtech.jts.geom.Coordinate[] jtsCoordArray = + new org.locationtech.jts.geom.Coordinate[jtsCoordList.size()]; + return geomFactory.createLineString( + jtsCoordList.toArray(jtsCoordArray)); + } + case Geography.PGVAL: { + Polygon polygon = geog.getPgVal(); + List> coordListList = polygon.getCoordListList(); + if (coordListList.isEmpty()) { + throw new RuntimeException("Polygon must at least contain one loop"); + } + + List rings = new ArrayList<>(); + for (int i = 0; i < coordListList.size(); ++i) { + List coordList = coordListList.get(i); + List jtsCoordList = + new ArrayList<>(); + for (int j = 0; j < coordList.size(); ++j) { + jtsCoordList.add(new org.locationtech.jts.geom.Coordinate( + coordList.get(j).x, coordList.get(j).y)); + } + org.locationtech.jts.geom.Coordinate[] jtsCoordArray = + new org.locationtech.jts.geom.Coordinate[jtsCoordList.size()]; + rings.add(geomFactory.createLinearRing( + jtsCoordList.toArray(jtsCoordArray))); + } + org.locationtech.jts.geom.LinearRing shell = rings.get(0); + if (rings.size() == 1) { + return geomFactory.createPolygon(shell); + } else { + rings.remove(0); + org.locationtech.jts.geom.LinearRing[] holesArrary = + new org.locationtech.jts.geom.LinearRing[rings.size() - 1]; + return geomFactory.createPolygon(shell, rings.toArray(holesArrary)); + } + } + default: + throw new RuntimeException("Unknown geography: " + + geog.getFieldValue().getClass()); + } + } } diff --git a/client/src/main/java/com/vesoft/nebula/encoder/SchemaProvider.java b/client/src/main/java/com/vesoft/nebula/encoder/SchemaProvider.java index 9f6940134..89fc2e591 100644 --- a/client/src/main/java/com/vesoft/nebula/encoder/SchemaProvider.java +++ b/client/src/main/java/com/vesoft/nebula/encoder/SchemaProvider.java @@ -22,6 +22,8 @@ public interface Field { public int offset(); public int nullFlagPos(); + + public int geoShape(); } public long getVersion(); diff --git a/client/src/main/java/com/vesoft/nebula/encoder/SchemaProviderImpl.java b/client/src/main/java/com/vesoft/nebula/encoder/SchemaProviderImpl.java index 6aebb6298..114b868f5 100644 --- a/client/src/main/java/com/vesoft/nebula/encoder/SchemaProviderImpl.java +++ b/client/src/main/java/com/vesoft/nebula/encoder/SchemaProviderImpl.java @@ -26,6 +26,7 @@ static class SchemaField implements Field { private final int size; private final int offset; private final int nullFlagPos; + private final int geoShape; public SchemaField(String name, int type, @@ -34,7 +35,8 @@ public SchemaField(String name, byte[] defaultValue, int size, int offset, - int nullFlagPos) { + int nullFlagPos, + int geoShape) { this.name = name; this.type = type; this.nullable = nullable; @@ -43,6 +45,7 @@ public SchemaField(String name, this.size = size; this.offset = offset; this.nullFlagPos = nullFlagPos; + this.geoShape = geoShape; } @Override @@ -84,6 +87,11 @@ public int offset() { public int nullFlagPos() { return nullFlagPos; } + + @Override + public int geoShape() { + return geoShape; + } } public SchemaProviderImpl(long ver) { @@ -169,7 +177,8 @@ public void addField(String name, int type, int fixedStrLen, boolean nullable, - byte[] defaultValue) { + byte[] defaultValue, + int geoShape) { int size = fieldSize(type, fixedStrLen); int offset = 0; @@ -190,7 +199,8 @@ public void addField(String name, defaultValue, size, offset, - nullFlagPos)); + nullFlagPos, + geoShape)); fieldNameIndex.put(name, fields.size() - 1); } @@ -241,6 +251,8 @@ public int fieldSize(int type, int fixedStrLimit) { + Byte.BYTES // minute + Byte.BYTES // sec + Integer.BYTES; // microsec + case GEOGRAPHY: + return 8; // wkb offset + wkb length default: throw new RuntimeException("Incorrect field type " + type); } diff --git a/client/src/test/java/com/vesoft/nebula/encoder/MetaCacheImplTest.java b/client/src/test/java/com/vesoft/nebula/encoder/MetaCacheImplTest.java index 483c73560..3e8a0cb2a 100644 --- a/client/src/test/java/com/vesoft/nebula/encoder/MetaCacheImplTest.java +++ b/client/src/test/java/com/vesoft/nebula/encoder/MetaCacheImplTest.java @@ -82,6 +82,15 @@ private Schema genNoDefaultVal() { new ColumnTypeDef(PropertyType.INT32)); columnDef.setNullable(true); columns.add(columnDef); + columnDef = new ColumnDef(("Col16").getBytes(), + new ColumnTypeDef(PropertyType.GEOGRAPHY)); + columns.add(columnDef); + columnDef = new ColumnDef(("Col17").getBytes(), + new ColumnTypeDef(PropertyType.GEOGRAPHY)); + columns.add(columnDef); + columnDef = new ColumnDef(("Col18").getBytes(), + new ColumnTypeDef(PropertyType.GEOGRAPHY)); + columns.add(columnDef); return new Schema(columns, null); } diff --git a/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java b/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java index 31ed6fb08..c728379a5 100644 --- a/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java +++ b/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java @@ -5,10 +5,15 @@ package test.java.com.vesoft.nebula.encoder; +import com.vesoft.nebula.Coordinate; import com.vesoft.nebula.Date; import com.vesoft.nebula.DateTime; +import com.vesoft.nebula.Geography; import com.vesoft.nebula.HostAddr; +import com.vesoft.nebula.LineString; import com.vesoft.nebula.NullType; +import com.vesoft.nebula.Point; +import com.vesoft.nebula.Polygon; import com.vesoft.nebula.Time; import com.vesoft.nebula.Value; import com.vesoft.nebula.encoder.MetaCacheImplTest; @@ -34,11 +39,19 @@ public class TestEncoder { final String allTypeValueExpectResult = "090cc001081000200000004000000000000000db0f494069" + "57148b0abf05405d0000000c0000004e6562756c61204772617068bb334e5e000000" + "00e40702140a1e2d00000000e40702140a1e2d00000000000000000000000000000000" - + "48656c6c6f20776f726c6421"; + + "48656c6c6f20776f726c6421010100000000000000006066409a999999997956400102" + + "000000030000000000000000000000000000000000f03f000000000000f03f00000000" + + "0000004000000000000008400000000000001c4001030000000200000004000000cdcc" + + "cccccc2c5bc0000000000080414000000000000059c00000000000404740cdcccccccc" + + "ac56c03333333333734140cdcccccccc2c5bc000000000008041400400000066666666" + + "660659c03333333333b344409a99999999b959c0cdcccccccccc424033333333333358" + + "c00000000000c0424066666666660659c03333333333b34440fe148c1853d00500"; private List getCols() { return Arrays.asList("Col01","Col02", "Col03", "Col04", "Col05", "Col06", - "Col07","Col08", "Col09", "Col10", "Col11", "Col12", "Col13", "Col14"); + "Col07","Col08", "Col09", "Col10", "Col11", "Col12", "Col13", "Col14", + // Purposely skip the col15 + "Col16", "Col17", "Col18"); } private List getValues() { @@ -61,8 +74,39 @@ private List getValues() { dateValue.setDVal(new Date((short)2020, (byte)2, (byte)20)); final Value nullVal = new Value(); nullVal.setNVal(NullType.__NULL__); + // geograph point + // POINT(179.0 89.9) + final Value geogPointVal = new Value(); + geogPointVal.setGgVal(Geography.ptVal(new Point(new Coordinate(179.0, 89.9)))); + // geography linestring + // LINESTRING(0 1, 1 2, 3 7) + final Value geogLineStringVal = new Value(); + List line = new ArrayList<>(); + line.add(new Coordinate(0, 1)); + line.add(new Coordinate(1, 2)); + line.add(new Coordinate(3, 7)); + geogLineStringVal.setGgVal(Geography.lsVal(new LineString(line))); + // geography polygon + // POLYGON((-108.7 35.0, -100.0 46.5, -90.7 34.9, -108.7 35.0), + // (-100.1 41.4, -102.9 37.6, -96.8 37.5, -100.1 41.4)) + final Value geogPolygonVal = new Value(); + List shell = new ArrayList<>(); + shell.add(new Coordinate(-108.7, 35.0)); + shell.add(new Coordinate(-100.0, 46.5)); + shell.add(new Coordinate(-90.7, 34.9)); + shell.add(new Coordinate(-108.7, 35.0)); + List hole = new ArrayList<>(); + hole.add(new Coordinate(-100.1, 41.4)); + hole.add(new Coordinate(-102.9, 37.6)); + hole.add(new Coordinate(-96.8, 37.5)); + hole.add(new Coordinate(-100.1, 41.4)); + List> rings = new ArrayList>(); + rings.add(shell); + rings.add(hole); + geogPolygonVal.setGgVal(Geography.pgVal(new Polygon(rings))); return Arrays.asList(true, 8, 16, 32, intVal, pi, e, strVal, fixVal, - timestampVal, dateValue, timeVal, datetimeValue, nullVal); + timestampVal, dateValue, timeVal, datetimeValue, nullVal, + geogPointVal, geogLineStringVal, geogPolygonVal); } public int getSpaceVidLen(String spaceName) throws RuntimeException { @@ -227,6 +271,10 @@ public void testEncodeVertexValue() { // File file = new File("encode_java.txt"); // FileOutputStream fileOutputStream = new FileOutputStream(file); // fileOutputStream.write(encodeStr); + System.out.println("hex vs expect"); + System.out.println(hexStr); + System.out.println("***********"); + System.out.println(allTypeValueExpectResult); Assert.assertArrayEquals(allTypeValueExpectResult.getBytes(), hexStr.substring(0, hexStr.length() - 16).getBytes()); } catch (Exception exception) { diff --git a/pom.xml b/pom.xml index 5bccb3416..5076510d8 100644 --- a/pom.xml +++ b/pom.xml @@ -9,6 +9,7 @@ UTF-8 + 1.16.1 @@ -157,4 +158,13 @@ + + + + org.locationtech.jts + jts-core + 1.16.1 + + + From bee451ef162e8b2fcae898ba435ff2d865e57310 Mon Sep 17 00:00:00 2001 From: jievince <38901892+jievince@users.noreply.github.com> Date: Tue, 9 Nov 2021 14:43:58 +0800 Subject: [PATCH 2/7] debug --- .../vesoft/nebula/encoder/RowWriterImpl.java | 17 +++++++++++++++-- .../com/vesoft/nebula/encoder/TestEncoder.java | 17 +++++++++-------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java b/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java index 9f42d199b..47c008245 100644 --- a/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java +++ b/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java @@ -20,7 +20,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.apache.commons.codec.binary.Hex; import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.io.ByteOrderValues; import org.locationtech.jts.io.WKBWriter; public class RowWriterImpl implements RowWriter { @@ -635,6 +637,7 @@ public void write(int index, DateTime v) { @Override public void write(int index, Geography v) { + System.out.println("write(index, geography), " + index); SchemaProvider.Field field = schema.field(index); PropertyType typeEnum = PropertyType.findByValue(field.type()); if (typeEnum == null) { @@ -650,8 +653,15 @@ public void write(int index, Geography v) { throw new RuntimeException("Value: " + v + "'s type is unexpected"); } org.locationtech.jts.geom.Geometry jtsGeom = convertGeographyToJTSGeometry(v); - byte[] wkb = new org.locationtech.jts.io.WKBWriter().write(jtsGeom); + byte[] wkb = new org.locationtech.jts.io + .WKBWriter(2, ByteOrderValues.LITTLE_ENDIAN) + .write(jtsGeom); write(index, wkb); + String hexedWKb = Hex.encodeHexString(wkb); + System.out.println("write(index, geography), wkb.size()=" + wkb.length + + ", wkb.content=" + wkb); + System.out.println("write(index, geography), hexedWKb.size()=" + hexedWKb.length() + + ", hexedWKb.content=" + hexedWKb); } @Override @@ -696,6 +706,7 @@ public void setValue(int index, Object value) { } else if (value instanceof DateTime) { write(index, (DateTime)value); } else if (value instanceof Geography) { + System.out.println("jie, " + index + ", setValue(index, Object) for geography"); write(index, (Geography)value); } else { throw new RuntimeException("Unsupported value object `" + value.getClass() + "\""); @@ -740,6 +751,7 @@ public void setValue(int index, Value value) { write(index, value.getDtVal()); break; case Value.GGVAL: + System.out.println("jie, " + index + ", setValue(index, value) for geography"); write(index, value.getGgVal()); break; default: @@ -858,7 +870,8 @@ public ByteBuffer processOutOfSpace() { if (typeEnum == null) { throw new RuntimeException("Incorrect field type " + field.type()); } - if (typeEnum != PropertyType.STRING) { + if (typeEnum != PropertyType.STRING + && typeEnum != PropertyType.GEOGRAPHY) { continue; } int offset = headerLen + numNullBytes + field.offset(); diff --git a/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java b/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java index c728379a5..7862a0a02 100644 --- a/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java +++ b/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java @@ -37,15 +37,16 @@ public class TestEncoder { private final NebulaCodecImpl codec = new NebulaCodecImpl(); final String allTypeValueExpectResult = "090cc001081000200000004000000000000000db0f494069" - + "57148b0abf05405d0000000c0000004e6562756c61204772617068bb334e5e000000" + + "57148b0abf0540750000000c0000004e6562756c61204772617068bb334e5e000000" + "00e40702140a1e2d00000000e40702140a1e2d00000000000000000000000000000000" - + "48656c6c6f20776f726c6421010100000000000000006066409a999999997956400102" - + "000000030000000000000000000000000000000000f03f000000000000f03f00000000" - + "0000004000000000000008400000000000001c4001030000000200000004000000cdcc" - + "cccccc2c5bc0000000000080414000000000000059c00000000000404740cdcccccccc" - + "ac56c03333333333734140cdcccccccc2c5bc000000000008041400400000066666666" - + "660659c03333333333b344409a99999999b959c0cdcccccccccc424033333333333358" - + "c00000000000c0424066666666660659c03333333333b34440fe148c1853d00500"; + + "81000000150000009600000039000000cf0000009100000048656c6c6f20776f726c6421" + + "010100000000000000006066409a99999999795640010200000003000000000000000000" + + "0000000000000000f03f000000000000f03f000000000000004000000000000008400000" + + "000000001c4001030000000200000004000000cdcccccccc2c5bc0000000000080414000" + + "000000000059c00000000000404740cdccccccccac56c03333333333734140cdcccccccc" + + "2c5bc000000000008041400400000066666666660659c03333333333b344409a99999999" + + "b959c0cdcccccccccc424033333333333358c00000000000c0424066666666660659c033" + + "33333333b34440"; private List getCols() { return Arrays.asList("Col01","Col02", "Col03", "Col04", "Col05", "Col06", From 9e66ca5f1ab572c9b2d03937ce14c96af15aa835 Mon Sep 17 00:00:00 2001 From: jievince <38901892+jievince@users.noreply.github.com> Date: Tue, 9 Nov 2021 17:15:01 +0800 Subject: [PATCH 3/7] remove debug log --- .../java/com/vesoft/nebula/encoder/RowWriterImpl.java | 8 -------- .../test/java/com/vesoft/nebula/encoder/TestEncoder.java | 4 ---- 2 files changed, 12 deletions(-) diff --git a/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java b/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java index 47c008245..4a3186fb3 100644 --- a/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java +++ b/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java @@ -637,7 +637,6 @@ public void write(int index, DateTime v) { @Override public void write(int index, Geography v) { - System.out.println("write(index, geography), " + index); SchemaProvider.Field field = schema.field(index); PropertyType typeEnum = PropertyType.findByValue(field.type()); if (typeEnum == null) { @@ -657,11 +656,6 @@ public void write(int index, Geography v) { .WKBWriter(2, ByteOrderValues.LITTLE_ENDIAN) .write(jtsGeom); write(index, wkb); - String hexedWKb = Hex.encodeHexString(wkb); - System.out.println("write(index, geography), wkb.size()=" + wkb.length - + ", wkb.content=" + wkb); - System.out.println("write(index, geography), hexedWKb.size()=" + hexedWKb.length() - + ", hexedWKb.content=" + hexedWKb); } @Override @@ -706,7 +700,6 @@ public void setValue(int index, Object value) { } else if (value instanceof DateTime) { write(index, (DateTime)value); } else if (value instanceof Geography) { - System.out.println("jie, " + index + ", setValue(index, Object) for geography"); write(index, (Geography)value); } else { throw new RuntimeException("Unsupported value object `" + value.getClass() + "\""); @@ -751,7 +744,6 @@ public void setValue(int index, Value value) { write(index, value.getDtVal()); break; case Value.GGVAL: - System.out.println("jie, " + index + ", setValue(index, value) for geography"); write(index, value.getGgVal()); break; default: diff --git a/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java b/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java index 7862a0a02..c163ebf1d 100644 --- a/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java +++ b/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java @@ -272,10 +272,6 @@ public void testEncodeVertexValue() { // File file = new File("encode_java.txt"); // FileOutputStream fileOutputStream = new FileOutputStream(file); // fileOutputStream.write(encodeStr); - System.out.println("hex vs expect"); - System.out.println(hexStr); - System.out.println("***********"); - System.out.println(allTypeValueExpectResult); Assert.assertArrayEquals(allTypeValueExpectResult.getBytes(), hexStr.substring(0, hexStr.length() - 16).getBytes()); } catch (Exception exception) { From 8823dcec3c58a4bd147cb1a72742d04a8fa10573 Mon Sep 17 00:00:00 2001 From: jievince <38901892+jievince@users.noreply.github.com> Date: Thu, 11 Nov 2021 15:48:38 +0800 Subject: [PATCH 4/7] let WKBWriter use machine byte order --- .../main/java/com/vesoft/nebula/encoder/RowWriterImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java b/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java index 4a3186fb3..65d8e1c6a 100644 --- a/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java +++ b/client/src/main/java/com/vesoft/nebula/encoder/RowWriterImpl.java @@ -653,7 +653,9 @@ public void write(int index, Geography v) { } org.locationtech.jts.geom.Geometry jtsGeom = convertGeographyToJTSGeometry(v); byte[] wkb = new org.locationtech.jts.io - .WKBWriter(2, ByteOrderValues.LITTLE_ENDIAN) + .WKBWriter(2, this.byteOrder == ByteOrder.BIG_ENDIAN + ? ByteOrderValues.BIG_ENDIAN + : ByteOrderValues.LITTLE_ENDIAN) .write(jtsGeom); write(index, wkb); } From 704766edf11dccefedd90fae93ae8599cfa2d4b0 Mon Sep 17 00:00:00 2001 From: jievince <38901892+jievince@users.noreply.github.com> Date: Thu, 11 Nov 2021 16:30:59 +0800 Subject: [PATCH 5/7] add null test for geo --- .../com/vesoft/nebula/encoder/MetaCacheImplTest.java | 10 +++++++--- .../java/com/vesoft/nebula/encoder/TestEncoder.java | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/client/src/test/java/com/vesoft/nebula/encoder/MetaCacheImplTest.java b/client/src/test/java/com/vesoft/nebula/encoder/MetaCacheImplTest.java index 3e8a0cb2a..f9b6e1d8e 100644 --- a/client/src/test/java/com/vesoft/nebula/encoder/MetaCacheImplTest.java +++ b/client/src/test/java/com/vesoft/nebula/encoder/MetaCacheImplTest.java @@ -11,6 +11,7 @@ import com.vesoft.nebula.meta.ColumnDef; import com.vesoft.nebula.meta.ColumnTypeDef; import com.vesoft.nebula.meta.EdgeItem; +import com.vesoft.nebula.meta.GeoShape; import com.vesoft.nebula.meta.PropertyType; import com.vesoft.nebula.meta.Schema; import com.vesoft.nebula.meta.SpaceDesc; @@ -83,13 +84,16 @@ private Schema genNoDefaultVal() { columnDef.setNullable(true); columns.add(columnDef); columnDef = new ColumnDef(("Col16").getBytes(), - new ColumnTypeDef(PropertyType.GEOGRAPHY)); + new ColumnTypeDef(PropertyType.GEOGRAPHY, 0, GeoShape.POINT)); columns.add(columnDef); columnDef = new ColumnDef(("Col17").getBytes(), - new ColumnTypeDef(PropertyType.GEOGRAPHY)); + new ColumnTypeDef(PropertyType.GEOGRAPHY, 0, GeoShape.LINESTRING)); columns.add(columnDef); columnDef = new ColumnDef(("Col18").getBytes(), - new ColumnTypeDef(PropertyType.GEOGRAPHY)); + new ColumnTypeDef(PropertyType.GEOGRAPHY, 0, GeoShape.POLYGON)); + columns.add(columnDef); + columnDef = new ColumnDef(("Col19").getBytes(), + new ColumnTypeDef(PropertyType.GEOGRAPHY, 0, GeoShape.ANY)); columns.add(columnDef); return new Schema(columns, null); } diff --git a/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java b/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java index c163ebf1d..75b63c128 100644 --- a/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java +++ b/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java @@ -52,7 +52,7 @@ private List getCols() { return Arrays.asList("Col01","Col02", "Col03", "Col04", "Col05", "Col06", "Col07","Col08", "Col09", "Col10", "Col11", "Col12", "Col13", "Col14", // Purposely skip the col15 - "Col16", "Col17", "Col18"); + "Col16", "Col17", "Col18", "Col19"); } private List getValues() { @@ -107,7 +107,7 @@ private List getValues() { geogPolygonVal.setGgVal(Geography.pgVal(new Polygon(rings))); return Arrays.asList(true, 8, 16, 32, intVal, pi, e, strVal, fixVal, timestampVal, dateValue, timeVal, datetimeValue, nullVal, - geogPointVal, geogLineStringVal, geogPolygonVal); + geogPointVal, geogLineStringVal, geogPolygonVal, nullVal); } public int getSpaceVidLen(String spaceName) throws RuntimeException { From 64cb87be30924448f965fad0783a6aae8111e33e Mon Sep 17 00:00:00 2001 From: jievince <38901892+jievince@users.noreply.github.com> Date: Thu, 11 Nov 2021 16:48:37 +0800 Subject: [PATCH 6/7] debug --- .../com/vesoft/nebula/encoder/MetaCacheImplTest.java | 9 +++++---- .../java/com/vesoft/nebula/encoder/TestEncoder.java | 10 +++++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/client/src/test/java/com/vesoft/nebula/encoder/MetaCacheImplTest.java b/client/src/test/java/com/vesoft/nebula/encoder/MetaCacheImplTest.java index f9b6e1d8e..aec661f4f 100644 --- a/client/src/test/java/com/vesoft/nebula/encoder/MetaCacheImplTest.java +++ b/client/src/test/java/com/vesoft/nebula/encoder/MetaCacheImplTest.java @@ -84,16 +84,17 @@ private Schema genNoDefaultVal() { columnDef.setNullable(true); columns.add(columnDef); columnDef = new ColumnDef(("Col16").getBytes(), - new ColumnTypeDef(PropertyType.GEOGRAPHY, 0, GeoShape.POINT)); + new ColumnTypeDef(PropertyType.GEOGRAPHY, (short)0, GeoShape.POINT)); columns.add(columnDef); columnDef = new ColumnDef(("Col17").getBytes(), - new ColumnTypeDef(PropertyType.GEOGRAPHY, 0, GeoShape.LINESTRING)); + new ColumnTypeDef(PropertyType.GEOGRAPHY, (short)0, GeoShape.LINESTRING)); columns.add(columnDef); columnDef = new ColumnDef(("Col18").getBytes(), - new ColumnTypeDef(PropertyType.GEOGRAPHY, 0, GeoShape.POLYGON)); + new ColumnTypeDef(PropertyType.GEOGRAPHY, (short)0, GeoShape.POLYGON)); columns.add(columnDef); columnDef = new ColumnDef(("Col19").getBytes(), - new ColumnTypeDef(PropertyType.GEOGRAPHY, 0, GeoShape.ANY)); + new ColumnTypeDef(PropertyType.GEOGRAPHY, (short)0, GeoShape.ANY)); + columnDef.setNullable(true); columns.add(columnDef); return new Schema(columns, null); } diff --git a/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java b/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java index 75b63c128..24ee9e19e 100644 --- a/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java +++ b/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java @@ -36,10 +36,10 @@ public class TestEncoder { private final MetaCacheImplTest cacheImplTest = new MetaCacheImplTest(); private final NebulaCodecImpl codec = new NebulaCodecImpl(); - final String allTypeValueExpectResult = "090cc001081000200000004000000000000000db0f494069" - + "57148b0abf0540750000000c0000004e6562756c61204772617068bb334e5e000000" + final String allTypeValueExpectResult = "090ce001081000200000004000000000000000db0f494069" + + "57148b0abf05407d0000000c0000004e6562756c61204772617068bb334e5e000000" + "00e40702140a1e2d00000000e40702140a1e2d00000000000000000000000000000000" - + "81000000150000009600000039000000cf0000009100000048656c6c6f20776f726c6421" + + "89000000150000009e00000039000000d700000091000000000000000000000048656c6c6f20776f726c6421" + "010100000000000000006066409a99999999795640010200000003000000000000000000" + "0000000000000000f03f000000000000f03f000000000000004000000000000008400000" + "000000001c4001030000000200000004000000cdcccccccc2c5bc0000000000080414000" @@ -272,6 +272,10 @@ public void testEncodeVertexValue() { // File file = new File("encode_java.txt"); // FileOutputStream fileOutputStream = new FileOutputStream(file); // fileOutputStream.write(encodeStr); + System.out.println("got vs expect:"); + System.out.println(hexStr); + System.out.println("**************"); + System.out.println(allTypeValueExpectResult); Assert.assertArrayEquals(allTypeValueExpectResult.getBytes(), hexStr.substring(0, hexStr.length() - 16).getBytes()); } catch (Exception exception) { From 3aa394f7f98e5ac453e6e7404cb70edc81768302 Mon Sep 17 00:00:00 2001 From: jievince <38901892+jievince@users.noreply.github.com> Date: Thu, 11 Nov 2021 17:39:18 +0800 Subject: [PATCH 7/7] format code --- .../vesoft/nebula/encoder/TestEncoder.java | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java b/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java index 24ee9e19e..0ed452b78 100644 --- a/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java +++ b/client/src/test/java/com/vesoft/nebula/encoder/TestEncoder.java @@ -36,17 +36,17 @@ public class TestEncoder { private final MetaCacheImplTest cacheImplTest = new MetaCacheImplTest(); private final NebulaCodecImpl codec = new NebulaCodecImpl(); - final String allTypeValueExpectResult = "090ce001081000200000004000000000000000db0f494069" - + "57148b0abf05407d0000000c0000004e6562756c61204772617068bb334e5e000000" - + "00e40702140a1e2d00000000e40702140a1e2d00000000000000000000000000000000" - + "89000000150000009e00000039000000d700000091000000000000000000000048656c6c6f20776f726c6421" - + "010100000000000000006066409a99999999795640010200000003000000000000000000" - + "0000000000000000f03f000000000000f03f000000000000004000000000000008400000" - + "000000001c4001030000000200000004000000cdcccccccc2c5bc0000000000080414000" - + "000000000059c00000000000404740cdccccccccac56c03333333333734140cdcccccccc" - + "2c5bc000000000008041400400000066666666660659c03333333333b344409a99999999" - + "b959c0cdcccccccccc424033333333333358c00000000000c0424066666666660659c033" - + "33333333b34440"; + final String allTypeValueExpectResult = "090ce001081000200000004000000000000000" + + "db0f49406957148b0abf05407d0000000c0000004e6562756c61204772617068bb334e5e" + + "00000000e40702140a1e2d00000000e40702140a1e2d0000000000000000000000000000" + + "000089000000150000009e00000039000000d70000009100000000000000000000004865" + + "6c6c6f20776f726c6421010100000000000000006066409a999999997956400102000000" + + "030000000000000000000000000000000000f03f000000000000f03f0000000000000040" + + "00000000000008400000000000001c4001030000000200000004000000cdcccccccc2c5b" + + "c0000000000080414000000000000059c00000000000404740cdccccccccac56c0333333" + + "3333734140cdcccccccc2c5bc000000000008041400400000066666666660659c0333333" + + "3333b344409a99999999b959c0cdcccccccccc424033333333333358c00000000000c042" + + "4066666666660659c03333333333b34440"; private List getCols() { return Arrays.asList("Col01","Col02", "Col03", "Col04", "Col05", "Col06", @@ -272,10 +272,6 @@ public void testEncodeVertexValue() { // File file = new File("encode_java.txt"); // FileOutputStream fileOutputStream = new FileOutputStream(file); // fileOutputStream.write(encodeStr); - System.out.println("got vs expect:"); - System.out.println(hexStr); - System.out.println("**************"); - System.out.println(allTypeValueExpectResult); Assert.assertArrayEquals(allTypeValueExpectResult.getBytes(), hexStr.substring(0, hexStr.length() - 16).getBytes()); } catch (Exception exception) {