#3145 Add IP Address Data Type (#3175)

* Add `ExprIpValue` and `IP` data type Signed-off-by: currantw <[email protected]> * Add support for casting (`cast(field_name to ip)`) and remove existing unused sorting syntax. Signed-off-by: currantw <[email protected]> * Update comparison logic to compare in IPv6 Signed-off-by: currantw <[email protected]> * Fix bug casting to IP Signed-off-by: currantw <[email protected]> * Fix failing tests Signed-off-by: currantw <[email protected]> * Assert that comparison only valid if same type, update tests accordingly Signed-off-by: currantw <[email protected]> * Add additional tests to increase code coverage Signed-off-by: currantw <[email protected]> * Integrate `cidrmatch` changes Signed-off-by: currantw <[email protected]> * Remove `OpenSearchIPType` data type Signed-off-by: currantw <[email protected]> * Fix more failing tests Signed-off-by: currantw <[email protected]> * Minor cleanup Signed-off-by: currantw <[email protected]> * Add new tests for IP data type to `SortCommandIT`, and update `weblogs` test data. Signed-off-by: currantw <[email protected]> * Fixing IT test failure. Signed-off-by: currantw <[email protected]> * Spotless and update test to sort in SQL Signed-off-by: currantw <[email protected]> * Fix broken link Signed-off-by: currantw <[email protected]> * Fix failing code coverage Signed-off-by: currantw <[email protected]> * Fix failing doctest Signed-off-by: currantw <[email protected]> * Fix failing `ip.rst` doctest Signed-off-by: currantw <[email protected]> * Fix test failure due to merge. Signed-off-by: currantw <[email protected]> * Fix spotless Signed-off-by: currantw <[email protected]> * Add missing `url` field Signed-off-by: currantw <[email protected]> * Address minor review comments. Signed-off-by: currantw <[email protected]> * Revert sort syntax changes Signed-off-by: currantw <[email protected]> * Minor doc update Signed-off-by: currantw <[email protected]> * FIx failing `ip.rst` doctest Signed-off-by: currantw <[email protected]> * Add `IPComparisonIT` tests for comparison operators, rename modules and weblogs test index to make plural for consistency. Signed-off-by: currantw <[email protected]> --------- Signed-off-by: currantw <[email protected]>
opensearch-project · Dec 19, 2024 · 3430deb · 3430deb
1 parent 8bfa2e9
commit 3430deb
Show file tree

Hide file tree

Showing 44 changed files with 803 additions and 400 deletions.
diff --git a/DEVELOPER_GUIDE.rst b/DEVELOPER_GUIDE.rst
@@ -405,7 +405,7 @@ Sample test class:
 Doctest
 >>>>>>>
 
-Python doctest library makes our document executable which keeps it up-to-date to source code. The doc generator aforementioned served as scaffolding and generated many docs in short time. Now the examples inside is changed to doctest gradually. For more details please read `Doctest <./dev/Doctest.md>`_.
+Python doctest library makes our document executable which keeps it up-to-date to source code. The doc generator aforementioned served as scaffolding and generated many docs in short time. Now the examples inside is changed to doctest gradually. For more details please read `testing-doctest <./docs/dev/testing-doctest.md>`_.
 
 
 Backports

diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java
@@ -12,6 +12,7 @@
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_DOUBLE;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_FLOAT;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_INT;
+import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_IP;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_LONG;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_SHORT;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_STRING;
@@ -54,6 +55,7 @@ public class Cast extends UnresolvedExpression {
           .put("time", CAST_TO_TIME.getName())
           .put("timestamp", CAST_TO_TIMESTAMP.getName())
           .put("datetime", CAST_TO_DATETIME.getName())
+          .put("ip", CAST_TO_IP.getName())
           .build();
 
   /** The source expression cast from. */

diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprIpValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprIpValue.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.data.model;
+
+import inet.ipaddr.IPAddress;
+import org.opensearch.sql.data.type.ExprCoreType;
+import org.opensearch.sql.data.type.ExprType;
+import org.opensearch.sql.utils.IPUtils;
+
+/** Expression IP Address Value. */
+public class ExprIpValue extends AbstractExprValue {
+  private final IPAddress value;
+
+  public ExprIpValue(String addressString) {
+    value = IPUtils.toAddress(addressString);
+  }
+
+  @Override
+  public String value() {
+    return value.toCanonicalString();
+  }
+
+  @Override
+  public ExprType type() {
+    return ExprCoreType.IP;
+  }
+
+  @Override
+  public int compare(ExprValue other) {
+    return IPUtils.compare(value, ((ExprIpValue) other).value);
+  }
+
+  @Override
+  public boolean equal(ExprValue other) {
+    return compare(other) == 0;
+  }
+
+  @Override
+  public String toString() {
+    return String.format("IP %s", value());
+  }
+
+  @Override
+  public IPAddress ipValue() {
+    return value;
+  }
+}
diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprValue.java
@@ -5,6 +5,7 @@
 
 package org.opensearch.sql.data.model;
 
+import inet.ipaddr.IPAddress;
 import java.io.Serializable;
 import java.time.Instant;
 import java.time.LocalDate;
@@ -102,6 +103,11 @@ default Double doubleValue() {
         "invalid to get doubleValue from value of type " + type());
   }
 
+  /** Get IP address value. */
+  default IPAddress ipValue() {
+    throw new ExpressionEvaluationException("invalid to get ipValue from value of type " + type());
+  }
+
   /** Get string value. */
   default String stringValue() {
     throw new ExpressionEvaluationException(

diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprValueUtils.java b/core/src/main/java/org/opensearch/sql/data/model/ExprValueUtils.java
@@ -5,6 +5,7 @@
 
 package org.opensearch.sql.data.model;
 
+import inet.ipaddr.IPAddress;
 import java.time.Instant;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
@@ -75,6 +76,10 @@ public static ExprValue timestampValue(Instant value) {
     return new ExprTimestampValue(value);
   }
 
+  public static ExprValue ipValue(String value) {
+    return new ExprIpValue(value);
+  }
+
   /** {@link ExprTupleValue} constructor. */
   public static ExprValue tupleValue(Map<String, Object> map) {
     LinkedHashMap<String, ExprValue> valueMap = new LinkedHashMap<>();
@@ -188,6 +193,10 @@ public static Map<String, ExprValue> getTupleValue(ExprValue exprValue) {
     return exprValue.tupleValue();
   }
 
+  public static IPAddress getIpValue(ExprValue exprValue) {
+    return exprValue.ipValue();
+  }
+
   public static Boolean getBooleanValue(ExprValue exprValue) {
     return exprValue.booleanValue();
   }

diff --git a/core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java b/core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java
@@ -45,6 +45,9 @@ public enum ExprCoreType implements ExprType {
   TIMESTAMP(STRING, DATE, TIME),
   INTERVAL(UNDEFINED),
 
+  /** IP Address. */
+  IP(STRING),
+
   /** Struct. */
   STRUCT(UNDEFINED),
 

diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java
@@ -835,6 +835,10 @@ public static FunctionExpression castTimestamp(Expression value) {
     return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_TIMESTAMP, value);
   }
 
+  public static FunctionExpression castIp(Expression value) {
+    return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_IP, value);
+  }
+
   public static FunctionExpression typeof(Expression value) {
     return compile(FunctionProperties.None, BuiltinFunctionName.TYPEOF, value);
   }

diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java
@@ -231,6 +231,7 @@ public enum BuiltinFunctionName {
   CAST_TO_TIME(FunctionName.of("cast_to_time")),
   CAST_TO_TIMESTAMP(FunctionName.of("cast_to_timestamp")),
   CAST_TO_DATETIME(FunctionName.of("cast_to_datetime")),
+  CAST_TO_IP(FunctionName.of("cast_to_ip")),
   TYPEOF(FunctionName.of("typeof")),
 
   /** Relevance Function. */

diff --git a/core/src/main/java/org/opensearch/sql/expression/ip/IPFunctions.java b/core/src/main/java/org/opensearch/sql/expression/ip/IPFunctions.java
@@ -6,21 +6,21 @@
 package org.opensearch.sql.expression.ip;
 
 import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN;
+import static org.opensearch.sql.data.type.ExprCoreType.IP;
 import static org.opensearch.sql.data.type.ExprCoreType.STRING;
 import static org.opensearch.sql.expression.function.FunctionDSL.define;
 import static org.opensearch.sql.expression.function.FunctionDSL.impl;
 import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling;
 
-import inet.ipaddr.AddressStringException;
-import inet.ipaddr.IPAddressString;
-import inet.ipaddr.IPAddressStringParameters;
+import inet.ipaddr.IPAddress;
 import lombok.experimental.UtilityClass;
 import org.opensearch.sql.data.model.ExprValue;
 import org.opensearch.sql.data.model.ExprValueUtils;
 import org.opensearch.sql.exception.SemanticCheckException;
 import org.opensearch.sql.expression.function.BuiltinFunctionName;
 import org.opensearch.sql.expression.function.BuiltinFunctionRepository;
 import org.opensearch.sql.expression.function.DefaultFunctionResolver;
+import org.opensearch.sql.utils.IPUtils;
 
 /** Utility class that defines and registers IP functions. */
 @UtilityClass
@@ -31,75 +31,30 @@ public void register(BuiltinFunctionRepository repository) {
   }
 
   private DefaultFunctionResolver cidrmatch() {
-
-    // TODO #3145: Add support for IP address data type.
     return define(
         BuiltinFunctionName.CIDRMATCH.getName(),
-        impl(nullMissingHandling(IPFunctions::exprCidrMatch), BOOLEAN, STRING, STRING));
+        impl(nullMissingHandling(IPFunctions::exprCidrMatch), BOOLEAN, IP, STRING));
   }
 
   /**
    * Returns whether the given IP address is within the specified inclusive CIDR IP address range.
    * Supports both IPv4 and IPv6 addresses.
    *
-   * @param addressExprValue IP address as a string (e.g. "198.51.100.14" or
-   *     "2001:0db8::ff00:42:8329").
-   * @param rangeExprValue IP address range in CIDR notation as a string (e.g. "198.51.100.0/24" or
+   * @param addressExprValue IP address (e.g. "198.51.100.14" or "2001:0db8::ff00:42:8329").
+   * @param rangeExprValue IP address range string in CIDR notation (e.g. "198.51.100.0/24" or
    *     "2001:0db8::/32")
    * @return true if the address is in the range; otherwise false.
    * @throws SemanticCheckException if the address or range is not valid, or if they do not use the
    *     same version (IPv4 or IPv6).
    */
   private ExprValue exprCidrMatch(ExprValue addressExprValue, ExprValue rangeExprValue) {
 
-    // TODO #3145: Update to support IP address data type.
-    String addressString = addressExprValue.stringValue();
-    String rangeString = rangeExprValue.stringValue();
-
-    final IPAddressStringParameters validationOptions =
-        new IPAddressStringParameters.Builder()
-            .allowEmpty(false)
-            .setEmptyAsLoopback(false)
-            .allow_inet_aton(false)
-            .allowSingleSegment(false)
-            .toParams();
-
-    // Get and validate IP address.
-    IPAddressString address =
-        new IPAddressString(addressExprValue.stringValue(), validationOptions);
-
-    try {
-      address.validate();
-    } catch (AddressStringException e) {
-      String msg =
-          String.format(
-              "IP address '%s' is not valid. Error details: %s", addressString, e.getMessage());
-      throw new SemanticCheckException(msg, e);
-    }
-
-    // Get and validate CIDR IP address range.
-    IPAddressString range = new IPAddressString(rangeExprValue.stringValue(), validationOptions);
-
-    try {
-      range.validate();
-    } catch (AddressStringException e) {
-      String msg =
-          String.format(
-              "CIDR IP address range '%s' is not valid. Error details: %s",
-              rangeString, e.getMessage());
-      throw new SemanticCheckException(msg, e);
-    }
-
-    // Address and range must use the same IP version (IPv4 or IPv6).
-    if (address.isIPv4() ^ range.isIPv4()) {
-      String msg =
-          String.format(
-              "IP address '%s' and CIDR IP address range '%s' are not compatible. Both must be"
-                  + " either IPv4 or IPv6.",
-              addressString, rangeString);
-      throw new SemanticCheckException(msg);
-    }
+    IPAddress address = addressExprValue.ipValue();
+    IPAddress range = IPUtils.toRange(rangeExprValue.stringValue());
 
-    return ExprValueUtils.booleanValue(range.contains(address));
+    return (IPUtils.compare(address, range.getLower()) < 0)
+            || (IPUtils.compare(address, range.getUpper()) > 0)
+        ? ExprValueUtils.LITERAL_FALSE
+        : ExprValueUtils.LITERAL_TRUE;
   }
 }
diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java
@@ -11,6 +11,7 @@
 import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE;
 import static org.opensearch.sql.data.type.ExprCoreType.FLOAT;
 import static org.opensearch.sql.data.type.ExprCoreType.INTEGER;
+import static org.opensearch.sql.data.type.ExprCoreType.IP;
 import static org.opensearch.sql.data.type.ExprCoreType.LONG;
 import static org.opensearch.sql.data.type.ExprCoreType.SHORT;
 import static org.opensearch.sql.data.type.ExprCoreType.STRING;
@@ -31,6 +32,7 @@
 import org.opensearch.sql.data.model.ExprDoubleValue;
 import org.opensearch.sql.data.model.ExprFloatValue;
 import org.opensearch.sql.data.model.ExprIntegerValue;
+import org.opensearch.sql.data.model.ExprIpValue;
 import org.opensearch.sql.data.model.ExprLongValue;
 import org.opensearch.sql.data.model.ExprShortValue;
 import org.opensearch.sql.data.model.ExprStringValue;
@@ -54,6 +56,7 @@ public static void register(BuiltinFunctionRepository repository) {
     repository.register(castToFloat());
     repository.register(castToDouble());
     repository.register(castToBoolean());
+    repository.register(castToIp());
     repository.register(castToDate());
     repository.register(castToTime());
     repository.register(castToTimestamp());
@@ -173,6 +176,13 @@ private static DefaultFunctionResolver castToBoolean() {
         impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN));
   }
 
+  private static DefaultFunctionResolver castToIp() {
+    return FunctionDSL.define(
+        BuiltinFunctionName.CAST_TO_IP.getName(),
+        impl(nullMissingHandling((v) -> new ExprIpValue(v.stringValue())), IP, STRING),
+        impl(nullMissingHandling((v) -> v), IP, IP));
+  }
+
   private static DefaultFunctionResolver castToDate() {
     return FunctionDSL.define(
         BuiltinFunctionName.CAST_TO_DATE.getName(),
-Original file line number
+Diff line change
@@ Expand Up / @@ -405,7 +405,7 @@ Sample test class: @@
     Doctest
     >>>>>>>
-    Python doctest library makes our document executable which keeps it up-to-date to source code. The doc generator aforementioned served as scaffolding and generated many docs in short time. Now the examples inside is changed to doctest gradually. For more details please read `Doctest <./dev/Doctest.md>`_.
+    Python doctest library makes our document executable which keeps it up-to-date to source code. The doc generator aforementioned served as scaffolding and generated many docs in short time. Now the examples inside is changed to doctest gradually. For more details please read `testing-doctest <./docs/dev/testing-doctest.md>`_.
     Backports
@@ Expand Down @@