Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add distance type #1732

Merged
merged 2 commits into from
Jul 15, 2022
Merged

add distance type #1732

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e26c65d
add distance type
kpango Jul 14, 2022
9e5a340
update NGT version
kpango Jul 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion charts/vald-helm-operator/crds/valdrelease.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,13 +221,26 @@ spec:
enum:
- l1
- l2
- ang
- angle
- ham
- hamming
- cos
- cosine
- poincare
- poinc
- lorentz
- loren
- jac
- jaccard
- spjac
- sparsejaccard
- norml2
- normalizedl2
- normang
- normalizedangle
- normcos
- normalizedcosine
- jaccard
enable_copy_on_write:
type: boolean
enable_in_memory_mode:
Expand Down
2 changes: 1 addition & 1 deletion charts/vald/values.schema.json
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions charts/vald/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1787,9 +1787,9 @@ agent:
# @schema {"name": "agent.ngt.bulk_insert_chunk_size", "type": "integer"}
# agent.ngt.bulk_insert_chunk_size -- bulk insert chunk size
bulk_insert_chunk_size: 10
# @schema {"name": "agent.ngt.distance_type", "type": "string", "enum": ["l1", "l2", "angle", "hamming", "cos", "cosine", "normalizedangle", "normalizedcosine", "jaccard"]}
# @schema {"name": "agent.ngt.distance_type", "type": "string", "enum": ["l1", "l2", "ang", "angle", "ham", "hamming", "cos", "cosine", "poincare", "poinc", "lorentz", "loren", "jac", "jaccard", "spjac", "sparsejaccard", "norml2", "normalizedl2", "normang", "normalizedangle", "normcos", "normalizedcosine"]}
# agent.ngt.distance_type -- distance type.
# it should be `l1`, `l2`, `angle`, `hamming`, `cosine`, `normalizedangle`, `normalizedcosine` or `jaccard`.
# it should be `l1`, `l2`, `angle`, `hamming`, `cosine`,`poincare`, `lorentz`, `jaccard`, `sparsejaccard`, `normalizedangle` or `normalizedcosine`.
# for further details about NGT libraries supported distance is https://github.com/yahoojapan/NGT/wiki/Command-Quick-Reference
# and vald agent's supported NGT distance type is https://pkg.go.dev/github.com/vdaas/vald/internal/core/algorithm/ngt#pkg-constants
distance_type: l2
Expand Down
2 changes: 1 addition & 1 deletion internal/config/ngt.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ type NGT struct {
BulkInsertChunkSize int `yaml:"bulk_insert_chunk_size" json:"bulk_insert_chunk_size,omitempty"`

// DistanceType represent the ngt index distance type
// it should be `l1`, `l2`, `angle`, `hamming`, `cosine`, `normalizedangle`, `normalizedcosine` or `jaccard`. for further details about NGT libraries supported distance is https://github.com/yahoojapan/NGT/wiki/Command-Quick-Reference and vald agent's supported NGT distance type is https://pkg.go.dev/github.com/vdaas/vald/internal/core/algorithm/ngt#pkg-constants
// it should be `l1`, `l2`, `angle`, `hamming`, `cosine`,`poincare`, `lorentz`, `jaccard`, `sparsejaccard`, `normalizedangle` or `normalizedcosine`. for further details about NGT libraries supported distance is https://github.com/yahoojapan/NGT/wiki/Command-Quick-Reference and vald agent's supported NGT distance type is https://pkg.go.dev/github.com/vdaas/vald/internal/core/algorithm/ngt#pkg-constants
DistanceType string `yaml:"distance_type" json:"distance_type,omitempty" info:"distance_type"`

// ObjectType represent the ngt index object type float or int
Expand Down
25 changes: 21 additions & 4 deletions internal/core/algorithm/ngt/ngt.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,12 +139,21 @@ const (
Hamming
// Cosine is cosine distance.
Cosine
// Poincare is poincare distance.
Poincare
// Lorentz is lorenz distance.
Lorentz
// Jaccard is jaccard distance.
Jaccard
// SparseJaccard is sparse jaccard distance.
SparseJaccard
// NormalizedL2 is l2 distance with normalization.
NormalizedL2
// NormalizedAngle is angle distance with normalization.
NormalizedAngle
// NormalizedCosine is cosine distance with normalization.
NormalizedCosine
// Jaccard is jaccard distance.
Jaccard

// -------------------------------------------------------------.

// -------------------------------------------------------------
Expand Down Expand Up @@ -178,12 +187,20 @@ func (d distanceType) String() string {
return "Hamming"
case Cosine:
return "Cosine"
case Poincare:
return "Poincare"
case Lorentz:
return "Lorentz"
case Jaccard:
return "Jaccard"
case SparseJaccard:
return "SparseJaccard"
case NormalizedL2:
return "NormalizedL2"
case NormalizedAngle:
return "NormalizedAngle"
case NormalizedCosine:
return "NormalizedCosine"
case Jaccard:
return "Jaccard"
}
return "Unknown"
}
Expand Down
44 changes: 36 additions & 8 deletions internal/core/algorithm/ngt/option.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,12 +119,20 @@ func WithDistanceTypeByString(dt string) Option {
d = Hamming
case "cosine", "cos":
d = Cosine
case "normalizedangle", "normalizedang", "nang", "nangle":
d = NormalizedAngle
case "normalizedcosine", "normalizedcos", "ncos", "ncosine":
d = NormalizedCosine
case "poincare", "poinc", "poi", "po", "pc":
d = Poincare
case "lorenz", "loren", "lor", "lo", "lz":
d = Lorentz
case "jaccard", "jac":
d = Jaccard
case "sparsejaccard", "sparsejac", "spjac", "sjc", "sj":
d = SparseJaccard
case "normalizedl2", "norml2", "nol2", "nl2":
d = NormalizedL2
case "normalizedangle", "normalizedang", "normang", "nang", "nangle":
d = NormalizedAngle
case "normalizedcosine", "normalizedcos", "normcos", "ncos", "ncosine":
d = NormalizedCosine
}
return WithDistanceType(d)
}
Expand Down Expand Up @@ -159,13 +167,13 @@ func WithDistanceType(t distanceType) Option {
err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String())
return errors.NewErrCriticalOption("distanceType", t, err)
}
case NormalizedAngle:
if C.ngt_set_property_distance_type_normalized_angle(n.prop, ebuf) == ErrorCode {
case Poincare:
if C.ngt_set_property_distance_type_poincare(n.prop, ebuf) == ErrorCode {
err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String())
return errors.NewErrCriticalOption("distanceType", t, err)
}
case NormalizedCosine:
if C.ngt_set_property_distance_type_normalized_cosine(n.prop, ebuf) == ErrorCode {
case Lorentz:
if C.ngt_set_property_distance_type_lorentz(n.prop, ebuf) == ErrorCode {
err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String())
return errors.NewErrCriticalOption("distanceType", t, err)
}
Expand All @@ -174,6 +182,26 @@ func WithDistanceType(t distanceType) Option {
err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String())
return errors.NewErrCriticalOption("distanceType", t, err)
}
case SparseJaccard:
if C.ngt_set_property_distance_type_sparse_jaccard(n.prop, ebuf) == ErrorCode {
err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String())
return errors.NewErrCriticalOption("distanceType", t, err)
}
case NormalizedL2:
if C.ngt_set_property_distance_type_normalized_l2(n.prop, ebuf) == ErrorCode {
err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String())
return errors.NewErrCriticalOption("distanceType", t, err)
}
case NormalizedAngle:
if C.ngt_set_property_distance_type_normalized_angle(n.prop, ebuf) == ErrorCode {
err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String())
return errors.NewErrCriticalOption("distanceType", t, err)
}
case NormalizedCosine:
if C.ngt_set_property_distance_type_normalized_cosine(n.prop, ebuf) == ErrorCode {
err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String())
return errors.NewErrCriticalOption("distanceType", t, err)
}
default:
err := errors.ErrUnsupportedDistanceType
n.PutErrorBuffer(ebuf)
Expand Down
12 changes: 12 additions & 0 deletions k8s/operator/helm/clusterrole.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,18 @@ rules:
- patch
- update
- watch
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- rbac.authorization.k8s.io
resources:
Expand Down
15 changes: 14 additions & 1 deletion k8s/operator/helm/crds/valdrelease.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,13 +221,26 @@ spec:
enum:
- l1
- l2
- ang
- angle
- ham
- hamming
- cos
- cosine
- poincare
- poinc
- lorentz
- loren
- jac
- jaccard
- spjac
- sparsejaccard
- norml2
- normalizedl2
- normang
- normalizedangle
- normcos
- normalizedcosine
- jaccard
enable_copy_on_write:
type: boolean
enable_in_memory_mode:
Expand Down
2 changes: 1 addition & 1 deletion versions/NGT_VERSION
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.14.6
1.14.7