Merge branch 'main' into cross-encoder

Signed-off-by: Henry Lindeman <[email protected]>
opensearch-project · May 21, 2024 · 7a134b0 · 7a134b0
2 parents b063f32 + ec7e023
commit 7a134b0
Show file tree

Hide file tree

Showing 12 changed files with 33 additions and 45 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -39,6 +39,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - use try-except-else block for handling unexpected exceptions during integration tests by @rawwar([#370](https://github.com/opensearch-project/opensearch-py-ml/pull/370))
 - Removed pandas version pin in nox tests by @rawwar ([#368](https://github.com/opensearch-project/opensearch-py-ml/pull/368))
 - Switch AL2 to AL2023 agent and DockerHub to ECR images in ml-models.JenkinsFile ([#377](https://github.com/opensearch-project/opensearch-py-ml/pull/377))
+- Refactored validators in ML Commons' client([#385](https://github.com/opensearch-project/opensearch-py-ml/pull/385))
 
 ### Fixed
 - Enable make_model_config_json to add model description to model config file by @thanawan-atc in ([#203](https://github.com/opensearch-project/opensearch-py-ml/pull/203))
@@ -53,6 +54,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Fix conditional job execution issue in model upload workflow by @thanawan-atc in ([#294](https://github.com/opensearch-project/opensearch-py-ml/pull/294))
 - fix bug in `MLCommonClient_client.upload_model` by @rawwar in ([#336](https://github.com/opensearch-project/opensearch-py-ml/pull/336))
 - fix lint issues on main by @rawwar in ([#374](https://github.com/opensearch-project/opensearch-py-ml/pull/374))
+- fix CVE vulnerability by @rawwar in ([#383](https://github.com/opensearch-project/opensearch-py-ml/pull/383))
 
 ## [1.1.0]
 

diff --git a/SECURITY.md b/SECURITY.md
@@ -1,3 +1,3 @@
 ## Reporting a Vulnerability
 
-If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to aws-security@amazon.com. Please do **not** create a public GitHub issue.
+If you discover a potential security issue in this project we ask that you notify OpenSearch Security directly via email to security@opensearch.org. Please do **not** create a public GitHub issue.
diff --git a/opensearch_py_ml/ml_commons/ml_commons_client.py b/opensearch_py_ml/ml_commons/ml_commons_client.py
@@ -25,7 +25,7 @@
 from opensearch_py_ml.ml_commons.model_connector import Connector
 from opensearch_py_ml.ml_commons.model_execute import ModelExecute
 from opensearch_py_ml.ml_commons.model_uploader import ModelUploader
-from opensearch_py_ml.ml_commons.validators.profile import validate_profile_input
+from opensearch_py_ml.ml_commons.validators import validate_profile_input
 
 
 class MLCommonClient:

diff --git a/opensearch_py_ml/ml_commons/model_access_control.py b/opensearch_py_ml/ml_commons/model_access_control.py
@@ -11,7 +11,7 @@
 from opensearchpy.exceptions import NotFoundError
 
 from opensearch_py_ml.ml_commons.ml_common_utils import ML_BASE_URI
-from opensearch_py_ml.ml_commons.validators.model_access_control import (
+from opensearch_py_ml.ml_commons.validators import (
     validate_create_model_group_parameters,
     validate_delete_model_group_parameters,
     validate_search_model_group_parameters,

diff --git a/...ommons/validators/model_access_control.py → opensearch_py_ml/ml_commons/validators.py b/...ommons/validators/model_access_control.py → opensearch_py_ml/ml_commons/validators.py
@@ -95,3 +95,11 @@ def validate_delete_model_group_parameters(model_group_id: str):
 
 def validate_search_model_group_parameters(query: dict):
     _validate_model_group_query(query)
+
+
+def validate_profile_input(path_parameter, payload):
+    if path_parameter is not None and not isinstance(path_parameter, str):
+        raise ValueError("path_parameter needs to be a string or None")
+
+    if payload is not None and not isinstance(payload, dict):
+        raise ValueError("payload needs to be a dictionary or None")
diff --git a/opensearch_py_ml/ml_commons/validators/__init__.py b/opensearch_py_ml/ml_commons/validators/__init__.py
diff --git a/opensearch_py_ml/ml_commons/validators/profile.py b/opensearch_py_ml/ml_commons/validators/profile.py
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -5,14 +5,15 @@ pandas>=1.5.2,<2
 matplotlib>=3.6.2,<4
 numpy>=1.24.0,<2
 opensearch-py>=2.2.0
-torch==2.0.1
-onnx
-accelerate
-sentence_transformers
-tqdm
-transformers
-deprecated
-mdutils
+torch>=2.0.1,<2.1.0
+onnx>=1.15.0
+accelerate>=0.27
+sentence_transformers>=2.5.0,<2.6
+tqdm>4.66.0,<5
+transformers>=4.36.0,<5
+deprecated>=1.2.14,<2
+mdutils>=1.6.0,<2
+pillow>10.0.0,<11
 
 #
 # Testing

diff --git a/requirements.txt b/requirements.txt
@@ -5,11 +5,10 @@ pandas>=1.5.2,<2
 matplotlib>=3.6.2,<4
 numpy>=1.24.0,<2
 opensearch-py>=2.2.0
-torch==2.0.1
-onnx
-accelerate
-sentence_transformers
-tqdm
-transformers
-deprecated
-requests
+torch>=2.0.1,<2.1.0
+onnx>=1.15.0
+accelerate>=0.27
+sentence_transformers>=2.5.0,<2.6
+tqdm>4.66.0,<5
+transformers>=4.36.0,<5
+deprecated>=1.2.14,<2
diff --git a/...s/test_model_access_control_validators.py → tests/ml_commons/test_validators.py b/...s/test_model_access_control_validators.py → tests/ml_commons/test_validators.py
@@ -7,7 +7,7 @@
 
 import pytest
 
-from opensearch_py_ml.ml_commons.validators.model_access_control import (
+from opensearch_py_ml.ml_commons.validators import (
     _validate_model_group_access_mode,
     _validate_model_group_add_all_backend_roles,
     _validate_model_group_backend_roles,

diff --git a/tests/ml_models/test_sentencetransformermodel_pytest.py b/tests/ml_models/test_sentencetransformermodel_pytest.py
@@ -463,7 +463,7 @@ def test_overwrite_description():
 def test_long_description():
     model_id = "sentence-transformers/gtr-t5-base"
     model_format = "TORCH_SCRIPT"
-    expected_model_description = "This is a sentence-transformers model: It maps sentences & paragraphs to a 768 dimensional dense vector space. The model was specifically trained for the task of sematic search."
+    expected_model_description = "This is a sentence-transformers model: It maps sentences & paragraphs to a 768 dimensional dense vector space. The model was specifically trained for the task of semantic search."
 
     clean_test_folder(TEST_FOLDER)
     test_model12 = SentenceTransformerModel(

diff --git a/utils/model_uploader/model_listing/pretrained_models_all_versions.json b/utils/model_uploader/model_listing/pretrained_models_all_versions.json
@@ -151,7 +151,7 @@
   {
     "name": "huggingface/cross-encoders/ms-marco-MiniLM-L-6-v2",
     "versions": {
-      "1.0.0": {
+      "1.0.2": {
         "format": [
           "onnx",
           "torch_script"
@@ -163,7 +163,7 @@
   {
     "name": "huggingface/cross-encoders/ms-marco-MiniLM-L-12-v2",
     "versions": {
-      "1.0.0": {
+      "1.0.2": {
         "format": [
           "onnx",
           "torch_script"