From 1a17642dd09ed2e7ec0f253680bb79a3d777197a Mon Sep 17 00:00:00 2001 From: Teddy Koker Date: Wed, 3 Feb 2021 18:36:22 -0500 Subject: [PATCH 01/11] add swav and simclr models to imageclassifier --- flash/vision/backbones.py | 15 +++++++++++++++ flash/vision/embedding/model_map.py | 4 ++++ 2 files changed, 19 insertions(+) diff --git a/flash/vision/backbones.py b/flash/vision/backbones.py index 89e68520666..0915969209d 100644 --- a/flash/vision/backbones.py +++ b/flash/vision/backbones.py @@ -17,6 +17,8 @@ import torchvision from pytorch_lightning.utilities.exceptions import MisconfigurationException +from flash.vision.embedding.model_map import load_simclr_imagenet, load_swav_imagenet + def torchvision_backbone_and_num_features(model_name: str, pretrained: bool = True) -> Tuple[nn.Module, int]: """ @@ -26,6 +28,10 @@ def torchvision_backbone_and_num_features(model_name: str, pretrained: bool = Tr (Sequential(...), 512) >>> torchvision_backbone_and_num_features('densenet121') # doctest: +ELLIPSIS (Sequential(...), 1024) + >>> torchvision_backbone_and_num_features('simclr-imagenet') # doctest: +ELLIPSIS + (Sequential(...), 1024) + >>> torchvision_backbone_and_num_features('swav-imagenet') # doctest: +ELLIPSIS + (Sequential(...), 1024) """ model = getattr(torchvision.models, model_name, None) if model is None: @@ -52,4 +58,13 @@ def torchvision_backbone_and_num_features(model_name: str, pretrained: bool = Tr num_features = model.classifier.in_features return backbone, num_features + elif model_name == "simclr-imagenet": + config = load_simclr_imagenet() + return config["model"], config["emb_size"] + + elif model_name == "swav-imagenet": + config = load_swav_imagenet() + return config["model"], config["emb_size"] + + raise ValueError(f"{model_name} is not supported yet.") diff --git a/flash/vision/embedding/model_map.py b/flash/vision/embedding/model_map.py index 4565440ea99..b747032cdce 100644 --- a/flash/vision/embedding/model_map.py +++ b/flash/vision/embedding/model_map.py @@ -23,6 +23,10 @@ ROOT_S3_BUCKET = "https://pl-bolts-weights.s3.us-east-2.amazonaws.com" +# TODO: move this stuff to backbones +# also, we should consider uploading plain pytorch weights so we don't need to rely on bolts to load these +# also mabye just use torchhub for the ssl lib + def load_simclr_imagenet(path_or_url: str = f"{ROOT_S3_BUCKET}/simclr/bolts_simclr_imagenet/simclr_imagenet.ckpt"): simclr = SimCLR.load_from_checkpoint(path_or_url, strict=False) model_config = {'model': simclr.encoder, 'emb_size': 2048} From e45e90d54828c1bd850a0a8c6a3ff783b62fafe9 Mon Sep 17 00:00:00 2001 From: Teddy Koker Date: Wed, 3 Feb 2021 18:38:14 -0500 Subject: [PATCH 02/11] pep8 --- flash/vision/backbones.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flash/vision/backbones.py b/flash/vision/backbones.py index 0915969209d..59108683f6d 100644 --- a/flash/vision/backbones.py +++ b/flash/vision/backbones.py @@ -66,5 +66,4 @@ def torchvision_backbone_and_num_features(model_name: str, pretrained: bool = Tr config = load_swav_imagenet() return config["model"], config["emb_size"] - raise ValueError(f"{model_name} is not supported yet.") From 743f619dcfc04db7ed2739af36eb8e62f3c0fba9 Mon Sep 17 00:00:00 2001 From: Teddy Koker Date: Wed, 3 Feb 2021 18:39:31 -0500 Subject: [PATCH 03/11] yapf --- flash/vision/embedding/model_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flash/vision/embedding/model_map.py b/flash/vision/embedding/model_map.py index b747032cdce..eb5e92b1441 100644 --- a/flash/vision/embedding/model_map.py +++ b/flash/vision/embedding/model_map.py @@ -22,11 +22,11 @@ ROOT_S3_BUCKET = "https://pl-bolts-weights.s3.us-east-2.amazonaws.com" - # TODO: move this stuff to backbones # also, we should consider uploading plain pytorch weights so we don't need to rely on bolts to load these # also mabye just use torchhub for the ssl lib + def load_simclr_imagenet(path_or_url: str = f"{ROOT_S3_BUCKET}/simclr/bolts_simclr_imagenet/simclr_imagenet.ckpt"): simclr = SimCLR.load_from_checkpoint(path_or_url, strict=False) model_config = {'model': simclr.encoder, 'emb_size': 2048} From faba990cf56b928eb2abc7306176c2cac1a08ee9 Mon Sep 17 00:00:00 2001 From: Teddy Koker Date: Wed, 3 Feb 2021 19:05:19 -0500 Subject: [PATCH 04/11] reorg --- flash/vision/backbones.py | 77 ++++++++++++++----- flash/vision/classification/model.py | 4 +- .../vision/embedding/image_embedder_model.py | 10 +-- flash/vision/embedding/model_map.py | 53 ------------- tests/vision/test_download.py | 4 +- 5 files changed, 65 insertions(+), 83 deletions(-) delete mode 100644 flash/vision/embedding/model_map.py diff --git a/flash/vision/backbones.py b/flash/vision/backbones.py index 59108683f6d..13c4c85e63f 100644 --- a/flash/vision/backbones.py +++ b/flash/vision/backbones.py @@ -11,13 +11,68 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from contextlib import suppress from typing import Tuple import torch.nn as nn import torchvision from pytorch_lightning.utilities.exceptions import MisconfigurationException +from pytorch_lightning.utilities import _BOLTS_AVAILABLE -from flash.vision.embedding.model_map import load_simclr_imagenet, load_swav_imagenet +if _BOLTS_AVAILABLE: + with suppress(TypeError): + from pl_bolts.models.self_supervised import SimCLR, SwAV + +ROOT_S3_BUCKET = "https://pl-bolts-weights.s3.us-east-2.amazonaws.com" + +MOBILENET_MODELS = ["mobilenet_v2"] +VGG_MODELS = ["vgg11", "vgg13", "vgg16", "vgg19"] +RESNET_MODELS = ["resnet18", "resnet34", "resnet50", "resnet101", "resnet152", "resnext50_32x4d", "resnext101_32x8d"] +DENSENET_MODELS = ["densenet121", "densenet169", "densenet161", "densenet161"] +TORCHVISION_MODELS = MOBILENET_MODELS + VGG_MODELS + RESNET_MODELS + DENSENET_MODELS + +BOLTS_MODELS = ["simclr-imagenet", "swav-imagenet"] + + +def backbone_and_num_features(model_name: str, *args, **kwargs) -> Tuple[nn.Module, int]: + if model_name in BOLTS_MODELS: + return bolts_backbone_and_num_features(model_name) + + if model_name in TORCHVISION_MODELS: + return torchvision_backbone_and_num_features(model_name, *args, **kwargs) + + raise ValueError(f"{model_name} is not supported yet.") + + +def bolts_backbone_and_num_features(model_name: str) -> Tuple[nn.Module, int]: + """ + >>> torchvision_backbone_and_num_features('simclr-imagenet') # doctest: +ELLIPSIS + (Sequential(...), 1024) + >>> torchvision_backbone_and_num_features('swav-imagenet') # doctest: +ELLIPSIS + (Sequential(...), 1024) + """ + + models = { + 'simclr-imagenet': load_simclr_imagenet, + 'swav-imagenet': load_swav_imagenet, + } + + # TODO: maybe we should plain pytorch weights so we don't need to rely on bolts to load these + # also mabye just use torchhub for the ssl lib + def load_simclr_imagenet(path_or_url: str = f"{ROOT_S3_BUCKET}/simclr/bolts_simclr_imagenet/simclr_imagenet.ckpt"): + simclr = SimCLR.load_from_checkpoint(path_or_url, strict=False) + return simclr.model, 2048 + + def load_swav_imagenet(path_or_url: str = f"{ROOT_S3_BUCKET}/swav/swav_imagenet/swav_imagenet.pth.tar"): + swav = SwAV.load_from_checkpoint(path_or_url, strict=True) + return swav.model, 3000 + + if not _BOLTS_AVAILABLE: + raise MisconfigurationException("Bolts isn't installed. Please, use ``pip install lightning-bolts``.") + if model_name in models: + return models[model_name]() + + raise ValueError(f"{model_name} is not supported yet.") def torchvision_backbone_and_num_features(model_name: str, pretrained: bool = True) -> Tuple[nn.Module, int]: @@ -28,42 +83,28 @@ def torchvision_backbone_and_num_features(model_name: str, pretrained: bool = Tr (Sequential(...), 512) >>> torchvision_backbone_and_num_features('densenet121') # doctest: +ELLIPSIS (Sequential(...), 1024) - >>> torchvision_backbone_and_num_features('simclr-imagenet') # doctest: +ELLIPSIS - (Sequential(...), 1024) - >>> torchvision_backbone_and_num_features('swav-imagenet') # doctest: +ELLIPSIS - (Sequential(...), 1024) """ model = getattr(torchvision.models, model_name, None) if model is None: raise MisconfigurationException(f"{model_name} is not supported by torchvision") - if model_name in ["mobilenet_v2", "vgg11", "vgg13", "vgg16", "vgg19"]: + if model_name in MOBILENET_MODELS + VGG_MODELS: model = model(pretrained=pretrained) backbone = model.features num_features = model.classifier[-1].in_features return backbone, num_features - elif model_name in [ - "resnet18", "resnet34", "resnet50", "resnet101", "resnet152", "resnext50_32x4d", "resnext101_32x8d" - ]: + elif model_name in RESNET_MODELS: model = model(pretrained=pretrained) # remove the last two layers & turn it into a Sequential model backbone = nn.Sequential(*list(model.children())[:-2]) num_features = model.fc.in_features return backbone, num_features - elif model_name in ["densenet121", "densenet169", "densenet161", "densenet161"]: + elif model_name in DENSENET_MODELS: model = model(pretrained=pretrained) backbone = nn.Sequential(*model.features, nn.ReLU(inplace=True)) num_features = model.classifier.in_features return backbone, num_features - elif model_name == "simclr-imagenet": - config = load_simclr_imagenet() - return config["model"], config["emb_size"] - - elif model_name == "swav-imagenet": - config = load_swav_imagenet() - return config["model"], config["emb_size"] - raise ValueError(f"{model_name} is not supported yet.") diff --git a/flash/vision/classification/model.py b/flash/vision/classification/model.py index 4a77264b82d..69a3fd8c859 100644 --- a/flash/vision/classification/model.py +++ b/flash/vision/classification/model.py @@ -19,7 +19,7 @@ from torch.nn import functional as F from flash.core.classification import ClassificationTask -from flash.vision.backbones import torchvision_backbone_and_num_features +from flash.vision.backbones import backbone_and_num_features from flash.vision.classification.data import ImageClassificationData, ImageClassificationDataPipeline @@ -57,7 +57,7 @@ def __init__( self.save_hyperparameters() - self.backbone, num_features = torchvision_backbone_and_num_features(backbone, pretrained) + self.backbone, num_features = backbone_and_num_features(backbone, pretrained) self.head = nn.Sequential( nn.AdaptiveAvgPool2d((1, 1)), diff --git a/flash/vision/embedding/image_embedder_model.py b/flash/vision/embedding/image_embedder_model.py index 7a504a2fc58..dd0fc7d6e3f 100644 --- a/flash/vision/embedding/image_embedder_model.py +++ b/flash/vision/embedding/image_embedder_model.py @@ -24,9 +24,8 @@ from flash.core import Task from flash.core.data import TaskDataPipeline from flash.core.data.utils import _contains_any_tensor -from flash.vision.backbones import torchvision_backbone_and_num_features +from flash.vision.backbones import backbone_and_num_features from flash.vision.classification.data import _default_valid_transforms, _pil_loader -from flash.vision.embedding.model_map import _load_bolts_model, _models class ImageEmbedderDataPipeline(TaskDataPipeline): @@ -115,12 +114,7 @@ def __init__( assert pooling_fn in [torch.mean, torch.max] self.pooling_fn = pooling_fn - if backbone in _models: - config = _load_bolts_model(backbone) - self.backbone = config['model'] - num_features = config['num_features'] - else: - self.backbone, num_features = torchvision_backbone_and_num_features(backbone, pretrained) + self.backbone, num_features = backbone_and_num_features(backbone, pretrained) if embedding_dim is None: self.head = nn.Identity() diff --git a/flash/vision/embedding/model_map.py b/flash/vision/embedding/model_map.py deleted file mode 100644 index eb5e92b1441..00000000000 --- a/flash/vision/embedding/model_map.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright The PyTorch Lightning team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from contextlib import suppress - -from pytorch_lightning.utilities import _BOLTS_AVAILABLE -from pytorch_lightning.utilities.exceptions import MisconfigurationException - -if _BOLTS_AVAILABLE: - with suppress(TypeError): - from pl_bolts.models.self_supervised import SimCLR, SwAV - -ROOT_S3_BUCKET = "https://pl-bolts-weights.s3.us-east-2.amazonaws.com" - -# TODO: move this stuff to backbones -# also, we should consider uploading plain pytorch weights so we don't need to rely on bolts to load these -# also mabye just use torchhub for the ssl lib - - -def load_simclr_imagenet(path_or_url: str = f"{ROOT_S3_BUCKET}/simclr/bolts_simclr_imagenet/simclr_imagenet.ckpt"): - simclr = SimCLR.load_from_checkpoint(path_or_url, strict=False) - model_config = {'model': simclr.encoder, 'emb_size': 2048} - return model_config - - -def load_swav_imagenet(path_or_url: str = f"{ROOT_S3_BUCKET}/swav/swav_imagenet/swav_imagenet.pth.tar"): - swav = SwAV.load_from_checkpoint(path_or_url, strict=True) - model_config = {'model': swav.model, 'num_features': 3000} - return model_config - - -_models = { - 'simclr-imagenet': load_simclr_imagenet, - 'swav-imagenet': load_swav_imagenet, -} - - -def _load_bolts_model(name): - if not _BOLTS_AVAILABLE: - raise MisconfigurationException("Bolts isn't installed. Please, use ``pip install lightning-bolts``.") - if name in _models: - return _models[name]() - raise MisconfigurationException("Currently, only `simclr-imagenet` and `swav-imagenet` are supported.") diff --git a/tests/vision/test_download.py b/tests/vision/test_download.py index 2ae3f06d971..f1448a419f7 100644 --- a/tests/vision/test_download.py +++ b/tests/vision/test_download.py @@ -13,9 +13,9 @@ # limitations under the License. import pytest -from flash.vision.embedding.model_map import _load_bolts_model +from flash.vision.backbones import bolts_backbone_and_num_features @pytest.mark.parametrize("name", ['simclr-imagenet', 'swav-imagenet']) def test_load_bolts(name): - _load_bolts_model(name) + bolts_backbone_and_num_features(name) From 3a4f2ce75da6a56d1a0f6510854f44d1f1c63fc9 Mon Sep 17 00:00:00 2001 From: Teddy Koker Date: Wed, 3 Feb 2021 19:07:05 -0500 Subject: [PATCH 05/11] pep8 --- flash/vision/backbones.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/flash/vision/backbones.py b/flash/vision/backbones.py index 13c4c85e63f..1b835c9ea41 100644 --- a/flash/vision/backbones.py +++ b/flash/vision/backbones.py @@ -52,11 +52,6 @@ def bolts_backbone_and_num_features(model_name: str) -> Tuple[nn.Module, int]: (Sequential(...), 1024) """ - models = { - 'simclr-imagenet': load_simclr_imagenet, - 'swav-imagenet': load_swav_imagenet, - } - # TODO: maybe we should plain pytorch weights so we don't need to rely on bolts to load these # also mabye just use torchhub for the ssl lib def load_simclr_imagenet(path_or_url: str = f"{ROOT_S3_BUCKET}/simclr/bolts_simclr_imagenet/simclr_imagenet.ckpt"): @@ -67,6 +62,10 @@ def load_swav_imagenet(path_or_url: str = f"{ROOT_S3_BUCKET}/swav/swav_imagenet/ swav = SwAV.load_from_checkpoint(path_or_url, strict=True) return swav.model, 3000 + models = { + 'simclr-imagenet': load_simclr_imagenet, + 'swav-imagenet': load_swav_imagenet, + } if not _BOLTS_AVAILABLE: raise MisconfigurationException("Bolts isn't installed. Please, use ``pip install lightning-bolts``.") if model_name in models: From c713052f4d101a3cdfb169d2e07f6c26c2fbe27b Mon Sep 17 00:00:00 2001 From: Teddy Koker Date: Wed, 3 Feb 2021 19:09:04 -0500 Subject: [PATCH 06/11] isort --- flash/vision/backbones.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flash/vision/backbones.py b/flash/vision/backbones.py index 1b835c9ea41..f73f77c802d 100644 --- a/flash/vision/backbones.py +++ b/flash/vision/backbones.py @@ -14,10 +14,10 @@ from contextlib import suppress from typing import Tuple -import torch.nn as nn import torchvision -from pytorch_lightning.utilities.exceptions import MisconfigurationException from pytorch_lightning.utilities import _BOLTS_AVAILABLE +from pytorch_lightning.utilities.exceptions import MisconfigurationException +from torch import nn as nn if _BOLTS_AVAILABLE: with suppress(TypeError): From 457933cd75ef51d5fd67c60da1cdbebbc0e24b52 Mon Sep 17 00:00:00 2001 From: Teddy Koker Date: Wed, 3 Feb 2021 19:17:02 -0500 Subject: [PATCH 07/11] fix doctest --- flash/vision/backbones.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flash/vision/backbones.py b/flash/vision/backbones.py index f73f77c802d..63dc15e2e4c 100644 --- a/flash/vision/backbones.py +++ b/flash/vision/backbones.py @@ -46,10 +46,10 @@ def backbone_and_num_features(model_name: str, *args, **kwargs) -> Tuple[nn.Modu def bolts_backbone_and_num_features(model_name: str) -> Tuple[nn.Module, int]: """ - >>> torchvision_backbone_and_num_features('simclr-imagenet') # doctest: +ELLIPSIS - (Sequential(...), 1024) - >>> torchvision_backbone_and_num_features('swav-imagenet') # doctest: +ELLIPSIS - (Sequential(...), 1024) + >>> bolts_backbone_and_num_features('simclr-imagenet') # doctest: +ELLIPSIS + (Sequential(...), 2048) + >>> bolts_backbone_and_num_features('swav-imagenet') # doctest: +ELLIPSIS + (Sequential(...), 3000) """ # TODO: maybe we should plain pytorch weights so we don't need to rely on bolts to load these From ef3d5c1c8389fc6104b04ddfeec0b561bbb3f748 Mon Sep 17 00:00:00 2001 From: Teddy Koker Date: Wed, 3 Feb 2021 19:18:16 -0500 Subject: [PATCH 08/11] fix pytest --- tests/vision/classification/test_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/vision/classification/test_model.py b/tests/vision/classification/test_model.py index b8d04fd47cb..c419a22a966 100644 --- a/tests/vision/classification/test_model.py +++ b/tests/vision/classification/test_model.py @@ -51,7 +51,7 @@ def test_init_train(tmpdir, backbone): def test_non_existent_backbone(): - with pytest.raises(MisconfigurationException): + with pytest.raises(ValueError): ImageClassifier(2, "i am never going to implement this lol") From 0be859bdc5acd210b577db0d327cd4b581b81511 Mon Sep 17 00:00:00 2001 From: Teddy Koker Date: Wed, 3 Feb 2021 19:28:25 -0500 Subject: [PATCH 09/11] simclr fix --- flash/vision/backbones.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flash/vision/backbones.py b/flash/vision/backbones.py index 63dc15e2e4c..c000a3cde44 100644 --- a/flash/vision/backbones.py +++ b/flash/vision/backbones.py @@ -56,7 +56,7 @@ def bolts_backbone_and_num_features(model_name: str) -> Tuple[nn.Module, int]: # also mabye just use torchhub for the ssl lib def load_simclr_imagenet(path_or_url: str = f"{ROOT_S3_BUCKET}/simclr/bolts_simclr_imagenet/simclr_imagenet.ckpt"): simclr = SimCLR.load_from_checkpoint(path_or_url, strict=False) - return simclr.model, 2048 + return simclr.encoder, 2048 def load_swav_imagenet(path_or_url: str = f"{ROOT_S3_BUCKET}/swav/swav_imagenet/swav_imagenet.pth.tar"): swav = SwAV.load_from_checkpoint(path_or_url, strict=True) From c81e9e735b97abd8bafd69c36e552464c0c981cc Mon Sep 17 00:00:00 2001 From: Teddy Koker Date: Wed, 3 Feb 2021 20:45:39 -0500 Subject: [PATCH 10/11] fix doctest --- flash/vision/backbones.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/flash/vision/backbones.py b/flash/vision/backbones.py index c000a3cde44..d733de9ee3f 100644 --- a/flash/vision/backbones.py +++ b/flash/vision/backbones.py @@ -56,11 +56,15 @@ def bolts_backbone_and_num_features(model_name: str) -> Tuple[nn.Module, int]: # also mabye just use torchhub for the ssl lib def load_simclr_imagenet(path_or_url: str = f"{ROOT_S3_BUCKET}/simclr/bolts_simclr_imagenet/simclr_imagenet.ckpt"): simclr = SimCLR.load_from_checkpoint(path_or_url, strict=False) - return simclr.encoder, 2048 + # remove the last two layers & turn it into a Sequential model + backbone = nn.Sequential(*list(simclr.encoder.children())[:-2]) + return backbone, 2048 def load_swav_imagenet(path_or_url: str = f"{ROOT_S3_BUCKET}/swav/swav_imagenet/swav_imagenet.pth.tar"): swav = SwAV.load_from_checkpoint(path_or_url, strict=True) - return swav.model, 3000 + # remove the last two layers & turn it into a Sequential model + backbone = nn.Sequential(*list(swav.model.children())[:-2]) + return backbone, 3000 models = { 'simclr-imagenet': load_simclr_imagenet, From ed34ba53b676b3a43134dbaaaa0f9cfec7e71ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Thu, 4 Feb 2021 19:53:57 +0100 Subject: [PATCH 11/11] Remove suppress --- flash/vision/backbones.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flash/vision/backbones.py b/flash/vision/backbones.py index d733de9ee3f..c492f0015db 100644 --- a/flash/vision/backbones.py +++ b/flash/vision/backbones.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from contextlib import suppress from typing import Tuple import torchvision @@ -20,8 +19,7 @@ from torch import nn as nn if _BOLTS_AVAILABLE: - with suppress(TypeError): - from pl_bolts.models.self_supervised import SimCLR, SwAV + from pl_bolts.models.self_supervised import SimCLR, SwAV ROOT_S3_BUCKET = "https://pl-bolts-weights.s3.us-east-2.amazonaws.com"