Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cve_feed: manage http request and add request log #1407

Merged
merged 5 commits into from
Dec 13, 2024
Merged

cve_feed: manage http request and add request log #1407

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8aff713
cve_feed: manage http request and add request log
kledo-lyft Dec 13, 2024
d27cce7
Wrap around the session
kledo-lyft Dec 13, 2024
2909248
Remove unnecessary newlines
kledo-lyft Dec 13, 2024
de77c8c
Fix unit-tests
kledo-lyft Dec 13, 2024
1a3288d
Merge branch 'master' into cve-feed-memory-debug
kledo-lyft Dec 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 25 additions & 23 deletions cartography/intel/cve/feed.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,30 +85,32 @@ def _call_cves_api(url: str, api_key: str | None, params: Dict[str, Any]) -> Dic
)
results: Dict[Any, Any] = dict()

while params["resultsPerPage"] > 0 or params["startIndex"] < totalResults:
try:
res = requests.get(
url, params=params, headers=headers, timeout=CONNECT_AND_READ_TIMEOUT,
)
res.raise_for_status()
except requests.exceptions.HTTPError:
logger.error(
f"Failed to get CVE data from NIST NVD API {res.status_code} : {res.text}",
)
retries += 1
if retries >= MAX_RETRIES:
raise
# Exponential backoff
sleep_time *= 2
with requests.Session() as session:
while params["resultsPerPage"] > 0 or params["startIndex"] < totalResults:
logger.info(f"Calling NIST NVD API at {url} with params {params}")
try:
res = session.get(
url, params=params, headers=headers, timeout=CONNECT_AND_READ_TIMEOUT,
)
res.raise_for_status()
data = res.json()
except requests.exceptions.HTTPError:
logger.error(
f"Failed to get CVE data from NIST NVD API {res.status_code} : {res.text}",
Comment on lines +98 to +99
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think the spikes of memory are falling in this exception case.
Spikes are not raising an explicit HTTP exception, instead a runtime one (I guess, but I haven't find any logs from kibana / OpenSearch).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

discussing offline since it's related to Lyft

)
retries += 1
if retries >= MAX_RETRIES:
raise
# Exponential backoff
sleep_time *= 2
time.sleep(sleep_time)
continue
_map_cve_dict(results, data)
totalResults = data["totalResults"]
params["resultsPerPage"] = data["resultsPerPage"]
params["startIndex"] += data["resultsPerPage"]
retries = 0
time.sleep(sleep_time)
continue
data = res.json()
_map_cve_dict(results, data)
totalResults = data["totalResults"]
params["resultsPerPage"] = data["resultsPerPage"]
params["startIndex"] += data["resultsPerPage"]
retries = 0
time.sleep(sleep_time)
return results


Expand Down
13 changes: 9 additions & 4 deletions tests/unit/cartography/intel/cve/test_feed.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from unittest.mock import Mock
from unittest.mock import patch

import pytest
import requests

from cartography.intel.cve.feed import _call_cves_api
Expand All @@ -18,9 +19,14 @@
API_KEY = "nvd_api_key"


@patch("cartography.intel.cve.feed.DEFAULT_SLEEP_TIME", 0)
@patch("cartography.intel.cve.feed.requests.get")
def test_call_cves_api(mock_get: Mock):
@pytest.fixture
def mock_get():
with patch("cartography.intel.cve.feed.requests.Session") as mock_session:
session_mock = mock_session.return_value.__enter__.return_value
yield session_mock.get


def test_call_cves_api(mock_get):
# Arrange
mock_response_1 = Mock()
mock_response_1.status_code = 200
Expand Down Expand Up @@ -140,7 +146,6 @@ def test_call_cves_api(mock_get: Mock):


@patch("cartography.intel.cve.feed.DEFAULT_SLEEP_TIME", 0)
@patch("cartography.intel.cve.feed.requests.get")
def test_call_cves_api_with_error(mock_get: Mock):
# Arrange
mock_response = Mock()
Expand Down
Loading