add cache clearing and testing

src/imf_reader/sdr/__init__.py

@@ -45,8 +45,15 @@
 sdr.fetch_exchange_rates("USD")
 ```
 
+Clear cached data
+
+```python
+sdr.clear_cache()
+```
+
 """
 
 from imf_reader.sdr.read_interest_rate import fetch_interest_rates
 from imf_reader.sdr.read_exchange_rate import fetch_exchange_rates
 from imf_reader.sdr.read_announcements import fetch_allocations_holdings
+from imf_reader.sdr.clear_cache import clear_cache

src/imf_reader/sdr/clear_cache.py

@@ -0,0 +1,23 @@
+from imf_reader.sdr.read_announcements import (
+    get_holdings_and_allocations_data,
+    get_latest_date,
+)
+from imf_reader.sdr.read_exchange_rate import fetch_exchange_rates
+from imf_reader.sdr.read_interest_rate import fetch_interest_rates
+from imf_reader.config import logger
+
+
+def clear_cache():
+    """Clear the cache for all lru_cache-decorated functions in the 3 sdr modules."""
+
+    # clear cache from read_announcements module
+    get_holdings_and_allocations_data.cache_clear()
+    get_latest_date.cache_clear()
+
+    # clear cache from read_exchange_rate module
+    fetch_exchange_rates.cache_clear()
+
+    # clear cache from read_interest_rate module
+    fetch_interest_rates.cache_clear()
+
+    logger.info("Cache cleared")

src/imf_reader/sdr/read_announcements.py

@@ -12,7 +12,7 @@
 from datetime import datetime
 
 from imf_reader.utils import make_request
-from imf_reader.config import logger, NoDataError
+from imf_reader.config import logger
 
 BASE_URL = "https://www.imf.org/external/np/fin/tad/"
 MAIN_PAGE_URL = "https://www.imf.org/external/np/fin/tad/extsdr1.aspx"
@@ -54,7 +54,10 @@ def format_date(month: int, year: int) -> str:
 
 
 @lru_cache
-def get_holdings_and_allocations_data(year: int, month: int):
+def get_holdings_and_allocations_data(
+    year: int,
+    month: int,
+):
     """Get sdr allocations and holdings data for a given month and year"""
 
     date = format_date(month, year)

tests/test_sdr/test_read_announcements.py

@@ -1,102 +1,240 @@
-import unittest
-from unittest.mock import patch, MagicMock
+from unittest.mock import patch, Mock
+import pytest
 import pandas as pd
-from bs4 import BeautifulSoup
-from imf_reader.sdr import read_announcements
-
-
-class TestReadAnnouncements(unittest.TestCase):
+from imf_reader import sdr
+from imf_reader.sdr.read_announcements import (
+    read_tsv,
+    clean_df,
+    format_date,
+    get_holdings_and_allocations_data,
+    get_latest_date,
+    fetch_allocations_holdings,
+    BASE_URL,
+    MAIN_PAGE_URL,
+)
+
+
+@pytest.fixture
+def input_df():
+    df = pd.DataFrame(
+        {
+            "SDR Allocations and Holdings": [
+                "for all members as of June 30, 2020",
+                "(in SDRs)",
+                "Members\tSDR Holdings\tSDR Allocations",
+                "Spain\t123\t456",
+                "Total\t321\t654",
+            ]
+        }
+    )
+    return df
+
+
+class TestReadAnnouncements:
     """Tests functions in the read_announcements module."""
 
+    @pytest.fixture(autouse=True)
+    def auto_clear_cache(self):
+        """Clear cache before each test."""
+        sdr.clear_cache()
+
     @patch("pandas.read_csv")
-    def test_read_tsv(self, mock_read_csv):
-        """Ensure read_tsv processes well-formated tsv correctly and raises ValueError on malformed data."""
-        # Mock successful TSV read
+    def test_read_tsv_success(self, mock_read_csv):
+        """Test read_tsv successfully processes a well-formatted TSV."""
         mock_read_csv.return_value = pd.DataFrame({"A": [1], "B": [2]})
-        result = read_announcements.read_tsv("mock_url")
-        self.assertTrue(isinstance(result, pd.DataFrame))
+        result = read_tsv("mock_url")
+        assert isinstance(result, pd.DataFrame)
+        assert result.equals(pd.DataFrame({"A": [1], "B": [2]}))
 
-        # Mock failure
+    @patch("pandas.read_csv")
+    def test_read_tsv_failure(self, mock_read_csv):
+        """Test read_tsv raises ValueError on malformed data."""
         mock_read_csv.side_effect = pd.errors.ParserError
-        with self.assertRaises(ValueError):
-            read_announcements.read_tsv("mock_url")
+        with pytest.raises(ValueError, match="SDR _data not available for this date"):
+            read_tsv("mock_url")
 
-    def test_clean_df_correct_format(self):
+    def test_clean_df_correct_format(self, input_df):
         """Test clean_df with the expected format."""
         # Mock input DataFrame
-        raw_data = pd.DataFrame({0: ["", "", "", "Country A\t$100\t$200"]})
-        expected_data = pd.DataFrame(
+
+        expected_df = pd.DataFrame(
             {
-                "entity": ["Country A", "Country A"],
-                "indicator": ["holdings", "allocations"],
-                "value": [100, 200],
+                "entity": ["Spain", "Total", "Spain", "Total"],
+                "indicator": ["holdings", "holdings", "allocations", "allocations"],
+                "value": [123, 321, 456, 654],
             }
         )
 
-        result = read_announcements.clean_df(raw_data)
-        pd.testing.assert_frame_equal(result, expected_data)
-
-    def test_clean_df_empty(self):
-        """Test clean_df with an empty DataFrame"""
-        input_df = pd.DataFrame()
-        with self.assertRaises(IndexError):
-            read_announcements.clean_df(input_df)
-
-    def test_format_date(self):
-        """Test format_date computes last day of a given month/year."""
-        self.assertEqual(read_announcements.format_date(2, 2024), "2024-2-29")
-        self.assertEqual(read_announcements.format_date(1, 2023), "2023-1-31")
+        result = clean_df(input_df)
+        pd.testing.assert_frame_equal(result, expected_df)
+
+    @pytest.mark.parametrize(
+        "month, year, expected",
+        [
+            (1, 2024, "2024-1-31"),  # January 2024
+            (2, 2024, "2024-2-29"),  # February (Leap year)
+            (2, 2023, "2023-2-28"),  # February (Non-leap year)
+            (4, 2024, "2024-4-30"),  # April
+            (12, 2023, "2023-12-31"),  # December
+        ],
+    )
+    def test_format_date_valid(self, month, year, expected):
+        """Test format_date returns the correct last day of the month."""
+        assert format_date(month, year) == expected
+
+    def test_format_date_invalid_month(self):
+        """Test format_date raises ValueError for invalid month input."""
+        with pytest.raises(ValueError):
+            format_date(0, 2024)  # Invalid month (0)
+
+        with pytest.raises(ValueError):
+            format_date(13, 2024)  # Invalid month (13)
 
     @patch("imf_reader.sdr.read_announcements.read_tsv")
     @patch("imf_reader.sdr.read_announcements.clean_df")
-    def test_get_holdings_and_allocations_data(self, mock_clean_df, mock_read_tsv):
-        """Test get_holdings_and_allocations_data caches data properly."""
-        mock_read_tsv.return_value = pd.DataFrame()
-        mock_clean_df.return_value = pd.DataFrame({"data": [1]})
+    def test_get_holdings_and_allocations_data_success(
+        self, mock_clean_df, mock_read_tsv, input_df
+    ):
+        """Test get_holdings_and_allocations_data successfully returns processed data."""
+        # Mock the read_tsv output
+        mock_read_tsv.return_value = input_df
+        # Mock the clean_df output
+        mock_clean_df.return_value = clean_df(input_df)
+
+        # Expected final output
+        expected_df = pd.DataFrame(
+            {
+                "entity": ["Spain", "Total", "Spain", "Total"],
+                "indicator": ["holdings", "holdings", "allocations", "allocations"],
+                "value": [123, 321, 456, 654],
+                "date": [pd.to_datetime("2024-02-29")] * 4,
+            }
+        )
 
-        result = read_announcements.get_holdings_and_allocations_data(2024, 11)
-        self.assertTrue("data" in result.columns)
+        # Call the function
+        result = get_holdings_and_allocations_data(2024, 2)
+
+        # Assertions
+        mock_read_tsv.assert_called_once_with(
+            f"{BASE_URL}extsdr2.aspx?date1key=2024-2-29&tsvflag=Y"
+        )
+        mock_clean_df.assert_called_once()
+        pd.testing.assert_frame_equal(result, expected_df)
+
+    @patch(
+        "imf_reader.sdr.read_announcements.read_tsv",
+        side_effect=ValueError("Data not available"),
+    )
+    def test_get_holdings_and_allocations_data_failure(self, mock_read_tsv):
+        """Test get_holdings_and_allocations_data raises ValueError when read_tsv fails."""
+        with pytest.raises(ValueError, match="Data not available"):
+            get_holdings_and_allocations_data(2024, 2)
 
     @patch("imf_reader.sdr.read_announcements.make_request")
-    @patch("bs4.BeautifulSoup")
-    def test_get_latest_date(self, mock_soup, mock_make_request):
-        """Test correct extraction of get_latest_date."""
-        # Simulate HTML content
-        html_content = """
+    def test_get_latest_date_success(self, mock_make_request):
+        """Test get_latest_date successfully returns the latest date."""
+
+        # Mock HTML content
+        mock_html_content = """
         <html>
             <body>
-                <table></table><table></table><table></table><table></table>
                 <table>
-                    <tr><td>Header</td></tr>
-                    <tr><td>November 30, 2024</td></tr>
+                    <tr></tr> <!-- Index 0 -->
+                    <tr><td>November 30, 2023</td></tr> <!-- Index 1: Latest date -->
+                </table>
+                <table></table>
+                <table></table>
+                <table></table>
+                <table>
+                    <tr></tr>
+                    <tr><td>November 30, 2023</td></tr>
                 </table>
             </body>
         </html>
         """
-        mock_make_request.return_value.content = html_content
-        mock_soup.return_value = BeautifulSoup(html_content, "html.parser")
+
+        # Mock the make_request response
+        mock_response = Mock()
+        mock_response.content = mock_html_content
+        mock_make_request.return_value = mock_response
 
         # Call the function
-        year, month = read_announcements.get_latest_date()
+        result = get_latest_date()
 
-        # Assert expected output
-        self.assertEqual((year, month), (2024, 11))
+        # Assertions
+        mock_make_request.assert_called_once_with(MAIN_PAGE_URL)
+        assert result == (2023, 11)  # Expected year and month
+
+    @patch("imf_reader.sdr.read_announcements.make_request")
+    def test_get_latest_date_invalid_html(self, mock_make_request):
+        """Test get_latest_date raises an error when HTML parsing fails."""
+
+        # Mock malformed HTML content
+        mock_response = Mock()
+        mock_response.content = "<html><body></body></html>"  # Missing tables
+        mock_make_request.return_value = mock_response
+
+        # Call the function and expect an IndexError
+        with pytest.raises(IndexError):
+            get_latest_date()
 
     @patch("imf_reader.sdr.read_announcements.get_latest_date")
+    @patch("imf_reader.sdr.read_announcements.clean_df")
+    @patch("imf_reader.sdr.read_announcements.read_tsv")
     @patch("imf_reader.sdr.read_announcements.get_holdings_and_allocations_data")
-    def test_fetch_allocations_holdings(self, mock_get_data, mock_get_latest_date):
-        """Ensure fetch_allocations_holdings fetches data for the provided date or the latest date."""
-        # Mock latest date and data fetch
-        mock_get_latest_date.return_value = (2024, 11)
-        mock_get_data.return_value = pd.DataFrame({"data": [1]})
+    def test_fetch_allocations_holdings_default_date(
+        self,
+        mock_get_holdings_and_allocations_data,
+        mock_read_tsv,
+        mock_clean_df,
+        mock_get_latest_date,
+        input_df,
+    ):
+        """Test fetch_allocations_holdings when no date is provided."""
+        # Mock get_latest_date to return a specific date
+        mock_get_latest_date.return_value = (2, 2024)
+
+        # Mock read_tsv
+        mock_read_tsv.return_value = input_df
+
+        # Mock clean_df to return a cleaned DataFrame
+        cleaned_df = pd.DataFrame(
+            {
+                "entity": ["Spain", "Total", "Spain", "Total"],
+                "indicator": ["holdings", "holdings", "allocations", "allocations"],
+                "value": [123, 321, 456, 654],
+                "date": [pd.to_datetime("2024-02-29")] * 4,
+            }
+        )
+        mock_clean_df.return_value = cleaned_df
 
-        result = read_announcements.fetch_allocations_holdings()
-        self.assertTrue("data" in result.columns)
+        # Mock get_holdings_and_allocations_data to return the final DataFrame
+        mock_get_holdings_and_allocations_data.return_value = cleaned_df
 
-        # Test with specific date
-        result = read_announcements.fetch_allocations_holdings((2023, 10))
-        mock_get_data.assert_called_with(2023, 10)
+        # Call the function
+        result = fetch_allocations_holdings()
 
+        # Assertions
+        mock_get_latest_date.assert_called_once()  # Ensure get_latest_date was called
+        mock_get_holdings_and_allocations_data.assert_called_once_with(
+            2, 2024
+        )  # Ensure the correct call
+        pd.testing.assert_frame_equal(
+            result, cleaned_df
+        )  # Ensure the result matches the cleaned data
 
-if __name__ == "__main__":
-    unittest.main()
+    @patch("imf_reader.sdr.read_announcements.get_holdings_and_allocations_data")
+    @patch("imf_reader.sdr.read_announcements.get_latest_date")
+    def test_fetch_allocations_holdings_failure(
+        self, mock_get_latest_date, mock_get_holdings_data
+    ):
+        """Test fetch_allocations_holdings raises ValueError when data fetching fails."""
+        # Mock get_latest_date to return a date
+        mock_get_latest_date.return_value = (2, 2024)
+
+        # Simulate a failure in get_holdings_and_allocations_data
+        mock_get_holdings_data.side_effect = ValueError("Data not available")
+
+        # Assertions
+        with pytest.raises(ValueError, match="Data not available"):
+            fetch_allocations_holdings()

tests/test_sdr/test_read_exchange_rate.py

@@ -2,6 +2,7 @@
 import pytest
 import requests
 import pandas as pd
+from imf_reader import sdr
 from imf_reader.sdr.read_exchange_rate import (
     preprocess_data,
     fetch_exchange_rates,
@@ -31,9 +32,9 @@ def input_df():
 class TestExchangeRateModule:
 
     @pytest.fixture(autouse=True)
-    def clear_cache(self):
+    def auto_clear_cache(self):
         """Clear cache before each test."""
-        fetch_exchange_rates.cache_clear()
+        sdr.clear_cache()
 
     @patch("requests.post")
     def test_get_exchange_rates_data_success(self, mock_post):
@@ -191,7 +192,6 @@ def test_fetch_exchange_rates(self, mock_parse_data, mock_get_data, input_df):
         """Test fetching exchange rates"""
         # Mock return values for the patched functions
         mock_get_data.return_value = input_df
-        mock_get_data.return_value = input_df
         mock_parse_data.return_value = pd.DataFrame(
             {"date": pd.to_datetime(["2023-11-30"]), "exchange_rate": [0.123]}
         )