From cbab6bd329f882ecf3e00c78503910aa4b91b242 Mon Sep 17 00:00:00 2001 From: lockefox Date: Sun, 20 Jan 2019 10:00:54 -0800 Subject: [PATCH 1/4] fixing ESI endpoint ROOT --- publicAPI/crest_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/publicAPI/crest_utils.py b/publicAPI/crest_utils.py index 2fa9b3d..c5d31ce 100644 --- a/publicAPI/crest_utils.py +++ b/publicAPI/crest_utils.py @@ -255,7 +255,8 @@ def fetch_crest_endpoint_get(): return data -ESI_BASE = 'https://esi.tech.ccp.is/latest/' +# ESI_BASE = 'https://esi.tech.ccp.is/latest/' +ESI_BASE = 'https://esi.evetech.net/latest/' def fetch_esi_endpoint( endpoint_name, esi_base=ESI_BASE, From a0378bea8cda99547cded8096bd471a42ba489c9 Mon Sep 17 00:00:00 2001 From: lockefox Date: Sun, 20 Jan 2019 10:03:25 -0800 Subject: [PATCH 2/4] removing branches from travis plan --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 093f465..21af4b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,10 +11,6 @@ script: - "python scripts/manage_api.py --testkey" - "python scripts/create_splitcache.py --regions=10000002 --type=34,35 --range=30 --db=publicAPI/cache/travis_splitcache.json --source=esi" - "python setup.py test" -branches: - only: - - master - - v1.2_withsplit after_success: - "pip install python-coveralls" - "coveralls" From 3cfe62475a5abda0d54dff3efe5fa044041d9e78 Mon Sep 17 00:00:00 2001 From: lockefox Date: Sun, 20 Jan 2019 17:58:35 -0800 Subject: [PATCH 3/4] deprecating split functionality + EMD --- publicAPI/crest_endpoint.py | 76 ++++++------- publicAPI/forecast_utils.py | 2 +- publicAPI/split_utils.py | 4 +- tests/test_forecast_utils.py | 1 + tests/test_split_utils.py | 213 ++++++++++++++++++----------------- 5 files changed, 152 insertions(+), 144 deletions(-) diff --git a/publicAPI/crest_endpoint.py b/publicAPI/crest_endpoint.py index 03c782e..8abeea0 100644 --- a/publicAPI/crest_endpoint.py +++ b/publicAPI/crest_endpoint.py @@ -15,7 +15,7 @@ import publicAPI.api_utils as api_utils import publicAPI.exceptions as exceptions import publicAPI.config as api_config -import publicAPI.split_utils as split_utils +# import publicAPI.split_utils as split_utils import prosper.common.prosper_logging as p_logging import prosper.common.prosper_config as p_config @@ -127,21 +127,21 @@ def get(self, return_type): ## Fetch CREST ## try: #LOGGER.info(api_config.SPLIT_INFO) - if args.get('typeID') in api_config.SPLIT_INFO: - self.logger.info('FORK: using split utility') - data = split_utils.fetch_split_history( - args.get('regionID'), - args.get('typeID'), - config=api_config.CONFIG, - logger=self.logger, - ) - else: - data = crest_utils.fetch_market_history( - args.get('regionID'), - args.get('typeID'), - config=api_config.CONFIG, - logger=LOGGER - ) + # if args.get('typeID') in api_config.SPLIT_INFO: + # self.logger.info('FORK: using split utility') + # data = split_utils.fetch_split_history( + # args.get('regionID'), + # args.get('typeID'), + # config=api_config.CONFIG, + # logger=self.logger, + # ) + # else: + data = crest_utils.fetch_market_history( + args.get('regionID'), + args.get('typeID'), + config=api_config.CONFIG, + logger=LOGGER + ) data = crest_utils.data_to_ohlc(data) except exceptions.ValidatorException as err: #pragma: no cover self.logger.error( @@ -304,28 +304,28 @@ def get(self, return_type): ## No cache, get data ## try: - if args.get('typeID') in api_config.SPLIT_INFO: - LOGGER.info('FORK: using split utility') - data = split_utils.fetch_split_history( - args.get('regionID'), - args.get('typeID'), - data_range=api_config.MAX_RANGE, - config=api_config.CONFIG, - logger=self.logger, - ) - data.sort_values( - by='date', - ascending=True, - inplace=True - ) - else: - data = forecast_utils.fetch_extended_history( - args.get('regionID'), - args.get('typeID'), - data_range=api_config.MAX_RANGE, - config=api_config.CONFIG, - logger=self.logger, - ) + # if args.get('typeID') in api_config.SPLIT_INFO: + # LOGGER.info('FORK: using split utility') + # data = split_utils.fetch_split_history( + # args.get('regionID'), + # args.get('typeID'), + # data_range=api_config.MAX_RANGE, + # config=api_config.CONFIG, + # logger=self.logger, + # ) + # data.sort_values( + # by='date', + # ascending=True, + # inplace=True + # ) + # else: + data = forecast_utils.fetch_extended_history( + args.get('regionID'), + args.get('typeID'), + data_range=api_config.MAX_RANGE, + config=api_config.CONFIG, + logger=self.logger, + ) data = forecast_utils.build_forecast( data, api_config.MAX_RANGE diff --git a/publicAPI/forecast_utils.py b/publicAPI/forecast_utils.py index eca6d37..ced0cb5 100644 --- a/publicAPI/forecast_utils.py +++ b/publicAPI/forecast_utils.py @@ -271,7 +271,7 @@ def fetch_market_history_emd( 'region_ids': region_id, 'type_ids': type_id, 'days': data_range, - 'char_name': config.get('GLOBAL', 'useragent_short') + 'char_name': 'lockefox', } headers = { 'User-Agent': config.get('GLOBAL', 'useragent') diff --git a/publicAPI/split_utils.py b/publicAPI/split_utils.py index 41b8e3b..801b601 100644 --- a/publicAPI/split_utils.py +++ b/publicAPI/split_utils.py @@ -15,6 +15,8 @@ import publicAPI.exceptions as exceptions HERE = path.abspath(path.dirname(__file__)) + +DeprecationWarning('PLEX split happened too far back for dependencies to work') class SplitInfo(object): """utility for managing split information""" def __init__(self, json_entry=None): @@ -293,7 +295,7 @@ def execute_split( def fetch_split_history( region_id, type_id, - fetch_source=api_config.SwitchCCPSource.EMD, + fetch_source=api_config.SwitchCCPSource.ESI, data_range=400, config=api_config.CONFIG, logger=logging.getLogger('publicAPI') diff --git a/tests/test_forecast_utils.py b/tests/test_forecast_utils.py index e3cb3eb..9bed982 100644 --- a/tests/test_forecast_utils.py +++ b/tests/test_forecast_utils.py @@ -20,6 +20,7 @@ CONFIG = helpers.get_config(CONFIG_FILENAME) +@pytest.mark.skip('EMD No longer reliable') def test_fetch_emd_history(config=CONFIG): """happypath test for `fetch_market_history_emd`""" data = forecast_utils.fetch_market_history_emd( diff --git a/tests/test_split_utils.py b/tests/test_split_utils.py index 0054650..fc5bd4c 100644 --- a/tests/test_split_utils.py +++ b/tests/test_split_utils.py @@ -159,6 +159,7 @@ def test_split_history_throws(): SPLIT_CACHE_FILE = path.join( ROOT, 'publicAPI', 'cache', TEST_CONFIG.get('TEST', 'splitcache_file') ) +@pytest.mark.skip('EMD No longer reliable') def test_fetch_cache_data(): """fetch data from cache and make sure shape is correct""" cache_data = split_utils.fetch_split_cache_data( @@ -177,7 +178,7 @@ def test_fetch_cache_fail(): int(TEST_CONFIG.get('TEST', 'bad_typeid')), #split_cache_file=SPLIT_CACHE_FILE ) - +@pytest.mark.skip('EMD No longer reliable') def test_execute_split_forward(): """check if execute_split works as expected""" split_obj = split_utils.SplitInfo(DEMO_SPLIT) @@ -205,6 +206,7 @@ def test_execute_split_forward(): vol_diff = abs(split_data[col_name] - (cache_data[col_name] * vol_mod)) assert vol_diff.max() < float(TEST_CONFIG.get('TEST', 'float_limit')) +@pytest.mark.skip('EMD No longer reliable') def test_execute_split_backwards(): """check if execute_split works as expected""" split_obj = split_utils.SplitInfo(DEMO_UNSPLIT) @@ -231,7 +233,7 @@ def test_execute_split_backwards(): vol_diff = abs(split_data[col_name] - (cache_data[col_name] * vol_mod)) assert vol_diff.max() < float(TEST_CONFIG.get('TEST', 'float_limit')) -@pytest.mark.incremental +# @pytest.mark.skip('EMD No longer reliable') class TestNoSplit: """validate behavior if there's no split to perform""" test_type_id = DEMO_UNSPLIT['type_id'] @@ -251,6 +253,7 @@ def test_future_split_esi(self): ) ) + @pytest.mark.skip('EMD No longer reliable') def test_future_split_emd(self): """valdiate with EMD source""" test_data_emd = split_utils.fetch_split_history( @@ -268,9 +271,11 @@ def test_future_split_emd(self): ) assert test_data_emd.equals(forecast_utils.parse_emd_data(emd_data_raw['result'])) + @pytest.mark.skip('EMD No longer reliable') def test_short_split(self): """make sure escaped if split was too far back""" - short_days = floor(DAYS_SINCE_SPLIT/2) + # short_days = floor(DAYS_SINCE_SPLIT/2) + short_days = 100 test_data_emd = split_utils.fetch_split_history( TEST_CONFIG.get('TEST', 'region_id'), DEMO_SPLIT['type_id'], @@ -392,104 +397,104 @@ class TestSplit: """test end-to-end behavior on fetch_split_history""" test_type_id = DEMO_SPLIT['type_id'] test_original_id = DEMO_SPLIT['original_id'] - def test_forward_happypath_esi(self): - """test a forward-split: ESI""" - split_obj = split_utils.SplitInfo(DEMO_SPLIT) - raw_esi_data1 = crest_utils.fetch_market_history( - TEST_CONFIG.get('TEST', 'region_id'), - self.test_type_id, - config=ROOT_CONFIG - ) - raw_esi_data2 = crest_utils.fetch_market_history( - TEST_CONFIG.get('TEST', 'region_id'), - self.test_original_id, - config=ROOT_CONFIG - ) - split_data = split_utils.fetch_split_history( - TEST_CONFIG.get('TEST', 'region_id'), - DEMO_SPLIT['type_id'], - fetch_source=api_utils.SwitchCCPSource.ESI, - config=ROOT_CONFIG - ) - #split_data.to_csv('split_data_esi.csv', index=False) - - ## Doctor data for testing ## - min_split_date = split_data.date.min() - raw_esi_data1 = prep_raw_data( - raw_esi_data1.copy(), - min_split_date - ) - raw_esi_data2 = prep_raw_data( - raw_esi_data2.copy(), - min_split_date - ) - - pre_split_data = split_data[split_data.date <= split_obj.date_str].reset_index() - pre_raw_data = raw_esi_data2[raw_esi_data2.date <= split_obj.date_str].reset_index() - post_split_data = split_data[split_data.date > split_obj.date_str].reset_index() - post_raw_data = raw_esi_data1[raw_esi_data1.date > split_obj.date_str].reset_index() - - ## Validate pre/post Split values ## - validate_plain_data( - post_raw_data, - post_split_data - ) - - validate_split_data( - pre_raw_data, - pre_split_data, - split_obj - ) - - def test_forward_happypath_emd(self): - """test a forward-split: emd""" - split_obj = split_utils.SplitInfo(DEMO_SPLIT) - raw_emd_data = forecast_utils.fetch_market_history_emd( - TEST_CONFIG.get('TEST', 'region_id'), - self.test_type_id, - data_range=TEST_CONFIG.get('TEST', 'history_count'), - config=ROOT_CONFIG - ) - raw_emd_data1 = forecast_utils.parse_emd_data(raw_emd_data['result']) - raw_emd_data = forecast_utils.fetch_market_history_emd( - TEST_CONFIG.get('TEST', 'region_id'), - self.test_original_id, - data_range=TEST_CONFIG.get('TEST', 'history_count'), - config=ROOT_CONFIG - ) - raw_emd_data2 = forecast_utils.parse_emd_data(raw_emd_data['result']) - - split_data = split_utils.fetch_split_history( - TEST_CONFIG.get('TEST', 'region_id'), - DEMO_SPLIT['type_id'], - api_utils.SwitchCCPSource.EMD, - config=ROOT_CONFIG - ) - - ## Doctor data for testing ## - min_split_date = split_data.date.min() - raw_emd_data1 = prep_raw_data( - raw_emd_data1.copy(), - min_split_date - ) - raw_emd_data2 = prep_raw_data( - raw_emd_data2.copy(), - min_split_date - ) - - pre_split_data = split_data[split_data.date <= split_obj.date_str].reset_index() - pre_raw_data = raw_emd_data2[raw_emd_data2.date <= split_obj.date_str].reset_index() - post_split_data = split_data[split_data.date > split_obj.date_str].reset_index() - post_raw_data = raw_emd_data1[raw_emd_data1.date > split_obj.date_str].reset_index() - - ## Validate pre/post Split values ## - validate_plain_data( - post_raw_data, - post_split_data - ) - - validate_split_data( - pre_raw_data, - pre_split_data, - split_obj - ) + # def test_forward_happypath_esi(self): + # """test a forward-split: ESI""" + # split_obj = split_utils.SplitInfo(DEMO_SPLIT) + # raw_esi_data1 = crest_utils.fetch_market_history( + # TEST_CONFIG.get('TEST', 'region_id'), + # self.test_type_id, + # config=ROOT_CONFIG + # ) + # raw_esi_data2 = crest_utils.fetch_market_history( + # TEST_CONFIG.get('TEST', 'region_id'), + # self.test_original_id, + # config=ROOT_CONFIG + # ) + # split_data = split_utils.fetch_split_history( + # TEST_CONFIG.get('TEST', 'region_id'), + # DEMO_SPLIT['type_id'], + # fetch_source=api_utils.SwitchCCPSource.ESI, + # config=ROOT_CONFIG + # ) + # #split_data.to_csv('split_data_esi.csv', index=False) + + # ## Doctor data for testing ## + # min_split_date = split_data.date.min() + # raw_esi_data1 = prep_raw_data( + # raw_esi_data1.copy(), + # min_split_date + # ) + # raw_esi_data2 = prep_raw_data( + # raw_esi_data2.copy(), + # min_split_date + # ) + + # pre_split_data = split_data[split_data.date <= split_obj.date_str].reset_index() + # pre_raw_data = raw_esi_data2[raw_esi_data2.date <= split_obj.date_str].reset_index() + # post_split_data = split_data[split_data.date > split_obj.date_str].reset_index() + # post_raw_data = raw_esi_data1[raw_esi_data1.date > split_obj.date_str].reset_index() + + # ## Validate pre/post Split values ## + # validate_plain_data( + # post_raw_data, + # post_split_data + # ) + + # validate_split_data( + # pre_raw_data, + # pre_split_data, + # split_obj + # ) + + # def test_forward_happypath_emd(self): + # """test a forward-split: emd""" + # split_obj = split_utils.SplitInfo(DEMO_SPLIT) + # raw_emd_data = forecast_utils.fetch_market_history_emd( + # TEST_CONFIG.get('TEST', 'region_id'), + # self.test_type_id, + # data_range=TEST_CONFIG.get('TEST', 'history_count'), + # config=ROOT_CONFIG + # ) + # raw_emd_data1 = forecast_utils.parse_emd_data(raw_emd_data['result']) + # raw_emd_data = forecast_utils.fetch_market_history_emd( + # TEST_CONFIG.get('TEST', 'region_id'), + # self.test_original_id, + # data_range=TEST_CONFIG.get('TEST', 'history_count'), + # config=ROOT_CONFIG + # ) + # raw_emd_data2 = forecast_utils.parse_emd_data(raw_emd_data['result']) + + # split_data = split_utils.fetch_split_history( + # TEST_CONFIG.get('TEST', 'region_id'), + # DEMO_SPLIT['type_id'], + # api_utils.SwitchCCPSource.EMD, + # config=ROOT_CONFIG + # ) + + # ## Doctor data for testing ## + # min_split_date = split_data.date.min() + # raw_emd_data1 = prep_raw_data( + # raw_emd_data1.copy(), + # min_split_date + # ) + # raw_emd_data2 = prep_raw_data( + # raw_emd_data2.copy(), + # min_split_date + # ) + + # pre_split_data = split_data[split_data.date <= split_obj.date_str].reset_index() + # pre_raw_data = raw_emd_data2[raw_emd_data2.date <= split_obj.date_str].reset_index() + # post_split_data = split_data[split_data.date > split_obj.date_str].reset_index() + # post_raw_data = raw_emd_data1[raw_emd_data1.date > split_obj.date_str].reset_index() + + # ## Validate pre/post Split values ## + # validate_plain_data( + # post_raw_data, + # post_split_data + # ) + + # validate_split_data( + # pre_raw_data, + # pre_split_data, + # split_obj + # ) From dfaf2c62f2866884b54c4d9c2f3d27e2ecee388f Mon Sep 17 00:00:00 2001 From: Lockefox Date: Sun, 20 Jan 2019 18:26:32 -0800 Subject: [PATCH 4/4] adding release notes to changelog --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index bd4c239..9ac2f53 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +prosper-api (1.4.0) stable; urgency=high + + * Fixes ESI ROOT endpoint + * Deprecates EMD routes due to bug #21 + * Deprecates plex-split functionality (>600 days since patch) + -- John Purcell Sun, 20 Jan 2019 18:30:00 -0700 + prosper-api (1.3.0) stable; urgency=low * Fixes serious test errors