From 61ab40b544189663a3a3d9d8a135b4f84d1f45d0 Mon Sep 17 00:00:00 2001 From: Tom Barnes Date: Tue, 20 Jun 2017 23:04:04 +0100 Subject: [PATCH 1/7] split speedtest data on tabs, not semi-colons --- gsheet_add.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gsheet_add.py b/gsheet_add.py index 58ce633..851d134 100644 --- a/gsheet_add.py +++ b/gsheet_add.py @@ -40,7 +40,7 @@ result_names = ['startdate', 'stopdate', 'provider', 'ip', 'speedtestserver', 'distance', 'pingtime', 'downloadspeed', 'uploadspeed', 'resultimg'] # 2016-04-26 02:59:03;2016-04-26 02:59:37;CenturyLink;97.116.3.36;US Internet (Minnetonka, MN);16.21 km;45.778 ms;45.73 Mbit/s;16.96 Mbit/s;http://www.speedtest.net/result/5278910900.png - result_list = result_string.split(";") + result_list = result_string.split("\t") # create the OAuth2 token From 75016fac963192274909ae833411afcd8f18d2e9 Mon Sep 17 00:00:00 2001 From: Tom Barnes Date: Sat, 1 Jul 2017 23:58:51 +0100 Subject: [PATCH 2/7] update README --- README.md | 9 ++++++- gsheet_add.py | 75 --------------------------------------------------- 2 files changed, 8 insertions(+), 76 deletions(-) delete mode 100644 gsheet_add.py diff --git a/README.md b/README.md index fe2a0c1..7c9462e 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,16 @@ - # raspberrypi-speedtest Inspired by Make magazine http://makezine.com/projects/send-ticket-isp-when-your-internet-drops/ +# About this fork + +This fork is customised for Virgin Media, an ISP in the UK, and their SuperHub 3.0 modem/router. + +It may be obvious, but this has absolutely nothing to do with Raspberry Pi. I'm using it on a desktop PC running Ubuntu. + +The rest of the README is from the upstream repository. There are ommissions in the setup stages - I may correct these at a later date. + # Setup ## Prep your Raspberry Pi for doing the speedtest, per the article diff --git a/gsheet_add.py b/gsheet_add.py deleted file mode 100644 index 851d134..0000000 --- a/gsheet_add.py +++ /dev/null @@ -1,75 +0,0 @@ -import ConfigParser -import fileinput -import gdata.spreadsheets.client -import gdata.spreadsheets.data -import gdata.gauth -import os - - -# see: https://pythonhosted.org/gdata/docs/auth.html#docs-auth - -# - https://developers.google.com/apps-script/guides/rest/api#general_procedure -# - https://developers.google.com/apps-script/guides/rest/quickstart/python#step_1_turn_on_the_api_name - - -config_filename = os.path.splitext(os.path.realpath(__file__))[0]+'.cfg' -config = ConfigParser.ConfigParser() -config.read(config_filename) - - -if not (config.has_option('oauth', 'client_id') and config.has_option('oauth', 'client_secret')): - raise ValueError('auth client_id and client_secret are required') - - -client_id=config.get('oauth', 'client_id') -client_secret=config.get('oauth', 'client_secret') - -access_token=config.get('oauth_token', 'access_token') -refresh_token=config.get('oauth_token', 'refresh_token') - -sheet_id=config.get('sheet_info', 'sheet_id') -tab_id=config.get('sheet_info', 'tab_id') - -connection_type=config.get('other_values', 'connection_type') - - - - -for result_string in fileinput.input(): - - result_names = ['startdate', 'stopdate', 'provider', 'ip', 'speedtestserver', 'distance', 'pingtime', 'downloadspeed', 'uploadspeed', 'resultimg'] - - # 2016-04-26 02:59:03;2016-04-26 02:59:37;CenturyLink;97.116.3.36;US Internet (Minnetonka, MN);16.21 km;45.778 ms;45.73 Mbit/s;16.96 Mbit/s;http://www.speedtest.net/result/5278910900.png - result_list = result_string.split("\t") - - - # create the OAuth2 token - token = gdata.gauth.OAuth2Token(client_id=client_id,client_secret=client_secret,scope='https://spreadsheets.google.com/feeds/',user_agent='rpi-speedtest-add',access_token=access_token,refresh_token=refresh_token) - - # create the spreadsheet client and authenticate - spr_client = gdata.spreadsheets.client.SpreadsheetsClient() - token.authorize(spr_client) - - #create a ListEntry. the first item of the list corresponds to the first 'header' row - entry = gdata.spreadsheets.data.ListEntry() - - entry.set_value('connectiontype', connection_type) - - for i in range(len(result_list)): - - clean_result = result_list[i].strip() - - for ending in [" km", " ms", " Mbit/s"]: - if clean_result.endswith(ending): - clean_result = clean_result[:-len(ending)] - - entry.set_value(result_names[i], clean_result) - - # add the ListEntry you just made - spr_client.add_list_entry(entry, sheet_id, tab_id) - -# print "" -# print entry -# print "" - - From 738d0a0544cee28d866b072861257b52b36e5f0d Mon Sep 17 00:00:00 2001 From: Tom Barnes Date: Sat, 1 Jul 2017 23:59:56 +0100 Subject: [PATCH 3/7] try to log the snr data from the superhub 3.0 --- gsheet_add.py | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100755 gsheet_add.py diff --git a/gsheet_add.py b/gsheet_add.py new file mode 100755 index 0000000..5f6fc7e --- /dev/null +++ b/gsheet_add.py @@ -0,0 +1,107 @@ +import ConfigParser +import fileinput +import requests +import gdata.spreadsheets.client +import gdata.spreadsheets.data +import gdata.gauth +import os +from dateutil.parser import parse +from datetime import datetime, timedelta + +# see: https://pythonhosted.org/gdata/docs/auth.html#docs-auth + +# - https://developers.google.com/apps-script/guides/rest/api#general_procedure +# - https://developers.google.com/apps-script/guides/rest/quickstart/python#step_1_turn_on_the_api_name + +config_filename = os.path.splitext(os.path.realpath(__file__))[0]+'.cfg' +config = ConfigParser.ConfigParser() +config.read(config_filename) + + +if not (config.has_option('oauth', 'client_id') and config.has_option('oauth', 'client_secret')): + raise ValueError('auth client_id and client_secret are required') + + +client_id=config.get('oauth', 'client_id') +client_secret=config.get('oauth', 'client_secret') + +access_token=config.get('oauth_token', 'access_token') +refresh_token=config.get('oauth_token', 'refresh_token') + +sheet_id=config.get('sheet_info', 'sheet_id') +tab_id=config.get('sheet_info', 'tab_id') + +connection_type=config.get('other_values', 'connection_type') + +def add_decimal_point(i): + return i[:2] + '.' + i[2:] + ' ' + +for result_string in fileinput.input(): + + result_names = ['startdate', 'stopdate', 'provider', 'ip', 'speedtestserver', 'distance', 'pingtime', 'downloadspeed', 'uploadspeed', 'resultimg', 'snrvalues'] + try: + snr_values = requests.get("http://192.168.0.1/walk?oids=1.3.6.1.4.1.4491.2.1.20.1.24.1.1;&_n=80212&_=1498924016542").json().values() + snr_values.remove('Finish') + except: + snr_values = [] + pass + snr_decimalised = map(add_decimal_point, snr_values) + snr_string = ','.join(snr_decimalised) + + if result_string.rst ip() == 'error': + log_file = open('error_log.txt','a') + log_file.write(datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ';' + '\'' + snr_string + '\n') + break + + # 2016-04-26 02:59:03;2016-04-26 02:59:37;CenturyLink;97.116.3.36;US Internet (Minnetonka, MN);16.21 km;45.778 ms;45.73 Mbit/s;16.96 Mbit/s;http://www.speedtest.net/result/5278910900.png + result_list = result_string.split("\t") + result_list.append('\'' + snr_string) + + # create the OAuth2 token + token = gdata.gauth.OAuth2Token(client_id=client_id,client_secret=client_secret,scope='https://spreadsheets.google.com/feeds/',user_agent='rpi-speedtest-add',access_token=access_token,refresh_token=refresh_token) + + # create the spreadsheet client and authenticate + spr_client = gdata.spreadsheets.client.SpreadsheetsClient() + token.authorize(spr_client) + + log_file = open('error_log.txt','r') + previous_errors = log_file.readlines() + for error in previous_errors: + error_list = error.split(';') + entry = gdata.spreadsheets.data.ListEntry() + entry.set_value('connectiontype', 'unavailable') + entry.set_value('startdate', error_list[0]) + entry.set_value('snrvalues', error_list[1]) + spr_client.add_list_entry(entry, sheet_id, tab_id) + + #create a ListEntry. the first item of the list corresponds to the first 'header' row + entry = gdata.spreadsheets.data.ListEntry() + + entry.set_value('connectiontype', connection_type) + + for i in range(len(result_list)): + + clean_result = result_list[i].strip() + + for ending in [" km", " ms", " Mbit/s"]: + if clean_result.endswith(ending): + clean_result = clean_result[:-len(ending)] + + print(result_names) + print(clean_result) + entry.set_value(result_names[i], clean_result) + + # add the ListEntry you just made + spr_client.add_list_entry(entry, sheet_id, tab_id) + + # print "" + # print entry + # print "" + + # close the file connection + log_file.close() + # empty the log file + open('error_log.txt', 'w').close() + + + From a55a95e105ee99ee0f01a016d34fbbad29c0aaf8 Mon Sep 17 00:00:00 2001 From: Tom Barnes Date: Sun, 2 Jul 2017 00:00:15 +0100 Subject: [PATCH 4/7] better handling of network errors --- run.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) mode change 100644 => 100755 run.sh diff --git a/run.sh b/run.sh old mode 100644 new mode 100755 index b8a6ebe..2435347 --- a/run.sh +++ b/run.sh @@ -1,6 +1,16 @@ -#!/bin/sh +#!/bin/bash + +set -o pipefail PATH=$PATH:/usr/local/bin cd `dirname $0` -../speedtest-cli-extras/bin/speedtest-csv | python gsheet_add.py + +curl "http://www.google.co.uk" --max-time 5 +result=$? + +if [ $result -ne 0 ]; then + echo "error" | python gsheet_add.py +else + ../speedtest-cli-extras/bin/speedtest-csv | python gsheet_add.py +fi \ No newline at end of file From 2d9c034a68837418aca53e2cb94a76230d1a68e5 Mon Sep 17 00:00:00 2001 From: Tom Barnes Date: Sun, 2 Jul 2017 00:04:00 +0100 Subject: [PATCH 5/7] formatting --- gsheet_add.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gsheet_add.py b/gsheet_add.py index 5f6fc7e..678d9fa 100755 --- a/gsheet_add.py +++ b/gsheet_add.py @@ -34,7 +34,7 @@ connection_type=config.get('other_values', 'connection_type') def add_decimal_point(i): - return i[:2] + '.' + i[2:] + ' ' + return i[:2] + '.' + i[2:] for result_string in fileinput.input(): @@ -46,7 +46,7 @@ def add_decimal_point(i): snr_values = [] pass snr_decimalised = map(add_decimal_point, snr_values) - snr_string = ','.join(snr_decimalised) + snr_string = ', '.join(snr_decimalised) if result_string.rst ip() == 'error': log_file = open('error_log.txt','a') From a523e71652acc9d3943e7fb3e5d16dfe0162f2a3 Mon Sep 17 00:00:00 2001 From: Tom Barnes Date: Sun, 2 Jul 2017 10:29:21 +0100 Subject: [PATCH 6/7] fix typo --- gsheet_add.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gsheet_add.py b/gsheet_add.py index 678d9fa..47a9c82 100755 --- a/gsheet_add.py +++ b/gsheet_add.py @@ -48,7 +48,7 @@ def add_decimal_point(i): snr_decimalised = map(add_decimal_point, snr_values) snr_string = ', '.join(snr_decimalised) - if result_string.rst ip() == 'error': + if result_string.rstrip() == 'error': log_file = open('error_log.txt','a') log_file.write(datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ';' + '\'' + snr_string + '\n') break From 4b072559dc991d2e4f3badcb03369c874324d4d7 Mon Sep 17 00:00:00 2001 From: Tom Barnes Date: Sun, 2 Jul 2017 10:39:15 +0100 Subject: [PATCH 7/7] remove duplication and better var names --- gsheet_add.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/gsheet_add.py b/gsheet_add.py index 47a9c82..43bf3b5 100755 --- a/gsheet_add.py +++ b/gsheet_add.py @@ -33,29 +33,33 @@ connection_type=config.get('other_values', 'connection_type') -def add_decimal_point(i): - return i[:2] + '.' + i[2:] +def add_decimal_point(unformatted_snr): + return unformatted_snr[:2] + '.' + unformatted_snr[2:] + +def unscientific_notarise(string): + return str('\'' + string) for result_string in fileinput.input(): result_names = ['startdate', 'stopdate', 'provider', 'ip', 'speedtestserver', 'distance', 'pingtime', 'downloadspeed', 'uploadspeed', 'resultimg', 'snrvalues'] try: - snr_values = requests.get("http://192.168.0.1/walk?oids=1.3.6.1.4.1.4491.2.1.20.1.24.1.1;&_n=80212&_=1498924016542").json().values() - snr_values.remove('Finish') + snr_list = requests.get("http://192.168.0.1/walk?oids=1.3.6.1.4.1.4491.2.1.20.1.24.1.1;&_n=80212&_=1498924016542").json().values() + snr_list.remove('Finish') except: - snr_values = [] + snr_list = [] pass - snr_decimalised = map(add_decimal_point, snr_values) + snr_decimalised = map(add_decimal_point, snr_list) snr_string = ', '.join(snr_decimalised) + snr_values = unscientific_notarise(snr_string) if result_string.rstrip() == 'error': log_file = open('error_log.txt','a') - log_file.write(datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ';' + '\'' + snr_string + '\n') + log_file.write(datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ';' + snr_values + '\n') break # 2016-04-26 02:59:03;2016-04-26 02:59:37;CenturyLink;97.116.3.36;US Internet (Minnetonka, MN);16.21 km;45.778 ms;45.73 Mbit/s;16.96 Mbit/s;http://www.speedtest.net/result/5278910900.png result_list = result_string.split("\t") - result_list.append('\'' + snr_string) + result_list.append(snr_values) # create the OAuth2 token token = gdata.gauth.OAuth2Token(client_id=client_id,client_secret=client_secret,scope='https://spreadsheets.google.com/feeds/',user_agent='rpi-speedtest-add',access_token=access_token,refresh_token=refresh_token)