Skip to content

Commit

Permalink
Merge pull request #667 from catchpoint/firefox_parser_fix
Browse files Browse the repository at this point in the history 
Fix Firefox log parser
  • Loading branch information
@lbartoli79
lbartoli79 authored Aug 13, 2024
2 parents 3e9509c + 9172c6b commit a6a7b82
Show file tree
Hide file tree
Showing 3 changed files with 73 additions and 36 deletions.
85 changes: 51 additions & 34 deletions internal/firefox.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
import json
from .desktop_browser import DesktopBrowser

def _get_location_uri(accuracy, lat, lng) -> str:
return f'data:application/json, {{ "status":"OK", "accuracy":{accuracy}, "location":{{ "lat":{lat}, "lng":{lng} }} }}'

class Firefox(DesktopBrowser):
"""Firefox"""
Expand Down Expand Up @@ -140,26 +142,47 @@ def start_firefox(self, job, task):
return
from selenium import webdriver # pylint: disable=import-error

capabilities = webdriver.DesiredCapabilities.FIREFOX.copy()
if 'ignoreSSL' in job and job['ignoreSSL']:
capabilities['acceptInsecureCerts'] = True
else:
capabilities['acceptInsecureCerts'] = False
if webdriver.__version__ >= "4.12":
service_args = ["--marionette-port", "2828"]
service = webdriver.FirefoxService(service_args=service_args, log_output=os.environ["MOZ_LOG_FILE"])

capabilities['moz:firefoxOptions'] = {
'binary': self.path,
'args': ['-profile', task['profile']],
'prefs': self.prepare_prefs(),
"log": {"level": "error"},
'env': {
"MOZ_LOG_FILE": os.environ["MOZ_LOG_FILE"],
"MOZ_LOG": os.environ["MOZ_LOG"]
}
}
service_args = ["--marionette-port", "2828"]
options = webdriver.FirefoxOptions()
options.binary_location = self.path
options.add_argument('--profile')
options.add_argument(f'{task["profile"]}')
options.log.level = 'error'
options.prefs = self.prepare_prefs()

capabilities = webdriver.DesiredCapabilities.FIREFOX.copy()
if 'ignoreSSL' in job and job['ignoreSSL']:
capabilities['acceptInsecureCerts'] = True
else:
capabilities['acceptInsecureCerts'] = False

for key, value in capabilities.items():
options.set_capability(key, value)
self.driver = webdriver.Firefox(options=options, service=service)
elif webdriver.__version__ <= "4.9":
capabilities = webdriver.DesiredCapabilities.FIREFOX.copy()
if 'ignoreSSL' in job and job['ignoreSSL']:
capabilities['acceptInsecureCerts'] = True
else:
capabilities['acceptInsecureCerts'] = False

self.driver = webdriver.Firefox(desired_capabilities=capabilities, service_args=service_args)
logging.debug(self.driver.capabilities)
capabilities['moz:firefoxOptions'] = {
'binary': self.path,
'args': ['-profile', task['profile']],
'prefs': self.prepare_prefs(),
"log": {"level": "error"},
'env': {
"MOZ_LOG_FILE": os.environ["MOZ_LOG_FILE"],
"MOZ_LOG": os.environ["MOZ_LOG"]
}
}
service_args = ["--marionette-port", "2828"]
self.driver = webdriver.Firefox(desired_capabilities=capabilities, service_args=service_args)
else:
raise Exception("Unsupported selenium version %s", webdriver.__version__)

self.driver.set_page_load_timeout(task['time_limit'])
if 'browserVersion' in self.driver.capabilities:
Expand Down Expand Up @@ -208,17 +231,13 @@ def launch(self, job, task):
ua_string += ' ' + task['AppendUA']
modified = True
if modified:
logging.debug(ua_string)
self.driver_set_pref('general.useragent.override', ua_string)
# Location
if 'lat' in self.job and 'lng' in self.job:
try:
lat = float(str(self.job['lat']))
lng = float(str(self.job['lng']))
location_uri = 'data:application/json,{{'\
'"status":"OK","accuracy":10.0,'\
'"location":{{"lat":{0:f},"lng":{1:f}}}'\
'}}'.format(lat, lng)
location_uri = _get_location_uri(10, lat, lng)
logging.debug('Setting location: %s', location_uri)
self.driver_set_pref('geo.wifi.uri', location_uri)
except Exception:
Expand Down Expand Up @@ -292,6 +311,8 @@ def close_browser(self, job, task):
if platform.system() == "Linux":
subprocess.call(['killall', '-9', 'firefox'])
subprocess.call(['killall', '-9', 'firefox-trunk'])
subprocess.call(['killall', '-9', 'firefox-nightly'])
subprocess.call(['killall', '-9', 'firefox-esr'])
os.environ["MOZ_LOG_FILE"] = ''
os.environ["MOZ_LOG"] = ''

Expand Down Expand Up @@ -326,7 +347,7 @@ def run_axe(self, task):
script += "'" + "', '".join(axe_cats) + "'"
script += ']}).then(results=>{return results;});'
except Exception as err:
logging.exception("Exception running Axe: %s", err.__str__())
logging.exception("Exception running Axe: %s", err)
if self.must_exit_now:
return
completed = False
Expand All @@ -349,7 +370,7 @@ def run_axe(self, task):
axe_info['incomplete'] = axe_results['incomplete']
task['page_data']['axe'] = axe_info
except Exception as err:
logging.exception("Exception running Axe: %s", err.__str__())
logging.exception("Exception running Axe: %s", err)
if not completed:
task['page_data']['axe_failed'] = 1
self.axe_time = monotonic() - start
Expand All @@ -376,7 +397,7 @@ def run_task(self, task):
logging.exception("Exception running task")
if command['record']:
self.wait_for_page_load()
if not task['combine_steps'] or not len(task['script']):
if not task['combine_steps'] or not task['script']:
self.on_stop_capture(task)
self.on_stop_recording(task)
recording = False
Expand All @@ -397,10 +418,9 @@ def run_task(self, task):
self.task = None

def alert_size(self, _alert_config, _task_dir, _prefix):
'''Checks the agents file size and alert on certain percentage over avg byte size'''
'''Checks the agents file size and alert on certain percentage over avg byte size'''
self.alert_desktop_results(_alert_config, 'Firefox', _task_dir, _prefix)


def wait_for_extension(self):
"""Wait for the extension to send the started message"""
if self.job['message_server'] is not None:
Expand Down Expand Up @@ -506,7 +526,7 @@ def run_js_file(self, file_name):
script = None
script_file_path = os.path.join(self.script_dir, file_name)
if os.path.isfile(script_file_path):
with open(script_file_path, 'r') as script_file:
with open(script_file_path, 'r', encoding='utf-8') as script_file:
script = script_file.read()
if self.driver is not None and script is not None:
try:
Expand All @@ -518,7 +538,7 @@ def run_js_file(self, file_name):
logging.debug(ret)
return ret

def get_sorted_requests_json(self, include_bodies):
def get_sorted_requests_json(self, _include_bodies):
return 'null'

def collect_browser_metrics(self, task):
Expand Down Expand Up @@ -962,10 +982,7 @@ def process_command(self, command):
parts = command['target'].split(',')
lat = float(parts[0])
lng = float(parts[1])
location_uri = 'data:application/json,{{'\
'"status":"OK","accuracy":{2:d},'\
'"location":{{"lat":{0:f},"lng":{1:f}}}'\
'}}'.format(lat, lng, accuracy)
location_uri = _get_location_uri(accuracy, lat, lng)
logging.debug('Setting location: %s', location_uri)
self.set_pref('geo.wifi.uri', location_uri)
except Exception:
Expand Down
22 changes: 21 additions & 1 deletion internal/support/firefox_log_parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,14 @@ def socket_thread_http_entry(self, msg):
socket = self.http['current_socket']
self.http['connections'][connection] = {'socket': socket}
del self.http['current_socket']
elif msg['message'].startswith('TlsHandshaker::SetupSSL '):
match = re.search(r'^TlsHandshaker::SetupSSL (?P<connection>[\w\d]+)',
msg['message'])
if match:
connection = match.groupdict().get('connection')
if connection in self.http['connections']:
if 'ssl_start' not in self.http['connections'][connection]:
self.http['connections'][connection]['ssl_start'] = msg['timestamp']
elif msg['message'].startswith('nsHttpConnection::SetupSSL '):
match = re.search(r'^nsHttpConnection::SetupSSL (?P<connection>[\w\d]+)',
msg['message'])
Expand Down Expand Up @@ -332,6 +340,17 @@ def socket_thread_http_entry(self, msg):
if byte_count > 0 and trans_id in self.http['requests'] and \
'start' not in self.http['requests'][trans_id]:
self.http['requests'][trans_id]['start'] = msg['timestamp']
elif msg['message'].startswith('nsHttpTransaction::OnSocketStatus ') and \
msg['message'].find(' status=4b0005 progress=') > -1:
match = re.search(r'^nsHttpTransaction::OnSocketStatus '
r'\[this=(?P<id>[\w\d]+) status=4b0005 progress=(?P<bytes>[\d+]+)',
msg['message'])
if match:
trans_id = match.groupdict().get('id')
byte_count = int(match.groupdict().get('bytes'))
if byte_count > 0 and trans_id in self.http['requests'] and \
'start' not in self.http['requests'][trans_id]:
self.http['requests'][trans_id]['start'] = msg['timestamp']
elif msg['message'].startswith('nsHttpTransaction::ProcessData '):
match = re.search(r'^nsHttpTransaction::ProcessData \[this=(?P<id>[\w\d]+)',
msg['message'])
Expand Down Expand Up @@ -446,14 +465,15 @@ def socket_transport_entry(self, msg):
port = match.groupdict().get('port')
self.http['sockets'][socket] = {'host': host, 'port': port}
# nsSocketTransport::SendStatus [this=143f4000 status=804b0007]
# nsSocketTransport::SendStatus [this=7fe074bd2a00 status=4B0007]
elif msg['message'].startswith('nsSocketTransport::SendStatus '):
match = re.search(r'^nsSocketTransport::SendStatus \['
r'this=(?P<socket>[\w\d]+) '
r'status=(?P<status>[\w\d]+)', msg['message'])
if match:
socket = match.groupdict().get('socket')
status = match.groupdict().get('status')
if status == '804b0007':
if status in ['804b0007', '4b0007']:
if socket not in self.http['sockets']:
self.http['sockets'][socket] = {}
if 'start' not in self.http['sockets'][socket]:
Expand Down
2 changes: 1 addition & 1 deletion wptagent.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1084,7 +1084,7 @@ def fix_selenium_version():
newer versions are going to use 4.8.3
"""
from internal.os_util import run_elevated
version = '4.8.3'
version = '4.18.1'
if sys.version_info[1] == 6:
version = '3.141.0'

Expand Down

0 comments on commit a6a7b82

Please sign in to comment.