Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Offline support for Network page #8332

Open
wants to merge 35 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 31 commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
fe11d5b
added initial implementation
hrajwade96 Sep 21, 2024
6c140b4
adding constants file
hrajwade96 Sep 29, 2024
0c1f414
updated NEXT_RELEASE_NOTES.md
hrajwade96 Sep 30, 2024
e7eeecb
Revert "updated NEXT_RELEASE_NOTES.md"
hrajwade96 Sep 30, 2024
c5ec6f6
used constants, added util for header size calc, added params
hrajwade96 Oct 2, 2024
131cde6
Merge branch 'master' into network_screen_offline_support
hrajwade96 Oct 2, 2024
2a9c4cb
param renamed, used _log
hrajwade96 Oct 2, 2024
4f473be
Merge remote-tracking branch 'origin/network_screen_offline_support' …
hrajwade96 Oct 2, 2024
ae58738
added copyright header
hrajwade96 Oct 2, 2024
2bfbe1c
setup changes implemented
hrajwade96 Oct 3, 2024
857cfbe
put class name in square brackets
hrajwade96 Oct 3, 2024
7150436
added isNullOrEmpty check
hrajwade96 Oct 3, 2024
da11a67
added exit offline cta, hiding other controls, removed json string
hrajwade96 Oct 4, 2024
941ec88
fix for request selection
hrajwade96 Oct 4, 2024
852b144
comments resolved
hrajwade96 Oct 8, 2024
433af38
removing getFullRequestData section
hrajwade96 Oct 8, 2024
4644b00
using updateOrAddAll for setting data
hrajwade96 Oct 8, 2024
6fd98d0
Wrapped with OfflineAwareControls, removed if-else.
hrajwade96 Oct 8, 2024
665ea0d
removed addAutoDisposeListener as we are using OfflineAwareControls
hrajwade96 Oct 8, 2024
b6df572
removed try-catch
hrajwade96 Oct 8, 2024
be63db4
code refactoring
hrajwade96 Oct 9, 2024
91505b5
added enum
hrajwade96 Oct 9, 2024
0715379
code refactoring, reverting changes on network service
hrajwade96 Oct 10, 2024
ade8b5f
using values from enum
hrajwade96 Oct 10, 2024
f21a4f6
delegating toJson using extension methods, code refactoring
hrajwade96 Oct 23, 2024
454ca92
socket data de-serialisation changes
hrajwade96 Oct 27, 2024
b7b9679
Merge branch 'master' into network_screen_offline_support
hrajwade96 Oct 27, 2024
ac726e8
used factory constructor, made enum private
hrajwade96 Nov 4, 2024
2ea3044
reorder params, used isEmpty
hrajwade96 Nov 4, 2024
e01bc3c
added tests, minor fixes
hrajwade96 Nov 17, 2024
e7b5c90
removed repetitive expect statements
hrajwade96 Nov 21, 2024
6ab0884
Merge remote-tracking branch 'upstream/master' into network_screen_of…
hrajwade96 Nov 22, 2024
4f31b92
minor fix
hrajwade96 Nov 22, 2024
04c8b41
added timelineMicrosOffset to offline data
hrajwade96 Nov 24, 2024
ab8c519
added timelineMicrosOffset to tests
hrajwade96 Nov 24, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import 'dart:convert';

import '../../screens/network/utils/http_utils.dart';
import '../../shared/http/http_request_data.dart';
import '../../shared/primitives/utils.dart';
import 'constants.dart';
Expand Down Expand Up @@ -131,7 +132,7 @@ class HarDataEntry {
NetworkEventKeys.text.name: e.requestBody,
},
NetworkEventKeys.headersSize.name:
_calculateHeadersSize(e.requestHeaders),
calculateHeadersSize(e.requestHeaders),
NetworkEventKeys.bodySize.name: _calculateBodySize(e.requestBody),
},
// Response
Expand All @@ -150,7 +151,7 @@ class HarDataEntry {
},
NetworkEventKeys.redirectURL.name: '',
NetworkEventKeys.headersSize.name:
_calculateHeadersSize(e.responseHeaders),
calculateHeadersSize(e.responseHeaders),
NetworkEventKeys.bodySize.name: _calculateBodySize(e.responseBody),
},
// Cache
Expand Down Expand Up @@ -263,27 +264,6 @@ class HarDataEntry {
}
}

int _calculateHeadersSize(Map<String, Object?>? headers) {
if (headers == null) return -1;

// Combine headers into a single string with CRLF endings
String headersString = headers.entries.map((entry) {
final key = entry.key;
var value = entry.value;
// If the value is a List, join it with a comma
if (value is List<String>) {
value = value.join(', ');
}
return '$key: $value\r\n';
}).join();

// Add final CRLF to indicate end of headers
headersString += '\r\n';

// Calculate the byte length of the headers string
return utf8.encode(headersString).length;
}

int _calculateBodySize(String? requestBody) {
if (requestBody.isNullOrEmpty) {
return 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import '../../shared/config_specific/logger/allowed_error.dart';
import '../../shared/globals.dart';
import '../../shared/http/http_request_data.dart';
import '../../shared/http/http_service.dart' as http_service;
import '../../shared/offline_data.dart';
import '../../shared/primitives/utils.dart';
import '../../shared/ui/filter.dart';
import '../../shared/ui/search.dart';
Expand All @@ -22,6 +23,7 @@ import 'har_network_data.dart';
import 'network_model.dart';
import 'network_screen.dart';
import 'network_service.dart';
import 'offline_network_data.dart';

/// Different types of Network Response which can be used to visualise response
/// on Response tab
Expand Down Expand Up @@ -49,10 +51,12 @@ class NetworkController extends DisposableController
with
SearchControllerMixin<NetworkRequest>,
FilterControllerMixin<NetworkRequest>,
OfflineScreenControllerMixin,
AutoDisposeControllerMixin {
NetworkController() {
_networkService = NetworkService(this);
_currentNetworkRequests = CurrentNetworkRequests();
_initHelper();
addAutoDisposeListener(
_currentNetworkRequests,
_filterAndRefreshSearchMatches,
Expand Down Expand Up @@ -168,6 +172,43 @@ class NetworkController extends DisposableController
@visibleForTesting
bool get isPolling => _pollingTimer != null;

void _initHelper() async {
if (offlineDataController.showingOfflineData.value) {
await maybeLoadOfflineData(
NetworkScreen.id,
createData: (json) => OfflineNetworkData.fromJson(json),
shouldLoad: (data) => !data.isEmpty,
loadData: (data) => loadOfflineData(data),
);
} else {
await startRecording();
}
}

Future<void> loadOfflineData(OfflineNetworkData offlineData) async {
final httpProfileData = offlineData.httpRequestData.mapToHttpProfileRequests;
final socketStatsData = offlineData.socketData.mapToSocketStatistics;

_currentNetworkRequests
..clear()
..updateOrAddAll(
requests: httpProfileData,
sockets: socketStatsData,
timelineMicrosOffset: DateTime.now().microsecondsSinceEpoch,
);
_filterAndRefreshSearchMatches();

// If a selectedRequestId is available, select it in offline mode.
if (offlineData.selectedRequestId != null) {
final selected = _currentNetworkRequests
.getRequest(offlineData.selectedRequestId ?? '');
if (selected != null) {
selectedRequest.value = selected;
resetDropDown();
}
}
}

@visibleForTesting
void processNetworkTrafficHelper(
List<SocketStatistic> sockets,
Expand Down Expand Up @@ -395,6 +436,30 @@ class NetworkController extends DisposableController
}
}

@override
OfflineScreenData prepareOfflineScreenData() {
final httpRequestData = <DartIOHttpRequestData>[];
final socketData = <Socket>[];
for (final request in _currentNetworkRequests.value) {
if (request is DartIOHttpRequestData) {
httpRequestData.add(request);
} else if (request is Socket) {
socketData.add(request);
}
}

final offlineData = OfflineNetworkData(
httpRequestData: httpRequestData,
socketData: socketData,
selectedRequestId: selectedRequest.value?.id,
);

return OfflineScreenData(
screenId: NetworkScreen.id,
data: offlineData.toJson(),
);
}

Future<void> _fetchFullDataBeforeExport() => Future.wait(
filteredData.value
.whereType<DartIOHttpRequestData>()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,15 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'package:devtools_shared/devtools_shared.dart';
import 'package:flutter/material.dart';
import 'package:vm_service/vm_service.dart';

import '../../shared/primitives/utils.dart';
import '../../shared/ui/search.dart';

abstract class NetworkRequest with ChangeNotifier, SearchableDataMixin {
abstract class NetworkRequest
with ChangeNotifier, SearchableDataMixin, Serializable {
String get method;

String get uri;
Expand Down Expand Up @@ -85,6 +87,15 @@ abstract class NetworkRequest with ChangeNotifier, SearchableDataMixin {
class Socket extends NetworkRequest {
Socket(this._socket, this._timelineMicrosBase);

factory Socket.fromJson(Map<String, Object?> json) {
return Socket(
SocketStatistic.parse(
json[SocketJsonKey.socket.name] as Map<String, Object?>,
)!,
json[SocketJsonKey.timelineMicrosBase.name] as int,
);
}

int _timelineMicrosBase;

SocketStatistic _socket;
Expand Down Expand Up @@ -180,4 +191,53 @@ class Socket extends NetworkRequest {

@override
int get hashCode => id.hashCode;

SocketStatistic get socketData => _socket;

@override
Map<String, Object?> toJson() {
return {
SocketJsonKey.timelineMicrosBase.name: _timelineMicrosBase,
SocketJsonKey.socket.name: _socket.toJson(),
};
}
}

extension on SocketStatistic {
Map<String, Object?> toJson() {
return {
SocketJsonKey.id.name: id,
SocketJsonKey.startTime.name: startTime,
SocketJsonKey.endTime.name: endTime,
//TODO verify if these timings are in correct format
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please address this TODO.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While testing I noticed 2 issues :

  1. The timings in socket stats are having discrepancy in online and offline mode - I thought this could be due to _timelineMicrosBase but I tried adding/ removing it there was still a discrepancy. Is it due to some time zone or formatting issue? I am trying to figure out.
  2. The Request and Response data is missing in offline mode - I think it's because full fetch is not called, if we want to call where can we ideally call it?

Attaching SS for 1st point :
Online mode
image

Offline mode
image

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another point is that in offline mode data of socket and http requests is grouped when it's populated.
I think we should show them in their original order as the requests came in? Perhaps we can pack them in a combined way instead, or other way is to reorder before rendering based on timestamps (once we fix the timings) ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually past 2 weeks I've been on vacation, and then unwell so got very little time to investigate.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The table should already sort the data by timestamp.

Copy link
Contributor Author

@hrajwade96 hrajwade96 Nov 22, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@kenzieschmoll can you help with first 2 points? also point 3) on page refresh when in offline mode, this is how it is looking like currently, it's clearing the list, and also not showing exit cta
what is the expected behaviour here?
Do we need to persist the data in storage?
Screenshot 2024-11-22 at 9 53 09 PM

Copy link
Contributor Author

@hrajwade96 hrajwade96 Nov 24, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think 1st point is due to value of _timelineMicrosBase being reset when entering offline mode, causing a discrepancy in timing calculation with this function :
int timelineMicrosecondsSinceEpoch(int micros) { return _timelineMicrosBase + micros; }

I fixed it by adding timelineMicrosOffset in offline data.

Copy link
Contributor Author

@hrajwade96 hrajwade96 Nov 24, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

2nd and 3rd points are remaining,
for (2) - I tried fetching full data with periodic timer

  void startPeriodicExport() {
    _periodicExportTimer = Timer.periodic(const Duration(milliseconds: 500), (
      _,
    ) async {
      try {
        unawaited(_fetchFullDataBeforeExport());
        debugPrint('Export data fetched successfully');
      } catch (e) {
        debugPrint('Error during periodic export fetch: $e');
      }
    });
  }

  void stopPeriodicExport() {
    _periodicExportTimer?.cancel();
    _periodicExportTimer = null;
  }

so that full data is always available before entering in offline mode. Or alternatively we can fetch each time new request data is added in the current list. Is this fine? or any better ideas?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

for (3) is there any simple way to add anther check for showing exit offline cta?
such as whenever it is disconnected, current check doesn't seem to work after page refresh

Copy link
Contributor Author

@hrajwade96 hrajwade96 Nov 27, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

BTW I will be unavailable here for the next 4 days (27th - 1st Dec)

SocketJsonKey.lastReadTime.name: lastReadTime,
SocketJsonKey.lastWriteTime.name: lastWriteTime,
SocketJsonKey.socketType.name: socketType,
SocketJsonKey.address.name: address,
SocketJsonKey.port.name: port,
SocketJsonKey.readBytes.name: readBytes,
SocketJsonKey.writeBytes.name: writeBytes,
};
}
}

enum SocketJsonKey {
id,
startTime,
endTime,
lastReadTime,
lastWriteTime,
socketType,
address,
port,
readBytes,
writeBytes,
timelineMicrosBase,
socket,
}

extension SocketExtension on List<Socket> {
List<SocketStatistic> get mapToSocketStatistics {
return map((socket) => socket._socket).toList();
}
}
31 changes: 13 additions & 18 deletions packages/devtools_app/lib/src/screens/network/network_screen.dart
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import '../../shared/analytics/analytics.dart' as ga;
import '../../shared/analytics/constants.dart' as gac;
import '../../shared/common_widgets.dart';
import '../../shared/config_specific/copy_to_clipboard/copy_to_clipboard.dart';
import '../../shared/globals.dart';
import '../../shared/http/curl_command.dart';
import '../../shared/http/http_request_data.dart';
import '../../shared/primitives/utils.dart';
Expand Down Expand Up @@ -106,19 +105,6 @@ class _NetworkScreenBodyState extends State<NetworkScreenBody>
void didChangeDependencies() {
super.didChangeDependencies();
if (!initController()) return;
kenzieschmoll marked this conversation as resolved.
Show resolved Hide resolved
unawaited(controller.startRecording());

cancelListeners();

addAutoDisposeListener(
serviceConnection.serviceManager.isolateManager.mainIsolate,
() {
if (serviceConnection.serviceManager.isolateManager.mainIsolate.value !=
null) {
unawaited(controller.startRecording());
}
},
);
}

@override
Expand All @@ -133,7 +119,13 @@ class _NetworkScreenBodyState extends State<NetworkScreenBody>
Widget build(BuildContext context) {
return Column(
children: [
_NetworkProfilerControls(controller: controller),
OfflineAwareControls(
controlsBuilder: (offline) => _NetworkProfilerControls(
controller: controller,
offline: offline,
),
gaScreen: gac.network,
),
const SizedBox(height: intermediateSpacing),
Expanded(
child: _NetworkProfilerBody(controller: controller),
Expand All @@ -148,12 +140,15 @@ class _NetworkScreenBodyState extends State<NetworkScreenBody>
class _NetworkProfilerControls extends StatefulWidget {
const _NetworkProfilerControls({
required this.controller,
required this.offline,
});

static const _includeTextWidth = 810.0;

final NetworkController controller;

final bool offline;

@override
State<_NetworkProfilerControls> createState() =>
_NetworkProfilerControlsState();
Expand All @@ -166,7 +161,6 @@ class _NetworkProfilerControlsState extends State<_NetworkProfilerControls>
@override
void initState() {
super.initState();

_recording = widget.controller.recordingNotifier.value;
addAutoDisposeListener(widget.controller.recordingNotifier, () {
setState(() {
Expand All @@ -183,7 +177,8 @@ class _NetworkProfilerControlsState extends State<_NetworkProfilerControls>
final hasRequests = widget.controller.filteredData.value.isNotEmpty;
return Row(
children: [
StartStopRecordingButton(
if (!widget.offline) ...[
StartStopRecordingButton(
recording: _recording,
onPressed: () async =>
await widget.controller.togglePolling(!_recording),
Expand Down Expand Up @@ -211,7 +206,7 @@ class _NetworkProfilerControlsState extends State<_NetworkProfilerControls>
gaScreen: gac.network,
gaSelection: gac.NetworkEvent.downloadAsHar.name,
),
const SizedBox(width: defaultSpacing),
const Spacer(),
// TODO(kenz): fix focus issue when state is refreshed
Expanded(
child: SearchField<NetworkController>(
Expand Down
Loading
Loading