diff --git a/src/ONYXKEYS.js b/src/ONYXKEYS.js index 4a255538c786..32a9c1c3185e 100755 --- a/src/ONYXKEYS.js +++ b/src/ONYXKEYS.js @@ -243,16 +243,14 @@ export default { // Experimental memory only Onyx mode flag IS_USING_MEMORY_ONLY_KEYS: 'isUsingMemoryOnlyKeys', - // The access token to be used with the Mapbox library - MAPBOX_ACCESS_TOKEN: 'mapboxAccessToken', + // Information about the onyx updates IDs that were received from the server + ONYX_UPDATES_FROM_SERVER: 'onyxUpdatesFromServer', - ONYX_UPDATES: { - // The ID of the last Onyx update that was applied to this client - LAST_UPDATE_ID: 'onyxUpdatesLastUpdateID', + // The last update ID that was applied to the client + ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT: 'OnyxUpdatesLastUpdateIDAppliedToClient', - // The ID of the previous Onyx update that was applied to this client - PREVIOUS_UPDATE_ID: 'onyxUpdatesPreviousUpdateID', - }, + // The access token to be used with the Mapbox library + MAPBOX_ACCESS_TOKEN: 'mapboxAccessToken', // Manual request tab selector SELECTED_TAB: 'selectedTab', diff --git a/src/libs/API.js b/src/libs/API.js index 9405fb8f3a51..d1c88f93b669 100644 --- a/src/libs/API.js +++ b/src/libs/API.js @@ -36,8 +36,9 @@ Request.use(Middleware.SaveResponseInOnyx); * @param {Object} [onyxData.optimisticData] - Onyx instructions that will be passed to Onyx.update() before the request is made. * @param {Object} [onyxData.successData] - Onyx instructions that will be passed to Onyx.update() when the response has jsonCode === 200. * @param {Object} [onyxData.failureData] - Onyx instructions that will be passed to Onyx.update() when the response has jsonCode !== 200. + * @param {Boolean} [prioritizeRequest] Whether or not the request should be prioritized at the front of the queue or placed onto the back of the queue */ -function write(command, apiCommandParameters = {}, onyxData = {}) { +function write(command, apiCommandParameters = {}, onyxData = {}, prioritizeRequest = false) { Log.info('Called API write', false, {command, ...apiCommandParameters}); // Optimistically update Onyx @@ -70,7 +71,7 @@ function write(command, apiCommandParameters = {}, onyxData = {}) { }; // Write commands can be saved and retried, so push it to the SequentialQueue - SequentialQueue.push(request); + SequentialQueue.push(request, prioritizeRequest); } /** diff --git a/src/libs/Middleware/SaveResponseInOnyx.js b/src/libs/Middleware/SaveResponseInOnyx.js index faecda08df7a..b347e45ab61c 100644 --- a/src/libs/Middleware/SaveResponseInOnyx.js +++ b/src/libs/Middleware/SaveResponseInOnyx.js @@ -1,6 +1,7 @@ import Onyx from 'react-native-onyx'; import CONST from '../../CONST'; import * as QueuedOnyxUpdates from '../actions/QueuedOnyxUpdates'; +import * as OnyxUpdates from '../actions/OnyxUpdates'; /** * @param {Promise} response @@ -14,6 +15,9 @@ function SaveResponseInOnyx(response, request) { return; } + // Save the update IDs to Onyx so they can be used to fetch incremental updates if the client gets out of sync from the server + OnyxUpdates.saveUpdateIDs(Number(responseData.lastUpdateID || 0), Number(responseData.previousUpdateID || 0)); + // For most requests we can immediately update Onyx. For write requests we queue the updates and apply them after the sequential queue has flushed to prevent a replay effect in // the UI. See https://github.com/Expensify/App/issues/12775 for more info. const updateHandler = request.data.apiRequestType === CONST.API_REQUEST_TYPE.WRITE ? QueuedOnyxUpdates.queueOnyxUpdates : Onyx.update; diff --git a/src/libs/Navigation/AppNavigator/AuthScreens.js b/src/libs/Navigation/AppNavigator/AuthScreens.js index ddd61da272e6..13216ea3b662 100644 --- a/src/libs/Navigation/AppNavigator/AuthScreens.js +++ b/src/libs/Navigation/AppNavigator/AuthScreens.js @@ -96,8 +96,8 @@ const propTypes = { /** Opt-in experimental mode that prevents certain Onyx keys from persisting to disk */ isUsingMemoryOnlyKeys: PropTypes.bool, - /** The last Onyx update ID that is stored in Onyx (used for getting incremental updates when reconnecting) */ - onyxUpdatesLastUpdateID: PropTypes.number, + /** The last Onyx update ID was applied to the client */ + lastUpdateIDAppliedToClient: PropTypes.number, ...windowDimensionsPropTypes, }; @@ -108,7 +108,7 @@ const defaultProps = { email: null, }, lastOpenedPublicRoomID: null, - onyxUpdatesLastUpdateID: 0, + lastUpdateIDAppliedToClient: null, }; class AuthScreens extends React.Component { @@ -120,7 +120,7 @@ class AuthScreens extends React.Component { componentDidMount() { NetworkConnection.listenForReconnect(); - NetworkConnection.onReconnect(() => App.reconnectApp(this.props.onyxUpdatesLastUpdateID)); + NetworkConnection.onReconnect(() => App.reconnectApp(this.props.lastUpdateIDAppliedToClient)); PusherConnectionManager.init(); Pusher.init({ appKey: CONFIG.PUSHER.APP_KEY, @@ -131,8 +131,7 @@ class AuthScreens extends React.Component { }); // If we are on this screen then we are "logged in", but the user might not have "just logged in". They could be reopening the app - // or returning from background. If so, we'll assume they have some app data already and we can call - // reconnectApp(onyxUpdatesLastUpdateID) instead of openApp(). + // or returning from background. If so, we'll assume they have some app data already and we can call reconnectApp() instead of openApp(). // Note: If a Guide has enabled the memory only key mode then we do want to run OpenApp as their app will not be rehydrated with // the correct state on refresh. They are explicitly opting out of storing data they would need (i.e. reports_) to take advantage of // the optimizations performed during ReconnectApp. @@ -140,7 +139,7 @@ class AuthScreens extends React.Component { if (shouldGetAllData) { App.openApp(); } else { - App.reconnectApp(this.props.onyxUpdatesLastUpdateID); + App.reconnectApp(this.props.lastUpdateIDAppliedToClient); } App.setUpPoliciesAndNavigate(this.props.session, !this.props.isSmallScreenWidth); @@ -329,8 +328,8 @@ export default compose( isUsingMemoryOnlyKeys: { key: ONYXKEYS.IS_USING_MEMORY_ONLY_KEYS, }, - onyxUpdatesLastUpdateID: { - key: ONYXKEYS.ONYX_UPDATES.LAST_UPDATE_ID, + lastUpdateIDAppliedToClient: { + key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, }, }), )(AuthScreens); diff --git a/src/libs/Network/SequentialQueue.js b/src/libs/Network/SequentialQueue.js index 93e3ba6bc207..ac71cec554b6 100644 --- a/src/libs/Network/SequentialQueue.js +++ b/src/libs/Network/SequentialQueue.js @@ -101,10 +101,11 @@ NetworkStore.onReconnection(flush); /** * @param {Object} request + * @param {Boolean} [front] whether or not the request should be placed in the front of the queue */ -function push(request) { +function push(request, front = false) { // Add request to Persisted Requests so that it can be retried if it fails - PersistedRequests.save([request]); + PersistedRequests.save([request], front); // If we are offline we don't need to trigger the queue to empty as it will happen when we come back online if (NetworkStore.isOffline()) { diff --git a/src/libs/Pusher/pusher.js b/src/libs/Pusher/pusher.js index 60587a68e173..43fde187d00b 100644 --- a/src/libs/Pusher/pusher.js +++ b/src/libs/Pusher/pusher.js @@ -136,7 +136,7 @@ function bindEventToChannel(channel, eventName, eventCallback = () => {}) { try { data = _.isObject(eventData) ? eventData : JSON.parse(eventData); } catch (err) { - Log.alert('[Pusher] Unable to parse JSON response from Pusher', {error: err, eventData}); + Log.alert('[Pusher] Unable to parse single JSON event data from Pusher', {error: err, eventData}); return; } if (data.id === undefined || data.chunk === undefined || data.final === undefined) { @@ -172,6 +172,9 @@ function bindEventToChannel(channel, eventName, eventCallback = () => {}) { error: err, eventData: chunkedEvent.chunks.join(''), }); + + // Using console.error is helpful here because it will print a usable stack trace to the console to debug where the error comes from + console.error(err); } delete chunkedDataEvents[data.id]; diff --git a/src/libs/actions/App.js b/src/libs/actions/App.js index ca03380368c2..fe2c908dfd0b 100644 --- a/src/libs/actions/App.js +++ b/src/libs/actions/App.js @@ -182,10 +182,9 @@ function openApp() { /** * Fetches data when the app reconnects to the network * @param {Number} [updateIDFrom] the ID of the Onyx update that we want to start fetching from - * @param {Number} [updateIDTo] the ID of the Onyx update that we want to fetch up to */ -function reconnectApp(updateIDFrom = 0, updateIDTo = 0) { - console.debug(`[OnyxUpdates] App reconnecting with updateIDFrom: ${updateIDFrom} and updateIDTo: ${updateIDTo}`); +function reconnectApp(updateIDFrom = 0) { + console.debug(`[OnyxUpdates] App reconnecting with updateIDFrom: ${updateIDFrom}`); getPolicyParamsForOpenOrReconnect().then((policyParams) => { const params = {...policyParams}; @@ -204,14 +203,72 @@ function reconnectApp(updateIDFrom = 0, updateIDTo = 0) { params.updateIDFrom = updateIDFrom; } - if (updateIDTo) { - params.updateIDTo = updateIDTo; - } - API.write('ReconnectApp', params, getOnyxDataForOpenOrReconnect()); }); } +/** + * Fetches data when the client has discovered it missed some Onyx updates from the server + * @param {Number} [updateIDFrom] the ID of the Onyx update that we want to start fetching from + * @param {Number} [updateIDTo] the ID of the Onyx update that we want to fetch up to + */ +function getMissingOnyxUpdates(updateIDFrom = 0, updateIDTo = 0) { + console.debug(`[OnyxUpdates] Fetching missing updates updateIDFrom: ${updateIDFrom} and updateIDTo: ${updateIDTo}`); + + API.write( + 'GetMissingOnyxMessages', + { + updateIDFrom, + updateIDTo, + }, + getOnyxDataForOpenOrReconnect(), + + // Set this to true so that the request will be prioritized at the front of the sequential queue + true, + ); +} + +// The next 40ish lines of code are used for detecting when there is a gap of OnyxUpdates between what was last applied to the client and the updates the server has. +// When a gap is detected, the missing updates are fetched from the API. + +// These key needs to be separate from ONYXKEYS.ONYX_UPDATES_FROM_SERVER so that it can be updated without triggering the callback when the server IDs are updated +let lastUpdateIDAppliedToClient = 0; +Onyx.connect({ + key: ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, + callback: (val) => (lastUpdateIDAppliedToClient = val), +}); + +Onyx.connect({ + key: ONYXKEYS.ONYX_UPDATES_FROM_SERVER, + callback: (val) => { + if (!val) { + return; + } + + const {lastUpdateIDFromServer, previousUpdateIDFromServer} = val; + console.debug('[OnyxUpdates] Received lastUpdateID from server', lastUpdateIDFromServer); + console.debug('[OnyxUpdates] Received previousUpdateID from server', previousUpdateIDFromServer); + console.debug('[OnyxUpdates] Last update ID applied to the client', lastUpdateIDAppliedToClient); + + // If the previous update from the server does not match the last update the client got, then the client is missing some updates. + // getMissingOnyxUpdates will fetch updates starting from the last update this client got and going to the last update the server sent. + if (lastUpdateIDAppliedToClient && previousUpdateIDFromServer && lastUpdateIDAppliedToClient < previousUpdateIDFromServer) { + console.debug('[OnyxUpdates] Gap detected in update IDs so fetching incremental updates'); + Log.info('Gap detected in update IDs from server so fetching incremental updates', true, { + lastUpdateIDFromServer, + previousUpdateIDFromServer, + lastUpdateIDAppliedToClient, + }); + getMissingOnyxUpdates(lastUpdateIDAppliedToClient, lastUpdateIDFromServer); + } + + if (lastUpdateIDFromServer > lastUpdateIDAppliedToClient) { + // Update this value so that it matches what was just received from the server + Onyx.merge(ONYXKEYS.ONYX_UPDATES_LAST_UPDATE_ID_APPLIED_TO_CLIENT, lastUpdateIDFromServer || 0); + } + }, +}); + /** * This promise is used so that deeplink component know when a transition is end. * This is necessary because we want to begin deeplink redirection after the transition is end. diff --git a/src/libs/actions/OnyxUpdates.js b/src/libs/actions/OnyxUpdates.js new file mode 100644 index 000000000000..e582016f0109 --- /dev/null +++ b/src/libs/actions/OnyxUpdates.js @@ -0,0 +1,22 @@ +import Onyx from 'react-native-onyx'; +import ONYXKEYS from '../../ONYXKEYS'; + +/** + * + * @param {Number} [lastUpdateID] + * @param {Number} [previousUpdateID] + */ +function saveUpdateIDs(lastUpdateID = 0, previousUpdateID = 0) { + // Return early if there were no updateIDs + if (!lastUpdateID) { + return; + } + + Onyx.merge(ONYXKEYS.ONYX_UPDATES_FROM_SERVER, { + lastUpdateIDFromServer: lastUpdateID, + previousUpdateIDFromServer: previousUpdateID, + }); +} + +// eslint-disable-next-line import/prefer-default-export +export {saveUpdateIDs}; diff --git a/src/libs/actions/PersistedRequests.js b/src/libs/actions/PersistedRequests.js index 99a2e406c15b..a71d034c83e1 100644 --- a/src/libs/actions/PersistedRequests.js +++ b/src/libs/actions/PersistedRequests.js @@ -15,9 +15,14 @@ function clear() { /** * @param {Array} requestsToPersist + * @param {Boolean} [front] whether or not the request should go in the front of the queue */ -function save(requestsToPersist) { - persistedRequests = persistedRequests.concat(requestsToPersist); +function save(requestsToPersist, front = false) { + if (persistedRequests.length) { + persistedRequests = front ? persistedRequests.unshift(...requestsToPersist) : persistedRequests.concat(requestsToPersist); + } else { + persistedRequests = requestsToPersist; + } Onyx.set(ONYXKEYS.PERSISTED_REQUESTS, persistedRequests); } diff --git a/src/libs/actions/User.js b/src/libs/actions/User.js index d07cb23c039c..e79d121caf37 100644 --- a/src/libs/actions/User.js +++ b/src/libs/actions/User.js @@ -18,6 +18,7 @@ import * as ReportActionsUtils from '../ReportActionsUtils'; import * as ErrorUtils from '../ErrorUtils'; import * as Session from './Session'; import * as PersonalDetails from './PersonalDetails'; +import * as OnyxUpdates from './OnyxUpdates'; let currentUserAccountID = ''; let currentEmail = ''; @@ -553,21 +554,7 @@ function subscribeToUserEvents() { updates = pushJSON; } else { updates = pushJSON.updates; - - // Not always we'll have the lastUpdateID and previousUpdateID properties in the pusher update - // until we finish the migration to reliable updates. So let's check it before actually updating - // the properties in Onyx - if (pushJSON.lastUpdateID && pushJSON.previousUpdateID) { - console.debug('[OnyxUpdates] Received lastUpdateID from pusher', pushJSON.lastUpdateID); - console.debug('[OnyxUpdates] Received previousUpdateID from pusher', pushJSON.previousUpdateID); - // Store these values in Onyx to allow App.reconnectApp() to fetch incremental updates from the server when a previous session is being reconnected to. - Onyx.multiSet({ - [ONYXKEYS.ONYX_UPDATES.LAST_UPDATE_ID]: Number(pushJSON.lastUpdateID || 0), - [ONYXKEYS.ONYX_UPDATES.PREVIOUS_UPDATE_ID]: Number(pushJSON.previousUpdateID || 0), - }); - } else { - console.debug('[OnyxUpdates] No lastUpdateID and previousUpdateID provided'); - } + OnyxUpdates.saveUpdateIDs(Number(pushJSON.lastUpdateID || 0), Number(pushJSON.previousUpdateID || 0)); } _.each(updates, (multipleEvent) => { PusherUtils.triggerMultiEventHandler(multipleEvent.eventType, multipleEvent.data);