eth.getTransaction sometimes never returns

[wbt]

Sample code:

        web3.eth.getTransaction(txHash, function(error, result) {
            if(error) {
                //This is NEVER REACHED in sample testing
                console.log(error)
            } else {
                //This is VERY RARELY REACHED
                console.log(result);
            }
        });

I observe the issue when this is called many times in parallel, e.g. fetching several thousand events and then calling it for each transaction cited by the events. The number of times the success condition is reached, and which transactions it is reached for, seems to vary randomly but the number is typically around the 2-12 range. This remains the case even when I wait for a couple days (without adding any new web3 requests) to see if the response may be just very slow. Unfortunately I do not have easily shareable reliable reproduction steps, as the set of success varies randomly from run to run.

I am already using the workaround of #3573 so that is not an adequate solution.

Expected behavior

In general, I expect the success branch to be reached. If there is some error that the success branch will never be reached, I expect the error branch to be reached.

Actual behavior

In most cases, neither the error nor the success branch are ever reached.

Environment

Windows 10
Node 9.3.0, npm 6.14.5
web3 1.2.7
geth 1.9.10

[wbt]

Here is a simplified set of reproduction steps that I hopes help illustrate the issue:

> npm uninstall truffle -g
> npm i truffle -g
> truffle version
Truffle v5.1.35 (core: 5.1.35)
Solidity v0.5.16 (solc-js)
Node v9.3.0
Web3.js v1.2.1
> mkdir bugexample
> cd bugexample
> truffle init
> npm i @truffle/hdwallet-provider
> npm i web3

New file contracts/Counter.sol:

// SPDX-License-Identifier: MIT
pragma solidity >=0.4.21 <0.7.0;
contract Counter {
    uint32 public value; //starts at 0 by default
    event UpdatedVal(uint32 indexed oldVal, uint32 indexed newVal);
    function addRows(uint32 stepSize) public {
        emit UpdatedVal(value, (value + stepSize));
        value += stepSize;
    }
}

New file migrations/2_deploy_contracts.js:

var Counter = artifacts.require("./Counter.sol");
module.exports = function(deployer) {
    deployer.deploy(Counter);
};

New file migrations/3_increment_counter.js:

//Separated from deployment migration so that if you have an error most of the
//way through, you can still save progress made prior to that issue.
var Counter = artifacts.require("./Counter.sol");
module.exports = async function(deployer) {
    try {
        let valToAdd = 100000;
        let batchSize = 10; //Errors out after several hundred if batch is too big
        let counterInstance = await Counter.deployed();
        let startingValue = await counterInstance.value.call();
        for(let i = 0; i<valToAdd; i+=batchSize) {
            console.log("Counter value estimate: "+startingValue.addn(i));
            let promises = [];
            for(let j = 0; j<batchSize; j++) {
                //Specifying gasLimit well below the Truffle default so several
                //of these can fit in the same block, accelerating migration time.
                promises.push(counterInstance.addRows(1, {gas: 46875}));
            }
            await Promise.all(promises);
        }
        let endingValue = await counterInstance.value.call();
        console.log("Counter value at end is "+endingValue);
    } catch (err) {
        console.log(err.message);
    }
};

New file fetchHistory.js:

const Web3 = require('web3');
const Counter = require('./build/contracts/Counter.json');
const TruffleContract= require("@truffle/contract");

let contract = TruffleContract(Counter);
let web3 = new Web3(new Web3.providers.WebsocketProvider('ws://127.0.0.1:8546'));
contract.setProvider(new Web3.providers.WebsocketProvider('ws://127.0.0.1:8546', {
    //Avoid CONNECTION ERROR: Couldn't connect to node on WS:
    //Frame size of xxxxxxx bytes exceeds maximum accepted frame size
    clientConfig: {maxReceivedFrameSize: 100000000, maxReceivedMessageSize: 100000000}
}));
let eventsReceived = 0;
let blockDetailsReceived = 0;
let successfulCompletions = 0;
let currentCounterValue;

let printTotals = function() {
    console.log("So far have received "+eventsReceived+" events and successfully "+
    "queried for "+blockDetailsReceived+" blocks and "+successfulCompletions+" transactions.");
    setTimeout(printTotals, 30000);
}
setTimeout(printTotals, 30000);

contract.deployed()
.then(function(counterInstance) {
    let eventHandler = function(eventReceived) {
        eventsReceived++;
        let blockInfo;
        web3.eth.getBlock(eventReceived.blockNumber).
        then(function (fetchedBlockInfo) {
            blockInfo = fetchedBlockInfo;
            blockDetailsReceived++;
            //console.log("Got details for block "+blockInfo.number);
            return web3.eth.getTransaction(eventReceived.transactionHash);
        }).then(function(txDetails) {
            successfulCompletions++;
            //console.log("Got details for transaction "+eventReceived.transactionHash+"; successful completions so far: "+successfulCompletions);
            if(successfulCompletions>(currentCounterValue-100)) { //limits output
                //console.log(txDetails.from+" triggered "+eventReceived.event+" in block with timestamp "+blockInfo.timestamp);
                console.log("Successful completions: ",successfulCompletions);
            }
        }).catch(function(error) {
            console.error(error);
        });
    };
    //Fetch and show current counter value; expect this many events.
    counterInstance.value.call().
    then(function (counterValue) {
        currentCounterValue = counterValue;
        console.log("Current counter value: "+counterValue);
        counterInstance.UpdatedVal({fromBlock: 0})
        .on("data", function(firedEvent) {
            eventHandler(firedEvent);
        }).on("error", function(error) {
            console.error("Error in receiving an event: ",error);
        });
    }).catch(function(error) {
        console.error(error);
    });
}).catch(function(error) {
    console.error(error);
});

Set up Geth 1.9.10 to run a webSocket interface on port 8546 (or change the port number in fetchHistory.js).
Set your truffle-config.js to point to that Geth node.

THEN:

> truffle migrate --reset
(this will take quite a while)
> node fetchHistory.js

If the migration is interrupted (e.g. because the Geth node is busy and gives an Invalid JSON RPC response: ""), you can restart the event-adding one with truffle migrate -f 3

In my example, the counter value reached 135817 including some interruptions and resumptions with that command. The specific number isn't important, only that it's large enough for the issue to show up. The actual result from node fetchHistory.js that I get is:

Current counter value: 135817
So far have received 135817 events and successfully queried for 0 blocks and 0 transactions.
So far have received 135817 events and successfully queried for 918 blocks and 0 transactions.
(last line repeats indefinitely, twice a minute, even if left to run for DAYS.)

while the expected repeating last line is one in which all three numbers match:

So far have received 135817 events and successfully queried for 135817 blocks and 135817 transactions.

[wbt]

FYI, I dove all the way down to the W3CWebSocket and I don’t see any “onMessage” calls that web3 isn’t handling.
This makes me think the issue is more likely in the blockchain node (e.g. geth) than in web3, though web3 could be more robust about having a timeout which might support resubmission. At an even more advanced level, if we can figure out what the limiting factor is, web3 could account for that possibly-configurable limit instead of overwhelming the node.

[github-actions]

This issue has been automatically marked as stale because it has not had recent activity. It will be closed in 7 days if no further activity occurs. Thank you for your contributions. If you believe this was a mistake, please comment.