Error: memory limit exceeded. Function invocation was interrupted.

[siddhant-mohan]

Environment details

• OS: mac latest
• Node.js version: 12.13.1
• npm version: 6.13.4
• @google-cloud/firestore version:

The output of the setLogFunction is:

I even increased the memory to 2GB but still it gives me same error if for a small cron job. I have even followed all the answers here but couldnt resolve. #768

Package.json file:

{
  "name": "functions",
  "description": "Cloud Functions for Firebase",
  "scripts": {
    "lint": "eslint .",
    "serve": "firebase serve --only v1",
    "shell": "firebase functions:shell",
    "start": "npm run shell",
    "deploy": "firebase deploy --only v1",
    "logs": "firebase functions:log"
  },
  "dependencies": {
    "body-parser": "^1.19.0",
    "camelcase": "^5.3.1",
    "es6-promise-pool": "^2.5.0",
    "express": "^4.17.1",
    "firebase-admin": "~8.0.0",
    "firebase-functions": "^3.6.1",
    "firebase-functions-helper": "^0.7.5",
    "glob": "^7.1.6",
    "react-native-uuid": "^1.4.9"
  },
  "engines": {
    "node": "10"
  },
  "devDependencies": {
    "eslint": "^5.12.0",
    "eslint-plugin-promise": "^4.0.1"
  },
  "private": true
}

my firebase function file:

'use strict';

const functions = require('firebase-functions');
const admin = require('firebase-admin');
const firebaseHelper = require("firebase-functions-helper");
const db = admin.firestore();
const firestore = admin.firestore();


const promisePool = require('es6-promise-pool');
const PromisePool = promisePool.PromisePool;
// Maximum concurrent account deletions.
const MAX_CONCURRENT = 2;
var FieldValue = require('firebase-admin').firestore.FieldValue;

const UUID = require("react-native-uuid");


/**
 * Run once a day at midnight, to cleanup the users
 * Manually run the task here https://console.cloud.google.com/cloudscheduler
 */
exports.createUserCategory = functions.runWith({
  memory: '2GB',
  timeoutSeconds: '540'
}).pubsub.schedule('every 5 minutes').timeZone('Asia/Kolkata').onRun(async context => {
  // Fetch all user details.
  const allCategoryData = await allCategory();
  // Use a pool so that we delete maximum `MAX_CONCURRENT` users in parallel.
  const promisePool = new PromisePool(() => populateCategory(allCategoryData), MAX_CONCURRENT);
  await promisePool.start();
  console.log('Push notif delivered');
});

/**
 * Deletes one inactive user from the list.
 */
async function populateCategory(allCategory) {

  if (allCategory.length > 0) {
    admin.firestore.setLogFunction(console.log);

    const category = allCategory.pop();


    const allUsersData = await allUsers();

    let maleUsers = allUsersData.filter(user => user.gender === 'Male');
    let femaleUsers = allUsersData.filter(user => user.gender === 'Female');

    console.log('category is ', category, maleUsers.length, femaleUsers.length);

    await createCategory(category, maleUsers, 'Male');
    await createCategory(category, femaleUsers, 'Female');

    return console.log('done for ', category);
  }
  else {
    return null;
  }
}

async function createCategory(category, allUsers, gender) {

  let userCount = allUsers.length;

  if (category === 'total_users') {
    console.log('category are ', category, allUsers.count, gender);

    const category_exist = await firestore.collection('categories')
      .where('name', '==', category)
      .where('gender', '==', gender)
      .get();
    if (category_exist.docs.length > 0) {
      let cat = category_exist.docs[0].data();
      let data = {count: userCount};
      await firestore.collection('categories').doc(cat.id).update(data);
      return console.log('category updated', category, gender);
    }
    else {
      let data = {
        id: UUID.v1(),
        gender: gender,
        count: userCount,
        name: category,
        featured: false,
        fieldName: category,
        featuredLabel: category
      };
      const category_exist = await firestore.collection('categories').doc(data.id).set(data);
      return console.log('category created', category, gender);
    }
  }
  // else if (category === 'country' || category === 'state') {
  //   let allUsersCount = allUsers.filter(user => user.country === category).length;
  //   const category_exist = await firestore.collection('categories')
  //     .where('name', '==', category)
  //     .where('gender', '==', gender)
  //     .get();
  //   if (category_exist.docs.length > 0) {
  //     let cat = category_exist.docs[0].data();
  //     cat.count = allUsersCount;
  //     await firestore.collection('categories').doc(cat.id).update(cat);
  //     return console.log('category updated', category, gender);
  //   }
  //   else {
  //     let data = {
  //       id: UUID.v1(),
  //       gender: gender,
  //       count: allUsers.length,
  //       name: category,
  //       featured: false,
  //       fieldName: category,
  //       featuredLabel: category
  //
  //   };
  //     const category_exist = await firestore.collection('categories').doc(data.id).set(data);
  //     return console.log('category created', category, gender);
  //   }
  // }
}

/**
 * Returns the list of all inactive users.
 */
async function allUsers() {

  let user_data = await firebaseHelper.firestore.queryData(db, 'users', []);

  user_data = Object.values(user_data);

  console.log('ia ', user_data.length);

  return user_data;
}

async function allCategory() {
  return ['total_users', 'selectedTags', 'country', 'state'];
}

[schmidt-sebastian]

@siddhant-mohan Please ignore my earlier comment about providing logs. Not sure how I missed them.

My only theory for how we could leak memory doesn't seem to apply here. When you issue hundreds of parallel operations, we can spin up multiple GRPC clients which greatly increase the amount of memory required. It looks like your function fails with just a single GRPC client, which should not be happening.

To look at this further, we probably need some investigation by our our backend team. They might have access to more detailed logs that explain your memory profile. Would you mind opening a ticket here: https://cloud.google.com/support-hub

[siddhant-mohan]

This is the response I got from the team. Can anyone help here? @schmidt-sebastian @yoshi-automation

Sorry for the late response, this case took us more time to analyze. I received feedback from the engineering team, and we took a look at the instances and there are instances showing sustained growth until they die. The instances start off at 180-300MB, then they grow steadily for 30-60 seconds, at which point they hit 2G and die off. So, the growth is entirely in your container, I recommend you to investigate your code for leaks, profiling if necessary.

This article explains the common memory leaks for Node JS and how to detect them. I believe this can help you !

[schmidt-sebastian]

Thanks for getting back to us. If all goes well, @bcoe knows where to route this form here :)

[mrousavy]

I'm also getting this error when trying to encode an Image to a Blurhash. I have no detailled error descriptions, just the out-of-memory error.

Any updates on this?

[schmidt-sebastian]

@bcoe Have you seen issues like this before? I wonder if we should route this to the Functions team.

[bcoe]

@mrousavy @schmidt-sebastian sorry for missing the initial @ on this issue. I have seen this behavior before, it seems like cloud functions are quick to terminate a process when it's close to a memory upper limit, vs., swapping like someone might be used to in other runtime environments.

I'm not sure of the best workaround, it might be worth opening an issue on the GCF issue tracker.

[jainabhishek14]

@siddhant-mohan Have you come across any solution of it? I am able to deploy using 2GB memory and 540s Timeout but not below that. And it is costing more than our budget. It would be helpful if somebody can help us.

[MorenoMdz]

Hey any updates to this? I am having issues with a callable function that creates at max about 20k documents. I tried to increase the memory allowance to 512mb, but the weird part is that the logs show it never got past 180mb of usage.

[jokasimr]

@MorenoMdz Same for me, the logs show no growth in memory usage beyond 150mb, and the runtime settings are 2GB. Still getting Error: memory limit exceeded. Function invocation was interrupted. even for fresh instances.

[jainabhishek14]

@jokasimr I have resolved this issue be eliminating global variables. This may be one of your problem also.

[jokasimr]

@jainabhishek14 Thanks, I'll look into it some more.

[MorenoMdz]

@MorenoMdz Same for me, the logs show no growth in memory usage beyond 150mb, and the runtime settings are 2GB. Still getting Error: memory limit exceeded. Function invocation was interrupted. even for fresh instances.

As Jainabhishek14 said, I also resolved it by refactoring my code. A few things to notice:

• Firebase logs are not very reliable, at least in my experience, this is feedback to the FB team, but the log is not near-realtime so and the memory usage is not as granular as it should. It looks like the crash happens before the log can register the memory usage, so one must assume the spike happened and the log didn't catch it.
• Having a lot of global variables might cause the memory allocation issue, as well as your code, keep in mind your code runtime, think about it as running in a week machine, in my case avoiding Promises inside loops is what helped me most as it reduced the runtime considerably.

In my scenario, I moved the creation of the subcollection items to a trigger, for example, when doc A is created, the trigger will do its job and create the related docs. The part for me is that it now takes quite longer than before for the new documents to be ready to read.

[34r7h]

ping