remove duplicate document from mongoose nodejs

Question

Great ones, I am new to Mongoose and I need your help. How can I remove duplicate documents? I tried the code below. But I get an error the forEach is not a function. The below is in my controller. Is this right? Or is it supposed to be in my schema?

The way I want it to work is, when data comes in, its stored in the database, and the aggregation removed the duplicate record based on newId.

Please assist

const addProductAgain = require("../models/AddProductAgain");
const dailyReport = require("../models/aggregatedProducts");
const { StatusCodes } = require("http-status-codes");
const { BadRequestError, UnauthenticatedError, NotFoundError } = require("../errors/index");

//POST REQUEST TO LOGIN A USER
const cronTest2 = async (req, res) => {
  console.log("this is cron job 1");
  const data = req.body;
  const cloudDBTwo = data.map((items) => ({
    productName: items.productName,
    shelfLife: items.shelfLife,
    productId: items.productId,
    todayDate: items.todayDate,
    timeEntered: items.timeEntered,
    timeToExpire: items.timeToExpire,
    timeProductExpired: items.timeProductExpired,
    productWeight: items.productWeight,
    panType: items.panType,
    createdBy: items.createdBy,
    expire: items.expire,
    isReturned: items.isReturned,
    isSoldOut: items.isSoldOut,
    RandSStatus: items.RandSStatus,
    location: items.location,
    newId: items._id.slice(-14),
  }));

  try {
    const cloudAddTheProduct = await dailyReport.insertMany(cloudDBTwo);
    console.log(cloudAddTheProduct);
  } catch (error) {
    console.log(error);
  }

//   REMOVES DUPLICATE DATA
  if (cloudAddTheProduct !== "") {
    try {
      await addProductAgain
        .aggregate([
          {
            $group: {
              _id: { newId: "$newId" },
              newId: { $addToSet: "$_id" },
              count: { $sum: 1 },
            },
          },
          {
            $match: {
              count: { $gt: 1 },
            },
          },
        ])
        .forEach(function (doc) {
          doc.newId.shift();
          db.addProductAgain.remove({
            _id: { $in: doc.newId },
          });
        });
    } catch (error) {
      console.log(error);
    }
  }

  res.status(StatusCodes.CREATED).send("saved");
};

module.exports = cronTest2;

Answer 1

while this code snippet may seem syntactically sound, it's worth noting that there could be an issue with using forEach alongside await in an asynchronous context.

In the MongoDB Node.js driver, forEach doesn't inherently handle asynchronous operations as it operates synchronously. Consequently, employing await within forEach might not yield the expected results and could potentially result in unpredictable behavior or errors.

To mitigate this, you might want to explore alternatives such as forEachAsync or map combined with Promise.all to correctly await asynchronous operations within a loop.

You could do some thing like this:

    if (cloudAddTheProduct !== "") {
    try {
        const aggregationResult = await addProductAgain.aggregate([
            {
                $group: {
                    _id: { newId: "$newId" },
                    newId: { $addToSet: "$_id" },
                    count: { $sum: 1 },
                },
            },
            {
                $match: {
                    count: { $gt: 1 },
                },
            },
        ]).toArray();

        await Promise.all(aggregationResult.map(async (doc) => {
            doc.newId.shift();
            await db.addProductAgain.deleteMany({
                _id: { $in: doc.newId },
            });
        }));
    } catch (error) {
        console.log(error);
    }
}

res.status(StatusCodes.CREATED).send("saved");