NodeJS: write to file responses of multiple GET requests in order

Question

I have an array of urls, and I need to call each one of them and write the response to file in a valid JSON in order. E.g., [resp1, resp2, resp3,...]. However, in the code below I get the responses written to file in random orders. E.g., [resp3]resp1,resp2,. How can I prevent that from happening within a foreach with many of urls (so that I cannot next them manually)?

fs.writeFile('my_file.json', "[", (err) => {
  if (err) throw err;
});
urls.forEach((url, index) => {
  https.get(url, (resp) => {
    let data = '';
    resp.on('data', (chunk) => {
      data += chunk;
    });
    resp.on('end', () => {
      let output = data;
      (index == urls.length) ? output += ']' : output += ',' 
      fs.appendFileSync('my_file.json', output, (err) => {
        if (err) throw err;
      });
    });
  }).on("error", (err) => {
    console.log("Error: " + err.message);
  });
});

Answer 1

Consider mapping each request to a Promise and then use Promise.all instead, and then write to the file only once:

const urlProms = urls.map(url => new Promise((resolve, reject) => {
  https.get(url, (resp) => {
    let data = '';
    resp.on('data', (chunk) => {
      data += chunk;
    });
    resp.on('end', () => resolve(data));
  })
  .on('error', reject);
}));
Promise.all(urlProms)
  .then((dataArr) => {
    fs.writeFile('my_file.json', JSON.stringify(dataArr), (err) => {
      if (err) throw err;
    });
  })
  .catch((err) => {
    // handle errors
  });

Manually constructing/concatenating JSON strings is rarely a good idea. If you had to do it manually, use async/await to wait for each iteration in order, to make sure everything gets inserted in the proper order, and that overlapping requests don't take up too much memory:

const getData = url => new Promise((resolve, reject) => {
  https.get(url, (resp) => {
    let data = '';
    resp.on('data', (chunk) => {
      data += chunk;
    });
    resp.on('end', () => resolve(data));
  })
  .on('error', reject);
});
(async () => {
  fs.writeFileSync('my_file.json', '[', (err) => {
    if (err) throw err;
  });
  for (let i = 0; i < urls.length; i++) {
    const data = await getData(urls[i]);
    fs.appendFileSync('my_file.json', data, (err) => {
      if (err) throw err;
    });
  }
  fs.appendFileSync('my_file.json', ']', (err) => {
    if (err) throw err;
  });
})();