Loop skips iteration after http request

Question

I can't figure out why this simple console (.Net Framework 4.5) app skips some iterations. As you can see in the code below i loop through the List of Fornitori and for each one of it i execute a post http request. If i have 1200 fornitori i expect 1200 requests but from the logs i get only 300. The strange thing is that i don't get any exception and i await the async method PostFornitoreAsync. I've tried to build a mock API myself and calling that instead the one i have to call and everything works but i can not figure out why the iterations are skipped even if there is some problem on the API side.

namespace Demoni___FornitoriPLM
{
    internal class Program
    {
        static IFornitoriRepository fornitoriRepository = new FornitoriRepository();
        static HttpClient httpClient = new HttpClient();
        const string fileName = "lastRun_FornitoriPLM.txt";
        static async Task Main(string[] args)
        {
            StringBuilder log = new StringBuilder("Fornitori elaborati:\n");
            try
            {
                httpClient.Timeout = TimeSpan.FromSeconds(60);
                ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12 | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls;
                httpClient.BaseAddress = new Uri(ConfigurationManager.AppSettings["Uri"]);
                httpClient.DefaultRequestHeaders.Accept.Clear();
                httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
                httpClient.DefaultRequestHeaders.Add("client_id", ConfigurationManager.AppSettings["ClientId"]);
                httpClient.DefaultRequestHeaders.Add("client_secret", ConfigurationManager.AppSettings["ClientSecret"]);

                //Aggiorno i fornitori dall'ultima volta che è girato il demone.
                string filePath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "lasportiva", fileName);
                DateTime lastRun = new DateTime();
                if (File.Exists(filePath))
                    lastRun = Convert.ToDateTime(File.ReadAllText(filePath));

                var fornitori = await fornitoriRepository.GetFornitoriAsync(lastRun);
                int contatoreLog = 1;
                foreach (var fornitore in fornitori)
                {
                    string logFornitore = $"{contatoreLog} chiamata - id fornitore (erpcode): {fornitore.Id}";
                    Console.WriteLine(logFornitore);
                    log.AppendLine(logFornitore);

                    FornitoreDTO fornitoreDaCreare = new FornitoreDTO
                    {
                       //here i map the fields
                    };

                    string logInizio = $"Inizio chiamata {contatoreLog}: {DateTime.Now}";
                    Console.WriteLine(logInizio);
                    log.AppendLine(logInizio);
                    string messaggioErrore = await PostFornitoreAsync(fornitoreDaCreare);
                    string logFine = $"Fine chiamata {contatoreLog}: {DateTime.Now}";
                    Console.WriteLine(logFine);
                    log.AppendLine(logFine);
                    if (messaggioErrore == null)
                    {
                        string logEsito = $"- {fornitoreDaCreare.erpcode} OK \n--------------------------------------------------------------------------------------";
                        Console.WriteLine(logEsito);
                        log.AppendLine(logEsito);
                    }
                    else
                    {
                        string logEsito = $"- {fornitoreDaCreare.erpcode} ERRORE:\n{messaggioErrore} \n--------------------------------------------------------------------------------------";
                        Console.WriteLine(logEsito);
                        log.AppendLine(logEsito);
                    }

                    contatoreLog++;
                }

                File.WriteAllText(filePath, DateTime.Now.ToString());
            }
            catch(Exception ex)
            {
                Log.InvioLog("Errore Demoni - FornitoriPLM", ex.ToString());
                Console.WriteLine($"ECCEZIONE: {ex}");
            }
            finally
            {
                Log.InvioLog("Demoni - FornitoriPLM", log.ToString());
            }
        }

        /// <summary>
        /// Se la richiesta POST è andata a buon fine ritorna null, altrimenti il messaggio d'errore
        /// </summary>
        /// <param name="fornitore">Fornitore da esportare.</param>
        private static async Task<string> PostFornitoreAsync(FornitoreDTO fornitore)
        {
            HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, ConfigurationManager.AppSettings["ResourceUri"]);
            string body = JsonConvert.SerializeObject(fornitore);
            request.Content = new StringContent(body, Encoding.UTF8, "application/json");

            //Questo andrebbe usato solo in dev ma tanto sul server la connessione è in vpn e con whitelist
            //System.Net.ServicePointManager.ServerCertificateValidationCallback = (senderX, certificate, chain, sslPolicyErrors) => { return true; };
            HttpResponseMessage response = await httpClient.SendAsync(request);
            Console.WriteLine($"STATUS CODE: {response.StatusCode}");
            if (!response.IsSuccessStatusCode)
            {
                string bodyResponse = await response.Content.ReadAsStringAsync();
                string messaggioErrore = $"Body Richiesta:\n{body}" +
                                  $"\n\nStatus Code Risposta: {response.StatusCode}\n\nBody Risposta:\n{bodyResponse}";
                return messaggioErrore;
            }

            Console.WriteLine($"{response.StatusCode} {await response.Content.ReadAsStringAsync()}");
            return null;
        }
    }
}

Answer 1

The issue proved to be more straightforward than initially anticipated. It was discovered that in the development environment, the "lastRun" date differed from that in the production environment, despite the initial assumption of their equivalence. Consequently, the elements within the "fornitori" list turned out to be different. I've made the wrong assumption that some elements of "fornitori" were skipped.

Answer 2

Additional to logging as is, you can use SemaphoreSlim to control the number of requests sent to external services at the same time and see if smaller chunks of requests resolve issues. Also you can use Polly to see if network fluctuations or short disconnection or server overload are more at the root of the issue, by letting it space out retry attempts. Something like Serilog can log the results.

In the code below both SemaphoreSlim as Polly are put into action simultaneously, you can tweak it as you want though to try them seperately and see what logs out of that.

You can use both tools beyond the purpose of uncovering issues, for preventing future occurrences of them. Whilst watching out that Polly can cause over-reliance on retries and mask underlying issues. Also SemaphoreSlim can overcomplicate issues with already complex processes. But logging them might help unravel the basic issue.

using System;
using System.Net.Http;
using System.Threading.Tasks;
using System.Threading;

///////////////////////
//       POLLY (1)   //
///////////////////////
using Polly;

///////////////////////
//     SERILOG (1)   //
///////////////////////
using Serilog;

class Program
{
    static HttpClient httpClient = new HttpClient();
    // Create a SemaphoreSlim for limiting concurrency. This prevents overwhelming the server with too many requests simultaneously.

    /////////////////////////////
    //    SEMAPHORESLIM (1)    //
    /////////////////////////////
    static SemaphoreSlim semaphore = new SemaphoreSlim(20); // Allows up to 20 concurrent tasks.

    static async Task Main(string[] args)
    {
        // Configure Serilog for logging. A simple console logger in this case.

        /////////////////////////
        //    SERILOG (2)      //
        /////////////////////////
        Log.Logger = new LoggerConfiguration()
            .WriteTo.Console()
            .CreateLogger();

        try
        {
            // INSERT YOUR HttpClient SETUP

            // Define a retry policy using Polly, specifying conditions for retries and the action on retry.
            // This policy handles HttpRequestException and any non-success status codes by waiting and retrying.

            /////////////////////////
            //     POLLY (2)       //
            /////////////////////////
            var retryPolicy = Policy
                .Handle<HttpRequestException>()
                .OrResult<HttpResponseMessage>(r => !r.IsSuccessStatusCode)
                .WaitAndRetryAsync(new[]
                {
                    TimeSpan.FromSeconds(1),
                    TimeSpan.FromSeconds(5),
                    TimeSpan.FromSeconds(10)
                }, (outcome, timespan, retryAttempt, context) =>
                {
                    // Log each retry attempt.

                    /////////////////////////
                    //    SERILOG (3)      //
                    /////////////////////////
                    Log.Information($"Request failed with {outcome.Result.StatusCode}. Waiting {timespan} before next retry. Retry attempt {retryAttempt}.");
                });

            // Example method to fetch a list of "fornitori". 
            // INSERT YOUR METHOD
            var fornitori = await GetFornitoriAsync(); // Your method to fetch fornitori

            foreach (var fornitore in fornitori)
            {
                // Wait to acquire the semaphore before proceeding. This limits concurrency.

                /////////////////////////////
                //    SEMAPHORESLIM (2)    //
                /////////////////////////////
                await semaphore.WaitAsync();
                
                // Process each fornitore asynchronously. Release the semaphore once the task completes.

                /////////////////////////////
                //    SEMAPHORESLIM (3)    //
                /////////////////////////////
                _ = ProcessFornitoreAsync(fornitore, retryPolicy).ContinueWith(t => semaphore.Release());
            }
        }
        catch (Exception ex)
        {
            // Log any exceptions that occur during the process.

            /////////////////////////
            //    SERILOG (4)      //
            /////////////////////////
            Log.Error($"An exception occurred: {ex.Message}");
        }
        finally
        {
            // Ensure that logging resources are properly flushed and closed.

            /////////////////////////
            //    SERILOG (5)      //
            /////////////////////////
            Log.CloseAndFlush();
        }
    }

    private static async Task ProcessFornitoreAsync(Fornitore fornitore, IAsyncPolicy<HttpResponseMessage> retryPolicy)
    {
        // Prepare the FornitoreDTO and HttpRequestMessage as before.
        FornitoreDTO fornitoreDaCreare = new FornitoreDTO
        {
            // Mapping logic here
        };

        var request = new HttpRequestMessage(HttpMethod.Post, "your/api/endpoint")
        {
            Content = new StringContent(JsonConvert.SerializeObject(fornitoreDaCreare), Encoding.UTF8, "application/json")
        };

        try
        {
            // Execute the HTTP request using the retry policy.

            /////////////////////////
            //     POLLY (3)       //
            /////////////////////////
            HttpResponseMessage response = await retryPolicy.ExecuteAsync(() => httpClient.SendAsync(request));

            // Log the outcome of the request.

            /////////////////////////
            //    SERILOG (6)      //
            /////////////////////////
            Log.Information($"Request completed with status code {response.StatusCode} for fornitore {fornitoreDaCreare.Id}");
            // Additional processing of the response can be done here.
        }
        catch (Exception ex)
        {
            // Log any exceptions that occur specifically during the processing of this fornitore.

            /////////////////////////
            //    SERILOG (7)      //
            /////////////////////////
            Log.Error($"An error occurred while processing fornitore {fornitoreDaCreare.Id}: {ex.Message}");
        }
    }
}