Overcoming SharePoint Online's Data Fetch Limit in .NET Core with Batch Processing

Question

  public async Task ProcessCandidateInformationList(string? SiteUrl, string? ClientID, string? ClientSecret, string? ConnectionString)
  {
      try
      {
          using (ClientContext context = new AuthenticationManager().GetACSAppOnlyContext(SiteUrl, ClientID, ClientSecret))
          {
                  List targetList = context.Web.Lists.GetByTitle("Candidate Information");
                  CamlQuery query = new CamlQuery
                  {
                      ViewXml = $"<View Scope='RecursiveAll'><RowLimit>4800</RowLimit><Query>                      </Query></View>"
                  };

                  ListItemCollection listItems;
 
                  do
                  {
                      listItems = targetList.GetItems(query);
                      context.Load(listItems);
                      context.Load(listItems, collection => collection.Include(item => item.AttachmentFiles));
                      context.ExecuteQuery();

                      foreach (var item in listItems)
                      {
                         Console.WritelIne(item);
                      }
                      query.ListItemCollectionPosition = listItems.ListItemCollectionPosition;
                  } while (listItems.ListItemCollectionPosition != null);
                  await connection.CloseAsync();
              }
          }
      }
      catch (Exception ex)
      {
          Console.WriteLine("An error occurred in ProcessCandidateInformationList: " + ex);
          _logger.Error("An error occurred in ProcessCandidateInformationList: " + ex);
      }
  }

Dealing with SharePoint Online data retrieval limits can be a headache for developers, especially when working on large-scale projects that require accessing extensive lists. The default threshold caps at 5000 items, posing a significant challenge. However, there's a silver lining: a batch processing method in .NET Core (versions 6 and 8) using C# that smartly navigates around this limitation, ensuring you can fetch all the data needed without a hitch.