I have a large List of objects (over 100k rows in a txt file). I have to do some data work with each item in the list I save the object to the database if it doesnt exist and if it does I increase a number column in the for the item and then update the database while saving the updated item.
If I use a regular foreach loop everything works out fine but it takes over 24 hours to finish.
The issue is that when I use a parallel loop the records do not match and sometimes I will get an error on the .Save() method that the item ID was changed.
THIS CODE WORKS
public List<SreckaIsplacena> UpisiUTabeleSerijski (List<SreckaTemp>srt)
{
_srecke = _glavniRepository.UcitajSamoaktivneSrecke().OrderByDescending(item => item.ID).ToList<Srecka>();
List<SreckaIsplacena> pomList = new List<SreckaIsplacena>();
SreckaIsplacena _isplacena;
foreach (SreckaTemp lt in srt)
{
string beznula = lt.sreIspBroj.TrimStart('0');
SreckeDobici srd = new SreckeDobici();
Srecka sr = (from s in _srecke
where s.Sifra == lt.sreSif && s.Serija == lt.sreSerija
select s).First();
List<SreckeDobici> srDob = _glavniRepository.DohvatiSreckeDobiciZaIsplatniBroj(sr, beznula);
if (srDob.Count == 1)
{
srd = srDob.ElementAt(0);
}
List<SreckaNagrade> sreckaNagrade = new List<SreckaNagrade>(_glavniRepository.DohvatiNagradeZaSrecku(sr.ID).OrderBy(item => item.SifraFox));
double iznos = lt.sreIznDob / lt.sreBrDob;
SreckaNagrade nag = (from sn in sreckaNagrade
where sn.Iznos == lt.sreIznDob
select sn).FirstOrDefault();
Odobrenje odo = new Odobrenje();
odo = odo.DohvatiOdobrenje(valutaGlavna.ID, lt.sreIsplatio).FirstOrDefault();
List<PorezSrecka> listaPoreza = _glavniRepository.UcitajPorezSrecka(valutaGlavna, odo, sr, nag.NagradaId);
List<SreckaIsplacena> sveIsplacene = _glavniRepository.DohvatiIsplaceneSreckeZaValutuIProdavacaSreckuNagraduNovo(valutaGlavna.ID, sr.ID, nag.NagradaId, lt.sreIsplatio);
if (sveIsplacene.Count > 1)
{
System.Windows.MessageBox.Show("Greska, ista srecka s istim dobitkom kod istog prodavaca u istoj valuti 2 put nadjena u bazi");
}
else if (sveIsplacene.Count == 1)
{
_isplacena = sveIsplacene.ElementAt(0);
_isplacena.BrojDobitaka = _isplacena.BrojDobitaka + lt.sreBrDob;
_isplacena.Update();
var index = pomList.FindIndex(r => r.ID == _isplacena.ID);
if (index != -1)
{
pomList[index] = _isplacena;
}
}
else if (sveIsplacene.Count==0)
{
_isplacena = new SreckaIsplacena();
decimal iz = Convert.ToDecimal(lt.sreIznDob);
_isplacena.BrojDobitaka = lt.sreBrDob;
_isplacena.Iznos = iz;
_isplacena.Nagrada = nag;
_isplacena.Prodavac = lt.sreIsplatio;
_isplacena.Valuta = valutaGlavna;
_isplacena.Srecka = sr;
_isplacena.Cijena = Convert.ToDecimal(sr.Cijena);
if (listaPoreza.Count == 1)
{
PorezSrecka ps = listaPoreza.ElementAt(0);
_isplacena.SreckaPorez = ps;
}
_isplacena.Save();
int ispID = _isplacena.ID;
if (ispID != 0)
{
srd.Sre_isplatio = lt.sreIsplatio;
srd.Sre_valuta = valutaGlavna;
srd.Update();
}
pomList.Add(_isplacena);
}
}
return pomList;
}
THIS CODE DOES NOT WORK
public List<SreckaIsplacena> UpisiSamoUTabele(ConcurrentBag<SreckaTemp> srt)
{
_srecke = _glavniRepository.UcitajSamoaktivneSrecke().OrderByDescending(item => item.ID).ToList<Srecka>();
List<SreckaIsplacena> pomList = new List<SreckaIsplacena>();
SreckaIsplacena _isplacena;
Parallel.ForEach(srt, (lt) =>
{
string beznula = lt.sreIspBroj.TrimStart('0');
SreckeDobici srd = new SreckeDobici();
Srecka sr = (from s in _srecke
where s.Sifra == lt.sreSif && s.Serija == lt.sreSerija
select s).First();
List<SreckeDobici> srDob = _glavniRepository.DohvatiSreckeDobiciZaIsplatniBroj(sr, beznula);
if (srDob.Count == 1)
{
srd = srDob.ElementAt(0);
}
List<SreckaNagrade> sreckaNagrade = new List<SreckaNagrade>(_glavniRepository.DohvatiNagradeZaSrecku(sr.ID).OrderBy(item => item.SifraFox));
SreckaNagrade nag = (from sn in sreckaNagrade
where sn.Iznos == lt.sreIznDob
select sn).FirstOrDefault();
Odobrenje odo = new Odobrenje();
odo = odo.DohvatiOdobrenje(valutaGlavna.ID, lt.sreIsplatio).FirstOrDefault();
List<PorezSrecka> listaPoreza = _glavniRepository.UcitajPorezSrecka(valutaGlavna, odo, sr, nag.NagradaId);
List<SreckaIsplacena> sveIsplacene = _glavniRepository.DohvatiIsplaceneSreckeZaValutuIProdavacaSreckuNagraduNovo(valutaGlavna.ID, sr.ID, nag.NagradaId, lt.sreIsplatio);
if (sveIsplacene.Count == 1)
{
_isplacena = sveIsplacene.ElementAt(0);
lock (_isplacena)
{
_isplacena.BrojDobitaka = _isplacena.BrojDobitaka + lt.sreBrDob;
_isplacena.Update();
var index = pomList.FindIndex(r => r.ID == _isplacena.ID);
if (index != -1)
{
pomList[index] = _isplacena;
}
}
}
else if (sveIsplacene.Count==0)
{
_isplacena = new SreckaIsplacena();
decimal iz = Convert.ToDecimal(lt.sreIznDob);
_isplacena.BrojDobitaka = lt.sreBrDob;
_isplacena.Iznos = iz;
_isplacena.Nagrada = nag;
_isplacena.Prodavac = lt.sreIsplatio;
_isplacena.Valuta = valutaGlavna;
_isplacena.Srecka = sr;
_isplacena.Cijena = Convert.ToDecimal(sr.Cijena);
lock(_isplacena)
{
if (listaPoreza.Count == 1)
{
PorezSrecka ps = listaPoreza.ElementAt(0);
_isplacena.SreckaPorez = ps;
}
_isplacena.Save();
int ispID = _isplacena.ID;
if (ispID != 0)
{
srd.Sre_isplatio = lt.sreIsplatio;
srd.Sre_valuta = valutaGlavna;
srd.Update();
}
pomList.Add(_isplacena);
}
}
});
return pomList;
}
I tried using ConcurrentBag instead of List but ran into an issue because ConcurrentBag does not have the FindIndex method (var index = pomList.FindIndex(r => r.ID == _isplacena.ID);). I know there is some way to take out the object and put back in but not sure how to do that and from what I read Lists should be ok if you use locking which I am.
Please help.