I'm using below given code to download content of a webpage.
using (var w = new System.Net.WebClient())
{
w.Encoding = System.Text.Encoding.UTF8;
w.Headers["User-Agent"] = "Mozilla / 5.0(Windows NT 6.3; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 59.0.3071.104 Safari / 537.36";
var HtmlMarkup = w.DownloadString("http://www.as.com/statistics/announcements.do?by=asxCode&asxCode=HLO&timeframe=Y&year=" + i);
}
The response i'm getting doesn't have any content. But when I browse the page the data load as normal.
The return string of the call is
<html><head>
<meta http-equiv="Pragma" content="no-cache"/>
<meta http-equiv="Expires" content="-1"/>
<meta http-equiv="CacheControl" content="no-cache"/>
<noscript>Please enable JavaScript to view the page content.</noscript>
<APM_DO_NOT_TOUCH>
<script language="javascript">
(function(){
var securemsg;
var packmsg;
var CryptoUtils;
try{(function(){try{var IS,lS,oS=1;for(var OS=0;OS<lS;++OS)oS+=3;IS=oS;window._===IS&&(window._=++IS)}catch(ZS){window._=IS}var __=window.sdkljshr489=!0;function i_(S){window.sdkljshr489&&S&&(__=!1);return __}function I_(){}i_(window[I_.name]===I_);i_("undefined"===window.vodsS0);window.vodsS0=null;i_(/\x3c/.test(function(){return"\x3c"})&/x3d/.test(function(){return"0";"x3d"}));var j_=/mobi/i.test(navigator.userAgent),L_=+new Date,o_=j_?3E4:3E3;
function O_(){return i_(L_+o_<(L_=+new Date))}(function(){var S={decrypt:function(S){try{return JSON.parse(function(S){S=S.split("l");var l="";for(var L=0;L<S.length;++L)l+=String.fromCharCode(S[L]);return l}(S))}catch(L){}}};return S={configuration:S.decrypt("123l34l97l99l116l105l118l101l34l58l34l110l111l34l44l34l100l101l98l117l103l103l105l110l103l34l58l34l110l111l34l44l34l109l111l100l117l108l101l49l34l58l34l101l110l97l98l108l101l100l34l44l34l109l111l100l117l108l101l50l34l58l34l101l110l97l98l108l101l100l34l44l34l109l111l100l117l108l101l51l34l58l34l101l110l97l98l108l101l100l34l44l34l109l111l100l117l108l101l52l34l58l34l101l110l97l98l108l101l100l34l125")}})();
window.oi={iI:"083bf9465983600168c93c2ae5d6089fc80f5d9cdecbe2a72745978b659d0db65c4b160ead8f315f1f237ad9f9e564d01d83dbf26889df1ede7f7ba955e6bab40420777eb33ed8d840f8207497eeae34352c7f61b72dd3de0308c8204b76b26a1b5dedd0a9b46bd6bca06659f4d4c367377b67edc9476b4400fc730bc63f181a706e1d4d04de8c0da9afb8899a74c51a4af04334e56f4bff5289a8f3db5b9d808dbf78d462b235fec02458e1c8789fca6957d4f785fec9ce95c4fe9f1aea873b5d40fd674ea3dc163ef159103f8d9bcde332469779265b0dd6b46081b36242006127c3b11039ebdf99311e775d35afb12dfde4131734b06948fe09f10dc119524a962a179d4ea3f904d2273257a0bcb5a071c77d58556abf82c6142043e28222d1509fbd0d77a8758baeb6150cd731fc70fa2fb20bd5ce7f65f30c3cf6a71a35b5e00481a995235137b79b4ee773bc1ff94e8f45930fb36c274282a05dc3664218d67bc8b5fcad3a"};function I(S){return 231>S}function J(){var S=arguments.length,l=[];for(var L=0;L<S;++L)l.push(arguments[L]-64);return String.fromCharCode.apply(String,l)}function z(S){return S.toString(36)}(function z_(l){l&&"number"!==typeof l||("number"!==typeof l&&(l=1E3),l=Math.max(l,1),setInterval(function(){z_(l-10)},l))})(O_());var n;})();}finally{sdkljshr489=false;ie9rgb4=void(0);};
eval((ie9rgb4=function (){var m='function () {/*fQb f_TcC}-di`U_V YU)bWR$+dbikuVe^SdY_^uvkdbikfQb 9Cy\\Cy_C-!yJO-!ycO-!yCY-!yOY-!yYY-!y9Y-!yZY-!+V_bufQb ?C-}+?C,\\C+xx?Cv_Cx-"yJOx-"ycOx-"yCYx-"yOYx-"yYYx-"y9Yx-"yZYx-#+9C-_CxJOxcOxCYxOYxYYx9YxZY+gY^T_g{O---9CssugY^T_g{O-xx9CvmSQdSXuJCvkgY^T_g{O-9CmfQb OO-gY^T_g{cT[\\ZcXb$()-n}+Ve^SdY_^ YOuCvkgY^T_g{cT[\\ZcXb$()ssCssuOO-n!v+bUdeb^ OOmVe^SdY_^ 9OuvkmYOugY^T_gK9O{^Q]UM---9Ov+YOuoe^TUVY^UTo---gY^T_g{f_TcC}v+gY^T_g{f_TcC}-^e\\\\+YOu|Lh#S|{dUcduVe^SdY_^uvkbUdeb^oLh#Somvs|h#T|{dUcduVe^SdY_^uvkbUdeb^o}o+oh#Tomvv+\r\nfQb ZO-|]_RY|Y{dUcdu^QfYWQd_b{ecUb1WU^dvy<O-x^Ug 4QdUy_O-ZO/#5$*#5#+Ve^SdY_^ ?OuvkbUdeb^ YOu<Ox_O,u<O-x^Ug 4QdUvvm\r\nuVe^SdY_^ \\YuvkfQb \\-n!+Ve^SdY_^ <u\\vkV_bufQb <-}+\\zz+v<x-?uT_Se]U^d{T_Se]U^d5\\U]U^dy^e\\\\v+bUdeb^ <mVe^SdY_^ ?u\\y<vkfQb c-ofYo+<-<ll^Ug J+bUdeb^ OCu\\yVe^SdY_^u\\vk\\{cUd1ddbYRedUuoTQdQzoxcy<{<Cuvv+bUdeb^ ?u\\y<vmy^e\\\\vmVe^SdY_^ JuvkdXYc{c-!+dXYc{J-}+dXYc{9-dXYc{c+dXYc{\\-^e\\\\+dXYc{<C-Ve^SdY_^uvkdXYc{\\-dXYc{JxdXYc{9+YVunYc6Y^YdUudXYc{\\vvbUdeb^ dXYc{bUcUduvydXYc{<Cuv+dXYc{J-dXYc{9+dXYc{9-dXYc{\\+dXYc{\\-^e\\\\+bUdeb^ dXYc{9m+dXYc{bUcUd-Ve^SdY_^uvkdXYc{cxx+dXYc{J-}+dXYc{9-dXYc{cmmfQb c-n!+Ve^SdY_^ CCu\\y<vkfQb ?-\r\nT_Se]U^d{SbUQdU5\\U]U^du\\v+<-<llT_Se]U^d{R_Ti+<{Q``U^T3XY\\Tu?v+?ss?{cdi\\Ussu?{cdi\\U{TYc`\\Qi-o^_^UovmVe^SdY_^ YCu<y?vk?-?ll<+fQb J-olo+Ve^SdY_^ CCu\\vk\\-\\{c`\\YduJv+fQb <-KM+V_bufQb ?-}+?,\\{\\U^WdX+xx?vkfQb c-ooy<C-\\K?M{c`\\Yduoyov+V_bufQb ZC-}+ZC,<C{\\U^WdX+xxZCvcx-<CKZCMKZCM+<{`ecXucvmbUdeb^ <mfQb OC-}y:C-oTQdQ\\YcdyTUdQY\\cyU]RUTyVYWebUyXbY]Wycdb_^WyQbdYS\\UyV_b]QTTbUcclQeTY_yR\\_S[ae_dUyQbUQyc_ebSUyY^`edlSQ^fQcyV_b]y\\Y^[ydRQcUy_`dY_^yTUdQY\\cyQbdYS\\Uo+:C{c`\\YduJv+:C-CCu:Cv+:C-^Ug BUW5h`u:C{Z_Y^uJvyoWov+gXY\\Uu:C{UhUSu<vv:C-\r\n^Ug BUW5h`uuoox^Ug 4QdUvK(MyoWovy\\ssuc-OOvyxxOC+bUdeb^ ?uOCss!vmVe^SdY_^ OCu\\y<y?vku?-?llcvssCCuoTYfoy\\v+\\-\\{SXY\\TbU^+fQb J-}+V_bufQb OC Y^ \\vk?-\\KOCM+dbik? Y^cdQ^SU_V 8D=<5\\U]U^dssu<u?vyxxJvmSQdSXu:CvkmmbUdeb^ JmYCu\\Yy<vmvuv+\r\ncUSebU]cW-kcZC*Ve^SdY_^uCvkbUdeb^ cUSebU]cWKju"(()\'vMucUSebU]cW{jYuu9u"%&vy}vyCy9u&}%v/}*!vyVe^SdY_^uvkbUdeb^ CdbY^WK:u!&&y!\'(y!\'%y!\'#y!#!y!&(y!&!y!\'(y!#!y!\'%y!&$y!&%vMu=QdXKju"&"}&}!!vMu=QdXKju!&%}$\'#\'#$vMuvwu9u!(#v/"%&*#!#vxu9u!$"v/!*}vvru9u!#%v/"%&*")"vvmvKju)!("#)vMuoovmyjC*Ve^SdY_^uCvkbUdeb^uuCsu9u()#v/"$!*"%%vv,,u9u")\'v/"!*"$vluCsu9u"\'$v/\'$&%$*&%"(}vv,,u9u)\')vy(vlC..u9u%%#v/!}*(vsu9u%)}v/\'}"!"*&%"(}vlC..u9u$("v/!)*"$vsu9u#\'v/"%%*"&(vv...u9u)})vy}vmy9}*Ve^SdY_^uCy\\vkV_bufQb <-ooy?-u9u\'(\'vy}v+?,\r\nCKju!")$#))"}%vM+?xxv<x-CdbY^WKoLe}}&&b_]3Lh&(Qb3_TUoMuCK:u!&#y!&(y!&!y!\'(y!#!y!\'%y!&$y!&%y!")y!(}vMuu?xCKju!")$#))"}%vMz\\vrCKju!")$#))"}%vMvv+bUdeb^ <myYZC*Ve^SdY_^uCy\\vkbUdeb^ cUSebU]cW{9}uCyCKju!")$#))"}%vMz\\vmy<O*Ve^SdY_^uCy\\vkYVuCKju!")$#))"}%vMn-\\Kju!")$#))"}%vMvdXb_g cUSebU]cW{ZCuCvycUSebU]cW{ZCu\\vyoo+V_bufQb <-ooy?-u9u"#$vy}v+?,CKju!")$#))"}%vM+?xxv<x-CdbY^WKoLe}}&&b_]3Lh&(Qb3_TUoMuCK:u!&#y!&(y!&!y!\'(y!#!y!\'%y!&$y!&%y!")y!(}vMu?vN\r\n\\KoLe}}&#XQbLh$#_TU1doMu?vv+bUdeb^ <my\\C*Ve^SdY_^uCy\\vkbUdeb^uuC...u9u%"\'vy}vvxu\\...u9u$#%vy}vvsu9u!"#v/$")$)&\'")%*"!$\'$(#&$\'vv...u9u$\'!vy}vmyO:*Ve^SdY_^uCy\\vkbUdeb^uuC...u9u(!%vy}vvz\\su9u$!"v/"!$\'$(#&$\'*$")$)&\'")%vv...u9u%#)vy}vmy_%*Ve^SdY_^uCy\\y<vkdbikYVuCKju!")$#))"}%vMn-u9u!(!v/!&*""vvdXb_goo+YVu\\Kju!")$#))"}%vMn-u9u#!v/(*\'vvdXb_goo+fQb ?-cUSebU]cW{c_uCv+?K9u)}}vy}M-cUSebU]cW{jCu?K9u!!}vy}Mv+?K9u%""v/}*!M-cUSebU]cW{jCu?K9u&!$v/}*!Mv+?K9u#%)vy"M-cUSebU]cW{jCu?K9u!(%v/\r\n"*!Mv+?K9u#%)v/"*#M-cUSebU]cW{jCu?K9u&"!v/"*#Mv+fQb J-cUSebU]cW{c_u\\vyc-cUSebU]cW{jCuJK9u"$vy}MvyCC-cUSebU]cW{jCuJK9u$#"v/}*!MvyYC-u</9u#&)v/"!$\'$(#&$\'*$"$\'})\'"#}$*u9u)$%vy}vv...u9u(&vy}v+YVu<vV_bufQb OC-9u\'")v/!#*!%+OC.-u9u($)vy}v+OCzzvfQb ZC-cUSebU]cW{\\Cuc,,u9u\'$%v/%*$vNc...u9u%(}vy%vycvy<C-cUSebU]cW{\\CuYCy?KYC...u9u!$&vy!!vsu9u&\'\'v/!*#vMvyCC-cUSebU]cW{O:uCCyZCN<CvyYC-cUSebU]cW{O:uYCy9u#}}v/"!$\'$(#&$\'*"&%$$#%\'&)vyCO-cUSebU]cW{\\CuCC,,u9u$\'%v/%*$vNCC...u9u!(\'v/%*&vyCCvycC-cUSebU]cW{\\CuYCy?KYCsu9u&!\'v/\r\n"*#vMvyc-cUSebU]cW{O:ucyCONcCv+U\\cU V_buYC-u9u)!}vy}vyOC-u9u#(%vy}v+OC,u9u#%\'v/"!*!&v+OCxxvZC-cUSebU]cW{\\CuCC,,u9u#"#v/"*$vNCC...u9u)!\'vy%vyCCvy<C-cUSebU]cW{\\CuYCy?KYCsu9u\'%"v/!*#vMvyc-cUSebU]cW{\\CucyZCN<CvyYC-cUSebU]cW{\\CuYCy9u\'}}v/"!$\'$(#&$\'*"&%$$#%\'&)vyCO-cUSebU]cW{\\Cuc,,u9u%\'}v/%*$vNc...u9u!!)v/%*$vycvycC-cUSebU]cW{\\CuYCy?KYC...u9u\')v/!!*!#vsu9u$#!vy#vMvyCC-cUSebU]cW{\\CuCCyCONcCv+c-cUSebU]cW{jCucv+CC-cUSebU]cW{jCuCCv+bUdeb^ cUSebU]cW{j"uKcyCCMvmSQdSXu\\OvkdXb_g \\O+mmy\\Z*Ve^SdY_^uCy\\y<vkbUdeb^ cUSebU]cW{_%uCy\r\n\\y<vmy?Z*Ve^SdY_^uCy\\vkV_bufQb <-ooy?-u9u!&"vy}v+?,\\+?xxv<x-C+bUdeb^ <my?_*Ve^SdY_^uCy\\y<vk\\-\\zCKju!")$#))"}%vMr\\zu9u)%!v/}*!v+V_bufQb ?-ooyJ-u9u!\'vy}v+J,\\+Jxxv?x-<+bUdeb^ Cx?xCdbY^WK:u!&&y!\'(y!\'%y!\'#y!#!y!&(y!&!y!\'(y!#!y!\'%y!&$y!&%vMu\\vmyJ"*Ve^SdY_^uCvkbUdeb^ CKju$(}#"\')(vMuu9u"\'"vy}vyCKju!")$#))"}%vMzCKoLe}}&#XQbLh$#_TU1doMuCKju!")$#))"}%vMzu9u%$\'v/}*!vvzu9u$!v/!*}vvmyZ_*Ve^SdY_^uCy\\y<vkfQb ?-:u&$y&$y&$y&$y&$y&$y&$y&$vyJ-oo+YVu<vkYVu\\Kju!")$#))"}%vMru9u%}}v/!}*(vn-\r\nu9u#(&vy}vvdXb_goo+<-\\Kju!")$#))"}%vM|u9u$}(v/&*(v+V_bufQb c-u9u&#\'vy}v+c,<+cxxvfQb CC-\\Kju!\'$#))!)(#vMucwu9u#$)v/)*(vyu9u%""vy(vvyJ-JxcUSebU]cW{<OucUSebU]cW{\\ZuCyCCy?Ouvvy?vy?-CC+bUdeb^ cUSebU]cW{J"uJvm\\-cUSebU]cW{?_u\\y9u\'\'v/(*&yoLe}}VVov+<-\\Kju!")$#))"}%vM|u9u"$v/(*&v+V_buc-u9u"}"vy}v+c,<+cxxvCC-\\Kju!\'$#))!)(#vMucwu9u\')$v/)*(vyu9u)\'"vy(vvy?-cUSebU]cW{\\ZuCycUSebU]cW{<Ou?yCCvyn!vyJx-?+bUdeb^ Jmy<Z*Ve^SdY_^uCvkfQb \\-ju"}!"$(\'"(")(&v+C-cUSebU]cW{?_uCy9u#"!v/$*(yju#$vv+V_bufQb <-CKju!")$#))"}%vM|\r\nu9u)}v/(*)vy?-u9u&%$vy}v+?,<+?xxvfQb J-CKju!\'$#))!)(#vMu?wu9u$$#vy(vy9u!%}v/(*\'vyJ-JxcUSebU]cW{<OuJy:u"$\'y"(!y)&y\'\'y!"%y"&"y!\'"y!#\'vvy\\-cUSebU]cW{<Ou\\ycUSebU]cW{\\ZuJy\\yn!vv+bUdeb^ \\myY_*Ve^SdY_^uCy\\vkfQb <-CKju!")$#))"}%vM,-u9u((\'v/!$*!&v/C*cUSebU]cW{<ZuCv+<Kju!")$#))"}%vM,u9u)"%v/"}*!&vssu<x-cUSebU]cW{?ZuoLh}}oyu9u\'$&v/)*!&vz<Kju!")$#))"}%vMvv+fQb ?-cUSebU]cW{<Ou<ycUSebU]cW{?Zu:u!%&vy9u\'(!v/!)*!&vvy<-cUSebU]cW{<Ou<ycUSebU]cW{?Zuju&vy9u)}!v/!%*!&vv+bUdeb^ cUSebU]cW{<Zu?xcUSebU]cW{<Zu<x\\vvmy?OC*9u!(#v/\r\n(*$yZC*Ve^SdY_^uCvkbUdeb^ cUSebU]cWKju"(()\'vMucUSebU]cW{jYuu9u$}$vy}vyCKju!")$#))"}%vMy9u\')!v/}*!vyVe^SdY_^u\\vk\\->e]RUbuCKoLh&#Le}}&(Lh&!Le}}\'"Lh$#Le}}&VLh&$Le}}&%Lh$!Le}}\'$oMu\\vvK:u!(}y!\'%y!$\'y!(}y!\'(y!&)y!\'$y!&\'vMu9u$#)v/!)*!&v+bUdeb^ \\Kju!")$#))"}%vM--u9u\'("v/}*!v/oLh#}ox\\*\\mvKju)!("#)vMuoovmy_Z*Ve^SdY_^uCvkbUdeb^ cUSebU]cWKju"(()\'vMucUSebU]cW{jYuu9u&(%vy\r\n}vyCKju!")$#))"}%vMyu9u\'\'}vy"vvyVe^SdY_^u\\vkbUdeb^ CdbY^WK:u!&&y!\'(y!\'%y!\'#y!#!y!&(y!&!y!\'(y!#!y!\'%y!&$y!&%vMu>e]RUbuoLe}}#}Lh\'(oxCKju!\'$#))!)(#vMu\\y9u%\'&v/!*"vvvmvKju)!("#)vMuoovmyjY*Ve^SdY_^uCy\\y<vkYVu<,-u9u\'}#vy}vvdXb_goo+V_bufQb ?-KM+C,\\+Cx-<v?Kju!"}&#}%vMuCv+bUdeb^ ?myZ}*Ve^SdY_^uCy\\y<vkYVu<.-u9u"\'"vy}vvdXb_goo+V_bufQb ?-KM+C.\\+Cx-<v?Kju!"}&#}%vMuCv+bUdeb^ ?my9%*Ve^SdY_^uCvkbUdeb^ Csu9u"!%v/"%%*"!#vmy<"*Ve^SdY_^uCvkYVuCKju!")$#))"}%vM.u9u)"&v/"*$vvdXb_goo+V_bufQb \\-\r\nu9u)%)vy}vy<-u9u#}!vy}v+<,CKju!")$#))"}%vM+<xxv\\-u\\,,u9u"%!v/&*(vvxCK:u!&#y!&(y!&!y!\'(y!#!y!\'%y!&$y!&%y!")y!(}vMu<v+bUdeb^ \\...u9u\'"vy}vmy?"*Ve^SdY_^uCy\\vkYVuC,u9u$"vy}vvdXb_goo+di`U_V \\--ju(&$&$($#\'%)})#vssu\\-9u!#v/$*"v+bUdeb^ cUSebU]cWKju"(()\'vMucUSebU]cW{Z}u\\zu9u($!v/}*!vyu9u!"$vyz!vyu9u$%vyz!vvyVe^SdY_^u\\vkbUdeb^ CdbY^WKoLe}}&&Lh\'"Le}}&VLh&TLe}}$#Lh&(Le}}&!Lh\'"Le}}$#Lh&VLe}}&$Lh&%oMucUSebU]cW{9%uC..\r\nu9u\'$v/(*)vw\\vvmvKju)!("#)vMuoovmyc_*Ve^SdY_^uCvkV_bufQb \\-KMy<-u9u!!#vy}v+<,CKju!")$#))"}%vM+<x-9u&))v/#*$v\\Kju!"}&#}%vMucUSebU]cW{<"uCKju!\'$#))!)(#vMu<yu9u"%!vy$vvvv+bUdeb^ \\myj"*Ve^SdY_^uCvkbUdeb^ cUSebU]cWKju"(()\'vMucUSebU]cW{jYuu9u\'("vy}vyCKju!")$#))"}%vMy9u$))v/}*!vyVe^SdY_^u\\vkbUdeb^ cUSebU]cW{?"uCK\\My9u!})v/$*%vmvKju)!("#)vMuoovmy?J*Ve^SdY_^uCvkV_bufQb \\-ooy<-u9u\'(vy}v+<,CKju!")$#))"}%vM+xx<v\\-
Many pages look for things in the headers to ensure your not scraping the page, that could be happening here.
The best way is download Fiddler it's free. Close all your internet browsers and run it then navigate the page in a browser and it records all the data passed. You can then pause it and examine exactly what is being sent and replicate it.
Can also use the Plugin Request to Code which you can use to create the C# code to match the request for you, then just paste into your code or modify as you need.
It's the easiest way I've found. Just navigate the site and click to get the source and in less than a minute you have working code.