IdHttp + IdCookieManager + ASP.NET WebForm Page [Close]

Question

First, sorry for my bad english!

I need to perform a query on the website www.nfe.fazenda.org.br. For best performance'm using the component TIdHTTP with TIdCookieManager.

This site uses of captcha for control access. So, i'm trying get the page and the captcha for obtaining the cookies.

The user enter the captcha code and key for NFe. So, i send to page with post.

But, I'm being redirected to an error page when I run the post.

Here my test code and ask you to help me. Thank you!

unit Forms.MainForm;

interface

uses
  Winapi.Windows, Winapi.Messages,
  System.SysUtils, System.Variants, System.Classes,
  Vcl.Forms, Vcl.Graphics, Vcl.Dialogs, Vcl.Controls, Vcl.ExtCtrls,
  Vcl.StdCtrls,
  IdBaseComponent, IdComponent, IdTCPConnection, IdTCPClient, IdHTTP,
  IdIOHandler, IdIOHandlerSocket, IdIOHandlerStack, IdSSL, IdSSLOpenSSL,
  IdCookieManager, IdCookie, IdURI,
  GIFImg, WinInet;

type
  TMainForm = class(TForm)
    mem: TMemo;
    IdHttp: TIdHTTP;
    IdSSLHandlerSocket: TIdSSLIOHandlerSocketOpenSSL;
    IdCookieManager: TIdCookieManager;
    panBottom: TPanel;
    btnGo: TButton;
    imgCaptcha: TImage;
    edtKey: TEdit;
    edtCode: TEdit;
    lblInit: TLabel;
    procedure FormShow(Sender: TObject);
    procedure lblInitClick(Sender: TObject);
    procedure btnGoClick(Sender: TObject);
  private
    Cookies: TIdCookies;
    viewState, eventValidate: string;
    procedure GetHiddenFieldValues(html: string);
    procedure p_Execute;
  end;

var
  MainForm: TMainForm;

const
  HOST         = 'http://www.nfe.fazenda.gov.br';
  URLIMG       = 'http://www.nfe.fazenda.gov.br/scripts/srf/intercepta/captcha.aspx?opt=image';
  URLGET       = 'http://www.nfe.fazenda.gov.br/portal/consulta.aspx?tipoConsulta=completa&tipoConteudo=XbSeqxE8pl8=';
  URLPOST      = 'http://www.nfe.fazenda.gov.br/portal/consultaCompleta.aspx?tipoConteudo=XbSeqxE8pl8=';
  CONTENT_TYPE = 'application/x-www-form-urlencoded';

implementation

{$R *.dfm}

procedure TMainForm.FormShow(Sender: TObject);
begin
  lblInitClick(Sender);
end;

procedure TMainForm.lblInitClick(Sender: TObject);
var
  response: TMemoryStream;
  gif: TGIFImage;
  html: string;
begin
  response := TMemoryStream.Create;
  gif := TGIFImage.Create;
  try
    html := IdHttp.Get(URLGET);
    mem.Text := html;
    GetHiddenFieldValues(html);

    IdHttp.Get(URLIMG, response);
    response.Position := 0;
    gif.LoadFromStream(response);
    imgCaptcha.Picture.Assign(gif);

    Cookies := IdCookieManager.CookieCollection;
  finally
    gif.Free;
    response.Free;
  end;
end;

procedure TMainForm.btnGoClick(Sender: TObject);
begin
  p_Execute;
end;

procedure TMainForm.GetHiddenFieldValues(html: string);
var
  nIni, nLen: integer;
  cVal: string;
const
  TAG_VIEWSTATE = '<input type="hidden" name="__VIEWSTATE" id="__VIEWSTATE" value="';
  TAG_EVENTVALIDATION = '<input type="hidden" name="__EVENTVALIDATION" id="__EVENTVALIDATION" value="';
begin
  nIni := Pos(TAG_VIEWSTATE, html);
  nLen := Length(TAG_VIEWSTATE);
  cVal := Copy(html,nIni+nLen, Length(html));
  cVal := Copy(cVal, 1, Pos('" />', cVal)-1);
  viewState := cVal;

  nIni := Pos(TAG_EVENTVALIDATION, html);
  nLen := Length(TAG_EVENTVALIDATION);
  cVal := Copy(html,nIni+nLen, Length(html));
  cVal := Copy(cVal, 1, Pos('" />', cVal)-1);
  eventValidate := cVal;
end;

procedure TMainForm.p_Execute;
var
  params: TStringList;
  Uri: TIdURI;
  nI: Integer;
begin
  params := TStringList.Create;
  Uri := TIdURI.Create(Cookies[0].Domain);
  try
    for nI := 0 to Pred(Cookies.Count) do
      begin
        IdCookieManager.AddServerCookie(Cookies[nI].ClientCookie, Uri);
        if nI = 0 then
          IdHttp.Request.CustomHeaders.Values['Cookie'] := Cookies[nI].ClientCookie
        else
          IdHttp.Request.CustomHeaders.Values['Cookie'] := IdHttp.Request.CustomHeaders.Values['Cookie'] + '; ' + Cookies[nI].ClientCookie;
      end;

    params.Add('__VIEWSTATE=' + viewState);
    params.Add('__EVENTVALIDATION=' + eventValidate);

    params.Add('__EVENTTARGET=');
    params.Add('__EVENTARGUMENT=');

    params.Add('ctl00$txtPalavraChave=');

    params.Add('ctl00$ContentPlaceHolder1$txtChaveAcessoCompleta=' + edtKey.Text);
    params.Add('ctl00$ContentPlaceHolder1$txtCaptcha=' + edtCode.Text);

    params.Add('ctl00$ContentPlaceHolder1$btnConsultar=Continuar');
    params.Add('hiddenInputToUpdateATBuffer_CommonToolkitScripts=1');

    IdHttp.Request.ContentType := CONTENT_TYPE;
    mem.Text := IdHttp.Post(URLPOST, params);
  finally
    params.Free;
    Uri.Free;
  end;
end;

end.

Answer 1

You are mismanaging the server's cookies. TIdURI.Create() expects a full URL, and AddServerCookie() requires a full URL so it can process Paths, differentiate between HTTP and HTTPS cookies, etc. But you are passing TIdURI only a domain name by itself, which is not enough.

Why are you re-adding cookies back into the TIdCookieManager when they already exist in the TIdCookieManager? And why are you setting the CustomHeaders.Values['Cookie'] property manually? Do not do those things. All you need to do is assign the TIdCookieManager to the TIdHTTP.CookieManager property and make sure that the TIdHTTP.AllowCookies property is set to True. That's it. TIdHTTP and TIdCookieManager will then do all of the hard work of receiving, managing, and sending cookies for you. As long as you use the same TIdCookieManager object for multiple HTTP requests (even if you don't use the same TIdHTTP object), then cookies will automatically persist from one request to the next as needed.

If you re-use the same TIdHTTP object then you don't have to worry about creating a TIdCookieManager at all, since TIdHTTP will create one internally if needed and it will be used for the lifetime of the TIdHTTP object.

Try this instead:

unit Forms.MainForm;

interface

uses
  Winapi.Windows, Winapi.Messages,
  System.SysUtils, System.Variants, System.Classes,
  Vcl.Forms, Vcl.Graphics, Vcl.Dialogs, Vcl.Controls, Vcl.ExtCtrls,
  Vcl.StdCtrls,
  IdBaseComponent, IdComponent, IdTCPConnection, IdTCPClient, IdHTTP,
  IdIOHandler, IdIOHandlerSocket, IdIOHandlerStack, IdSSL, IdSSLOpenSSL,
  GIFImg;

type
  TMainForm = class(TForm)
    mem: TMemo;
    IdHttp: TIdHTTP;
    IdSSLHandlerSocket: TIdSSLIOHandlerSocketOpenSSL;
    panBottom: TPanel;
    btnGo: TButton;
    imgCaptcha: TImage;
    edtKey: TEdit;
    edtCode: TEdit;
    lblInit: TLabel;
    procedure FormShow(Sender: TObject);
    procedure lblInitClick(Sender: TObject);
    procedure btnGoClick(Sender: TObject);
  private
    viewState, eventValidate: string;
    procedure GetHiddenFieldValues(html: string);
    procedure p_Execute;
  end;

var
  MainForm: TMainForm;

const
  HOST         = 'http://www.nfe.fazenda.gov.br';
  URLIMG       = HOST+'/scripts/srf/intercepta/captcha.aspx?opt=image';
  URLGET       = HOST+'/portal/consulta.aspx?tipoConsulta=completa&tipoConteudo=XbSeqxE8pl8=';
  URLPOST      = HOST+'/portal/consultaCompleta.aspx?tipoConteudo=XbSeqxE8pl8=';

implementation

{$R *.dfm}

procedure TMainForm.FormShow(Sender: TObject);
begin
  lblInitClick(Sender);
end;

procedure TMainForm.lblInitClick(Sender: TObject);
var
  response: TMemoryStream;
  gif: TGIFImage;
  html: string;
begin
  html := IdHttp.Get(URLGET);
  mem.Text := html;
  GetHiddenFieldValues(html);

  gif := TGIFImage.Create;
  try
    response := TMemoryStream.Create;
    try
      IdHttp.Get(URLIMG, response);
      response.Position := 0;
      gif.LoadFromStream(response);
    finally
      response.Free;
    end;
    imgCaptcha.Picture.Assign(gif);
  finally
    gif.Free;
  end;
end;

procedure TMainForm.btnGoClick(Sender: TObject);
begin
  p_Execute;
end;

procedure TMainForm.GetHiddenFieldValues(html: string);
var
  nIni, nLen: integer;
  cVal: string;
const
  TAG_VIEWSTATE = '<input type="hidden" name="__VIEWSTATE" id="__VIEWSTATE" value="';
  TAG_EVENTVALIDATION = '<input type="hidden" name="__EVENTVALIDATION" id="__EVENTVALIDATION" value="';
begin
  nIni := Pos(TAG_VIEWSTATE, html);
  nLen := Length(TAG_VIEWSTATE);
  cVal := Copy(html,nIni+nLen, Length(html));
  cVal := Copy(cVal, 1, Pos('" />', cVal)-1);
  viewState := cVal;

  nIni := Pos(TAG_EVENTVALIDATION, html);
  nLen := Length(TAG_EVENTVALIDATION);
  cVal := Copy(html,nIni+nLen, Length(html));
  cVal := Copy(cVal, 1, Pos('" />', cVal)-1);
  eventValidate := cVal;
end;

procedure TMainForm.p_Execute;
var
  params: TStringList;
begin
  params := TStringList.Create;
  try    
    params.Add('__VIEWSTATE=' + viewState);
    params.Add('__EVENTVALIDATION=' + eventValidate);

    params.Add('__EVENTTARGET=');
    params.Add('__EVENTARGUMENT=');

    params.Add('ctl00$txtPalavraChave=');

    params.Add('ctl00$ContentPlaceHolder1$txtChaveAcessoCompleta=' + edtKey.Text);
    params.Add('ctl00$ContentPlaceHolder1$txtCaptcha=' + edtCode.Text);

    params.Add('ctl00$ContentPlaceHolder1$btnConsultar=Continuar');
    params.Add('hiddenInputToUpdateATBuffer_CommonToolkitScripts=1');

    mem.Text := IdHttp.Post(URLPOST, params);
  finally
    params.Free;
  end;
end;

end.

Now, with that said, there are other problems:

1) You are posting your params to http://www.nfe.fazenda.gov.br/portal/consultaCompleta.aspx?tipoConteudo=XbSeqxE8pl8=, but when I go to the login URL with a web browser and look at the HTML, I see that it actually wants the form posted to http://www.nfe.fazenda.gov.br/consulta.aspx?tipoConsulta=completa&tipoConteudo=XbSeqxE8pl8= instead. You are missing the tipoConsulta=completa portion.

2) You are parsing out the actual __VIEWSTATE and __EVENTVALIDATION values from the HTML, but are sending blank __EVENTTARGET and __EVENTARGUMENT values. Those values are blank in the HTML, but they are actually filled in dynamically via client-side scripts.

3) there are other <input> fields in the HTML that you are not posting.

A web browser posts every <input> field that has a non-empty value assigned to it, whether that value is assigned statically or dynamically. You need to do the same in your application. The HTTP server is likely to expect all of those values to be sent. Use a packet sniffer, such as Wireshark or Fiddler, to see what a web browser actually posts, and then replicate that same behavior in your code.

Answer 2

Thank you very much!
The problem is solved!

With your information and Wireshark I realized I needed a new "GET". Solved!

unit Forms.MainForm;

interface

uses
  Winapi.Windows, Winapi.Messages,
  System.SysUtils, System.Variants, System.Classes,
  Vcl.Forms, Vcl.Graphics, Vcl.Dialogs, Vcl.Controls, Vcl.ExtCtrls,
  Vcl.StdCtrls,
  IdBaseComponent, IdComponent, IdTCPConnection, IdTCPClient, IdHTTP,
  IdIOHandler, IdIOHandlerSocket, IdIOHandlerStack, IdSSL, IdSSLOpenSSL,
  IdCookieManager, IdCookie, IdURI,
  GIFImg;

type
  TMainForm = class(TForm)
    mem: TMemo;
    IdHttp: TIdHTTP;
    IdSSLHandlerSocket: TIdSSLIOHandlerSocketOpenSSL;
    panBottom: TPanel;
    btnGo: TButton;
    imgCaptcha: TImage;
    edtKey: TEdit;
    edtCode: TEdit;
    lblInit: TLabel;
    procedure FormShow(Sender: TObject);
    procedure lblInitClick(Sender: TObject);
    procedure btnGoClick(Sender: TObject);
  private
    viewState, eventValidate: string;
    procedure GetHiddenFieldValues(html: string);
    procedure p_Execute;
  end;

var
  MainForm: TMainForm;

const
  HOST         = 'http://www.nfe.fazenda.gov.br';
  URLIMG       = HOST + '/scripts/srf/intercepta/captcha.aspx?opt=image';
  URLGET       = HOST + '/portal/consulta.aspx?tipoConsulta=completa&tipoConteudo=XbSeqxE8pl8=';
  URLPOST      = HOST + '/portal/consulta.aspx?tipoConsulta=completa&tipoConteudo=XbSeqxE8pl8%3d';
  URLGETRESULT = HOST + '/portal/consultaCompleta.aspx?tipoConteudo=XbSeqxE8pl8=';
  CONTENT_TYPE = 'application/x-www-form-urlencoded';

implementation

{$R *.dfm}

procedure TMainForm.FormShow(Sender: TObject);
begin
  lblInitClick(Sender);
end;

procedure TMainForm.lblInitClick(Sender: TObject);
var
  response: TMemoryStream;
  gif: TGIFImage;
  html: string;
begin
  html := IdHttp.Get(URLGET);
  mem.Text := html;
  GetHiddenFieldValues(html);

  response := TMemoryStream.Create;
  gif := TGIFImage.Create;
  try
    IdHttp.Get(URLIMG, response);
    response.Position := 0;
    gif.LoadFromStream(response);
    imgCaptcha.Picture.Assign(gif);
  finally
    gif.Free;
    response.Free;
  end;
end;

procedure TMainForm.btnGoClick(Sender: TObject);
begin
  p_Execute;
end;

procedure TMainForm.GetHiddenFieldValues(html: string);
var
  nIni, nLen: integer;
  cVal: string;
const
  TAG_VIEWSTATE = '<input type="hidden" name="__VIEWSTATE" id="__VIEWSTATE" value="';
  TAG_EVENTVALIDATION = '<input type="hidden" name="__EVENTVALIDATION" id="__EVENTVALIDATION" value="';
begin
  nIni := Pos(TAG_VIEWSTATE, html);
  nLen := Length(TAG_VIEWSTATE);
  cVal := Copy(html,nIni+nLen, Length(html));
  cVal := Copy(cVal, 1, Pos('" />', cVal)-1);
  viewState := cVal;

  nIni := Pos(TAG_EVENTVALIDATION, html);
  nLen := Length(TAG_EVENTVALIDATION);
  cVal := Copy(html,nIni+nLen, Length(html));
  cVal := Copy(cVal, 1, Pos('" />', cVal)-1);
  eventValidate := cVal;
end;

procedure TMainForm.p_Execute;
var
  params: TStringList;
begin
  params := TStringList.Create;
  try
    params.Add('__VIEWSTATE=' + viewState);
    params.Add('__EVENTVALIDATION=' + eventValidate);

    params.Add('__EVENTTARGET=');
    params.Add('__EVENTARGUMENT=');

    params.Add('ctl00$txtPalavraChave=');

    params.Add('ctl00$ContentPlaceHolder1$txtChaveAcessoCompleta=' + Trim(edtKey.Text));

    params.Add('ctl00$ContentPlaceHolder1$txtCaptcha=' + Trim(edtCode.Text));

    params.Add('ctl00$ContentPlaceHolder1$btnConsultar=Continuar');
    params.Add('hiddenInputToUpdateATBuffer_CommonToolkitScripts=1');

    mem.Text := IdHttp.Post(URLPOST, params);

    mem.Text := IdHttp.Get(URLGETRESULT);
  finally
    params.Free;
  end;
end;

end.