How to get the html in selenium of current page

Question

How to get the html in selenium of current page

629 Views Asked by Denis At 28 November 2021 at 05:15

I try to get HTML code of the page, but driver.page_source always return HTML + JavaScript.

I know it is many ways described earlier, but in my case they don't work.

The page is - https://qe.com.qa/en/companymoreinformationsearch?CompanyCode=QNBK

Original Q&A

There are 2 best solutions below

Denis On 28 November 2021 at 06:06

innerHTML = driver.execute_script("return document.documentElement.outerHTML")

**undetected Selenium** · Accepted Answer · 2021-11-28T11:52:44.893000

To extract the HTML along with the JavaScript you need to induce WebDriverWait for the visibility_of_all_elements_located() with id as moreinfo-heading and you can use the following Locator Strategy:

Code Block:

Using driver.page_source:

driver.get("https://qe.com.qa/en/companymoreinformationsearch?CompanyCode=QNBK")
WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "#moreinfo-heading")))
print(driver.page_source)

As an alternative, you can also use document.documentElement.outerHTML and document.documentElement.innerHTML as follows:

Using document.documentElement.outerHTML:

driver.get("https://qe.com.qa/en/companymoreinformationsearch?CompanyCode=QNBK")
print(driver.execute_script("return document.documentElement.outerHTML"))

Using document.documentElement.innerHTML:

driver.get("https://qe.com.qa/en/companymoreinformationsearch?CompanyCode=QNBK")
print(driver.execute_script("return document.documentElement.innerHTML"))

Console Output:

getSessionId: function() {
                return '';
            },
            getSiteAdminURL: function() {
                return 'https://qe.com.qa/group/guest/~/control_panel/manage?p_p_lifecycle=0&p_p_state=maximized&p_p_mode=view';
            },
            getSiteGroupId: function() {
                return '20181';
            },
            getURLControlPanel: function() {
                return '/en/group/control_panel?refererPlid=1002057';
            },
            getURLHome: function() {
                return 'https\x3a\x2f\x2fqe\x2ecom\x2eqa\x2fweb\x2fguest';
            },
            getUserEmailAddress: function() {
                return '';
            },
            getUserId: function() {
                return '20158';
            },
            getUserName: function() {
                return '';
            },
            isAddSessionIdToURL: function() {
                return false;
            },
            isImpersonated: function() {
                return false;
            },
            isSignedIn: function() {
                return false;
            },
            isStateExclusive: function() {
                return false;
            },
            isStateMaximized: function() {
                return false;
            },
            isStatePopUp: function() {
                return false;
            }
        };

        var themeDisplay = Liferay.ThemeDisplay;

        Liferay.AUI = {



            getAvailableLangPath: function() {
                return 'available_languages.jsp?browserId=other&themeId=mainsitetheme_WAR_mainsitetheme&colorSchemeId=01&minifierType=js&languageId=en_US&b=7210&t=1635431572526';
            },
            getCombine: function() {
                return true;
            },
            getComboPath: function() {
                return '/combo/?browserId=other&minifierType=&languageId=en_US&b=7210&t=1620065756342&';
            },
            getDateFormat: function() {
                return '%m/%d/%Y';
            },
            getEditorCKEditorPath: function() {
                return '/o/frontend-editor-ckeditor-web';
            },
            getFilter: function() {
                var filter = 'raw';



                        filter = 'min';



                return filter;
            },
            getFilterConfig: function() {
                var instance = this;

                var filterConfig = null;

                if (!instance.getCombine()) {
                    filterConfig = {
                        replaceStr: '.js' + instance.getStaticResourceURLParams(),
                        searchExp: '\\.js$'
                    };
                }

                return filterConfig;
            },
            getJavaScriptRootPath: function() {
                return '/o/frontend-js-web';
            },
            getLangPath: function() {
                return 'aui_lang.jsp?browserId=other&themeId=mainsitetheme_WAR_mainsitetheme&colorSchemeId=01&minifierType=js&languageId=en_US&b=7210&t=1620065756342';
            },
            getPortletRootPath: function() {
                return '/html/portlet';
            },
            getStaticResourceURLParams: function() {
                return '?browserId=other&minifierType=&languageId=en_US&b=7210&t=1620065756342';
            }
        };

        Liferay.authToken = '8LGXsu8b';



        Liferay.currentURL = '\x2fen\x2fcompanymoreinformationsearch\x3fCompanyCode\x3dQNBK';
        Liferay.currentURLEncoded = '\x252Fen\x252Fcompanymoreinformationsearch\x253FCompanyCode\x253DQNBK';
    // ]]>
</script>

<script src="/o/js_loader_config?t=1635431593465" type="text/javascript"></script>
<script data-senna-track="permanent" src="/combo?browserId=other&amp;minifierType=js&amp;languageId=en_US&amp;b=7210&amp;t=1620065756342&amp;/o/frontend-js-web/loader/config.js&amp;/o/frontend-js-web/loader/loader.js&amp;/o/frontend-js-web/aui/aui/aui.js&amp;/o/frontend-js-web/aui/aui-base-html5-shiv/aui-base-html5-shiv.js&amp;/o/frontend-js-web/liferay/browser_selectors.js&amp;/o/frontend-js-web/liferay/modules.js&amp;/o/frontend-js-web/liferay/aui_sandbox.js&amp;/o/frontend-js-web/misc/svg4everybody.js&amp;/o/frontend-js-web/aui/arraylist-add/arraylist-add.js&amp;/o/frontend-js-web/aui/arraylist-filter/arraylist-filter.js&amp;/o/frontend-js-web/aui/arraylist/arraylist.js&amp;/o/frontend-js-web/aui/array-extras/array-extras.js&amp;/o/frontend-js-web/aui/array-invoke/array-invoke.js&amp;/o/frontend-js-web/aui/attribute-base/attribute-base.js&amp;/o/frontend-js-web/aui/attribute-complex/attribute-complex.js&amp;/o/frontend-js-web/aui/attribute-core/attribute-core.js&amp;/o/frontend-js-web/aui/attribute-observable/attribute-observable.js&amp;/o/frontend-js-web/aui/attribute-extras/attribute-extras.js&amp;/o/frontend-js-web/aui/base-base/base-base.js&amp;/o/frontend-js-web/aui/base-pluginhost/base-pluginhost.js&amp;/o/frontend-js-web/aui/classnamemanager/classnamemanager.js&amp;/o/frontend-js-web/aui/datatype-xml-format/datatype-xml-format.js&amp;/o/frontend-js-web/aui/datatype-xml-parse/datatype-xml-parse.js&amp;/o/frontend-js-web/aui/dom-base/dom-base.js&amp;/o/frontend-js-web/aui/dom-core/dom-core.js&amp;/o/frontend-js-web/aui/dom-screen/dom-screen.js&amp;/o/frontend-js-web/aui/dom-style/dom-style.js&amp;/o/frontend-js-web/aui/event-base/event-base.js&amp;/o/frontend-js-web/aui/event-custom-base/event-custom-base.js&amp;/o/frontend-js-web/aui/event-custom-complex/event-custom-complex.js&amp;/o/frontend-js-web/aui/event-delegate/event-delegate.js&amp;/o/frontend-js-web/aui/event-focus/event-focus.js&amp;/o/frontend-js-web/aui/event-hover/event-hover.js&amp;/o/frontend-js-web/aui/event-key/event-key.js&amp;/o/frontend-js-web/aui/event-mouseenter/event-mouseenter.js&amp;/o/frontend-js-web/aui/event-mousewheel/event-mousewheel.js" type="text/javascript"></script>
<script data-senna-track="permanent" src="/combo?browserId=other&amp;minifierType=js&amp;languageId=en_US&amp;b=7210&amp;t=1620065756342&amp;/o/frontend-js-web/aui/event-outside/event-outside.js&amp;/o/frontend-js-web/aui/event-resize/event-resize.js&amp;/o/frontend-js-web/aui/event-simulate/event-simulate.js&amp;/o/frontend-js-web/aui/event-synthetic/event-synthetic.js&amp;/o/frontend-js-web/aui/intl/intl.js&amp;/o/frontend-js-web/aui/io-base/io-base.js&amp;/o/frontend-js-web/aui/io-form/io-form.js&amp;/o/frontend-js-web/aui/io-queue/io-queue.js&amp;/o/frontend-js-web/aui/io-upload-iframe/io-upload-iframe.js&amp;/o/frontend-js-web/aui/io-xdr/io-xdr.js&amp;/o/frontend-js-web/aui/json-parse/json-parse.js&amp;/o/frontend-js-web/aui/json-stringify/json-stringify.js&amp;/o/frontend-js-web/aui/node-base/node-base.js&amp;/o/frontend-js-web/aui/node-core/node-core.js&amp;/o/frontend-js-web/aui/node-event-delegate/node-event-delegate.js&amp;/o/frontend-js-web/aui/node-event-simulate/node-event-simulate.js&amp;/o/frontend-js-web/aui/node-focusmanager/node-focusmanager.js&amp;/o/frontend-js-web/aui/node-pluginhost/node-pluginhost.js&amp;/o/frontend-js-web/aui/node-screen/node-screen.js&amp;/o/frontend-js-web/aui/node-style/node-style.js&amp;/o/frontend-js-web/aui/oop/oop.js&amp;/o/frontend-js-web/aui/plugin/plugin.js&amp;/o/frontend-js-web/aui/pluginhost-base/pluginhost-base.js&amp;/o/frontend-js-web/aui/pluginhost-config/pluginhost-config.js&amp;/o/frontend-js-web/aui/querystring-stringify-simple/querystring-stringify-simple.js&amp;/o/frontend-js-web/aui/queue-promote/queue-promote.js&amp;/o/frontend-js-web/aui/selector-css2/selector-css2.js&amp;/o/frontend-js-web/aui/selector-css3/selector-css3.js&amp;/o/frontend-js-web/aui/selector-native/selector-native.js&amp;/o/frontend-js-web/aui/selector/selector.js&amp;/o/frontend-js-web/aui/widget-base/widget-base.js&amp;/o/frontend-js-web/aui/widget-htmlparser/widget-htmlparser.js&amp;/o/frontend-js-web/aui/widget-skin/widget-skin.js&amp;/o/frontend-js-web/aui/widget-uievents/widget-uievents.js&amp;/o/frontend-js-web/aui/yui-throttle/yui-throttle.js&amp;/o/frontend-js-web/aui/aui-base-core/aui-base-core.js" type="text/javascript"></script>
<script data-senna-track="permanent" src="/combo?browserId=other&amp;minifierType=js&amp;languageId=en_US&amp;b=7210&amp;t=1620065756342&amp;/o/frontend-js-web/aui/aui-base-lang/aui-base-lang.js&amp;/o/frontend-js-web/aui/aui-classnamemanager/aui-classnamemanager.js&amp;/o/frontend-js-web/aui/aui-component/aui-component.js&amp;/o/frontend-js-web/aui/aui-debounce/aui-debounce.js&amp;/o/frontend-js-web/aui/aui-delayed-task-deprecated/aui-delayed-task-deprecated.js&amp;/o/frontend-js-web/aui/aui-event-base/aui-event-base.js&amp;/o/frontend-js-web/aui/aui-event-input/aui-event-input.js&amp;/o/frontend-js-web/aui/aui-form-validator/aui-form-validator.js&amp;/o/frontend-js-web/aui/aui-node-base/aui-node-base.js&amp;/o/frontend-js-web/aui/aui-node-html5/aui-node-html5.js&amp;/o/frontend-js-web/aui/aui-selector/aui-selector.js&amp;/o/frontend-js-web/aui/aui-timer/aui-timer.js&amp;/o/frontend-js-web/liferay/dependency.js&amp;/o/frontend-js-web/liferay/dom_task_runner.js&amp;/o/frontend-js-web/liferay/events.js&amp;/o/frontend-js-web/liferay/language.js&amp;/o/frontend-js-web/liferay/lazy_load.js&amp;/o/frontend-js-web/liferay/liferay.js&amp;/o/frontend-js-web/liferay/util.js&amp;/o/frontend-js-web/liferay/global.bundle.js&amp;/o/frontend-js-web/liferay/portal.js&amp;/o/frontend-js-web/liferay/portlet.js&amp;/o/frontend-js-web/liferay/workflow.js&amp;/o/frontend-js-web/liferay/form.js&amp;/o/frontend-js-web/liferay/form_placeholders.js&amp;/o/frontend-js-web/liferay/icon.js&amp;/o/frontend-js-web/liferay/menu.js&amp;/o/frontend-js-web/liferay/notice.js&amp;/o/frontend-js-web/liferay/poller.js" type="text/javascript"></script>






    <script data-senna-track="temporary" src="/o/js_bundle_config?t=1635431612919" type="text/javascript"></script>


<script data-senna-track="temporary" type="text/javascript">
    // <![CDATA[








    // ]]>
</script>
...

How to get the html in selenium of current page

There are 2 best solutions below

Related Questions in PYTHON

Related Questions in HTML

Related Questions in SELENIUM

Related Questions in INNERHTML

Related Questions in OUTERHTML

Trending Questions

Popular # Hahtags

Popular Questions