java htmlunit使用说明

1.抓取网页

WebClient webClient = new WebClient();
HtmlPage page = webClient.getPage("http://www.baidu.com");
System.out.println(page.asText());
webClient.close();

2.常用设置

//启动js
webClient.getOptions().setJavaScriptEnabled(true);
//关闭css渲染
webClient.getOptions().setCssEnabled(false);
//启动重定向
webClient.getOptions().setRedirectEnabled(true);
//启动cookie管理
webClient.setCookieManager(new CookieManager());
//启动ajax代理
webClient.setAjaxController(new NicelyResynchronizingAjaxController());
//js运行时错误,是否抛出异常
webClient.getOptions().setThrowExceptionOnScriptError(false);

3.模拟不同浏览器

//谷歌
WebClient webClient = new WebClient(BrowserVersion.CHROME);
//火狐
//WebClient webClient = new WebClient(BrowserVersion.FIREFOX_45);
//edge
//WebClient webClient = new WebClient(BrowserVersion.EDGE);

4.查找特定元素

String url = "http://weixin.sogou.com/";
HtmlPage page = client.getPage(url);
HtmlInput search = (HtmlInput)page.getElementByName("query");

5.通过XPath获取元素

文档地址学习地址http://www.w3school.com.cn/xpath/xpath_syntax.asp

6.提交表单

HtmlPage page = (HtmlPage)webClient.getPage("http://www.baidu.com/");
//获取搜索输入框并提交搜索内容
HtmlInput input = (HtmlInput)page.getHtmlElementById("kw");
System.out.println(input.toString());
input.setValueAttribute("指尖快递");
System.out.println(input.toString());
//获取搜索按钮并点击
HtmlInput btn = (HtmlInput)page.getHtmlElementById("su");
HtmlPage page2 = btn.click();

7.获取页面中的a标签

java.util.List<HtmlAnchor> achList=page.getAnchors();
for(HtmlAnchor ach:achList){
System.out.println(ach.getHrefAttribute());
}

8.使用代理


WebClient webClient = new WebClient(BrowserVersion.CHROME,"http://127.0.0.1",8087);
DefaultCredentialsProvider credentialsProvider = (DefaultCredentialsProvider) webClient.getCredentialsProvider();
credentialsProvider.addCredentials("username","password");



已标记关键词 清除标记
下面代码中 能获取到百度页面的Form 表单, 但是获取不到另外一个 网页的表单, 此时该如何处理? ps:获取另外一个网页的源码保存html后,用浏览器打开,能看到 form 表单。 package com.xttx.cn.fetchpro.fetchImp; import java.io.IOException; import java.net.URL; import java.util.List; import com.gargoylesoftware.htmlunit.BrowserVersion; import com.gargoylesoftware.htmlunit.CollectingAlertHandler; import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException; import com.gargoylesoftware.htmlunit.HttpWebConnection; import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController; import com.gargoylesoftware.htmlunit.WebClient; import com.gargoylesoftware.htmlunit.WebRequest; import com.gargoylesoftware.htmlunit.html.HtmlForm; import com.gargoylesoftware.htmlunit.html.HtmlPage; import com.xttx.cn.fetchpro.exception.PageNotFoundException; public class SimulationFetch { protected static WebClient webClient = null; public WebClient getWebClient() { return webClient; } public void setWebClient(WebClient webClient) { this.webClient = webClient; } static WebRequest request = null; public static void main(String[] args) throws FailingHttpStatusCodeException, IOException, PageNotFoundException, InterruptedException { // TODO Auto-generated method stub BrowserVersion.FIREFOX_24.setBrowserLanguage("zh-CN"); BrowserVersion.FIREFOX_24.setSystemLanguage("zh-CN"); BrowserVersion.FIREFOX_24.setUserLanguage("zh-CN"); BrowserVersion.FIREFOX_24.setUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0"); BrowserVersion.FIREFOX_24.setBrowserVersion(46.0f); BrowserVersion.FIREFOX_24.setCpuClass("x64"); webClient = new WebClient(BrowserVersion.FIREFOX_24); webClient.setWebConnection(new HttpWebConnection(webClient)); webClient.getCache().clear(); webClient.getCookieManager().clearCookies(); webClient.getOptions().setJavaScriptEnabled(true); webClient.setJavaScriptTimeout(60*1000); webClient.getOptions().setCssEnabled(true); webClient.getOptions().setActiveXNative(true); webClient.getOptions().setPopupBlockerEnabled(true); webClient.getOptions().setRedirectEnabled(true); webClient.getOptions().setTimeout(10000); webClient.getOptions().setDoNotTrackEnabled(true); webClient.getCookieManager().setCookiesEnabled(true); webClient.getOptions().setThrowExceptionOnFailingStatusCode(true); webClient.getOptions().setThrowExceptionOnScriptError(false); webClient.getOptions().setUseInsecureSSL(true); webClient.getOptions().setSSLInsecureProtocol("TLSv1.2"); webClient.setAjaxController(new NicelyResynchronizingAjaxController()); webClient.setAlertHandler(new CollectingAlertHandler()); HtmlPage loginPageWithForm =(HtmlPage)webClient.getPage(new WebRequest(new URL("https://login.youzan.com/sso/index?service=kdt&from_source=pzshouye"))); List<HtmlForm> htmlForm0 = loginPageWithForm.getForms(); HtmlPage loginPageNoForm = (HtmlPage)webClient.getPage(new WebRequest(new URL("https://www.baidu.com/"))); HtmlForm htmlForm = loginPageNoForm.getForms().get(0); System.out.println("---"); } }
©️2020 CSDN 皮肤主题: 大白 设计师:CSDN官方博客 返回首页