java 使用htmlunit模拟登录爬取新浪微博页面
16lz
2021-01-22
import java.io.IOException;
import java.net.MalformedURLException;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlInput;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
public class SinaLoginTest {
public static void main(String[] args) throws FailingHttpStatusCodeException, MalformedURLException, IOException, InterruptedException {
WebClient client = new WebClient(BrowserVersion.FIREFOX_24);
client.getOptions().setJavaScriptEnabled(true); //默认执行js,如果不执行js,则可能会登录失败,因为用户名密码框需要js来绘制。
client.getOptions().setCssEnabled(false);
client.setAjaxController(new NicelyResynchronizingAjaxController());
client.getOptions().setThrowExceptionOnScriptError(false);
HtmlPage page = client.getPage("http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.3.16)");
//System.out.println(page.asText());
HtmlInput pwd = page.getHtmlElementById("password");
HtmlInput btn = page.getFirstByXPath(".//*[@id='vForm']/div[3]/ul/li[6]/div[2]/input");
ln.setAttribute("value", "此处改为你的用户名");
pwd.setAttribute("value", "此处改为你的密码");
HtmlPage page2 = btn.click();
//登录完成,现在可以爬取任意你想要的页面了。
System.out.println("\n\n\n");
//System.out.println(page2.asText());
HtmlPage page3 = client.getPage("http://weibo.com/friends?leftnav=1&wvr=5&isfriends=1&step=2");
System.out.println(" : " + page3.asXml());
client.closeAllWindows();
}
import java.net.MalformedURLException;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlInput;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
public class SinaLoginTest {
public static void main(String[] args) throws FailingHttpStatusCodeException, MalformedURLException, IOException, InterruptedException {
WebClient client = new WebClient(BrowserVersion.FIREFOX_24);
client.getOptions().setJavaScriptEnabled(true); //默认执行js,如果不执行js,则可能会登录失败,因为用户名密码框需要js来绘制。
client.getOptions().setCssEnabled(false);
client.setAjaxController(new NicelyResynchronizingAjaxController());
client.getOptions().setThrowExceptionOnScriptError(false);
HtmlPage page = client.getPage("http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.3.16)");
//System.out.println(page.asText());
//登录
HtmlInput pwd = page.getHtmlElementById("password");
HtmlInput btn = page.getFirstByXPath(".//*[@id='vForm']/div[3]/ul/li[6]/div[2]/input");
ln.setAttribute("value", "此处改为你的用户名");
pwd.setAttribute("value", "此处改为你的密码");
HtmlPage page2 = btn.click();
//登录完成,现在可以爬取任意你想要的页面了。
System.out.println("\n\n\n");
//System.out.println(page2.asText());
HtmlPage page3 = client.getPage("http://weibo.com/friends?leftnav=1&wvr=5&isfriends=1&step=2");
System.out.println(" : " + page3.asXml());
client.closeAllWindows();
}
}
<注>取回html(对于page3,就是page3.asXml())后只需要用正则取出你想要的内容即可。
更多相关文章
- 我正在尝试使用带有post方法的AJAX将用户名和密码发送到php文件
- 用于检查用户名可用性的Javascript帖子无效
- PHP中对用户密码进行加密
- php扫马、内网查mysql数据库账号密码(内网webshell渗透)
- 如何使用用户名作为子域名创建子域?
- jQuery PHP:检查用户名是否已经存在
- PHP万能密码登陆
- 从mysql中选择加密密码
- Zend数据库适配器-未捕获异常-堆栈跟踪显示用户名和密码