[转载].Net解析html文档类库HtmlAgilityPack完整使用说明--采集软件开发尤其好用 - 项目实战 - IT工作生活这点事。Just Such So!

1、获取网页title：doc.DocumentNode.SelectSingleNode(“//title”).InnerText;
解释：XPath中“//title”表示所有title节点。SelectSingleNode用于获取满足条件的唯一的节点。
2、获取所有的超链接：doc.DocumentNode.Descendants(“a”)
3、获取name为kw的input，也就是相当于getElementsByName()：
var kwBox = doc.DocumentNode.SelectSingleNode(“//input[@name=’kw’]”);

解释：”//input[@name=’kw’]”也是XPath的语法，表示：name属性等于kw的input标签。

//li/h3/a[@href]：所有li下面的h3包含a超级链接有href属性才符合。有的a可能是支持的js事件

//div[starts-with(@class,’content_single’)]：所有符合条件的div，并且它的class是由字符串content_single 开头的。

HtmlAgilityPack.HtmlDocument doccc = hw.Load(url);//是你需要解析的url

ArrayList ImagePtahs = GetHrefs(doccc);

//向指定地址发送请求

HttpWebRequest HttpWReq = (HttpWebRequest)WebRequest.Create(url);

HttpWReq.Proxy = proxyObject;

HttpWReq.Timeout = 10000;

HttpWebResponse HttpWResp = (HttpWebResponse)HttpWReq.GetResponse();

StreamReader sr = new StreamReader(HttpWResp.GetResponseStream(), System.Text.Encoding.GetEncoding(“UTF-8”));

//注意上面的编码了吗？？

HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();

doc.Load(sr);

int res = CheckIsGoodProxy(doc); //这是我解析的函数，还没到那一步。不解释了。

sr.Close();

HttpWResp.Close();

HttpWReq.Abort();

HtmlAgilityPack.HtmlWeb.PreRequestHandler handler = delegate(HttpWebRequest request)

{

request.Headers[HttpRequestHeader.AcceptEncoding] = “gzip, deflate”;

request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;

request.CookieContainer = new System.Net.CookieContainer();

return true;

};

webClient.PreRequest += handler;

HtmlDocument doc = webClient.Load(this.getUrl());

{

try

{

Images = new ArrayList();

HtmlNodeCollection hrefs = _doc.DocumentNode.SelectNodes(“//li/h3/a[@href]”);

HtmlNodeCollection hrefs2 = _doc.DocumentNode.SelectNodes(“//div[starts-with(@class,’content_single’)]”);

if (hrefs == null)

return new ArrayList();

foreach (HtmlNode href in hrefs)

{

//  Images.Add(href.Attributes[“src”].Value);

string hreff = href.Attributes[“href”].Value;// 排除  博海拾贝第二百零二期】吃完薯条寂寞了

string title = href.Attributes[“title”].Value;

if (title.IndexOf(“邪恶”) >= 0)

{

continue;

}

if (title.IndexOf(“恶搞”) >= 0)

{

continue;

}

if (title.IndexOf(“雷人”) >= 0)

{

continue;

}

///执行数据保存的逻辑

}

}

catch (Exception ex)

{

ShowLogMsg(“出错了：”+ex.Message+ex.StackTrace);

return new ArrayList();

}

}