通過(guò)線程，HttpWebResponse，正則獲取句子迷中的句子

2019-11-10 19:24:49

字體：大中小

供稿：網(wǎng)友

之前一直在關(guān)注句子迷這個(gè)網(wǎng)站，在里面可以找到自己喜歡的名人的金典語(yǔ)錄。 這里寫(xiě)圖片描述每一次都去打開(kāi)，再去翻頁(yè)查看太煩，有沒(méi)有什么方法可以把網(wǎng)站中的句子直接抓出來(lái)，保存到本地txt文件中？ 然后在網(wǎng)上查看了一些資料，自己動(dòng)手寫(xiě)了一個(gè)控制臺(tái)應(yīng)用程序。

注：代碼存在點(diǎn)問(wèn)題，多次發(fā)起請(qǐng)求，線程會(huì)被占用，目前沒(méi)有解決；求救中

具體實(shí)現(xiàn)方式：

步驟1：通過(guò)HttpWebResponse發(fā)送一個(gè)請(qǐng)求，將整個(gè)HTML頁(yè)面請(qǐng)求過(guò)來(lái)，將全部數(shù)據(jù)讀入StreamReader中，下圖html代碼這里寫(xiě)圖片描述 步驟2：將讀入流中的數(shù)據(jù)進(jìn)行處理，只取包含句子的文本，這里面需要查詢頁(yè)面的中html。在取文本的過(guò)程中，使用正則表達(dá)式，取出想要的數(shù)據(jù)

///正則表達(dá)式 Regex regText = new Regex(@"<div/s+class/=/""views-field-phpcode-1/"">([/S/s]*?)</div>", RegexOptions.IgnoreCase);

步驟3：對(duì)上面的標(biāo)簽，在進(jìn)一步進(jìn)行提取

Regex objRegExp = new Regex("<(.|/n)+?>");

具體代碼如下：

class PRogram { static ReaderWriterLock writeLock = new ReaderWriterLock(); const int LOCK = 1000; //申請(qǐng)讀寫(xiě)時(shí)間 const int SLEEP = 100; //線程掛起時(shí)間 static void Main(string[] args) { Console.WriteLine("-------------------句子迷文檔下載----------"); Console.WriteLine("操作API："); Console.WriteLine("注釋1：查詢的的作者名,以逗號(hào)（英文）隔開(kāi)，例子如下："); Console.WriteLine(" 魯迅,胡適,顧城"); Console.WriteLine("注釋2：保存的盤(pán)符，例子如下："); Console.WriteLine(" X:////句子迷 "); Console.WriteLine("-------------------文檔結(jié)束----------"); Console.WriteLine("請(qǐng)輸入需要保存的盤(pán):"); directoryName1 = Console.ReadLine(); Console.WriteLine("請(qǐng)輸入作者姓名:"); string writers = Console.ReadLine(); string[] strWriter = writers.Split(','); //string[] strWriter = { "魯迅", "胡適", "belle", "stars" }; for (int i = 0; i < strWriter.Length; i++) { Thread thread = new Thread(DownLoad1); thread.Name = strWriter[i]; thread.Start(); } Console.ReadKey(); } #region 通過(guò)網(wǎng)頁(yè)獲取句子迷中的名言名句 /// <summary> /// 通過(guò)網(wǎng)頁(yè)獲取句子迷中的名言名句 /// </summary> /// <param name="category"></param> public static void DownLoad1() { string url = string.Empty; bool flag = true;//遇到取文件異常就直接跳出 for (int pageSize = 0; pageSize < 10; pageSize++) { try { string writerName = Thread.CurrentThread.Name; ; //查詢作者列子 //if (pageSize == 0) // url = "http://www.juzimi.com/writer/徐志摩"; //else // url = "http://www.juzimi.com/writer/徐志摩?page=" + pageSize; //第一版查詢功能 //if (pageSize == 0) // url = "http://www.juzimi.com/writer/"+writerName; //else // url = "http://www.juzimi.com/writer/" + writerName + "?page=" + pageSize; if (pageSize == 0) url = "http://www.juzimi.com/search/node/" + writerName + "%20type:sentence"; else url = "http://www.juzimi.com/search/node/" + writerName + "%20type%3Asentence?page=" + pageSize; //創(chuàng)建http鏈接 var request = (HttpWebRequest)WebRequest.Create(url); //request.Timeout = 1000 * 10; //5s過(guò)期 var response = (HttpWebResponse)request.GetResponse(); Stream stream = response.GetResponseStream(); StreamReader sr = new StreamReader(stream); string content = sr.ReadToEnd(); var list = GetHtmlTextList(content); if (list.Count == 0) { Console.WriteLine("時(shí)間：" + DateTime.Now + " 當(dāng)前網(wǎng)址：" + url + "：未找到相關(guān)信息；" + "當(dāng)前線程：" + Thread.CurrentThread.ManagedThreadId); flag = false; break; } try { //文件保存文件夾 string directoryName = string.IsNullOrEmpty(directoryName1) ? "X://句子迷" : directoryName1; //文件名 string fileName = writerName; Write(directoryName, fileName, list); Console.WriteLine("時(shí)間：" + DateTime.Now + " 當(dāng)前網(wǎng)址：" + url + "句子信息下載完成！" + "當(dāng)前線程：" + Thread.CurrentThread.ManagedThreadId); } catch (Exception e) { Console.WriteLine("時(shí)間：" + DateTime.Now + " 當(dāng)前網(wǎng)址：" + url + " 錯(cuò)誤信息：" + e.Message + "當(dāng)前線程：" + Thread.CurrentThread.ManagedThreadId); continue; } } catch (Exception ex) { if (ex.ToString().Contains("404")) { Console.WriteLine("時(shí)間：" + DateTime.Now + " 當(dāng)前網(wǎng)址：" + url + " 錯(cuò)誤信息：" + ex.Message + "當(dāng)前線程：" + Thread.CurrentThread.ManagedThreadId); flag = false; break; } else { Console.WriteLine("時(shí)間：" + DateTime.Now + " 當(dāng)前網(wǎng)址：" + url + " 錯(cuò)誤信息：" + ex.Message + "當(dāng)前線程：" + Thread.CurrentThread.ManagedThreadId); } } if (!flag) { break; } } } /// <summary> /// 保存句子迷中的句子 /// </summary> /// <param name="path">保存路勁地址</param> /// /// <param name="path">句子迷中的句子</param> public static void Write(string path, string fileName, List<string> strBook) { writeLock.AcquireWriterLock(LOCK); FileStream fs = null; //判斷文件夾是否存在 if (!Directory.Exists(path)) Directory.CreateDirectory(path); //判斷集體文件保存路徑是否存在 string filePalce = path + "http://" + fileName + ".txt"; if (!File.Exists(filePalce)) { File.Create(filePalce); fs = new FileStream(filePalce, FileMode.Create); } else { fs = new FileStream(filePalce, FileMode.Append); } StreamWriter sw = new StreamWriter(fs); foreach (var item in strBook) { //開(kāi)始寫(xiě)入 sw.Write(item + "/r/n/r/n"); } Thread.Sleep(SLEEP); //清空緩沖區(qū) sw.Flush(); //關(guān)閉流 sw.Close(); fs.Close(); writeLock.ReleaseWriterLock(); } /// <summary> /// 取得HTML中所有圖片的 URL。 /// </summary> /// <param name="sHtmlText">HTML代碼</param> /// <returns>圖片的URL列表</returns> public static List<string> GetHtmlTextList(string sHtmlText) { // 定義正則表達(dá)式用來(lái)匹配 text 標(biāo)簽 Regex regText = new Regex(@"<div/s+class/=/""views-field-phpcode-1/"">([/S/s]*?)</div>", RegexOptions.IgnoreCase); // 搜索匹配的字符串 MatchCollection matches = regText.Matches(sHtmlText); List<string> sUrlList = new List<string>(); // 取得匹配項(xiàng)列表 foreach (Match match in matches) { sUrlList.Add(replceHtml(match.Value)); } return sUrlList; } /// <summary> /// 將取出來(lái)的含有html的標(biāo)簽替換掉，只留下里面的值 /// </summary> /// <param name="strHtml"></param> /// <returns></returns> public static string replceHtml(string strHtml) { Regex objRegExp = new Regex("<(.|/n)+?>"); return objRegExp.Replace(strHtml, ""); } #endregion