巧用Webbrowser实现网络数据采集

 在上一篇文章：《巧用C# webbrowser实现动态网页爬虫机器人》中，给出了一个综合利用webbrowser控件，MSHTML DOM,正则表达式，以及线程阻塞技术爬取动态网页连接的框架和方法。（注：所谓动态网页是指：目标网页的URL不能从当前网页中直接得到，而是要依赖JavaScript从新定向和生成。这种动态网页，咱们有的时候能够经过查看页面<script>元素的内容找到蛛丝马迹，当更多时候，生成目标网页的URL的JS函数的源码再也不当前网页，这时候就显得无能为力了）。《巧用C# webbrowser实现动态网页爬虫机器人》一文中的框架方法也一样适用于静态网页，能够说这个框架方法在某种程度上具备必定的通用性。此篇博文是对上一篇博文的一个补充，实现的功能是：从一级索引页获取到二级索引页首页的连接，在二级索引页首页获取当前页面中指向内容页面的URL连接，而且进行翻页获取内容页源码，同时完成二级索引页首页到次页的翻页，并以此类推。（注意：索引页，内容页的定义请见：《巧用C# webbrowser实现动态网页爬虫机器人》）

    网络数据采集是网络挖掘的一个重要步骤。属于预处理工做范畴。通常分为两步，网络信息采集（即下载网页源码）和网络信息解析（即对网页源码进行解析，提取出BOI (block of interest)）。本文是对《巧用C# webbrowser实现动态网页爬虫机器人》的补充，完成网络信息采集功能。

下面给出关键模块代码：

 

public class ArticlePage
    {
        public string title;
        public string url;
        public string rawtext;
        public ArticlePage()
        {
            title = string.Empty;
            url = string.Empty;
            rawtext = string.Empty;
        }
    }

 public bool mysignal1;//btnworkflow按钮是否被点击
        public bool mysignal2;
        public bool mysignal3;
        public bool loading;//工做流按钮与webbrowser进行交互的通讯按钮
        public bool subloading;
        public bool subloadingPer;

public Form1()
{
            InitializeComponent();
            mysignal1 = false;
            mysignal2 = false;
            mysignal3 = false;
            loading = true;
            subloading = true;
            subloadingPer = true;
 }

 

 private void webBrowser1_DocumentCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
      {
            if (webBrowser1.ReadyState ==WebBrowserReadyState.Complete)
            {
                if (mysignal1)
                {
                    if (!mysignal2&&!mysignal3)
                    {
                        loading = false;
                    }
                    else
                    {
                        if (mysignal2&&!mysignal3)
                        {
                            subloading = false;
                        }
                       
                        if (mysignal3)
                        {
                            subloadingPer = false;
                        }

                    }
                   
                   
                }
               
               
            }
                        

           
               
           


        }

private void btnworkflow_Click(object sender, EventArgs e)
        {
mysignal1 = true;
            string dirCurPageUrl = string.Empty;//用于恢复目录页当前页的网页视图
            List<ArticlePage> arListCurrentPage;
            foreach (string s in issuesMap)
            {

                loading = true;
                string tmpurl = s;
                webBrowser1.Navigate(tmpurl);
              
                while (loading == true)
                {
                    Application.DoEvents();
                }
              
                
                    arListCurrentPage = GetArticlePageInfoFromCurrentDirpage();
                     if (arListCurrentPage != null)
                     {
                         dirCurPageUrl = webBrowser1.Url.ToString();
                         mysignal3 = true;
                         foreach ( ArticlePage ap in arListCurrentPage)
                         {
                             webBrowser1.Navigate(ap.url);
                             subloadingPer = true;
                             while(subloadingPer)
                             {
                                 Application.DoEvents();
                             }
                             ap.rawtext = webBrowser1.DocumentText;
                         }
                         InsertTitleUrlToDataBase(arListCurrentPage);
                         mysignal3 = false;
                         webBrowser1.Navigate(dirCurPageUrl);
                         loading = true;
                         while (loading == true)
                         {
                             Application.DoEvents();
                         }
                     }


                  

                    mysignal2 = true;
                    while (AnchorNextPage())
                    {
                        subloading = true;
                        while(subloading)
                        {
                            Application.DoEvents();

                        }
                        arListCurrentPage = GetArticlePageInfoFromCurrentDirpage();
                        if (arListCurrentPage != null)
                        {
                            dirCurPageUrl = webBrowser1.Url.ToString();
                            mysignal3 = true;
                            foreach (ArticlePage ap in arListCurrentPage)
                            {
                                webBrowser1.Navigate(ap.url);
                                subloadingPer = true;
                                while (subloadingPer)
                                {
                                    Application.DoEvents();
                                }
                                ap.rawtext = webBrowser1.DocumentText;
                            }
                            InsertTitleUrlToDataBase(arListCurrentPage);
                            mysignal3 = false;
                            webBrowser1.Navigate(dirCurPageUrl);
                            subloading = true;
                            while (subloading)
                            {
                                Application.DoEvents();

                            }


                        }
                       

                       

                    }
                    mysignal2 = false;
                   
                    //得到当前页面的下一页连接
               
                  
            }
               

                  
        }

private void InsertTitleUrlToDataBase(List<ArticlePage> arlist)
        {
            DataBaseManipulation dm = new DataBaseManipulation();
            string conStr = "server=(local);database=xxxxx;uid=sa;pwd=xxx";
            dm.ConstructConnection(conStr);
            foreach (ArticlePage article in arlist)
            {
dm.InsertToDataBase(article, "xxx");
            }




        }

 public void InsertToDataBase(ArticlePage article,string table )
        { //插入字符串
           string  sqlcommand=string.Format("insert  into {0}(ArticlePageUrl,ArticlePageTitle,ArticlePageSource)values(@ArticlePageUrl,@ArticlePageTitle,@ArticlePageSource)",table);
         
            //数据库参数构造与赋初值
           SqlParameter ArticlePageTitle = new SqlParameter("@ArticlePageTitle", SqlDbType.VarChar, 400);
           ArticlePageTitle.Value = article.title;
           SqlParameter ArticlePageUrl = new SqlParameter("@ArticlePageUrl", SqlDbType.VarChar, 400);
           ArticlePageUrl.Value = article.url;
           SqlParameter ArticlePageSource = new SqlParameter("@ArticlePageSource", SqlDbType.Text);
           ArticlePageSource.Value = article.rawtext;
           SqlCommand cmd = new SqlCommand(sqlcommand, connection);
           cmd.Parameters.Add(ArticlePageTitle);
           cmd.Parameters.Add(ArticlePageUrl);
          cmd.Parameters.Add(ArticlePageSource);
         
            //打开数据库链接
           OpenConnection();

        
           try
           {  
               //执行cmd操做
               cmd.ExecuteNonQuery();
           }
           catch (System.Exception e)
           {   //输出错误到记事本中
               StreamWriter sw = new StreamWriter("D:\\myerror.txt", true, Encoding.Default);
               sw.Write(e.Message);
               sw.Close();
              //一旦发生错误程序就中止运行，等待用户发现
               Console.Read();
              
           }

            //关闭数据库链接
           CloseConnection();


        }





    }
}

 

本地数据库（保存爬取的信息）视图：

 

 

小结与发散：笔者于本科毕设作过新闻类网页正文提取的课题，请见《新闻类网页正文提取系列》，而且从各大新闻门户网站的不一样版面提取了四万余条新闻做为语料库，该语料库已经提供给网友下载，语料库说明以及下载地址见：《献给热衷于天然语言处理的业余爱好者的语料库》。 可是毕设当时，我对于动态网页提取方法并无一个清晰的认识，因此能够提取的新闻版面上限制很大（必须是静态新闻网页才能提取）。有兴趣，而且对语料库有需求的网友，能够参考个人此篇博客，以及上一篇博客《巧用C# webbrowser实现动态网页爬虫机器人》，还有正文提取系列的一些博文，本身动手配置一个自动获取新闻语料库的小型爬虫。