用C#+Selenium+ChromeDriver 爬取网页，完美模拟真实的用户浏览行为

背景

 Selenium是一个用于Web应用程序测试的工具。Selenium测试直接运行在浏览器中，就像真正的用户在操作一样。而对于爬虫来说，使用Selenium操控浏览器来爬取网上的数据那么肯定是爬虫中的杀手武器。这里，我将介绍selenium + 谷歌浏览器的一般使用。

需求

在平常的爬虫开发中，有时候网页是一堆js堆起来的代码，涉及很多异步计算，如果是普通的http 控制台请求，那么得到的源文件是一堆js ，需要自己在去组装数据，很费力；但是采用Selenium+ChromeDriver可以达到所见即所得的完美效果。

实现方式

项目结构：为了方便使用，用的winform程序，附nuget包

 

 

 以下是form1.cs的代码，这里就只放关键方法代码了。需要安装最新的chrome浏览器+代码中使用的chromedriver是 v2.9.248315

 1 private void crawlingWebFunc() 2   { 3    SetText("\r\n开始尝试..."); 4    List<testfold> surls = new List<testfold>(); 5    string path = System.Environment.CurrentDirectory + "\\图片url\\"; 6    DirectoryInfo root = new DirectoryInfo(path); 7    DirectoryInfo[] dics = root.GetDirectories(); 8    foreach (var itemdic in dics) 9    {10     string txt = "";11     StreamReader sr = new StreamReader(itemdic.FullName + "\\data.txt");12     while (!sr.EndOfStream)13     {14      string str = sr.ReadLine();15      txt += str;// + "\n";16     }17     sr.Close();18     surls.Add(new testfold() { key = itemdic.FullName, picurl = txt });19    }20 21    ChromeDriverService service = ChromeDriverService.CreateDefaultService(System.Environment.CurrentDirectory);22    // service.HideCommandPromptWindow = true;23 24    ChromeOptions options = new ChromeOptions();25    options.AddArguments("--test-type", "--ignore-certificate-errors");26    options.AddArgument("enable-automation");27    // options.AddArgument("headless");28    // options.AddArguments("--proxy-server=:password@yourProxyServer.com:8080");29 30    using (IWebDriver driver = new OpenQA.Selenium.Chrome.ChromeDriver(service, options, TimeSpan.FromSeconds(120)))31    {32     driver.Url = "https://www.1688.com/";33     Thread.Sleep(200);34     try35     {36      int a = 1;37      foreach (var itemsurls in surls)38      {39       SetText("\r\n第" + a.ToString() + "个");40       driver.Navigate().GoToUrl(itemsurls.picurl);41       //登录42       if (driver.Url.Contains("login.1688.com"))43       {44        SetText("\r\n需要登录，开始尝试...");45        trylogin(driver); //尝试登录完成46            //再试试47        driver.Navigate().GoToUrl("https://s.1688.com/youyuan/index.htm?tab=imageSearch&imageType=oss&imageAddress=cbuimgsearch/eWXC7XHHPN1607529600000&spm=");48 49        if (driver.Url.Contains("login.1688.com"))50        {51         //没办法退出52         SetText("\r\n退出，换ip重试...");53         return;54        }55       }56 57       //鼠标放上去的内容因为页面自带只能显示一个的原因 没办法做到全部显示 然后在下载 只能是其他方式下载58       // var elements = document.getElementsByClassName('hover-container');59       // Array.prototype.forEach.call(elements, function(element) {60       // element.style.display = "block";61       // console.log(element);62       // });63 64       // IJavaScriptExecutor js = (IJavaScriptExecutor)driver;65 66       // var sss = js.ExecuteScript(" var elements = document.getElementsByClassName('hover-container'); Array.prototype.forEach.call(elements, function(element) { console.log(element); element.setAttribute(\"class\", \"测试title\"); element.style.display = \"block\"; console.log(element); });");67 68       Thread.Sleep(500);69       var responseModel = Write(itemsurls.key, driver.PageSource, Pagetypeenum.列表);70       Thread.Sleep(500);71       int i = 1;72       foreach (var offer in responseModel?.data?.offerList ?? new List<OfferItemModel>())73       {74        driver.Navigate().GoToUrl(offer.information.detailUrl);75        string responseDatadetail = driver.PageSource;76        Write(itemsurls.key, driver.PageSource, Pagetypeenum.详情);77        SetText("\r\n第" + a.ToString() + "-" + i.ToString() + "个");78        Thread.Sleep(500);79        i++;80       }81      }82     }83     catch (Exception ex)84     {85      CloseChromeDriver(driver);86      throw;87     }88    }89   }

 1 #region 异常 退出chromedriver 2  3   [DllImport("user32.dll", EntryPoint = "FindWindow")] 4   private extern static IntPtr FindWindow(string lpClassName, string lpWindowName); 5  6   [DllImport("user32.dll", EntryPoint = "SendMessage")] 7   public static extern int SendMessage(IntPtr hWnd, int Msg, int wParam, int lParam); 8  9   public const int SW_HIDE = 0;10   public const int SW_SHOW = 5;11 12   [DllImport("user32.dll", EntryPoint = "ShowWindow")]13   public static extern int ShowWindow(IntPtr hwnd, int nCmdShow);14 15   /// <summary>16   /// 获取窗口句柄17   /// </summary>18   /// <returns></returns>19   public IntPtr GetWindowHandle()20   {21    string name = (Environment.CurrentDirectory + "\\chromedriver.exe");22    IntPtr hwd = FindWindow(null, name);23    return hwd;24   }25 26   /// <summary>27   /// 关闭chromedriver窗口28   /// </summary>29   public void CloseWindow()30   {31    try32    {33     IntPtr hwd = GetWindowHandle();34     SendMessage(hwd, 0x10, 0, 0);35    }36    catch { }37   }38 39   /// <summary>40   /// 退出chromedriver41   /// </summary>42   /// <param name="driver"></param>43   public void CloseChromeDriver(IWebDriver driver)44   {45    try46    {47     driver.Quit();48     driver.Dispose();49    }50    catch { }51    CloseWindow();52   }53 54