C# 网络爬虫源码

using System;using System.Collections.Generic;using System.ComponentModel;using System.Data;using System.Drawing;using System.Linq;using System.Text;using System.Windows.Forms;using NWebCrawlerLib;using System.Diagnostics;namespace NWebCrawler{ public partial class MainForm : Form { #region Fields private PerformanceCounter m_cpuCounter; private PerformanceCounter m_ramCounter; private Downloader m_downloader; #endregion #region Properties // number of bytes downloaded private int nByteCount; private int ByteCount { get { return nByteCount; } set { nByteCount = value; this.statusStrip.Text = Commas(nByteCount / 1024 1) " KB"; } } // number of Uri's found private int nURLCount; private int URLCount { get { return nURLCount; } set { nURLCount = value; this.statusBarPanelURLs.Text = Commas(nURLCount) " URL found"; } } // available memory private float nFreeMemory; private float FreeMemory { get { return nFreeMemory; } set { nFreeMemory = value; this.statusBarPanelMem.Text = nFreeMemory " MB Available"; } } // CPU usage private int nCPUUsage; private int CPUUsage { get { return nCPUUsage; } set { nCPUUsage = value; this.statusBarPanelCPU.Text = "CPU usage " nCPUUsage "%"; } } // number of files downloaded private int nFileCount; private int FileCount { get { return nFileCount; } set { nFileCount = value; this.statusBarPanelFiles.Text = Commas(nFileCount) " file(s) downloaded"; } } #endregion public MainForm() { InitializeComponent(); m_cpuCounter = new PerformanceCounter(); m_cpuCounter.CategoryName = "Processor"; m_cpuCounter.CounterName = "% Processor Time"; m_cpuCounter.InstanceName = "_Total"; m_ramCounter = new PerformanceCounter("Memory", "Available MBytes"); m_downloader = new Downloader(); m_downloader.StatusChanged = new DownloaderStatusChangedEventHandler(DownloaderStatusChanged); } #region Helpers string Commas(int nNum) { string str = nNum.ToString(); int nIndex = str.Length; while (nIndex > 3) { str = str.Insert(nIndex - 3, ","); nIndex -= 3; } return str; } void ShowSettingsDialog() { SettingsForm dialog = new SettingsForm(); if (dialog.ShowDialog() == DialogResult.OK) { } } #endregion #region UI Events private void exitToolStripMenuItem_Click(object sender, EventArgs e) { this.Close(); } private void settingsToolStripMenuItem_Click(object sender, EventArgs e) { ShowSettingsDialog(); } private void buttonResume_Click(object sender, EventArgs e) { m_downloader.Resume(); } private void buttonPause_Click(object sender, EventArgs e) { m_downloader.Suspend(); } private void buttonStop_Click(object sender, EventArgs e) { m_downloader.Abort(); } private void buttonSettings_Click(object sender, EventArgs e) { ShowSettingsDialog(); } #endregion private void buttonGo_Click(object sender, EventArgs e) { m_downloader.InitSeeds(new string[] { txtSeed.Text }); m_downloader.Start(); } delegate void UpdateDataGridCallback(Downloader d); private void UpdateDataGrid(Downloader d) { try { if (this.dataGridThreads.InvokeRequired) { UpdateDataGridCallback callback = new UpdateDataGridCallback(UpdateDataGrid); this.Invoke(callback, new object[] { d }); } else { dataGridThreads.DataSource = typeof(CrawlerThread[]); dataGridThreads.DataSource = d.Crawlers; dataGridThreads.Columns["Name"].Width = 100; dataGridThreads.Columns["Status"].Width = 100; } } catch (ObjectDisposedException) { } } delegate void UpdateStatusStripCallback(); private void UpdateStatusStrip() { if (this.statusStrip.InvokeRequired) { UpdateStatusStripCallback callback = new UpdateStatusStripCallback(UpdateStatusStrip); this.Invoke(callback, new object[] { }); } else { statusBarPanelCPU.Text = string.Format("CPU: {0:0.00}%", m_cpuCounter.NextValue()); statusBarPanelMem.Text = string.Format("Mem: {0:0.00}MB", m_ramCounter.NextValue()); statusBarPanelURLs.Text = string.Format("URLs: {0}", m_downloader.UrlsQueueFrontier.Count.ToString()); statusBarPanelFiles.Text = string.Format("Files: {0}", m_downloader.CrawleHistroy.Count.ToString()); statusBarPanelSpeed.Text = string.Format("Speed: {0:0.00}KB/sec", m_downloader.GetDownloadSpeed()); statusBarPanelByteCount.Text = string.Format("Total size: {0:0.00}MB", 1.0 * m_downloader.TotalSize / 1024 / 1024); } } private void DownloaderStatusChanged(object sender, DownloaderStatusChangedEventArgs e) { Downloader d = (Downloader)sender; UpdateDataGrid(d); } private void timer_Tick(object sender, EventArgs e) { UpdateStatusStrip(); } private void MainForm_FormClosing(object sender, FormClosingEventArgs e) { m_downloader.Abort(); m_downloader.Dump("dump.txt"); } }}