C#使用WebRequest抓取网页代码

该静态函数执行一个 HTTP GET 操作，并将回应作为 String 返回。

/// <summary>
/// Get a response as a string, given a uri string.
/// </summary>
/// <param name="uriArg">Specifies a uri such as "http://www.google.com" or @"file://X:\dir\myFile.html"</param>
/// <returns>web response as a string.</returns>
/// <example>
/// try
/// {
///    string uri = "http://www.google.zzz"; // note bad uri with zzz to exercise exception.
///    string s = GetWebPageResponse( uri ); 
///    Console.WriteLine( s );
/// }
/// catch ( WebException ex )
/// {
///    // wex.Message could be something like: The remote server returned an error: (404) Not Found.
///    string s = string.Format( "Request: {0}\nResult: {1}", uri, ex.Message );
///    Console.WriteLine( s );
/// }
/// </example>
static string GetWebPageResponse(string uriArg)
{
    Stream responseStream = WebRequest.Create(uriArg).GetResponse().GetResponseStream();

    StreamReader reader = new StreamReader(responseStream);

    return reader.ReadToEnd();
}

/// <summary>
/// Similar to GetWebPageResponse(string uriArg), but uses a user/pw to log in.
/// </summary>
/// <param name="uriArg">e.g. "http://192.168.2.1"</param>
/// <param name="userArg">e.g. "root"</param>
/// <param name="pwArg">e.g. "admin"</param>
/// <returns>string containing the http response.</returns>
/// <example>
/// // Example to get a response with DHCP table from my LinkSys router.
/// string s = GetWebPageResponse( "http://192.168.2.1/DHCPTable.htm", "root", "admin" );
/// </example>
static string GetWebPageResponse(string uriArg, string userArg, string pwArg)
{
    Uri uri = new Uri(uriArg);
    HttpWebRequest req = (HttpWebRequest)WebRequest.Create(uri);
    CredentialCache creds = new CredentialCache();

    // See http://msdn.microsoft.com/en-us/library/system.directoryservices.protocols.authtype.aspx for list of types.
    const string authType = "basic";

    creds.Add(uri, authType, new NetworkCredential(userArg, pwArg));
    req.PreAuthenticate = true;
    req.Credentials = creds.GetCredential(uri, authType);

    Stream responseStream = req.GetResponse().GetResponseStream();

    StreamReader reader = new StreamReader(responseStream);

    return reader.ReadToEnd();
}

标签： Google

版权申明：本站文章部分自网络，如有侵权，请联系：west999com@outlook.com
特别注意：本站所有转载文章言论不代表本站观点！
本站所提供的图片等素材，版权归原作者所有，如需使用，请与原作者联系。