Build a Multi-Provider Async Methods Search Page

Extending the concepts presented in a previous article, Peter shows how to use Asynchronous Page Tasks to combine the results from seven different RSS Search providers simultaneously, and display the results in a pageable GridView.

using System;
using System.Data;
using System.Configuration;
using System.Collections;
using System.Web;
using System.Web.Security;
using System.Web.UI;
using System.Web.UI.WebControls;
using System.Web.UI.WebControls.WebParts;
using System.Web.UI.HtmlControls;
using System.Text;
using System.Net;
using System.IO;
using System.Collections.Generic ;

public partial class samples_MultiWebRequest : System.Web.UI.Page
{
    int total = 0;
    Dictionary<string, DataRow> dctFeeds = new Dictionary<string, DataRow>();
    protected void Page_Load(object sender, EventArgs e)
    {
        this.txtSearch.Attributes.Add("onfocus", "javascript:this.value='';");      
    }    

    protected override void OnPreRenderComplete(EventArgs e)
    {
       base.OnPreRenderComplete(e);
        // processing is complete, bind control
       if (Page.IsPostBack && GridView1.PageIndex == 0 && Session["dt"] == null)
           Bind();      
    }  

    IAsyncResult BeginAsyncWork1(Object sender, EventArgs e, AsyncCallback cb, object state)
    {
        // get the search url for this request
       string fullUrl = (string)state;        
       HttpWebRequest req1 = (HttpWebRequest)WebRequest.Create(fullUrl);
       req1.Method = "GET"; 
       IAsyncResult  result1=  req1.BeginGetResponse(cb,req1); 
       return result1;
    }

    
    void EndAsyncWork1(IAsyncResult asyncResult)
    {
        HttpWebRequest req1 = (HttpWebRequest)asyncResult.AsyncState;
        WebResponse response = (WebResponse)req1.EndGetResponse(asyncResult);
        Stream streamResponse = response.GetResponseStream();
        DataSet  ds1 = new DataSet();
        ds1.ReadXml(streamResponse);
        streamResponse.Close();
        ProcessItems(ds1);         
    }
   
    void TimeoutHandler(IAsyncResult asyncResult)
    {
        this.lblMessage.Text = "<strong>Async Request timed out!</strong>";
    }

    private void ProcessItems( DataSet ds)
    {
        if (ds.Tables.Count > 2 && ds.Tables[2].Rows.Count>0)
        {
            // some feeds don't always end up with items in table 3, so....
            int tbl = 2;
            if (ds.Tables[tbl].Columns["description"] == null)
                tbl = 3;
            foreach (DataRow row in ds.Tables[tbl].Rows)
            {
                if (!dctFeeds.ContainsKey((string)row["link"]))
                {
                    dctFeeds.Add((string)row["link"], row);
                    total++;
                }
            }
        }
        
    }
    
    private void Bind ()
    {
          DataSet ds = new DataSet();
          DataTable dt = new DataTable();
          dt.Columns.Add("title", typeof(String));
          dt.Columns.Add("link", typeof(string));
          dt.Columns.Add("description", typeof(string));
          DataRow newRow;
          foreach(DataRow row in dctFeeds.Values)
          {
              newRow = dt.NewRow();
              newRow["title"] = row["title"];
              newRow["link"] = row["link"];
              newRow["description"] = row["description"];
              dt.Rows.Add(newRow);
          }
          lblMessage.Text = "Retrieved " + total.ToString() + " results.";
          Session["dt"] = dt;
          this.GridView1.DataSource = dt;
          GridView1.DataBind();
    }

   
    protected void Button1_Click(object sender, EventArgs e)
    {
        Session["dt"] = null;
        // Register each separate WebRequest as an AsyncTask, 
        // specifying that they should be run in parallel:
        PageAsyncTask task=null;   

        string[] urls = 
        {
        "http://search.live.com/feeds/results.aspx?format=rss&count=50&q="+this.txtSearch.Text,
        "http://blogsearch.google.com/blogsearch_feeds?hl=en&spell=1&oi=spell&ie=utf-8&num=100&output=rss&q="+this.txtSearch.Text,
        "http://search.msn.com/results.aspx?count=200&format=rss&q=" +this.txtSearch.Text,
        "http://lab.msdn.microsoft.com/search/data.ashx?query="+this.txtSearch.Text,
        "http://digg.com/rss_search?area=all&type=both&age=7&search="+this.txtSearch.Text,
         "http://forums.asp.net/search/Searchrss.aspx?q="+this.txtSearch.Text,
        "http://feeds.technorati.com/search/" +this.txtSearch.Text
        };

        foreach (string s in urls)
        {
            task = new PageAsyncTask(new BeginEventHandler(this.BeginAsyncWork1), new EndEventHandler(this.EndAsyncWork1), new EndEventHandler(this.TimeoutHandler), s, true);
            Page.RegisterAsyncTask(task);
        }     
        Page.ExecuteRegisteredAsyncTasks();    
    }
    
    protected void GridView1_PageIndexChanged(object sender, EventArgs e)
    {       
    }
    protected void GridView1_PageIndexChanging(object sender, GridViewPageEventArgs e)
    {
       GridView1.PageIndex = e.NewPageIndex;
       GridView1.DataSource = (DataTable)Session["dt"];
       GridView1.DataBind();
    }
}

Hi Paeter,

Excellent stuff on your site always.

I have a requirement, processing say 100s of requests with username as a query param, which I should process,to get the data from the database, which gets the results qucikly (less than a sec).

I need to scale out, will this pageAsync works for me. can I increase thread pool size.

Need your comments on this.