Example: Cloud Parallel Web Downloader

This example illustrates doing I/O tasks in parallel using the workers in the cluster

This example is from the MBrace Starter Kit.

 1: 
 2: 
 3: 
 4: 
 5: 
 6: 
 7: 
 8: 
 9: 
10: 
11:

// Cloud parallel url-downloader
open System.Net

let cluster = Config.GetCluster() 
let fs = cluster.Store.CloudFileSystem

let urls = 
    [| ("bing", "http://bing.com")
       ("yahoo", "http://yahoo.com")
       ("google", "http://google.com")
       ("msn", "http://msn.com") |]

Next define a cloud workflow to download a file and wave it into cloud storage.

 1: 
 2: 
 3: 
 4: 
 5: 
 6: 
 7: 
 8: 
 9: 
10: 
11: 
12: 
13: 
14: 
15: 
16: 
17: 
18: 
19: 
20: 
21: 
22: 
23: 
24: 
25: 
26: 
27: 
28: 
29:

let download (name: string, uri: string) = 
    local {
        let webClient = new WebClient()
        let! text = webClient.AsyncDownloadString(Uri(uri)) |> Cloud.OfAsync
        do! CloudFile.Delete(sprintf "pages/%s.html" name)
        let! file = CloudFile.WriteAllText(path = sprintf "pages/%s.html" name, text = text)
        return file
    }

let filesTask = 
    urls 
    |> Array.map download
    |> Cloud.ParallelBalanced
    |> cluster.CreateProcess

// Check on progress...
filesTask.ShowInfo()

// Get the result of the job
let files = filesTask.Result

// Read the files we just downloaded
let contentsOfFiles = 
    files
    |> Array.map (fun file ->
        local { let text = fs.File.ReadAllText(file.Path)
                return (file.Path, text.Length) })
    |> Cloud.ParallelBalanced
    |> cluster.Run

In this example, you've seen how cloud tasks can perform I/O to web data sources. Continue with further samples to learn more about the MBrace programming model.

Note, you can use the above techniques from both scripts and compiled projects. To see the components referenced by this script, see ThespianCluster.fsx or AzureCluster.fsx.

namespace System

namespace System.IO

namespace MBrace

namespace MBrace.Core

namespace MBrace.Flow

namespace System.Net

val cluster : MBrace.Thespian.ThespianCluster

Full name: 200-cloud-parallel-web-download-example.cluster

module Config

val GetCluster : unit -> MBrace.Thespian.ThespianCluster

Full name: Config.GetCluster

Gets or creates a new Thespian cluster session.

val fs : MBrace.Core.Internals.CloudFileSystem

Full name: 200-cloud-parallel-web-download-example.fs

property MBrace.Runtime.MBraceClient.Store: MBrace.Core.Internals.CloudStoreClient

property MBrace.Core.Internals.CloudStoreClient.CloudFileSystem: MBrace.Core.Internals.CloudFileSystem

val urls : (string * string) []

Full name: 200-cloud-parallel-web-download-example.urls

val download : name:string * uri:string -> 'a

Full name: 200-cloud-parallel-web-download-example.download

val name : string

Multiple items
val string : value:'T -> string

Full name: Microsoft.FSharp.Core.Operators.string

--------------------
type string = String

Full name: Microsoft.FSharp.Core.string

val uri : string

Multiple items
type WebClient =
  inherit Component
  new : unit -> WebClient
  member BaseAddress : string with get, set
  member CachePolicy : RequestCachePolicy with get, set
  member CancelAsync : unit -> unit
  member Credentials : ICredentials with get, set
  member DownloadData : address:string -> byte[] + 1 overload
  member DownloadDataAsync : address:Uri -> unit + 1 overload
  member DownloadFile : address:string * fileName:string -> unit + 1 overload
  member DownloadFileAsync : address:Uri * fileName:string -> unit + 1 overload
  member DownloadString : address:string -> string + 1 overload
  ...

Full name: System.Net.WebClient

--------------------
WebClient() : unit

Multiple items
type Uri =
  new : uriString:string -> Uri + 5 overloads
  member AbsolutePath : string
  member AbsoluteUri : string
  member Authority : string
  member DnsSafeHost : string
  member Equals : comparand:obj -> bool
  member Fragment : string
  member GetComponents : components:UriComponents * format:UriFormat -> string
  member GetHashCode : unit -> int
  member GetLeftPart : part:UriPartial -> string
  ...

Full name: System.Uri

--------------------
Uri(uriString: string) : unit
Uri(uriString: string, uriKind: UriKind) : unit
Uri(baseUri: Uri, relativeUri: string) : unit
Uri(baseUri: Uri, relativeUri: Uri) : unit

val sprintf : format:Printf.StringFormat<'T> -> 'T

Full name: Microsoft.FSharp.Core.ExtraTopLevelOperators.sprintf

val filesTask : MBrace.Runtime.CloudProcess<obj []>

Full name: 200-cloud-parallel-web-download-example.filesTask

type Array =
  member Clone : unit -> obj
  member CopyTo : array:Array * index:int -> unit + 1 overload
  member GetEnumerator : unit -> IEnumerator
  member GetLength : dimension:int -> int
  member GetLongLength : dimension:int -> int64
  member GetLowerBound : dimension:int -> int
  member GetUpperBound : dimension:int -> int
  member GetValue : [<ParamArray>] indices:int[] -> obj + 7 overloads
  member Initialize : unit -> unit
  member IsFixedSize : bool
  ...

Full name: System.Array

val map : mapping:('T -> 'U) -> array:'T [] -> 'U []

Full name: Microsoft.FSharp.Collections.Array.map

member MBrace.Runtime.MBraceClient.CreateProcess : workflow:MBrace.Core.Cloud<'T> * ?cancellationToken:MBrace.Core.ICloudCancellationToken * ?faultPolicy:MBrace.Core.FaultPolicy * ?target:MBrace.Core.IWorkerRef * ?additionalResources:MBrace.Core.Internals.ResourceRegistry * ?taskName:string -> MBrace.Runtime.CloudProcess<'T>

member MBrace.Runtime.CloudProcess.ShowInfo : unit -> unit

val files : obj []

Full name: 200-cloud-parallel-web-download-example.files

property MBrace.Runtime.CloudProcess.Result: obj []

val contentsOfFiles : obj

Full name: 200-cloud-parallel-web-download-example.contentsOfFiles

val file : obj

property MBrace.Core.Internals.CloudFileSystem.File: MBrace.Core.Internals.CloudFileClient

member MBrace.Core.Internals.CloudFileClient.ReadAllText : path:string * ?encoding:Text.Encoding -> string

type Path =
  static val DirectorySeparatorChar : char
  static val AltDirectorySeparatorChar : char
  static val VolumeSeparatorChar : char
  static val InvalidPathChars : char[]
  static val PathSeparator : char
  static member ChangeExtension : path:string * extension:string -> string
  static member Combine : [<ParamArray>] paths:string[] -> string + 3 overloads
  static member GetDirectoryName : path:string -> string
  static member GetExtension : path:string -> string
  static member GetFileName : path:string -> string
  ...

Full name: System.IO.Path

member MBrace.Runtime.MBraceClient.Run : workflow:MBrace.Core.Cloud<'T> * ?cancellationToken:MBrace.Core.ICloudCancellationToken * ?faultPolicy:MBrace.Core.FaultPolicy * ?target:MBrace.Core.IWorkerRef * ?additionalResources:MBrace.Core.Internals.ResourceRegistry * ?taskName:string -> 'T

MBrace.Core and MBrace.Azure

Example: Cloud Parallel Web Downloader