Creating and Using Cloud Files

let fs = cluster.Store.CloudFileSystem

let root = fs.Path.DefaultDirectory

let (++) path1 path2 = fs.Path.Combine(path1, path2)

let ls path = fs.File.Enumerate(path)

let rec lsRec path = 
    seq { yield! fs.File.Enumerate(path)
          for d in fs.Directory.Enumerate(path) do 
              yield! lsRec path }

let mkdir path = fs.Directory.Create(path)

let rmdir path = fs.Directory.Delete(path)

let rmdirRec path = fs.Directory.Delete(path,recursiveDelete=true)

let randdir() = fs.Path.GetRandomDirectoryName()

let randfile() = fs.Path.GetRandomFilePath()

let rm path = fs.File.Delete path

let cat path = fs.File.ReadAllText path

let catLines path = fs.File.ReadAllLines path

let catBytes path = fs.File.ReadAllBytes path

let write path text = fs.File.WriteAllText(path, text)

let writeLines path lines = fs.File.WriteAllLines(path, lines)

let writeBytes path bytes = fs.File.WriteAllBytes(path, bytes)

mkdir (root ++ "data")

write (root ++ "data" ++ "humpty.txt") "All the king's horses and all the king's men"
writeLines (root ++ "data" ++ "spider.txt") [ for i in 0 .. 1000 -> "Incy wincy spider climed up the water spout" ]

ls (root ++ "data")

cat (root ++ "data" ++ "spider.txt") 
catLines (root ++ "data" ++ "spider.txt") 

rmdirRec (root ++ "data")

let localTmpFile = 
    let path = Path.GetTempFileName()
    let lines = 
        [ for i in 1 .. 1000 do 
             let time = DateTime.Now.Date.AddSeconds(float i)
             let text = sprintf "click user%d %s" (i%10) (time.ToString())
             yield text ]
    File.WriteAllLines(path, lines)
    path

let cloudFile = fs.File.Upload(localTmpFile, sprintf "%s/tmp/%s" root (Path.GetFileName localTmpFile))     

File.Delete localTmpFile

let lines = 
    cloud {
        let lines = fs.File.ReadAllLines cloudFile.Path
        let users = [ for line in lines -> line.Split(' ').[1] ]
        return users |> Seq.distinct |> Seq.toList
    } 
    |> cluster.Run

let dataDir = root ++ "data"

mkdir dataDir

let cloudFiles = 
    [ for i in 1 .. 100 ->
        local { 
            let lines = 
                [ for j in 1 .. 100000 -> 
                    "file " + string i + ", item " + string (i*100+j) + ", " + string (j+i*100) ] 
            let nm = dataDir + "/file" + string i
            do! CloudFile.Delete(nm)
            let file = fs.File.WriteAllLines(nm, lines)
            return file.Path
        } ]
   |> Cloud.ParallelBalanced
   |> cluster.Run

let sumOfLengthsOfLines =
    cloudFiles
    |> CloudFlow.OfCloudFileByLine
    |> CloudFlow.map (fun line -> line.Split(',').[2] |> int)
    |> CloudFlow.sum
    |> cluster.Run

rmdirRec (root ++ "data")