Thursday, 4 April 2019

Find the square root of sum of squares of each numbers from a file using Spark with Scala

// Exclude all characters from each line and find the square root of sum of squares of each numbers


// given input file has character and numbers separated by comma
$ cat charsAndNumbers.txt
1,a,b,c,2,3,4
2,3,4,x,y,z
s,t,u,5,2
m,n,8,10
5,2,1,a,x,y
7,a,x,2,6,h


scala.math.sqrt( (1*1) + (2*2) + (3*3) + (4*4)
 + (2*2) + (3*3) + (4*4)
 + (5*5) + (2*2)
 + (8*8) + (10*10)
 + (5*5) + (2*2) + (1*1)
 + (7*7) + (2*2) + (6*6))

scala> val r1 = sc.textFile("/home/hadoop/Desktop/vow/charsAndNumbers.txt")

scala> r1.foreach(println)
1,a,b,c,2,3,4
2,3,4,x,y,z
s,t,u,5,2
m,n,8,10
5,2,1,a,x,y
7,a,x,2,6,h



val r2 = r1.map(x => {
                       val fields = x.split(",")
   var s = 0
   for(f <- fields)
{
  try
  {
s = s + (f.toInt * f.toInt)
  }
  catch
  {
case ex: Exception  => {
}
  }
}
s
                   })


scala> r2.collect
res1: Array[Int] = Array(30, 29, 29, 164, 30, 89)

scala> (1*1) + (2*2) + (3*3) + (4*4)
res2: Int = 30

scala> (2*2) + (3*3) + (4*4)
res3: Int = 29

scala> (5*5) + (2*2)
res4: Int = 29

scala> (8*8) + (10*10)
res5: Int = 164

scala> (5*5) + (2*2) + (1*1)
res6: Int = 30

scala> (7*7) + (2*2) + (6*6)
res7: Int = 89



scala> r2.foreach(println)
30
29
29
164
30
89


scala> val result = r2.reduce(_+_)
result: Int = 371


scala> val finalResult = scala.math.sqrt(result)
finalResult: Double = 19.261360284258224

scala> scala.math.sqrt( (1*1) + (2*2) + (3*3) + (4*4)
     |  + (2*2) + (3*3) + (4*4)
     |  + (5*5) + (2*2)
     |  + (8*8) + (10*10)
     |  + (5*5) + (2*2) + (1*1)
     |  + (7*7) + (2*2) + (6*6))
res8: Double = 19.261360284258224



No comments:

Post a Comment

Flume - Simple Demo

// create a folder in hdfs : $ hdfs dfs -mkdir /user/flumeExa // Create a shell script which generates : Hadoop in real world <n>...