Sunday, 10 May 2020

Numpy Sample Programs

import numpy
import numpy as np

print(np.__version__)
1.15.4


import numpy as np
arr = np.array([1,2,3,4,5,6,7,8,9,10])
print(type(arr))
print(arr)

for a in arr:
    print("a : {}".format(a))
<class 'numpy.ndarray'>
[ 1  2  3  4  5  6  7  8  9 10]
a : 1
a : 2
a : 3
a : 4
a : 5
a : 6
a : 7
a : 8
a : 9
a : 10



import numpy as np
arr = np.array([range(1,10,2)])

print(type(arr))

<class 'numpy.ndarray'>


print(arr)

[[1 3 5 7 9]]



for a in arr:
    print(a)
[1 3 5 7 9]


#Tuple to ndarray
myTuple = tuple((1,6,3,2,6,7,2,2))
import numpy as np
arr = np.array(myTuple)
print(arr)

[1 6 3 2 6 7 2 2]



1Dimension Arrray:

import numpy as np
arr1D = np.array(7)
print(arr1D.shape)

print(arr1D)
7

print(arr1D.ndim)
0

2D array:
arr = np.array([1,2,3,4,5,6])
print(arr.shape)

(6,)

print(arr2D.ndim)
2


arr2D = np.array([[1,2,3], [10,11,12]])
print(arr2D)



[[ 1  2  3]
 [10 11 12]]
(2, 3)

 


import numpy as np
arr3D=np.array([[  [1,2,3],[6,7,8]], [[5,3,2],[6,2,1] ] ])
print(arr3D)
print(arr3D.shape)
              
  
[[[1 2 3]
  [6 7 8]]

 [[5 3 2]
  [6 2 1]]]
(2, 2, 3)

print(arr3D.ndim)
3


import numpy as np

arr = np.array([5,7,3,2,6,33,2,1,2,3,4,5,6,7,8,9,2,3,5,6,2,12,1,3,5,3])
print(len(arr))
print(arr[0])
print(arr[1])
print(arr[-1])
print(arr[1:2])
print(arr[3:20])
print(arr[3:20:2])
print(arr[3:-1])
print(arr[3:-1:3])



26
5
7
3
[7]
[ 2  6 33  2  1  2  3  4  5  6  7  8  9  2  3  5  6]
[ 2 33  1  3  5  7  9  3  6]
[ 2  6 33  2  1  2  3  4  5  6  7  8  9  2  3  5  6  2 12  1  3  5]
[ 2  2  3  6  9  5 12  5]




import numpy as np
arr = np.array([1, 2, 3, 4])
print(arr[0] + arr[-1])


5


import numpy as np
arr = np.array([[1,2,3,4,5], [6,7,8,9,10]])
print('3rd element on 2st dim: ', arr[1, 2])


8


import numpy as np
arr = np.array([[1,2,3,4,5,3,2,3,4], [6,7,8,9,10,3,2,5,3]])
print('5th element on 1st dim: ', arr[0, 4])


5th element on 1st dim:  5



import numpy as np
arr = np.array([[[3,2,1], [6,-5,-4]], [[9,8,7], [5,4,3]]])
print(arr[0, 1, 2])
print(arr[1,0,1])


-4
8


import numpy as np
arr = np.array([[1,2,22,453,5], [3,7,8,32,3233]])
print('Last element from 2nd dim: ', arr[1, -1])
print('Last element from 1st dim: ', arr[0, -1])

Last element from 2nd dim:  3233
Last element from 1st dim:  5


import numpy as np
arr = np.array([1, 2, 3, 4, 5, 6, 7])
print(arr[1:5])
print(arr[:3])
print(arr[:7:2])
print(arr[:-1:2])
print(arr[::2])


[2 3 4 5]
[1 2 3]
[1 3 5 7]
[1 3 5]
[1 3 5 7]



import numpy as np
arr = np.array([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]])
print(arr[1, 1:4])
print(arr[0,2:-1])
print(arr[0,:4])
print(arr[1,-1])
print(arr[1,1:-1:2])


[7 8 9]
[3 4]
[1 2 3 4]
10
[7 9]



import numpy as np

arr = np.array([6,34,3,2])
print(arr.dtype)  #data type 

int64


arr = np.array(['silva', 'kalai', 'Raj'])
print(arr.dtype)

<U5


arr = np.array([1, 2, 3, 4], dtype='S') #defined data type
print(arr)
print(arr.dtype)

[b'1' b'2' b'3' b'4']
|S1

arr = np.array([1.3, 2.31, 3.12])
print(arr.dtype)  #float

float64


newarr = arr.astype('i') #float into integer
print(newarr)
print(newarr.dtype)

[1 2 3]
int32

arr = np.array([12, 0, 34])
print(arr)

[12  0 34]


newarr = arr.astype(bool) #integer into bool
print(newarr)
print(newarr.dtype)

[ True False  True]
bool




Copy Example:

import numpy as np

arr = np.array([6, 2, 7, 4, 5])
x = arr.copy()
arr[0] = 3333 #making changes in the source

print(arr)

[6, 2, 7, 4, 5]


print(x) #source changes not reflected here


[333, 2, 7, 4, 5]





import numpy as np

arr = np.array([1, 2, 3, 4, 5])
x = arr.view()
arr[0] = 42 #source element got changed

print(arr)
print(x)  #target will reflect the source 


[42  2  3  4  5]
[42  2  3  4  5]



import numpy as np

arr = np.array(['a','e','i','o','u'])

x = arr.copy()
y = arr.view()

print(x.base)
print(y.base)

#The copy returns None.
#The view returns the original array.




import numpy as np
arr = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
print(arr.shape)  # 2 D with 4 elements

(2, 4)


import numpy as np
arr = np.array([['a','b','c','d','e'], ['f','g','h','i','j'], ['k','l','m','n','o']  ])
print(arr.shape) # 3 D with 5 elements

(3, 5)




Reshape exa:
import numpy as np

arr = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,13,14,15,16])

newarr = arr.reshape(8, 2) #8 arrays 2 elements
print(newarr)

[[ 1  2]
 [ 3  4]
 [ 5  6]
 [ 7  8]
 [ 9 10]
 [11 12]
 [13 14]
 [15 16]]
 
 
newarr = arr.reshape(2, 8) #2 arrays 8 elements
print(newarr)

[[ 1  2  3  4  5  6  7  8]
 [ 9 10 11 12 13 14 15 16]]

newarr = arr.reshape(4, 4) #4 arrays 4 elements
print(newarr)

[[ 1  2  3  4]
 [ 5  6  7  8]
 [ 9 10 11 12]
 [13 14 15 16]]
 
 
1 D to 3 D:

import numpy as np
arr = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12])
newarr = arr.reshape(2, 3, 2) #2 arrays that contains 3 arrays, each with 2 elements:
print(newarr)

[[[ 1  2]
  [ 3  4]
  [ 5  6]]

 [[ 7  8]
  [ 9 10]
  [11 12]]]
  
  
#flattening array
  
import numpy as np
arr = np.array([[1, 2, 3], [4, 5, 6]])
print(arr)

[[1 2 3]
 [4 5 6]]
 
 
newarr = arr.reshape(-1) #flattening
print(newarr)
[1 2 3 4 5 6]



Iteration - Looping:
--------------------

import numpy as np
arr = np.array(['a','b','c','d','e','f'])
for x in arr:
  print(x)


a
b
c
d
e
f

#iteration loop through 
import numpy as np
arr = np.array([[1, 2, 3], [4, 5, 6], [7,8,9]])
for x in arr:
  print(x)

[1 2 3]
[4 5 6]
[7 8 9]

#flattening
for x in arr:
for y in x:
    print(y)
            
1
2
3
4
5
6
7
8
9



import numpy as np

arr = np.array([ [[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]], [[13,14, 15], [16, 17, 18]]  ])

for x in arr:
  print("x represents the 2-D array:")
  print(x)


x represents the 2-D array:
[[1 2 3]
 [4 5 6]]
 
x represents the 2-D array:
[[ 7  8  9]
 [10 11 12]]
 
x represents the 2-D array:
[[13 14 15]
 [16 17 18]]
 
 
 
import numpy as np

arr = np.array([ [[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]], [[13,14, 15], [16, 17, 18]]  ])

for x in arr:
  for y in x:
    for z in y:
      print(z)
  
#flattening
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18


#nd iteration : nditer

import numpy as np

arr = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])

for x in arr:
print(x)

[[1 2]
 [3 4]]
[[5 6]
 [7 8]]
 
for x in np.nditer(arr):
  print(x)

1
2
3
4
5
6
7
8



#join / Concatenate


import numpy as np

arr1 = np.array([1, 2, 3,7,8,9])
arr2 = np.array([4, 5, 6,10,11,12])
arr = np.concatenate((arr1, arr2))
print(arr)

[ 1  2  3  7  8  9  4  5  6 10 11 12]



import numpy as np

arr1 = np.array([ [1, 2], [3,7],[8,9]])
arr2 = np.array([[4, 5], [6,10],[11,12]])
arr = np.concatenate((arr1, arr2), axis=1)
print(arr)


[[ 1  2  4  5]
 [ 3  7  6 10]
 [ 8  9 11 12]]
 
 
 
import numpy as np
arr1 = np.array([1, 2, 3,4,5,6,7,8])
arr2 = np.array([9,10,11,12,13,14,15,16])
arr = np.stack((arr1, arr2), axis=1)
print(arr)


[[ 1  9]
 [ 2 10]
 [ 3 11]
 [ 4 12]
 [ 5 13]
 [ 6 14]
 [ 7 15]
 [ 8 16]]
 
 
import numpy as np
arr1 = np.array([1, 2, 3,4,5,6,7,8])
arr2 = np.array([9,10,11,12,13,14,15,16])
arr3 = np.array([17,18,19,20,21,22,23,24])
arr = np.stack((arr1, arr2,arr3), axis=1)
print(arr)

[[ 1  9 17]
 [ 2 10 18]
 [ 3 11 19]
 [ 4 12 20]
 [ 5 13 21]
 [ 6 14 22]
 [ 7 15 23]
 [ 8 16 24]]


split example:


import numpy as np
arr = np.array([1, 2, 3, 4, 5, 6,7,8,9,10,11,12,13,14,15,16])
newarr = np.array_split(arr, 3)
print(newarr)


[array([1, 2, 3, 4, 5, 6]), array([ 7,  8,  9, 10, 11]), array([12, 13, 14, 15, 16])]



import numpy as np
arr = np.array([1, 2, 3, 4, 5, 6,7,8,9,10,11,12,13,14,15,16])
newarr = np.array_split(arr, 4)
print(newarr)


[array([1, 2, 3, 4]), array([5, 6, 7, 8]), array([ 9, 10, 11, 12]), array([13, 14, 15, 16])]



import numpy as np
arr = np.array([1, 2, 3, 4, 5, 6,7,8,9,10,11,12,13,14,15,16])
newarr = np.array_split(arr,8)
print(newarr)

[array([1, 2]), array([3, 4]), array([5, 6]), array([7, 8]), array([ 9, 10]), array([11, 12]), array([13, 14]), array([15, 16])] 


import numpy as np
arr = np.array([1, 2, 3, 4, 5, 6,7,8,9,10,11,12,13,14,15,16])
newarr = np.array_split(arr,2)
print(newarr)
print(newarr[0])
print(newarr[1])

[array([1, 2, 3, 4, 5, 6, 7, 8]), array([ 9, 10, 11, 12, 13, 14, 15, 16])]


#split and slice 
import numpy as np
arr = np.array([1, 2, 3, 4, 5, 6,7,8,9,10,11,12,13,14,15,16])
newarr = np.array_split(arr,2)
print(newarr)

print(newarr[0])
print(newarr[1]) 

print(newarr[0][3:5])
print(newarr[1][3:5])


[array([1, 2, 3, 4, 5, 6, 7, 8]), array([ 9, 10, 11, 12, 13, 14, 15, 16])]
[1 2 3 4 5 6 7 8]
[ 9 10 11 12 13 14 15 16]
[4 5]
[12 13]



import numpy as np
arr = np.array([[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12], [13,14],[15,16],[17,18]])
newarr = np.array_split(arr, 3)
print(newarr)


[array([[1, 2],
       [3, 4],
       [5, 6]]), 
 array([[ 7,  8],
       [ 9, 10],
       [11, 12]]), 
 array([[13, 14],
       [15, 16],
       [17, 18]])]
   
#find the index position start with 0
   
import numpy as np
arr = np.array([1, 2, 3, 4, 5, 4, 4])  #0,1,2,3,4,5,6
x = np.where(arr == 4)
print(x)


(array([3, 5, 6]),)




#Even number's position index starts from 0

import numpy as np
arr = np.array([2222,1111,3333,4444,8888,223,224,999,1000])
x = np.where(arr%2 == 0)
print(x)

(array([0, 3, 4, 6, 8]),)


#Odd number's position index starts from 0
import numpy as np
arr = np.array([2222,1111,3333,4444,8888,223,224,999,1000])
x = np.where(arr%2 == 1)
print(x)


(array([1, 2, 5, 7]),)



#Find the indexes where the value 'n' should be inserted:


import numpy as np
arr = np.array([636, 173,285, 492])

x = np.searchsorted(arr, 177)
print(x)

2


x = np.searchsorted(arr, 77)
print(x)

0

x = np.searchsorted(arr, 377)
print(x)

3


x = np.searchsorted(arr, 500)
print(x)

4

x = np.searchsorted(arr, 1000)
print(x)

4


import numpy as np
arr = np.array([66, 17, 38, 69])
x = np.searchsorted(arr, 27, side='left')
print(x)

2

x = np.searchsorted(arr, 27, side='right')
print(x)

2



import numpy as np
arr = np.array([400, 200, 350, 100])
x = np.searchsorted(arr, [22, 444, 266,133,277,342,479])
print(x)


#sort
import numpy as np
arr = np.array([6,3,66,22,33,100,10,-1,-2,-3,0,34,1000,98])
print(np.sort(arr))

[  -3   -2   -1    0    3    6   10   22   33   34   66   98  100 1000]



import numpy as np
arr = np.array(["zeebra","elephant","lion","tiger","deer","cheetah","fox"])
print(np.sort(arr))

['cheetah' 'deer' 'elephant' 'fox' 'lion' 'tiger' 'zeebra']


import numpy as np
arr = np.array([True, False,False,True,True,True,False])
print(np.sort(arr))


[False False False  True  True  True  True]


#sort an array
import numpy as np
arr = np.array([[3, 2, 4], [5, 0, 1],[6,3,2], [5,2,1], [6,-1,-2],[0,1,-33] ])
print(np.sort(arr))

[[  2   3   4]
 [  0   1   5]
 [  2   3   6]
 [  1   2   5]
 [ -2  -1   6]
 [-33   0   1]]
 
 
Filter: 


import numpy as np
arr = np.array([41, 42, 43, 44])
x = arr[[True, True, True, True]]
print(x)

[41 42 43 44]

x = arr[[False,False,False,False]]
print(x)

[]

x = arr[[False,False,True,True]]
print(x)

[43 44]

x = arr[[True,True,False,False]]
print(x)

[41 42]



import numpy as np

arr = np.array([100,1000,800,200,700,500,600,300,200,0,900])
filter_arr = arr > 550
newarr = arr[filter_arr]
print(filter_arr)
print(newarr)

[False  True  True False  True False  True False False False  True]
[1000  800  700  600  900]



Even number filtering:

import numpy as np
arr = np.array([100,1000,800,200,700,500,600,300,200,0,900])
filter_arr = arr %2 == 0
newarr = arr[filter_arr]
print(filter_arr)
print(newarr)

[ True  True  True  True  True  True  True  True  True  True  True]
[ 100 1000  800  200  700  500  600  300  200    0  900]


Odd number filtering:

import numpy as np
arr = np.array([100,1000,800,200,700,500,600,300,200,0,900])
filter_arr = arr %2 == 1
newarr = arr[filter_arr]
print(filter_arr)
print(newarr)

[False False False False False False False False False False False]
[]

#odd number filtering
import numpy as np
arr = np.array([100,55,77,200,700,399,600,241,200,0,887])
filter_arr = arr %2 == 1
newarr = arr[filter_arr]
print(filter_arr)
print(newarr) 

[False  True  True False False  True False  True False False  True]
[ 55  77 399 241 887]


import numpy as np
arr = np.array([100,55,77,200,700,399,600,241,200,0,887])
filter_arr = arr %2 == 0
newarr = arr[filter_arr]
print(filter_arr)
print(newarr)

#even number filtering
[ True False False  True  True False  True False  True  True False]
[100 200 700 600 200   0]

Flume - Simple Demo

// create a folder in hdfs : $ hdfs dfs -mkdir /user/flumeExa // Create a shell script which generates : Hadoop in real world <n>...