In [1]: # demonstrating working of Decision Tree based on ID3 model
import pandas as pd
from pandas import DataFrame
from collections import Counter # to hold count of each element
In [5]: df_tennis=pd.read_csv('play_tennis.csv')
In [16]: df_tennis.head()
Out[16]:
day outlook temp humidity wind play
0 D1 Sunny Hot High Weak No
1 D2 Sunny Hot High Strong No
2 D3 Overcast Hot High Weak Yes
3 D4 Rain Mild High Weak Yes
4 D5 Rain Cool Normal Weak Yes
In [17]: df_tennis
Out[17]:
day outlook temp humidity wind play
0 D1 Sunny Hot High Weak No
1 D2 Sunny Hot High Strong No
2 D3 Overcast Hot High Weak Yes
3 D4 Rain Mild High Weak Yes
4 D5 Rain Cool Normal Weak Yes
5 D6 Rain Cool Normal Strong No
6 D7 Overcast Cool Normal Strong Yes
7 D8 Sunny Mild High Weak No
8 D9 Sunny Cool Normal Weak Yes
9 D10 Rain Mild Normal Weak Yes
10 D11 Sunny Mild Normal Strong Yes
11 D12 Overcast Mild High Strong Yes
12 D13 Overcast Hot Normal Weak Yes
13 D14 Rain Mild High Strong No
In [6]: df_tennis.keys()[4]
Out[6]: 'wind'
In [7]: # function to compute entropy of individual attribute
def entropy(probs):
import math
return sum([-prob*math.log(prob,2)for prob in probs])
In [8]: # function to compute entropy of given attribute w.r.t. target attribute
def entropy_of_list(a_list):
cnt=Counter(x for x in a_list)
num_instances=len(a_list)
print('\n Number of instances of the current sub class is {0}:'
.format(num_instances))
# to convert into binary form we use .format
probs=[x/num_instances for x in cnt.values()]
print('\n Classes:',min(cnt),max(cnt))
print('\n Probabilities of Class {0} is {1}:'.format(min(cnt),min(probs)))
print('\n Probabilities of Class {0} is {1}:'.format(max(cnt),max(probs)))
return entropy(probs)
In [9]: # wind---strong----yes
# wind---strong---no
# lets make independent and dependent variable i.e. X & Y
# here Y is binary (Play: Yes/No)
print('\n Input dataset for entropy calculation:\n',df_tennis['play'])
Input dataset for entropy calculation:
0 No
1 No
2 Yes
3 Yes
4 Yes
5 No
6 Yes
7 No
8 Yes
9 Yes
10 Yes
11 Yes
12 Yes
13 No
Name: play, dtype: object
In [10]: total_entropy=entropy_of_list(df_tennis['play'])
print('\n Total Entropy of Play Tennis Set is:',total_entropy)
Number of instances of the current sub class is 14:
Classes: No Yes
Probabilities of Class No is 0.35714285714285715:
Probabilities of Class Yes is 0.6428571428571429:
Total Entropy of Play Tennis Set is: 0.9402859586706309
Information Gain = Entropy before splitting - Entropy after splitting IG(S, a) = H(S) – H(S | a)
H(S | a) = sum v in a Sa(v)/S * H(Sa(v)) where
IG(S, a) is the information for the dataset S for the variable a for a random variable H(S) is the
entropy for the dataset before any change H(S | a) is the conditional entropy for the dataset
given the variable a
In [11]: def information_gain(df,split_attribute_name,target_attribute_name):
print("information gain calculation of",split_attribute_name)
df_split=df.groupby(split_attribute_name)
nobs=len(df.index*1.0)
print("NOBS",nobs)
df_agg_ent= df_split.agg({target_attribute_name:
[entropy_of_list,lambda x:len(x)/nobs]})
print('FEATURE',df_agg_ent)
df_agg_ent.columns=['Entropy','PropObservations']
new_entropy=sum(df_agg_ent['Entropy']*df_agg_ent['PropObservations'])
old_entropy=entropy_of_list(df[target_attribute_name])
return old_entropy - new_entropy
NOBS= number of observations .agg function allows you to apply function along one axis
In [12]: print('Information Gain for Outlook is:'
+str(information_gain(df_tennis,'outlook','play')))
information gain calculation of outlook
NOBS 14
Number of instances of the current sub class is 4:
Classes: Yes Yes
Probabilities of Class Yes is 1.0:
Probabilities of Class Yes is 1.0:
Number of instances of the current sub class is 5:
Classes: No Yes
Probabilities of Class No is 0.4:
Probabilities of Class Yes is 0.6:
Number of instances of the current sub class is 5:
Classes: No Yes
Probabilities of Class No is 0.4:
Probabilities of Class Yes is 0.6:
FEATURE play
entropy_of_list <lambda_0>
outlook
Overcast 0.000000 0.285714
Rain 0.970951 0.357143
Sunny 0.970951 0.357143
Number of instances of the current sub class is 14:
Classes: No Yes
Probabilities of Class No is 0.35714285714285715:
Probabilities of Class Yes is 0.6428571428571429:
Information Gain for Outlook is:0.2467498197744391
In [13]: print('Information Gain for Outlook is:'
+str(information_gain(df_tennis,'temp','play')),"\n")
information gain calculation of temp
NOBS 14
Number of instances of the current sub class is 4:
Classes: No Yes
Probabilities of Class No is 0.25:
Probabilities of Class Yes is 0.75:
Number of instances of the current sub class is 4:
Classes: No Yes
Probabilities of Class No is 0.5:
Probabilities of Class Yes is 0.5:
Number of instances of the current sub class is 6:
Classes: No Yes
Probabilities of Class No is 0.3333333333333333:
Probabilities of Class Yes is 0.6666666666666666:
FEATURE play
entropy_of_list <lambda_0>
temp
Cool 0.811278 0.285714
Hot 1.000000 0.285714
Mild 0.918296 0.428571
Number of instances of the current sub class is 14:
Classes: No Yes
Probabilities of Class No is 0.35714285714285715:
Probabilities of Class Yes is 0.6428571428571429:
Information Gain for Outlook is:0.029222565658954647
In [14]: print('Information Gain for Outlook is:'
+str(information_gain(df_tennis,'humidity','play')))
information gain calculation of humidity
NOBS 14
Number of instances of the current sub class is 7:
Classes: No Yes
Probabilities of Class No is 0.42857142857142855:
Probabilities of Class Yes is 0.5714285714285714:
Number of instances of the current sub class is 7:
Classes: No Yes
Probabilities of Class No is 0.14285714285714285:
Probabilities of Class Yes is 0.8571428571428571:
FEATURE play
entropy_of_list <lambda_0>
humidity
High 0.985228 0.5
Normal 0.591673 0.5
Number of instances of the current sub class is 14:
Classes: No Yes
Probabilities of Class No is 0.35714285714285715:
Probabilities of Class Yes is 0.6428571428571429:
Information Gain for Outlook is:0.15183550136234136
In [15]: print('Information Gain for Outlook is:'
+str(information_gain(df_tennis,'wind','play')))
information gain calculation of wind
NOBS 14
Number of instances of the current sub class is 6:
Classes: No Yes
Probabilities of Class No is 0.5:
Probabilities of Class Yes is 0.5:
Number of instances of the current sub class is 8:
Classes: No Yes
Probabilities of Class No is 0.25:
Probabilities of Class Yes is 0.75:
FEATURE play
entropy_of_list <lambda_0>
wind
Strong 1.000000 0.428571
Weak 0.811278 0.571429
Number of instances of the current sub class is 14:
Classes: No Yes
Probabilities of Class No is 0.35714285714285715:
Probabilities of Class Yes is 0.6428571428571429:
Information Gain for Outlook is:0.04812703040826927
