In [14]:
warnings.filterwarnings('ignore')
# Since there's way too many outliers on the dependant variable (games with many reviews compared to the vast majority which have less reviews)
# we're going to use the Inter Quartile Range method for identifying these outliers and removing them
# Finding the IQR
percentile25 = games_df['Total_User_Reviews'].quantile(0.25)
percentile75 = games_df['Total_User_Reviews'].quantile(0.75)
iqr = percentile75 - percentile25
# Finding the upper and lower limits
upper_limit = percentile75 + 1.5 * iqr
lower_limit = percentile25 - 1.5 * iqr
# Finding outliers
outliers_high = games_df[games_df['Total_User_Reviews'] > upper_limit]
outliers_low = games_df[games_df['Total_User_Reviews'] < lower_limit]
# Trimming outliers
trimmed_df = games_df[(games_df['Total_User_Reviews'] < upper_limit) & (games_df['Total_User_Reviews'] > lower_limit)]
trimmed_df.reset_index(drop=True,inplace=True)
# Compare the plots after trimming
plt.figure(figsize=(16, 8))
plt.subplot(2, 2, 1)
sns.distplot(games_df['Total_User_Reviews'])
plt.title('Original Distribution')
plt.subplot(2, 2, 2)
sns.boxplot(games_df['Total_User_Reviews'])
plt.title('Original Boxplot')
plt.subplot(2, 2, 3)
sns.distplot(trimmed_df['Total_User_Reviews'])
plt.title('Trimmed Distribution')
plt.subplot(2, 2, 4)
sns.boxplot(trimmed_df['Total_User_Reviews'])
plt.title('Trimmed Boxplot')
plt.tight_layout()
plt.show()
print("Original Dataset Statistics:")
print(f"Previous dataset size: {len(games_df)}")
print(f"mean: {games_df['Total_User_Reviews'].mean():.2f}")
print(f"max: {games_df['Total_User_Reviews'].max()}")
print(f"min: {games_df['Total_User_Reviews'].min()}")
print(f"std: {games_df['Total_User_Reviews'].std():.2f}")
print("\nTrimmed Dataset Statistics:")
print(f"Current dataset size: {len(trimmed_df)}")
print(f"mean: {trimmed_df['Total_User_Reviews'].mean():.2f}")
print(f"max: {trimmed_df['Total_User_Reviews'].max()}")
print(f"min: {trimmed_df['Total_User_Reviews'].min()}")
print(f"std: {trimmed_df['Total_User_Reviews'].std():.2f}")
# Number of outliers removed
num_outliers_removed = len(games_df) - len(trimmed_df)
print("\nNumber of outliers removed:", num_outliers_removed, f"({num_outliers_removed/len(games_df)})")
Original Dataset Statistics: Previous dataset size: 53392 mean: 1590.59 max: 7428921.0 min: 0.0 std: 38589.05 Trimmed Dataset Statistics: Current dataset size: 44516 mean: 36.90 max: 279.0 min: 0.0 std: 55.90 Number of outliers removed: 8876 (0.1662421336529817)
