{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "id": "yZo7KUYK3zVM" }, "outputs": [], "source": [ "import pandas as pd\n", "from sklearn.cluster import KMeans\n", "from sklearn.metrics import silhouette_score\n", "from sklearn.preprocessing import StandardScaler\n", "import matplotlib.pyplot as plt\n", "\n", "college_attitude = pd.read_csv(\"./input/corpus1_appraisal_attitude.csv\")\n", "college_engagement = pd.read_csv(\"./input/corpus1_appraisal_engagement.csv\")\n", "college_graduation = pd.read_csv(\"./input/corpus1_appraisal_graduation.csv\")\n", "\n", "college_attitude['category'] = 'attitude'\n", "college_engagement['category'] = 'engagement'\n", "college_graduation['category'] = 'graduation'\n", "\n", "combined_data = pd.concat([college_attitude[['feature', 'frequency', 'category']],\n", " college_engagement[['feature', 'frequency', 'category']],\n", " college_graduation[['feature', 'frequency', 'category']]],\n", " ignore_index=True)\n", "\n", "combined_data.fillna(0, inplace=True)\n", "\n", "combined_data['norm_freq_attitude'] = 0\n", "combined_data['norm_freq_engagement'] = 0\n", "combined_data['norm_freq_graduation'] = 0\n", "\n", "combined_data.loc[combined_data['category'] == 'attitude', 'norm_freq_attitude'] = combined_data['frequency']\n", "combined_data.loc[combined_data['category'] == 'engagement', 'norm_freq_engagement'] = combined_data['frequency']\n", "combined_data.loc[combined_data['category'] == 'graduation', 'norm_freq_graduation'] = combined_data['frequency']\n", "\n", "combined_data.drop(columns=['frequency'], inplace=True)\n", "\n", "X = combined_data[['norm_freq_attitude', 'norm_freq_engagement', 'norm_freq_graduation']]\n", "\n", "scaler = StandardScaler()\n", "X_standardized = scaler.fit_transform(X)\n", "\n", "wcss = []\n", "sil_scores = []\n", "\n", "for k in range(2, 11):\n", " kmeans = KMeans(n_clusters=k, random_state=42)\n", " kmeans.fit(X_standardized)\n", " wcss.append(kmeans.inertia_)\n", "\n", " sil_score = silhouette_score(X_standardized, kmeans.labels_)\n", " sil_scores.append(sil_score)\n", "\n", "plt.figure(figsize=(12, 6))\n", "plt.subplot(1, 2, 1)\n", "plt.plot(range(2, 11), wcss, marker='o', linestyle='-', color='b')\n", "plt.title('Elbow method')\n", "plt.xlabel('Number of clusters (k)')\n", "plt.ylabel('WCSS (Inertia)')\n", "\n", "plt.subplot(1, 2, 2)\n", "plt.plot(range(2, 11), sil_scores, marker='o', linestyle='-', color='g')\n", "plt.title('Silhouette score')\n", "plt.xlabel('Number of clusters (k)')\n", "plt.ylabel('Silhouette score')\n", "\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "j6IZZ6Pk3zVO" }, "outputs": [], "source": [ "import pandas as pd\n", "from sklearn.decomposition import PCA\n", "from sklearn.cluster import KMeans\n", "from sklearn.preprocessing import StandardScaler\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "\n", "college_attitude = pd.read_csv(\"./input/corpus1_appraisal_attitude.csv\")\n", "college_engagement = pd.read_csv(\"./input/corpus1_appraisal_engagement.csv\")\n", "college_graduation = pd.read_csv(\"./input/corpus1_appraisal_graduation.csv\")\n", "\n", "college_attitude['category'] = 'attitude'\n", "college_engagement['category'] = 'engagement'\n", "college_graduation['category'] = 'graduation'\n", "\n", "combined_data = pd.concat([college_attitude[['feature', 'frequency', 'category']],\n", " college_engagement[['feature', 'frequency', 'category']],\n", " college_graduation[['feature', 'frequency', 'category']]],\n", " ignore_index=True)\n", "\n", "combined_data.fillna(0, inplace=True)\n", "\n", "combined_data['norm_freq_attitude'] = 0\n", "combined_data['norm_freq_engagement'] = 0\n", "combined_data['norm_freq_graduation'] = 0\n", "\n", "combined_data.loc[combined_data['category'] == 'attitude', 'norm_freq_attitude'] = combined_data['frequency']\n", "combined_data.loc[combined_data['category'] == 'engagement', 'norm_freq_engagement'] = combined_data['frequency']\n", "combined_data.loc[combined_data['category'] == 'graduation', 'norm_freq_graduation'] = combined_data['frequency']\n", "\n", "combined_data.drop(columns=['frequency'], inplace=True)\n", "\n", "scaler = StandardScaler()\n", "scaled_features = scaler.fit_transform(combined_data[['norm_freq_attitude', 'norm_freq_engagement', 'norm_freq_graduation']])\n", "\n", "pca = PCA(n_components=2)\n", "pca_components = pca.fit_transform(scaled_features)\n", "\n", "pca_df = pd.DataFrame(pca_components, columns=['PCA Component 1', 'PCA Component 2'])\n", "\n", "optimal_k = 5\n", "kmeans = KMeans(n_clusters=optimal_k, random_state=42)\n", "pca_df['cluster'] = kmeans.fit_predict(pca_df)\n", "\n", "pca_df['feature'] = combined_data['feature']\n", "pca_df['category'] = combined_data['category']\n", "\n", "plt.figure(figsize=(10, 8))\n", "sns.scatterplot(\n", " x='PCA Component 1', y='PCA Component 2',\n", " hue='cluster', palette='Set1', data=pca_df, s=100, edgecolor='black'\n", ")\n", "\n", "for i, term in enumerate(pca_df['feature']):\n", " plt.annotate(term, (pca_components[i, 0], pca_components[i, 1]), fontsize=8, alpha=0.7, color='black')\n", "\n", "plt.title(f'K-means Clustering with k={optimal_k} (PCA Components)')\n", "plt.xlabel('PCA Component 1')\n", "plt.ylabel('PCA Component 2')\n", "plt.legend(title='Cluster')\n", "plt.show()\n", "\n", "pca_df.to_csv('college_clustered_features.csv', index=False)\n", "\n", "print(\"Clustered features saved to 'college_clustered_features.csv'.\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "i03Ipx1E3zVO" }, "outputs": [], "source": [ "import pandas as pd\n", "from sklearn.decomposition import PCA\n", "from sklearn.preprocessing import StandardScaler\n", "\n", "min_length = min(len(college_attitude), len(college_engagement), len(college_graduation))\n", "\n", "combined_data = pd.DataFrame({\n", " 'norm_freq_attitude': college_attitude['frequency'][:min_length],\n", " 'norm_freq_engagement': college_engagement['frequency'][:min_length],\n", " 'norm_freq_graduation': college_graduation['frequency'][:min_length]\n", "})\n", "\n", "combined_data.fillna(0, inplace=True)\n", "\n", "scaler = StandardScaler()\n", "scaled_data = scaler.fit_transform(combined_data)\n", "\n", "n_components = min(len(combined_data.columns), 2)\n", "pca = PCA(n_components=n_components)\n", "pca_components = pca.fit_transform(scaled_data)\n", "\n", "factor_loadings = pd.DataFrame(\n", " pca.components_.T,\n", " columns=[f\"PC{i}\" for i in range(1, n_components + 1)],\n", " index=combined_data.columns\n", ")\n", "\n", "print(\"Factor Loadings for all PCA components:\")\n", "print(factor_loadings)\n", "\n", "factor_loadings.to_csv('college_pca_factor_loadings.csv', index=True)\n", "\n", "print(\"Factor loadings saved to 'college_pca_factor_loadings.csv'.\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "FY601ocH3zVO" }, "outputs": [], "source": [ "explained_variance_ratio = pca.explained_variance_ratio_\n", "cumulative_variance = explained_variance_ratio.cumsum()\n", "\n", "print(\"Explained variance by each component:\")\n", "for i, ev in enumerate(explained_variance_ratio, start=1):\n", " print(f\"PC{i}: {ev:.4f}\")\n", "\n", "print(\"\\nCumulative explained variance:\")\n", "for i, cv in enumerate(cumulative_variance, start=1):\n", " print(f\"PC{i}: {cv:.4f}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "TkVirgAM3zVP" }, "outputs": [], "source": [ "import pandas as pd\n", "\n", "df = pd.read_csv('./output/college_clustered_features.csv')\n", "\n", "print(df.head())\n", "\n", "for cluster in df['cluster'].unique():\n", " print(f\"Top 20 Features for Cluster {cluster}:\\n\")\n", "\n", " cluster_data = df[df['cluster'] == cluster]\n", "\n", " cluster_data_sorted = cluster_data.sort_values(by=['PCA Component 1', 'PCA Component 2'], ascending=False)\n", "\n", " print(cluster_data_sorted[['feature', 'category', 'PCA Component 1', 'PCA Component 2']].head(20))\n", " print(\"\\n\" + \"-\"*40 + \"\\n\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "dlDDERfz3zVP" }, "outputs": [], "source": [ "import pandas as pd\n", "import scipy.stats as stats\n", "\n", "data = pd.read_csv('./output/college_clustered_features.csv')\n", "\n", "def perform_anova(data, feature_category):\n", "\n", " filtered_data = data[data['category'] == feature_category]\n", "\n", " groups = [filtered_data[filtered_data['cluster'] == cluster]['PCA Component 1'] for cluster in filtered_data['cluster'].unique()]\n", "\n", " f_statistic, p_value = stats.f_oneway(*groups)\n", "\n", " print(f\"ANOVA result for {feature_category}:\\n\")\n", " print(f\"F-statistic: {f_statistic:.4f}\")\n", " print(f\"P-value: {p_value:.4f}\")\n", "\n", " if p_value < 0.05:\n", " print(\"There is a statistically significant difference between clusters for this feature category.\")\n", " else:\n", " print(\"No statistically significant difference between clusters for this feature category.\\n\")\n", "\n", "perform_anova(data, 'attitude')\n", "perform_anova(data, 'engagement')\n", "perform_anova(data, 'graduation')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "6ZdHYcvC3zVP" }, "outputs": [], "source": [ "import pandas as pd\n", "from scipy.stats import f_oneway\n", "\n", "pca_df = pd.read_csv('./output/college_clustered_features.csv')\n", "\n", "feature_values_by_cluster = [pca_df.loc[pca_df['cluster'] == cluster, 'PCA Component 1'].values\n", " for cluster in pca_df['cluster'].unique()]\n", "\n", "f_stat, p_value = f_oneway(*feature_values_by_cluster)\n", "print(f\"ANOVA results: F-statistic = {f_stat}, p-value = {p_value}\")\n", "\n", "if p_value < 0.05:\n", " print(\"The differences between clusters are statistically significant.\")\n", "else:\n", " print(\"The differences between clusters are not statistically significant.\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "3-K85twF3zVP" }, "outputs": [], "source": [ "import seaborn as sns\n", "import matplotlib.pyplot as plt\n", "\n", "plt.figure(figsize=(10, 6))\n", "sns.boxplot(x='cluster', y='PCA Component 1', data=pca_df, palette='Set2')\n", "\n", "plt.title('Boxplot of PCA Component 1 by Cluster', fontsize=14)\n", "plt.xlabel('Cluster', fontsize=12)\n", "plt.ylabel('PCA Component 1', fontsize=12)\n", "plt.xticks(fontsize=10)\n", "plt.yticks(fontsize=10)\n", "plt.grid(axis='y', linestyle='--', alpha=0.7)\n", "\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "NBj-uDPl3zVP" }, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "\n", "file_path = './output/college_clustered_features.csv'\n", "df = pd.read_csv(file_path)\n", "\n", "cluster_means = df.groupby(['cluster', 'category'])['PCA Component 1'].mean().reset_index()\n", "\n", "df = pd.merge(df, cluster_means, on=['cluster', 'category'], suffixes=('', '_predicted'))\n", "\n", "df['residuals'] = df['PCA Component 1'] - df['PCA Component 1_predicted']\n", "\n", "print(df[['cluster', 'category', 'feature', 'PCA Component 1', 'PCA Component 1_predicted', 'residuals']].head())" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "hYUyUWVl3zVP" }, "outputs": [], "source": [ "import pandas as pd\n", "\n", "data = pd.read_csv('./output/college_clustered_features.csv')\n", "\n", "cluster_means = data.groupby('cluster')['PCA Component 1'].mean()\n", "\n", "data = data.merge(cluster_means, on='cluster', suffixes=('', '_predicted'))\n", "\n", "data['residuals'] = data['PCA Component 1'] - data['PCA Component 1_predicted']\n", "\n", "data['abs_residuals'] = data['residuals'].abs()\n", "\n", "threshold = 2\n", "outliers = data[data['abs_residuals'] > threshold]\n", "\n", "print(outliers)\n", "\n", "print(data.head())" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "cOaLCnIJ3zVQ" }, "outputs": [], "source": [ "pip install scikit-posthocs" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "qeh8OZl33zVQ" }, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import scipy.stats as stats\n", "\n", "file_path = './output/college_clustered_features.csv'\n", "df = pd.read_csv(file_path)\n", "\n", "contingency_table = pd.crosstab(df['category'], df['cluster'])\n", "\n", "chi2_stat, p_value, dof, expected = stats.chi2_contingency(contingency_table)\n", "\n", "residuals = (contingency_table - expected) / np.sqrt(expected)\n", "\n", "print(\"Chi-square Test Results:\")\n", "print(f\"Chi-square Statistic: {chi2_stat:.4f}\")\n", "print(f\"P-value: {p_value:.4f}\")\n", "print(f\"Degrees of Freedom: {dof}\")\n", "print(\"\\nExpected Frequencies:\")\n", "print(pd.DataFrame(expected, index=contingency_table.index, columns=contingency_table.columns))\n", "print(\"\\nPearson Residuals:\")\n", "print(residuals)\n", "\n", "residuals.to_csv('./output/pearson_residuals.csv')\n", "\n", "import seaborn as sns\n", "import matplotlib.pyplot as plt\n", "\n", "plt.figure(figsize=(10, 6))\n", "sns.heatmap(residuals, annot=True, cmap=\"coolwarm\", fmt=\".2f\", cbar_kws={'label': 'Pearson Residuals'})\n", "plt.xlabel(\"Cluster\")\n", "plt.ylabel(\"Category\")\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Qa-j9fiH3zVQ" }, "outputs": [], "source": [ "import pandas as pd\n", "import seaborn as sns\n", "import matplotlib.pyplot as plt\n", "\n", "file_path = './output/all_pearson_residuals_college.csv'\n", "df = pd.read_csv(file_path)\n", "\n", "melted_df = df.melt(\n", " id_vars=['feature', 'category'],\n", " value_vars=['0', '1', '2', '3', '4'],\n", " var_name='cluster',\n", " value_name='residual'\n", ")\n", "\n", "melted_df['cluster'] = pd.to_numeric(melted_df['cluster'])\n", "\n", "top_features = (\n", " melted_df.groupby('feature')['residual']\n", " .apply(lambda x: x.abs().max())\n", " .nlargest(50)\n", " .index\n", ")\n", "filtered_df = melted_df[melted_df['feature'].isin(top_features)]\n", "\n", "heatmap_data = filtered_df.pivot_table(\n", " index='feature',\n", " columns=['category', 'cluster'],\n", " values='residual'\n", ")\n", "\n", "plt.figure(figsize=(14, 10))\n", "sns.heatmap(\n", " heatmap_data,\n", " annot=True,\n", " fmt=\".2f\",\n", " cmap='coolwarm',\n", " cbar_kws={'label': 'Pearson residuals'}\n", ")\n", "plt.xlabel('Category and cluster')\n", "plt.ylabel('Appraisal resource')\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "_RysrecV3zVQ" }, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "\n", "data = pd.read_csv('./output/college_clustered_features.csv')\n", "\n", "contingency_table = pd.crosstab([data['feature'], data['category']], data['cluster'])\n", "\n", "expected_frequencies = np.outer(contingency_table.sum(axis=1), contingency_table.sum(axis=0)) / contingency_table.sum().sum()\n", "\n", "residuals = (contingency_table.values - expected_frequencies) / np.sqrt(expected_frequencies)\n", "\n", "residuals_df = pd.DataFrame(residuals, index=contingency_table.index, columns=contingency_table.columns)\n", "\n", "high_residuals = residuals_df[residuals_df > 2]\n", "low_residuals = residuals_df[residuals_df < -2]\n", "\n", "print(\"High Pearson Residuals (over-represented features):\")\n", "print(high_residuals.dropna(how='all'))\n", "\n", "print(\"\\nLow Pearson Residuals (under-represented features):\")\n", "print(low_residuals.dropna(how='all'))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "dvVMMXNH3zVQ" }, "outputs": [], "source": [ "import pandas as pd\n", "from sklearn.cluster import KMeans\n", "from sklearn.metrics import silhouette_score\n", "from sklearn.preprocessing import StandardScaler\n", "import matplotlib.pyplot as plt\n", "\n", "research_attitude = pd.read_csv(\"./input/corpus2_appraisal_attitude.csv\")\n", "research_engagement = pd.read_csv(\"./input/corpus2_appraisal_engagement.csv\")\n", "research_graduation = pd.read_csv(\"./input/corpus2_appraisal_graduation.csv\")\n", "\n", "research_attitude['category'] = 'attitude'\n", "research_engagement['category'] = 'engagement'\n", "research_graduation['category'] = 'graduation'\n", "\n", "combined_data_research = pd.concat(\n", " [research_attitude[['feature', 'frequency', 'category']],\n", " research_engagement[['feature', 'frequency', 'category']],\n", " research_graduation[['feature', 'frequency', 'category']]],\n", " ignore_index=True\n", ")\n", "\n", "combined_data_research.fillna(0, inplace=True)\n", "\n", "combined_data_research['norm_freq_attitude_research'] = 0\n", "combined_data_research['norm_freq_engagement_research'] = 0\n", "combined_data_research['norm_freq_graduation_research'] = 0\n", "\n", "combined_data_research.loc[combined_data_research['category'] == 'attitude', 'norm_freq_attitude_research'] = combined_data_research['frequency']\n", "combined_data_research.loc[combined_data_research['category'] == 'engagement', 'norm_freq_engagement_research'] = combined_data_research['frequency']\n", "combined_data_research.loc[combined_data_research['category'] == 'graduation', 'norm_freq_graduation_research'] = combined_data_research['frequency']\n", "\n", "combined_data_research.drop(columns=['frequency'], inplace=True)\n", "\n", "X = combined_data_research[['norm_freq_attitude_research', 'norm_freq_engagement_research', 'norm_freq_graduation_research']]\n", "\n", "scaler = StandardScaler()\n", "X_scaled = scaler.fit_transform(X)\n", "\n", "wcss = []\n", "sil_scores = []\n", "\n", "for k in range(2, 11):\n", " kmeans = KMeans(n_clusters=k, random_state=42)\n", " kmeans.fit(X_scaled)\n", " wcss.append(kmeans.inertia_)\n", "\n", " sil_score = silhouette_score(X_scaled, kmeans.labels_)\n", " sil_scores.append(sil_score)\n", "\n", "plt.figure(figsize=(12, 6))\n", "\n", "plt.subplot(1, 2, 1)\n", "plt.plot(range(2, 11), wcss, marker='o', linestyle='-', color='b')\n", "plt.title('Elbow method')\n", "plt.xlabel('Number of clusters (k)')\n", "plt.ylabel('WCSS (Inertia)')\n", "\n", "plt.subplot(1, 2, 2)\n", "plt.plot(range(2, 11), sil_scores, marker='o', linestyle='-', color='g')\n", "plt.title('Silhouette score')\n", "plt.xlabel('Number of clusters (k)')\n", "plt.ylabel('Silhouette Score')\n", "\n", "plt.tight_layout()\n", "plt.show()\n", "\n", "best_k = range(2, 11)[sil_scores.index(max(sil_scores))]\n", "print(f\"The best k is: {best_k}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "zUNQS6hi3zVQ" }, "outputs": [], "source": [ "import pandas as pd\n", "from sklearn.decomposition import PCA\n", "from sklearn.cluster import KMeans\n", "from sklearn.preprocessing import StandardScaler\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "\n", "research_attitude = pd.read_csv(\"./input/corpus2_appraisal_attitude.csv\")\n", "research_engagement = pd.read_csv(\"./input/corpus2_appraisal_engagement.csv\")\n", "research_graduation = pd.read_csv(\"./input/corpus2_appraisal_graduation.csv\")\n", "\n", "research_attitude['category'] = 'attitude'\n", "research_engagement['category'] = 'engagement'\n", "research_graduation['category'] = 'graduation'\n", "\n", "combined_data_research = pd.concat(\n", " [research_attitude[['feature', 'frequency', 'category']],\n", " research_engagement[['feature', 'frequency', 'category']],\n", " research_graduation[['feature', 'frequency', 'category']]],\n", " ignore_index=True\n", ")\n", "\n", "combined_data_research.fillna(0, inplace=True)\n", "\n", "combined_data_research['norm_freq_attitude_research'] = 0\n", "combined_data_research['norm_freq_engagement_research'] = 0\n", "combined_data_research['norm_freq_graduation_research'] = 0\n", "\n", "combined_data_research.loc[combined_data_research['category'] == 'attitude', 'norm_freq_attitude_research'] = combined_data_research['frequency']\n", "combined_data_research.loc[combined_data_research['category'] == 'engagement', 'norm_freq_engagement_research'] = combined_data_research['frequency']\n", "combined_data_research.loc[combined_data_research['category'] == 'graduation', 'norm_freq_graduation_research'] = combined_data_research['frequency']\n", "\n", "combined_data_research.drop(columns=['frequency'], inplace=True)\n", "\n", "scaler = StandardScaler()\n", "scaled_features = scaler.fit_transform(\n", " combined_data_research[['norm_freq_attitude_research', 'norm_freq_engagement_research', 'norm_freq_graduation_research']]\n", ")\n", "\n", "pca = PCA(n_components=2)\n", "pca_components = pca.fit_transform(scaled_features)\n", "\n", "pca_df = pd.DataFrame(pca_components, columns=['PCA Component 1', 'PCA Component 2'])\n", "\n", "optimal_k = 5\n", "kmeans = KMeans(n_clusters=optimal_k, random_state=42)\n", "pca_df['cluster'] = kmeans.fit_predict(pca_df)\n", "\n", "pca_df['feature'] = combined_data_research['feature']\n", "pca_df['category'] = combined_data_research['category']\n", "\n", "plt.figure(figsize=(10, 8))\n", "sns.scatterplot(\n", " x='PCA Component 1', y='PCA Component 2',\n", " hue='cluster', palette='Set1', data=pca_df, s=100, edgecolor='black'\n", ")\n", "\n", "for i, term in enumerate(pca_df['feature']):\n", " plt.annotate(term, (pca_components[i, 0], pca_components[i, 1]), fontsize=8, alpha=0.7, color='black')\n", "\n", "plt.xlabel('PCA component 1')\n", "plt.ylabel('PCA component 2')\n", "plt.legend(title='Cluster')\n", "plt.show()\n", "\n", "pca_df.to_csv('research_clustered_features.csv', index=False)\n", "\n", "print(\"Clustered features saved to 'research_clustered_features.csv'.\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "F02vhk0C3zVR" }, "outputs": [], "source": [ "import pandas as pd\n", "from sklearn.decomposition import PCA\n", "from sklearn.preprocessing import StandardScaler\n", "\n", "combined_data_research = pd.DataFrame({\n", " 'norm_freq_attitude_research': research_attitude['frequency'],\n", " 'norm_freq_engagement_research': research_engagement['frequency'],\n", " 'norm_freq_graduation_research': research_graduation['frequency']\n", "})\n", "\n", "combined_data_research.fillna(0, inplace=True)\n", "\n", "scaler = StandardScaler()\n", "scaled_data = scaler.fit_transform(combined_data_research)\n", "\n", "n_components = min(len(combined_data_research.columns), 2)\n", "pca = PCA(n_components=n_components)\n", "pca_components = pca.fit_transform(scaled_data)\n", "\n", "factor_loadings = pd.DataFrame(\n", " pca.components_.T,\n", " columns=[f\"PC{i}\" for i in range(1, n_components + 1)],\n", " index=combined_data_research.columns\n", ")\n", "\n", "print(\"Factor Loadings for all PCA components:\")\n", "print(factor_loadings)\n", "\n", "factor_loadings.to_csv('research_pca_factor_loadings.csv', index=True)\n", "\n", "print(\"Factor loadings saved to 'research_pca_factor_loadings.csv'.\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "I-3srXSG3zVR" }, "outputs": [], "source": [ "explained_variance_ratio = pca.explained_variance_ratio_\n", "cumulative_variance = explained_variance_ratio.cumsum()\n", "\n", "print(\"Explained variance by each component:\")\n", "for i, ev in enumerate(explained_variance_ratio, start=1):\n", " print(f\"PC{i}: {ev:.4f}\")\n", "\n", "print(\"\\nCumulative explained variance:\")\n", "for i, cv in enumerate(cumulative_variance, start=1):\n", " print(f\"PC{i}: {cv:.4f}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "5o-Bfqrk3zVR" }, "outputs": [], "source": [ "import pandas as pd\n", "\n", "df = pd.read_csv('./output/research_clustered_features.csv')\n", "\n", "print(df.head())\n", "\n", "for cluster in df['cluster'].unique():\n", " print(f\"Top 20 Features for Cluster {cluster}:\\n\")\n", "\n", " cluster_data = df[df['cluster'] == cluster]\n", "\n", " cluster_data_sorted = cluster_data.sort_values(by=['PCA Component 1', 'PCA Component 2'], ascending=False)\n", "\n", " print(cluster_data_sorted[['feature', 'category', 'PCA Component 1', 'PCA Component 2']].head(20)) # You can add 'PCA Component 2' or other columns if needed\n", " print(\"\\n\" + \"-\"*40 + \"\\n\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "AvHz1GWm3zVR" }, "outputs": [], "source": [ "import pandas as pd\n", "import scipy.stats as stats\n", "\n", "data = pd.read_csv('./output/research_clustered_features.csv')\n", "\n", "def perform_anova(data, feature_category):\n", "\n", " filtered_data = data[data['category'] == feature_category]\n", "\n", " groups = [filtered_data[filtered_data['cluster'] == cluster]['PCA Component 1'].dropna() for cluster in filtered_data['cluster'].unique()]\n", "\n", " if len(groups) < 2 or any(len(group) < 2 for group in groups):\n", " print(f\"Skipping ANOVA for {feature_category} due to insufficient group sizes.\\n\")\n", " return\n", "\n", " try:\n", " f_statistic, p_value = stats.f_oneway(*groups)\n", " print(f\"ANOVA result for {feature_category}:\\n\")\n", " print(f\"F-statistic: {f_statistic:.4f}\")\n", " print(f\"P-value: {p_value:.4f}\")\n", "\n", " if p_value < 0.05:\n", " print(\"There is a statistically significant difference between clusters for this feature category.\\n\")\n", " else:\n", " print(\"No statistically significant difference between clusters for this feature category.\\n\")\n", " except Exception as e:\n", " print(f\"Error performing ANOVA for {feature_category}: {e}\\n\")\n", "\n", "perform_anova(data, 'attitude')\n", "perform_anova(data, 'engagement')\n", "perform_anova(data, 'graduation')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "8Fff_2FJ3zVR" }, "outputs": [], "source": [ "import pandas as pd\n", "from scipy.stats import f_oneway\n", "\n", "pca_df = pd.read_csv('./output/research_clustered_features.csv')\n", "\n", "feature_values_by_cluster = [pca_df.loc[pca_df['cluster'] == cluster, 'PCA Component 1'].values\n", " for cluster in pca_df['cluster'].unique()]\n", "\n", "f_stat, p_value = f_oneway(*feature_values_by_cluster)\n", "print(f\"ANOVA results: F-statistic = {f_stat}, p-value = {p_value}\")\n", "\n", "if p_value < 0.05:\n", " print(\"The differences between clusters are statistically significant.\")\n", "else:\n", " print(\"The differences between clusters are not statistically significant.\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "WFvbVLqs3zVR" }, "outputs": [], "source": [ "pip install statsmodels" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "7O3-jlCe3zVR" }, "outputs": [], "source": [ "import seaborn as sns\n", "import matplotlib.pyplot as plt\n", "\n", "plt.figure(figsize=(10, 6))\n", "sns.boxplot(x='cluster', y='PCA Component 1', data=pca_df, palette='Set2')\n", "\n", "plt.title('Boxplot of PCA Component 1 by Cluster', fontsize=14)\n", "plt.xlabel('Cluster', fontsize=12)\n", "plt.ylabel('PCA Component 1', fontsize=12)\n", "plt.xticks(fontsize=10)\n", "plt.yticks(fontsize=10)\n", "plt.grid(axis='y', linestyle='--', alpha=0.7)\n", "\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "jxU8jQbw3zVS" }, "outputs": [], "source": [ "import pandas as pd\n", "\n", "data = pd.read_csv('./output/research_clustered_features.csv')\n", "\n", "cluster_means = data.groupby('cluster')['PCA Component 1'].mean()\n", "\n", "data = data.merge(cluster_means, on='cluster', suffixes=('', '_predicted'))\n", "\n", "data['residuals'] = data['PCA Component 1'] - data['PCA Component 1_predicted']\n", "\n", "data['abs_residuals'] = data['residuals'].abs()\n", "\n", "threshold = 2\n", "outliers = data[data['abs_residuals'] > threshold]\n", "\n", "print(outliers)\n", "\n", "print(data.head())" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "RZ2ISVrR3zVS" }, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import scipy.stats as stats\n", "\n", "file_path = './output/research_clustered_features.csv'\n", "df = pd.read_csv(file_path)\n", "\n", "contingency_table = pd.crosstab(df['category'], df['cluster'])\n", "\n", "chi2_stat, p_value, dof, expected = stats.chi2_contingency(contingency_table)\n", "\n", "residuals = (contingency_table - expected) / np.sqrt(expected)\n", "\n", "print(\"Chi-square Test Results:\")\n", "print(f\"Chi-square Statistic: {chi2_stat:.4f}\")\n", "print(f\"P-value: {p_value:.4f}\")\n", "print(f\"Degrees of Freedom: {dof}\")\n", "print(\"\\nExpected Frequencies:\")\n", "print(pd.DataFrame(expected, index=contingency_table.index, columns=contingency_table.columns))\n", "print(\"\\nPearson Residuals:\")\n", "print(residuals)\n", "\n", "residuals.to_csv('./output/pearson_residuals_research.csv')\n", "\n", "import seaborn as sns\n", "import matplotlib.pyplot as plt\n", "\n", "plt.figure(figsize=(10, 6))\n", "sns.heatmap(residuals, annot=True, cmap=\"coolwarm\", fmt=\".2f\", cbar_kws={'label': 'Pearson Residuals'})\n", "plt.xlabel(\"Cluster\")\n", "plt.ylabel(\"Category\")\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "oRz8uHkc3zVS" }, "outputs": [], "source": [ "import pandas as pd\n", "import seaborn as sns\n", "import matplotlib.pyplot as plt\n", "\n", "file_path = './output/all_pearson_residuals_research.csv'\n", "df = pd.read_csv(file_path)\n", "\n", "melted_df = df.melt(\n", " id_vars=['feature', 'category'],\n", " value_vars=['0', '1', '2', '3', '4'],\n", " var_name='cluster',\n", " value_name='residual'\n", ")\n", "\n", "melted_df['cluster'] = pd.to_numeric(melted_df['cluster'])\n", "\n", "top_features = (\n", " melted_df.groupby('feature')['residual']\n", " .apply(lambda x: x.abs().max())\n", " .nlargest(50)\n", " .index\n", ")\n", "filtered_df = melted_df[melted_df['feature'].isin(top_features)]\n", "\n", "heatmap_data = filtered_df.pivot_table(\n", " index='feature',\n", " columns=['category', 'cluster'],\n", " values='residual'\n", ")\n", "\n", "plt.figure(figsize=(14, 10))\n", "sns.heatmap(\n", " heatmap_data,\n", " annot=True,\n", " fmt=\".2f\",\n", " cmap='coolwarm',\n", " cbar_kws={'label': 'Pearson residuals'}\n", ")\n", "plt.xlabel('Category and cluster')\n", "plt.ylabel('Appraisal resource')\n", "plt.tight_layout()\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "W6ibx2903zVS" }, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "\n", "data = pd.read_csv('./output/research_clustered_features.csv')\n", "\n", "contingency_table = pd.crosstab([data['feature'], data['category']], data['cluster'])\n", "\n", "expected_frequencies = np.outer(contingency_table.sum(axis=1), contingency_table.sum(axis=0)) / contingency_table.sum().sum()\n", "\n", "residuals = (contingency_table.values - expected_frequencies) / np.sqrt(expected_frequencies)\n", "\n", "residuals_df = pd.DataFrame(residuals, index=contingency_table.index, columns=contingency_table.columns)\n", "\n", "high_residuals = residuals_df[residuals_df > 2]\n", "low_residuals = residuals_df[residuals_df < -2]\n", "\n", "print(\"High Pearson Residuals (over-represented features):\")\n", "print(high_residuals.dropna(how='all'))\n", "\n", "print(\"\\nLow Pearson Residuals (under-represented features):\")\n", "print(low_residuals.dropna(how='all'))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Yu3Y5kSv3zVS" }, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "\n", "def calculate_llr(df, N1, N2):\n", "\n", " df['frequency'] = df['frequency'].replace(0, 1e-10)\n", " df['reference_frequency'] = df['reference_frequency'].replace(0, 1e-10)\n", "\n", " df['e1'] = (df['frequency'] + df['reference_frequency']) * N1 / (N1 + N2)\n", " df['e2'] = (df['frequency'] + df['reference_frequency']) * N2 / (N1 + N2)\n", "\n", " df['LLR'] = 2 * (\n", " df['frequency'] * np.log(df['frequency'] / df['e1']) +\n", " df['reference_frequency'] * np.log(df['reference_frequency'] / df['e2'])\n", " )\n", "\n", " df['LLR'] = df['LLR'].replace([np.inf, -np.inf], 0).fillna(0)\n", " return df\n", "\n", "college_appraisal = pd.read_csv(\"./input/corpus1_stance_features.csv\")\n", "research_appraisal = pd.read_csv(\"./input/corpus2_stance_features.csv\")\n", "\n", "N1 = college_appraisal['frequency'].sum()\n", "N2 = research_appraisal['frequency'].sum()\n", "\n", "college_appraisal = college_appraisal.rename(columns={'frequency': 'frequency_college'})\n", "research_appraisal = research_appraisal.rename(columns={'frequency': 'frequency_research'})\n", "\n", "college_appraisal['feature'] = college_appraisal['feature'].str.lower().str.strip()\n", "research_appraisal['feature'] = research_appraisal['feature'].str.lower().str.strip()\n", "\n", "college_appraisal = college_appraisal.drop_duplicates(subset='feature')\n", "research_appraisal = research_appraisal.drop_duplicates(subset='feature')\n", "\n", "merged_df = pd.merge(\n", " college_appraisal, research_appraisal, on='feature', how='outer'\n", ").fillna(0)\n", "\n", "merged_df = merged_df.groupby('feature', as_index=False).sum()\n", "\n", "merged_df = merged_df.rename(columns={\n", " 'frequency_college': 'frequency',\n", " 'frequency_research': 'reference_frequency'\n", "})\n", "\n", "llr_results = calculate_llr(merged_df, N1, N2)\n", "\n", "top_30 = llr_results.sort_values('LLR', ascending=False).head(30)\n", "\n", "plt.figure(figsize=(12, 6))\n", "sns.barplot(data=top_30, x='LLR', y='feature', palette='viridis')\n", "plt.xlabel(\"Log-Likelihood Ratio (LLR)\")\n", "plt.ylabel(\"Appraisal marker\")\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "EXZwRyOD3zVS" }, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "\n", "def calculate_llr(df, N1, N2):\n", "\n", " df['frequency'] = df['frequency'].replace(0, 1e-10)\n", " df['reference_frequency'] = df['reference_frequency'].replace(0, 1e-10)\n", "\n", " df['e1'] = (df['frequency'] + df['reference_frequency']) * N1 / (N1 + N2)\n", " df['e2'] = (df['frequency'] + df['reference_frequency']) * N2 / (N1 + N2)\n", "\n", " df['LLR'] = 2 * (\n", " df['frequency'] * np.log(df['frequency'] / df['e1']) +\n", " df['reference_frequency'] * np.log(df['reference_frequency'] / df['e2'])\n", " )\n", "\n", " df['LLR'] = df['LLR'].replace([np.inf, -np.inf], 0).fillna(0)\n", " return df\n", "\n", "college_appraisal = pd.read_csv(\"./input/corpus1_stance_features.csv\")\n", "research_appraisal = pd.read_csv(\"./input/corpus2_stance_features.csv\")\n", "\n", "N1 = college_appraisal['frequency'].sum()\n", "N2 = research_appraisal['frequency'].sum()\n", "\n", "college_appraisal = college_appraisal.rename(columns={'frequency': 'frequency_college'})\n", "research_appraisal = research_appraisal.rename(columns={'frequency': 'frequency_research'})\n", "\n", "college_appraisal['feature'] = college_appraisal['feature'].str.lower().str.strip()\n", "research_appraisal['feature'] = research_appraisal['feature'].str.lower().str.strip()\n", "\n", "college_appraisal = college_appraisal.drop_duplicates(subset='feature')\n", "research_appraisal = research_appraisal.drop_duplicates(subset='feature')\n", "\n", "merged_df = pd.merge(\n", " college_appraisal, research_appraisal, on='feature', how='outer'\n", ").fillna(0)\n", "\n", "merged_df = merged_df.groupby('feature', as_index=False).sum()\n", "\n", "merged_df = merged_df.rename(columns={\n", " 'frequency_research': 'frequency',\n", " 'frequency_college': 'reference_frequency'\n", "})\n", "\n", "llr_results = calculate_llr(merged_df, N1, N2)\n", "\n", "top_30 = llr_results.sort_values('LLR', ascending=False).head(30)\n", "\n", "plt.figure(figsize=(12, 6))\n", "sns.barplot(data=top_30, x='LLR', y='feature', palette='viridis')\n", "plt.xlabel(\"Log-Likelihood Ratio (LLR)\")\n", "plt.ylabel(\"Appraisal marker\")\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "46SuR7h_3zVS" }, "outputs": [], "source": [ "pip install matplotlib_venn" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ayD9f4RY3zVS" }, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "from scipy.stats import levene\n", "\n", "college_appraisal = pd.read_csv(\"./input/corpus1_stance_features.csv\")\n", "research_appraisal = pd.read_csv(\"./input/corpus2_stance_features.csv\")\n", "\n", "total_frequency_college = college_appraisal['frequency'].sum()\n", "college_appraisal['norm_freq'] = college_appraisal['frequency'] / total_frequency_college\n", "\n", "sd_college = college_appraisal['norm_freq'].std()\n", "cv_college = sd_college / college_appraisal['norm_freq'].mean()\n", "\n", "total_frequency_research = research_appraisal['frequency'].sum()\n", "research_appraisal['norm_freq'] = research_appraisal['frequency'] / total_frequency_research\n", "\n", "stat, p_value = levene(college_appraisal['norm_freq'], research_appraisal['norm_freq'])\n", "\n", "print(f\"Standard Deviation (College): {sd_college}\")\n", "print(f\"Coefficient of Variation (College): {cv_college}\")\n", "print(f\"Levene's Test Statistic: {stat}, p-value: {p_value}\")\n", "\n", "plt.figure(figsize=(12, 6))\n", "sns.histplot(college_appraisal['norm_freq'], kde=True, color='blue', label='College Corpus', bins=30)\n", "sns.histplot(research_appraisal['norm_freq'], kde=True, color='green', label='Research Corpus', bins=30, alpha=0.7)\n", "plt.legend()\n", "plt.title('Normalized Frequency Distribution for College and Research Corpora')\n", "plt.xlabel('Normalized Frequency')\n", "plt.ylabel('Density')\n", "plt.show()\n", "\n", "print(\"Normalized frequencies calculated and Levene's Test results displayed.\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "zhgZ2-5r3zVS" }, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "from scipy.stats import levene\n", "\n", "college_appraisal = pd.read_csv(\"./input/corpus1_stance_features.csv\")\n", "research_appraisal = pd.read_csv(\"./input/corpus2_stance_features.csv\")\n", "\n", "total_frequency_college = college_appraisal['frequency'].sum()\n", "college_appraisal['norm_freq'] = college_appraisal['frequency'] / total_frequency_college\n", "\n", "total_frequency_research = research_appraisal['frequency'].sum()\n", "research_appraisal['norm_freq'] = research_appraisal['frequency'] / total_frequency_research\n", "\n", "sd_research = research_appraisal['norm_freq'].std()\n", "cv_research = sd_research / research_appraisal['norm_freq'].mean()\n", "\n", "stat, p_value = levene(college_appraisal['norm_freq'], research_appraisal['norm_freq'])\n", "\n", "print(f\"Standard Deviation (Research): {sd_research}\")\n", "print(f\"Coefficient of Variation (Research): {cv_research}\")\n", "print(f\"Levene's Test Statistic: {stat}, p-value: {p_value}\")\n", "\n", "plt.figure(figsize=(12, 6))\n", "sns.histplot(research_appraisal['norm_freq'], kde=True, color='green', label='Research Corpus', bins=30)\n", "sns.histplot(college_appraisal['norm_freq'], kde=True, color='blue', label='College Corpus', bins=30, alpha=0.7)\n", "plt.legend()\n", "plt.title('Normalized Frequency Distribution for Research and College Corpora')\n", "plt.xlabel('Normalized Frequency')\n", "plt.ylabel('Density')\n", "plt.show()\n", "\n", "print(\"Normalized frequencies calculated for the Research corpus and Levene's Test results displayed.\")" ] } ], "metadata": { "kernelspec": { "display_name": "venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.9" }, "colab": { "provenance": [] } }, "nbformat": 4, "nbformat_minor": 0 }