{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "## importing libarys etc\n", "\n", "import numpy as np\n", "import pandas as pd\n", "\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "import statsmodels.api as sm\n", "import statsmodels.formula.api \n", "from statsmodels.formula.api import ols\n", "from scipy import stats\n", "import os\n", "\n", "import properscoring as ps\n", "\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
tracerC24C26C28C30d15N
Mix
1-32.664220-34.429345-35.190276-36.2671334.538464
2-32.411763-33.936547-34.607865-35.8108112.620254
3-32.590501-34.355420-35.117930-36.1965614.530275
4-33.360635-34.979022-35.643818-36.8095163.692467
5-33.511746-35.148417-36.051318-37.0087034.721978
..................
146-31.979427-33.069509-34.011537-35.1233900.495186
147-32.722253-34.404809-35.074936-36.2380903.754634
148-32.144412-33.438957-34.235622-35.4017701.435607
149-32.627095-34.133296-34.709495-35.9812472.008588
150-32.170129-33.538249-34.313570-35.4796631.947104
\n", "

150 rows × 5 columns

\n", "
" ], "text/plain": [ "tracer C24 C26 C28 C30 d15N\n", "Mix \n", "1 -32.664220 -34.429345 -35.190276 -36.267133 4.538464\n", "2 -32.411763 -33.936547 -34.607865 -35.810811 2.620254\n", "3 -32.590501 -34.355420 -35.117930 -36.196561 4.530275\n", "4 -33.360635 -34.979022 -35.643818 -36.809516 3.692467\n", "5 -33.511746 -35.148417 -36.051318 -37.008703 4.721978\n", ".. ... ... ... ... ...\n", "146 -31.979427 -33.069509 -34.011537 -35.123390 0.495186\n", "147 -32.722253 -34.404809 -35.074936 -36.238090 3.754634\n", "148 -32.144412 -33.438957 -34.235622 -35.401770 1.435607\n", "149 -32.627095 -34.133296 -34.709495 -35.981247 2.008588\n", "150 -32.170129 -33.538249 -34.313570 -35.479663 1.947104\n", "\n", "[150 rows x 5 columns]" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "### MAKING VIRTUAL MIXTURES## \n", "\n", "##generation of sediment mixtures for single mixture generated from mean source values..DO NOT EDIT!!!!\n", "\n", "\n", "#uploading data\n", "os.chdir(r\"C:\\Users\\terry\\Documents\\Rhine data\")\n", "source_data=pd.read_csv(\"source_values_less_is_more.csv\")\n", "\n", "source_data=source_data.iloc[:,1:]\n", "\n", "\n", "##uploading concentrations of FA and grouping forests\n", "\n", "FA_conc=pd.read_csv(\"source_data_rhine_conc_less_is_more.csv\")\n", "\n", "FA_conc=FA_conc.iloc[:,2:]\n", "\n", "source_data.columns=source_data.columns.str.strip()\n", "FA_conc.columns=FA_conc.columns.str.strip()\n", "\n", "\n", "\n", "source_data.columns,Fa_conc = source_data.columns.str.strip(),FA_conc.columns.str.strip()\n", "\n", "\n", "#select tracers and landuse\n", "source_data= source_data[['Land-use', 'd15N','C24','C26','C28','C30']]\n", "FA_conc= FA_conc[['Land-use', 'd15N','C24','C26','C28','C30']]\n", "\n", "#group sources(i,e forests)\n", "source_data.loc[source_data['Land-use'].isin(['Coniferous forest','Broad-leaved forest']),'Land-use'] = 'Forest'\n", "FA_conc.loc[FA_conc['Land-use'].isin(['Coniferous forest','Broad-leaved forest']),'Land-use'] = 'Forest'\n", "\n", "source_data.loc[source_data['Land-use'].isin(['Arable']),'Land-use'] = 'Non-irrigated arable land'\n", "FA_conc.loc[FA_conc['Land-use'].isin(['Arable']),'Land-use'] = 'Non-irrigated arable land'\n", "\n", "\n", "\n", "\n", "\n", "## generation of sediment mixtures from mean source value\n", "\n", "\n", "\n", "tracer_mean=source_data.groupby('Land-use').mean()\n", "FA_conc_mean=FA_conc.groupby('Land-use').mean()\n", "\n", "## select seed to make reproducable\n", "np.random.seed(123)\n", "\n", "##selecting number of mixtures\n", "number_of_mix=150\n", "amlist=[]\n", "for q in range(1,(number_of_mix+1)):\n", " \n", "##formatting table\n", "\n", "##selecting tracers\n", " tracers=source_data.columns[1:]\n", " len(source_data['Land-use'].unique())\n", " amtracers=[]\n", " \n", " ###generate list of n replicates traecrs (n=number of sources)\n", " for i in tracers:\n", " amtracers.append([i]*len(source_data['Land-use'].unique()))\n", " flat_list_tracer = [item for sublist in amtracers for item in sublist]\n", "\n", " ###generate list of n replicates land uses(n= number of tracers/ number of unique sources)\n", " Landuse=source_data['Land-use'].unique()\n", " length=len(flat_list_tracer)/(len(source_data['Land-use'].unique()))\n", " listsource=[Landuse]*int(length)\n", " flat_list_source = [item for sublist in listsource for item in sublist]\n", " \n", " list_prop=list(np.random.dirichlet(np.ones(3),size=1))*int(length)\n", " flat_list_prop = [item for sublist in list_prop for item in sublist]\n", " AM=pd.DataFrame(flat_list_tracer)\n", " \n", " AM['Land-use'],AM['Prop']=flat_list_source,flat_list_prop\n", " AM.rename(columns={0:'tracer'}, inplace=True)\n", " AM['Mix']=q\n", " amlist.append(AM)\n", "amlist \n", "AM=pd.concat(amlist)\n", "\n", "AM['conc'] = np.nan\n", "for idx, row in AM.iterrows():\n", " AM.loc[idx,'conc'] = FA_conc_mean.loc[row['Land-use'],row['tracer']]\n", "\n", "\n", "AM['signal'] = np.nan\n", "for idx, row in AM.iterrows():\n", " AM.loc[idx,'signal'] = tracer_mean.loc[row['Land-use'],row['tracer']]\n", "\n", "AM['Mass']= AM['conc']*AM['Prop']\n", "AM['Value']= AM['signal']*AM['Mass']\n", "\n", "pvt_df = AM.pivot_table(values=['Value','Mass',],index=['Mix'],columns=['tracer'],aggfunc='sum')\n", "\n", "mixture_AM = pvt_df['Value']/pvt_df['Mass']\n", "\n", "\n", "\n", "AM=AM[['Land-use','tracer','conc','Mix','Prop','Value','signal']].rename(columns={'Land-use':'Source','tracer':'Tracer'})\n", "\n", "\n", "\n", "#mixture_AM.to_csv(\"auto_generated_sediment.cauto_generated_sedimentsv\") ##sediment mixtures \n", "#AM_.to_csv(\"auto_am (seed123).csv\") \n", "\n", "## use will need to create source and discr for MixSIAR manually \n", "mixture_AM" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Forest 0.019876\n", "Arable 0.077073\n", "Pasture 0.063379\n", "dtype: float64" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "### CRPS calculations\n", "\n", "\n", "## importing libarys etc\n", "\n", "import numpy as np\n", "import pandas as pd\n", "\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "import statsmodels.api as sm\n", "import statsmodels.formula.api \n", "from statsmodels.formula.api import ols\n", "from scipy import stats\n", "import os\n", "import properscoring as ps\n", "\n", "## use R script for preformatting of MixSIAR output\n", "\n", "##CRPS RESUTLS\n", "\n", "## change working directory to correct tracer comb\n", "os.chdir(r\"xxxxxxxxxxxxxx\")\n", "df=pd.read_csv(\"results_d15N+C28.csv\")\n", "\n", "\n", "\n", "df.loc[df['Source'].isin(['Non-irrigated arable land']),'Source'] = 'Arable'\n", "AM= pd.read_csv(r\"\\Users\\terry\\Documents\\full_tracer_combos\\auto_am_with seed(123)_d15N.csv\")\n", "## AM is AM generated when making the mixture above\n", "\n", "##makes list of distrubtion values of each source , each row is differnet mix\n", "\n", "fd=[]\n", "Forestdf=df[df['Source']== 'Forest']\n", "\n", "for i in df['Mix'].unique():\n", " fd.append(pd.DataFrame(Forestdf[(Forestdf['Mix']==i)]['value']).stack().values)\n", "\n", " \n", "arable=[]\n", "arabledf=df[df['Source']== 'Arable']\n", "\n", "for i in df['Mix'].unique():\n", " arable.append(pd.DataFrame(arabledf[(arabledf['Mix']==i)]['value']).stack().values)\n", "\n", " \n", "pasture=[]\n", "pasturedf=df[df['Source']== 'Pastures']\n", "\n", "\n", "for i in df['Mix'].unique():\n", " pasture.append(pd.DataFrame(pasturedf[(pasturedf['Mix']==i)]['value']).stack().values)\n", "\n", "\n", "#getting proporrtions\n", "AM3=AM.loc[(AM['Tracer']=='C24')&(AM['Source']=='Forest')]\n", "AM4=AM.loc[(AM['Tracer']=='C24')&(AM['Source']=='Non-irrigated arable land')]\n", "AM5=AM.loc[(AM['Tracer']=='C24')&(AM['Source']=='Pastures')]\n", "\n", "\n", "\n", "##creating of forest prop and list of dis values\n", "f=pd.Series(AM3['Prop']).reset_index(drop=True)\n", "listfdf=pd.DataFrame(f)\n", "listfdf['disvalues']=fd\n", "\n", "##creating of arable prop and list of dis values\n", "\n", "a=pd.Series(AM4['Prop']).reset_index(drop=True)\n", "arable_df=pd.DataFrame(a)\n", "arable_df['disvalues']=arable\n", "\n", "\n", "##creating of pasture prop and list of dis values\n", "\n", "p=pd.Series(AM5['Prop']).reset_index(drop=True)\n", "pasture_df=pd.DataFrame(p)\n", "pasture_df['disvalues']=pasture\n", "\n", "\n", "##iterating through rows and applying crps function\n", "\n", "\n", "##forest\n", "for_res=[]\n", "\n", "for i in range(0,len(listfdf)):\n", " a=ps.crps_ensemble(listfdf['Prop'][i], listfdf['disvalues'][i], weights=None, issorted=False,axis=-1)\n", " for_res.append(a)\n", "forestdf=pd.DataFrame(for_res).rename(columns={0:'Forest'})\n", "\n", "###arable\n", "arable_res=[]\n", "for i in range(0,len(listfdf)):\n", " b=ps.crps_ensemble(arable_df['Prop'][i], arable_df['disvalues'][i], weights=None, issorted=False,axis=-1)\n", " arable_res.append(b)\n", "arabledf=pd.DataFrame(arable_res).rename(columns={0:'Arable'})\n", "\n", "\n", "#pastuers\n", "pasture_res=[]\n", "for i in range(0,len(listfdf)):\n", " c=ps.crps_ensemble(pasture_df['Prop'][i], pasture_df['disvalues'][i], weights=None, issorted=False,axis=-1)\n", " pasture_res.append(c)\n", "pasturedf=pd.DataFrame(pasture_res).rename(columns={0:'Pasture'})\n", "\n", "\n", "CRPS_CD=pd.concat([forestdf, arabledf, pasturedf],axis=1)\n", "\n", "\n", "CRPS_CD['Fprop']=AM3['Prop'].reset_index(drop=True)\n", "CRPS_CD['Aprop']=AM4['Prop'].reset_index(drop=True)\n", "CRPS_CD['Pprop']=AM5['Prop'].reset_index(drop=True)\n", "\n", "\n", "#CRPS_CD.to_csv('CRPS_d15N+C28.csv')\n", "CRPS_CD.iloc[:,:3].median()" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
C24 C26 C28 C30 d15N
C241.0000000.0277350.0000930.0004430.000071
C260.0277351.0000000.0007200.0011520.000071
C280.0000930.0007201.0000000.0011520.000071
C300.0004430.0011520.0011521.0000000.000071
d15N0.0000710.0000710.0000710.0000711.000000
" ], "text/plain": [ "" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "###similar space test for non scaled\n", "\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "import statsmodels.api as sm\n", "import statsmodels.formula.api \n", "from statsmodels.formula.api import ols\n", "from scipy import stats\n", "import os\n", "import itertools\n", "import sklearn\n", "from sklearn.preprocessing import StandardScaler\n", "from sklearn.decomposition import PCA\n", "import numpy as np\n", "import pandas as pd\n", "import scipy\n", "\n", "\n", "os.chdir(r\"xxxxxxx\")\n", "source_data=pd.read_csv(\"source_values_less_is_more.csv\")\n", "source_data=source_data.iloc[:,1:]\n", "source_data.loc[source_data['Land-use'].isin(['Coniferous forest','Broad-leaved forest']),'Land-use'] = 'Forest'\n", "source_data.loc[source_data['Land-use'].isin(['Non-irrigated arable land']),'Land-use'] = 'Arable'\n", "\n", "\n", "##testing similar space for non scaled tracers\n", "\n", "\n", "df=source_data\n", "\n", "\n", "df_f=df[df['Land-use']=='Forest']\n", "df_a= df[df['Land-use']=='Arable']\n", "df_p= df[df['Land-use']=='Pastures']\n", " \n", "\n", " ##forest\n", "df= df_f.iloc[:,1:]\n", "pd.options.display.float_format = '{:,.4f}'.format\n", "\n", "dct = {x: {y: stats.kruskal(df[x], df[y]).pvalue for y in df} for x in df}\n", "mat1 = pd.DataFrame(dct).style.apply(lambda x: [\"background: red\" if v > 0.05 else \"\" for v in x], axis = 1)\n", "\n", "#arable\n", "df= df_a.iloc[:,1:]\n", "pd.options.display.float_format = '{:,.4f}'.format\n", "\n", "dct = {x: {y: stats.kruskal(df[x], df[y]).pvalue for y in df} for x in df}\n", "mat2 = pd.DataFrame(dct).style.apply(lambda x: [\"background: red\" if v > 0.05 else \"\" for v in x], axis = 1)\n", "\n", "#pasture\n", "df= df_p.iloc[:,1:]\n", "pd.options.display.float_format = '{:,.4f}'.format\n", "\n", "dct = {x: {y: stats.kruskal(df[x], df[y]).pvalue for y in df} for x in df}\n", "mat3 = pd.DataFrame(dct).style.apply(lambda x: [\"background: red\" if v > 0.05 else \"\" for v in x], axis = 1)\n", "\n", "\n", "mat1\n", "#mat1.to_excel( 'nonscaled forest KW.xlsx')\n", "#mat2.to_excel( 'nonscaled arable KW.xlsx')\n", "#mat3.to_excel( 'nonscaled Pasture KW.xlsx')" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
$^{15}$N C24 C26 C28 C30
$^{15}$N1.0000000.0007200.0000710.0000710.003477
C240.0007201.0000000.1076620.1076620.200381
C260.0000710.1076621.0000000.8954850.001023
C280.0000710.1076620.8954851.0000000.001023
C300.0034770.2003810.0010230.0010231.000000
" ], "text/plain": [ "" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "###similar space test for scaled tracers\n", "\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "import statsmodels.api as sm\n", "import statsmodels.formula.api \n", "from statsmodels.formula.api import ols\n", "from scipy import stats\n", "import os\n", "import itertools\n", "import sklearn\n", "from sklearn.preprocessing import StandardScaler\n", "from sklearn.decomposition import PCA\n", "import numpy as np\n", "import pandas as pd\n", "import scipy\n", "\n", "\n", "os.chdir(r\"xxxxxxxx\")\n", "source_data=pd.read_csv(\"source_values_less_is_more.csv\")\n", "source_data=source_data.iloc[:,1:]\n", "source_data.loc[source_data['Land-use'].isin(['Coniferous forest','Broad-leaved forest']),'Land-use'] = 'Forest'\n", "source_data.loc[source_data['Land-use'].isin(['Non-irrigated arable land']),'Land-use'] = 'Arable'\n", "\n", "\n", "source_data=source_data.loc[:,['Land-use','d15N','C24','C26','C28','C30']]\n", "\n", "y=source_data.iloc[:,1:]\n", "from sklearn import preprocessing\n", "\n", "x = y.values #returns a numpy array\n", "min_max_scaler = preprocessing.MinMaxScaler()\n", "x_scaled = min_max_scaler.fit_transform(x)\n", "df = pd.DataFrame(x_scaled)\n", "\n", "df.insert(loc=0, column='Land-use', value=source_data['Land-use'])\n", "\n", "\n", "df\n", "\n", "df.columns=['Land-use','$^{15}$N','C24','C26','C28','C30']\n", "df_f=df[df['Land-use']=='Forest']\n", "df_a= df[df['Land-use']=='Arable']\n", "df_p= df[df['Land-use']=='Pastures']\n", " \n", "\n", "##forest\n", "df= df_f.iloc[:,1:]\n", "pd.options.display.float_format = '{:,.4f}'.format\n", "\n", "\n", "\n", "dct = {x: {y: stats.kruskal(df[x], df[y]).pvalue for y in df} for x in df}\n", "mat1 = pd.DataFrame(dct).style.apply(lambda x: [\"background: red\" if v > 0.01 else \"\" for v in x], axis = 1)\n", "\n", "#arable\n", "df= df_a.iloc[:,1:]\n", "pd.options.display.float_format = '{:,.4f}'.format\n", "\n", "dct = {x: {y: stats.kruskal(df[x], df[y]).pvalue for y in df} for x in df}\n", "mat2 = pd.DataFrame(dct).style.apply(lambda x: [\"background: red\" if v > 0.01 else \"\" for v in x], axis = 1)\n", "\n", "#pasture\n", "df= df_p.iloc[:,1:]\n", "pd.options.display.float_format = '{:,.4f}'.format\n", "\n", "dct = {x: {y: stats.kruskal(df[x], df[y]).pvalue for y in df} for x in df}\n", "mat3 = pd.DataFrame(dct).style.apply(lambda x: [\"background: red\" if v > 0.01 else \"\" for v in x], axis = 1)\n", "\n", "#mat1.to_excel( 'scaled forest KW.xlsx')\n", "#mat2.to_excel( 'scaled arable KW.xlsx')\n", "#mat3.to_excel( 'scaled Pasture KW.xlsx')\n", "mat1" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.5" } }, "nbformat": 4, "nbformat_minor": 4 }