#!/usr/bin/env python
# coding: utf-8

# In[ ]:


# importing all the required modules

from importlib import reload
import nltk
from bs4 import BeautifulSoup
from nltk.tokenize import sent_tokenize, word_tokenize
from sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer
import os,sys
import errno
import string
from nltk.corpus import reuters 
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
from nltk import FreqDist
from nltk.text import TextCollection
import collections
import numpy as npc
from nltk.tokenize import sent_tokenize, word_tokenize
from sklearn.feature_extraction.text import TfidfVectorizer
import pandas as pd
import numpy as np
from sklearn.neighbors import KNeighborsClassifier
from prettytable import PrettyTable
import time
from datetime import timedelta
from sklearn.metrics import recall_score,precision_score,average_precision_score,f1_score,accuracy_score,roc_auc_score,mean_absolute_error
from sklearn.preprocessing import label_binarize,LabelEncoder
from imblearn.metrics import geometric_mean_score
from sklearn import metrics
from sklearn.metrics.cluster import homogeneity_score,completeness_score
import statistics
import math
import sklearn.metrics 
from sklearn.model_selection import KFold
import csv
from sklearn.model_selection import train_test_split
from collections import Counter
import math
from scipy.spatial import distance
import statistics
from scipy.stats import pearsonr,entropy,mode,hmean
from scipy.io import arff
from sklearn.metrics import pairwise_distances
import pandas as pd
from sklearn.datasets import fetch_20newsgroups
from sklearn.ensemble import RandomForestClassifier
nltk.download('reuters')
nltk.download('stopwords')
nltk.download('punkt')
nltk.download('wordnet')
from sklearn.neighbors import NearestNeighbors
from sklearn.neighbors import NearestCentroid
import csv
from sklearn import svm
import warnings
import random
from statistics import mode
from sklearn.linear_model import LogisticRegression
from sklearn import preprocessing
from sklearn.model_selection import StratifiedKFold
from sklearn.base import BaseEstimator, ClassifierMixin
import operator
random.seed(0)
warnings.filterwarnings('ignore')
def loadNonIRDatasets(dataset):
    documents,labels=[],[]
    fname=dataset+'.csv'
    with open(fname, 'r') as file:
        reader = csv.reader(file, delimiter = ',')
        for row in reader:
            if len(row)>0:
                if '?' in row:
                    continue
                doc=row[0:len(row)-1]
                try:
                    doc=[abs(float(d)) for d in doc]
                except:
                    continue
                documents.append(doc)
                #print(row)
                labels.append(row[len(row)-1])

    return documents,labels
def add_noise(filename, noise_level=0.05):
    #The noise schema randomly chose 5% of the samples from each data set and
    #replaced their values of each attribute with values sampled uniformly at random from a range 
    #between the minimum and maximum values of the corresponding attribute.
    #fname=dataset+'.csv'
    df = pd.read_csv(filename+'.csv')
    # Get the list of numeric columns
    numeric_columns = df.select_dtypes(include=['float64']).columns
    
    # Iterate over numeric columns
    for col in numeric_columns:
        # Calculate the number of samples to add noise to
        num_samples = int(len(df) * noise_level)
        
        # Select random samples
        random_samples = np.random.choice(df.index, size=num_samples, replace=False)
        
        # Generate noise values within the range of each attribute
        min_val = df[col].min()
        max_val = df[col].max()
        noise_values = np.random.uniform(min_val, max_val, size=num_samples)
        
        # Replace values with noise in the selected samples
        df.loc[random_samples, col] = noise_values

    noisy_filename= filename +'-an-nn.csv'
    df.to_csv(noisy_filename, index=False)


def Euclidean(a, b):#distance
    return distance.euclidean(a,b)

def PrintDetails(metric="smtp",measure= "Accuracy",Arr=None,kList=list(range(1,16)),train_time=0,test_time=0,std=[]):
    x = PrettyTable()
    #kList=[1,3,5,9,15,30,45]#,70,90,120]
    if std==[]:
        x.field_names = ["k",measure,"Train Time","Test time","Total Time"]
    else:
        x.field_names = ["k",measure,"std","Train Time","Test time","Total Time"]
        
    print("metric",metric)
    print("measure",measure)
    tables.write(("metric\t"+str(metric)+"\n").encode())
    tables.write(("measure\t"+str(measure)+"\n").encode())
    average=0
    total_time=train_time+test_time
    for i in range(0,len(Arr)):
        print(i,len(Arr))
        tr_time=str(timedelta(seconds=(train_time[i])))
        ts_time=str(timedelta(seconds=(test_time[i])))
        
        tot_time=str(timedelta(seconds=(total_time[i])))
        if std==[]:
            x.add_row([str(kList[i]),round(Arr[i],4),tr_time,ts_time,tot_time]) 
        else:
            x.add_row([str(kList[i]),round(Arr[i],4),round(std[i],4),tr_time,ts_time,tot_time]) 
            
        average+=Arr[i]
    if std==[]:
        x.add_row(["Average",round((average/len(Arr)),4),str(timedelta(seconds=np.mean(train_time))),
                   str(timedelta(seconds=np.mean(test_time))),str(timedelta(seconds=np.mean(total_time)))]) 
        x.add_row(["Std Dev.",round(np.std(Arr),4),np.std(train_time),
                   np.std(test_time),np.std(total_time)])   
    else:
        x.add_row(["Average",round((average/len(Arr)),4),round(np.mean(std),4),str(timedelta(seconds=np.mean(train_time))),
                   str(timedelta(seconds=np.mean(test_time))),str(timedelta(seconds=np.mean(total_time)))]) 
        x.add_row(["Std Dev.",round(np.std(Arr),4),round(np.std(std),4),
                   round(np.std(train_time),10),
                   round(np.std(test_time),10),
                   round(np.std(total_time),10)])
        
    x.padding_width =1
    tables.write(str(x).encode())
    print(x)
        

def classify(k=3,metric="euclidean",termOccurance=None, docOccurance=None):
    accuracy=[]
    precision=[]
    Recall=[]
    fMeasure=[]
    mprecision=[]
    mRecall=[]
    mfMeasure=[]
    roc=[]
    y_pred=[]
    mae=[]
    averageMeanPrecison=[]
    kList=list(range(1,16))
    func=globals()[metric]
    train_time=[]
    test_time=[]
    class_dist=[]
    label_dist=[]
    classes=list(set(train_labels))
    p=1
    clss=list(classes)
    for c in range(0,len(classes)):
        #print(c)
        #print(train_labels[0])
        ind=[i for i in range(0,len(train_labels)) if classes[c]==train_labels[i]]
        class_dist.append(train_data[ind,:])
        #print()
        label_dist.append([train_labels[i] for i in ind])
    
    for k in kList:
        time1=time.time()
        pred=[]
        print("k",k)
        distances_class=[]
        neighs_class=[]
        for c in range(0,len(classes)):
            if len(class_dist[c])<k:
                    classifier = KNeighborsClassifier(n_neighbors=len(class_dist[c]),metric=func).fit(class_dist[c],label_dist[c])
            else:
                classifier = KNeighborsClassifier(n_neighbors=k,metric=func).fit(class_dist[c],label_dist[c])
            d,n=classifier.kneighbors(test_data,return_distance=True)
            distances_class.append(d)
            neighs_class.append(n)
        classifier1 = KNeighborsClassifier(n_neighbors=k,metric=func).fit(train_data,train_labels)
        distances,neighs=classifier1.kneighbors(test_data,return_distance=True)
        time2=time.time()
        for t in range(0,len(test_data)):
            cl_rdist=[]
            cl_mean=[]
            for c in range(0,len(classes)):
                neigh_labels=[train_labels[i] for i in neighs[t] if train_labels[i]==classes[c]]
                neigh_vals=[train_data[i] for i in neighs[t] if train_labels[i]==classes[c]]
                dist=[distances[t][j] for j,i in enumerate(neighs[t]) if train_labels[i]==classes[c]]
                total_d=0
                f1,f2=False,False
                if len(neigh_vals)>0:
                    r_mean=np.nanmean(neigh_vals,axis=0)
                    m2=func(test_data[t,:],r_mean)#+np.nanmin(dist)
                    total_d+=m2  
                    #cl_rdist.append(total_d) 
                    f1=True
                if len(neighs_class[c])>0:
                    neigh_vals=[class_dist[c][i] for i in neighs_class[c][t]]
                    r1_mean=np.nanmean(neigh_vals,axis=0)
                    m2=func(test_data[t,:],r1_mean)+np.nanmin(distances_class[c][t])
                    total_d+=m2
                    f2=True
                if f1==True :
                    cl_rdist.append(total_d)   
                else:
                    cl_rdist.append(np.inf)  
                
            cl_rdist=np.array(cl_rdist)
            class_weight = np.array(cl_rdist**-1)
            membership = class_weight/ np.nansum(class_weight)
            maxInd=np.argmax(membership)
            pred.append(clss[maxInd])  

        y_pred.append(pred)    
        time3=time.time()
        train_time.append(time2-time1)
        test_time.append(time3-time2)

    classes=list(set(labels))
    class_dict={}
    for i in range(0,len(classes)):
        class_dict[classes[i]]=i
    print(class_dict)
    num_testLabels=[]
    for i in range(0,len(test_labels)):
        num_testLabels.append(class_dict[test_labels[i]])
       
    for i in range(0,len(y_pred)):
        test_labels1=list(test_labels)

        #Accuracy
       
        accuracy.append(accuracy_score(test_labels, y_pred[i]))
        
        #Precison
        precision.append(precision_score(test_labels, y_pred[i],average='macro'))
         
        #Recall
        Recall.append(recall_score(test_labels, y_pred[i],average='macro'))
           
        #f measure
        fMeasure.append(f1_score(test_labels, y_pred[i],average='macro'))
        
        #Precison
        mprecision.append(precision_score(test_labels, y_pred[i],average='micro'))
         
        #Recall
        mRecall.append(recall_score(test_labels, y_pred[i],average='micro'))
           
        #f measure
        mfMeasure.append(f1_score(test_labels, y_pred[i],average='micro'))
      
        test = label_binarize(test_labels1, classes=categories).reshape((-1))
        pred = label_binarize( y_pred[i], classes=categories).reshape((-1))
        
        #roc
        try:
            roc.append(roc_auc_score(test,pred))
        except:
            roc.append(-1)
        #Average Mean Precision
        #y_val_true, val_pred = y_val_true.reshape((-1)), val_pred.reshape((-1))
        averageMeanPrecison.append(average_precision_score(test, pred))
        #MAE
        num_pred=[]
        for j in range(0,len(y_pred[i])):
            num_pred.append(class_dict[y_pred[i][j]])
        mae.append(mean_absolute_error(num_testLabels, num_pred))
        
        #PrintDetails(metric=metric,time=str(timedelta(seconds=(time2-time1))),measure="Average Mean Precison",Arr=averageMeanPrecison)

    #Accuracy
    PrintDetails(metric=metric,measure="Accuracy",Arr=accuracy,train_time=train_time,test_time=test_time)
    
    #Precison
    PrintDetails(metric=metric,measure="Precision",Arr=precision,train_time=train_time,test_time=test_time)
    #Precison
    PrintDetails(metric=metric,measure="micro Precision",Arr=mprecision,train_time=train_time,test_time=test_time)
    #Recall
    PrintDetails(metric=metric,measure="Recall",Arr=Recall,train_time=train_time,test_time=test_time)
     #Recall
    PrintDetails(metric=metric,measure="micro Recall",Arr=mRecall,train_time=train_time,test_time=test_time)
    #f measure
    PrintDetails(metric=metric,measure="F Measure",Arr=fMeasure,train_time=train_time,test_time=test_time)
    
    #f measure
    PrintDetails(metric=metric,measure="micro F Measure",Arr=mfMeasure,train_time=train_time,test_time=test_time)
    
    #roc
    PrintDetails(metric=metric,measure="ROC",Arr=roc,train_time=train_time,test_time=test_time)
    #Average Mean Precision
    PrintDetails(metric=metric,measure="Average Mean Precison",Arr=averageMeanPrecison,train_time=train_time,test_time=test_time)
    #MAE
    PrintDetails(metric=metric,measure="MAE",Arr=mae,train_time=train_time,test_time=test_time)
  
    
    accuracy_avg.append(accuracy)
    precision_avg.append(precision)
    recall_avg.append(Recall)
    macroFMeasure_avg.append(fMeasure)
    mprecision_avg.append(mprecision)
    mrecall_avg.append(mRecall)
    mFMeasure_avg.append(mfMeasure)
    roc_avg.append(roc)
    AMP_avg.append(averageMeanPrecison)
    mae_avg.append(mae)
    avg_test_time.append(test_time)
    avg_train_time.append(train_time)
    
import glob
datasets=[]
for file_name in glob.glob("Noisy Dataets\\"+'*.csv'):
    file=os.path.split(file_name)[1]
    datasets.append(os.path.splitext(file)[0])

    
for dataset in datasets:
    fname="Noisy Dataets\\"+dataset
    print(fname.encode())
    arr,labels=loadNonIRDatasets(fname)
    print(len(arr))
    arr=np.array(arr)
    labels=np.array(labels)
    labels=list(labels)
    le = LabelEncoder()
    labels=le.fit_transform(labels)

    classes=categories=list(set(labels))
    measures=["Euclidean"]
    global w
    w=0
    global Weights
    print("arr",arr.shape)
    for met in measures:
        n_split=1
        labels=np.array(labels)
        path = "LMGL-FkNN\\Noisy Datasets"
        if not os.path.exists(path):
            os.makedirs(path)
        fname=path+'\\table_'+dataset+'_'+met+'-LMGL-FkNN.txt'
        tables = open(fname, 'wb')
        k_splits=5
        accuracy_avg=[]
        precision_avg=[]
        recall_avg=[]
        macroFMeasure_avg=[]
        mprecision_avg=[]
        mrecall_avg=[]
        mFMeasure_avg=[]
        roc_avg=[]
        AMP_avg=[]
        mae_avg=[]
        avg_test_time=[]
        avg_train_time=[]
        split_list=[]
        X = arr.copy()
        y = labels.copy()
        for i in range(0,5):
            X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)
            split_list.append([X_train, X_test, y_train, y_test])
            X = X_train.copy()
            y = y_train.copy()
        for i in range(0,5):
            xTrain, xTest = split_list[i][0],split_list[i][1]
            yTrain, yTest = split_list[i][2],split_list[i][3]
            train_data=np.array(xTrain)
            test_data=np.array(xTest)
            train_labels=np.array(yTrain)
            test_labels=np.array(yTest)
            allData=arr
            termOcc=[]
            docOcc=[]
            tables.write("\n*************".encode())
            tables.write((" Split\t"+str(n_split)).encode())
            tables.write(" ***********\n".encode())
            n_split=n_split+1
            k=1
            print("###############")
            Weights=np.ones(train_data.shape[0])
            classify(k=k,metric=met,termOccurance=termOcc,docOccurance=docOcc)
        list1=list(range(1,16))
        tables.write("\n************* Average Results ***********\n".encode())
        acc_std=np.std(np.array(accuracy_avg),axis=0)
        accuracy_avg=np.mean(np.array(accuracy_avg),axis=0)
        
        pr_std=np.std(np.array(precision_avg),axis=0)
        precision_avg=np.mean(np.array(precision_avg),axis=0)
        
        mpr_std=np.std(np.array(mprecision_avg),axis=0)
        mprecision_avg=np.mean(np.array(mprecision_avg),axis=0)
        
        rcl_std=np.std(np.array(recall_avg),axis=0)
        recall_avg=np.mean(np.array(recall_avg),axis=0)
        
        mrcl_std=np.std(np.array(mrecall_avg),axis=0)
        mrecall_avg=np.mean(np.array(mrecall_avg),axis=0)
        
        F_std=np.std(np.array(macroFMeasure_avg),axis=0)
        macroFMeasure_avg=np.mean(np.array(macroFMeasure_avg),axis=0)
        
        mF_std=np.std(np.array(mFMeasure_avg),axis=0)
        mFMeasure_avg=np.mean(np.array(mFMeasure_avg),axis=0)
        
        roc_std=np.std(np.array(roc_avg),axis=0)
        roc_avg=np.mean(np.array(roc_avg),axis=0)
        
        amp_std=np.std(np.array(AMP_avg),axis=0)
        AMP_avg=np.mean(np.array(AMP_avg),axis=0)
        
        mae_std=np.std(np.array(mae_avg),axis=0)
        mae_avg=np.mean(np.array(mae_avg),axis=0)
        
        avg_train_time=np.mean(np.array(avg_train_time),axis=0)
        avg_test_time=np.mean(np.array(avg_test_time),axis=0)
        #Accuracy
        PrintDetails(metric=met,measure="Accuracy",Arr=accuracy_avg,kList=list1,train_time=avg_train_time,test_time=avg_test_time,std=acc_std)
        
        #Precision
        PrintDetails(metric=met,measure="Precision",Arr=precision_avg,kList=list1,train_time=avg_train_time,test_time=avg_test_time,std=pr_std)
        
        #Precision
        PrintDetails(metric=met,measure="Micro Precision",Arr=mprecision_avg,kList=list1,train_time=avg_train_time,test_time=avg_test_time,std=mpr_std)


        #Recall
        PrintDetails(metric=met,measure="Recall",Arr=recall_avg,kList=list1,train_time=avg_train_time,test_time=avg_test_time,std=rcl_std)
        
         #Recall
        PrintDetails(metric=met,measure="Micro Recall",Arr=mrecall_avg,kList=list1,train_time=avg_train_time,test_time=avg_test_time,std=mrcl_std)


        #f measure
        PrintDetails(metric=met,measure="F Measure",Arr=macroFMeasure_avg,kList=list1,train_time=avg_train_time,test_time=avg_test_time,std=F_std)
        
       
       
        #f measure
        PrintDetails(metric=met,measure="Micro F Measure",Arr=mFMeasure_avg,kList=list1,train_time=avg_train_time,test_time=avg_test_time,std=mF_std)


        #roc
        PrintDetails(metric=met,measure="ROC",Arr=roc_avg,kList=list1,train_time=avg_train_time,test_time=avg_test_time,std=roc_std)

        #average Mean precision
        PrintDetails(metric=met,measure="AMP",Arr=AMP_avg,kList=list1,train_time=avg_train_time,test_time=avg_test_time,std=amp_std)
         
        #MAE
        PrintDetails(metric=met,measure="MAE",Arr=mae_avg,kList=list1,train_time=avg_train_time,test_time=avg_test_time,std=mae_std)
        
        tables.close()
        print("###############")





# In[ ]:


test=[1,2,3,4]
print(test[0:-1])


# In[ ]:


import numpy as np
from sklearn.metrics import roc_auc_score
y_true = np.array([0, 2, 0, 0, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
y_scores = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
roc_auc_score(y_true, y_scores)


# In[ ]:





# In[ ]: