ANN-NIDS-EVAL.py

import datetime
import glob
import os
import re

import numpy
import pandas
import tensorflow as tf
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder, StandardScaler

# Import all csv
csvpath = "C:/Users/thetr/Downloads/MachineLearningCVE"
csv = [file for file in glob.glob(csvpath + "**/*.csv", recursive=True)]
# Add all csv files to panda dataframe
dataset = [pandas.read_csv(f) for f in csv]
# Combine all tables into one
dataset = pandas.concat([d for d in dataset]).drop_duplicates(keep=False)
dataset.reset_index(drop=True, inplace=True)
# Remove white spaces
col_names = [col.replace(' ', '') for col in dataset.columns]
dataset.columns = col_names
# Remove unusual characters using regex
label_names = dataset['Label'].unique()
label_names = [re.sub("[^a-zA-Z ]+", "", l) for l in label_names]
label_names = [re.sub("[\s\s]", '_', l) for l in label_names]
label_names = [lab.replace("__", "_") for lab in label_names]
# Replacing labels using label_names to replace the ones in the dataset
labels = dataset['Label'].unique()
for i in range(0, len(label_names)):
    dataset['Label'] = dataset['Label'].replace({labels[i]: label_names[i]})
# Remove null values
dataset.dropna(inplace=True)
# Removing non-finite values
labl = dataset['Label']
dataset = dataset.loc[:, dataset.columns != 'Label'].astype('float64')
dataset = dataset.replace([numpy.inf, -numpy.inf], numpy.nan)
dataset = dataset.merge(labl, how='outer', left_index=True, right_index=True)
dataset.dropna(inplace=True)
# Seperating dataset into labels and features
labels = dataset['Label']
features = dataset.loc[:, dataset.columns != 'Label'].astype('float64')
# Scaling data using RobustScaler
scaler = StandardScaler()
scaler.fit(features)
features = scaler.transform(features)
# Label encoding
labelenc = LabelEncoder()
labelenc.fit(labels)
labels = labelenc.transform(labels)
# Clear variables
dataset = None
labl = None
#Load model
model = tf.keras.models.load_model('final_model.h5')
# Configure tensorboard with callback from training logs
log_dir = os.path.join("train_logs")
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
# Evaluate model accuracy
model.evaluate(features, labels, verbose=2)