data cleaning.py

import pandas as pd 
import numpy as np 
import dask.dataframe as dd

trips_by_distance = pd.read_csv('Trips_by_Distance.csv')
trips_full_data = pd.read_csv('Trips_Full Data.csv')

#removes rows with missing values
trips_full_data_cleaned = trips_full_data.dropna()
trips_by_distance = trips_by_distance.dropna()
#removes duplicate rows
trips_full_data_cleaned = trips_full_data_cleaned.drop_duplicates()
trips_by_distance_cleaned = trips_by_distance.drop_duplicates()

 
columns_with_nan_by_distance = trips_by_distance_cleaned.columns[trips_by_distance_cleaned.isnull().all()]
trips_by_distance_cleaned = trips_by_distance_cleaned.drop(columns=columns_with_nan_by_distance)

columns_with_nan = trips_full_data_cleaned.columns[trips_full_data_cleaned.isnull().all()]
trips_full_data_cleaned = trips_full_data_cleaned.drop(columns=columns_with_nan)

#convert data to datetime format
trips_full_data_cleaned['Date'] = pd.to_datetime(trips_full_data_cleaned['Date'], errors='coerce')