#Parallel Processing with Dask.py

#Parallel Processing with Dask 

import dask.dataframe as dd

#This loads dataset 
ddf = dd.read_csv("Trips_Full_Data.csv")

#This measures time for aggregation in Dask with default partitions
start_time = time.time()
result_dask_default = ddf.groupby('Trip Type')['Distance'].sum().compute()
end_time = time.time()

dask_time_default = end_time - start_time
print(f"Time taken for parallel processing with Dask (default partitions): {dask_time_default:.2f} seconds")

#This adjusts the number of partitions and measure time
num_partitions = [2, 4, 8, 16]
for n in num_partitions:
    ddf_repartitioned = ddf.repartition(npartitions=n)
    start_time = time.time()
    result_dask_repartitioned = ddf_repartitioned.groupby('Trip Type')['Distance'].sum().compute()
    end_time = time.time()

    dask_time_repartitioned = end_time - start_time
    print(f"Time taken for parallel processing with Dask ({n} partitions): {dask_time_repartitioned:.2f} seconds")