diff --git a/LoadAllData.m b/LoadAllData.m new file mode 100644 index 0000000..f4abc67 --- /dev/null +++ b/LoadAllData.m @@ -0,0 +1,17 @@ +function [AllDataMem] = LoadAllData(FileName) +%LOADALLDATA Summary of this function goes here + +Contents = ncinfo(FileName); +%% store the file content info into variable + +for idx = 1: 8 + AllData(idx,:,:,:) = ncread(FileName, Contents.Variables(idx).Name); + fprintf('Loading %s\n', Contents.Variables(idx).Name); % display loading information +end + +AllDataMem = whos('AllData').bytes/1000000; +fprintf('Memory used for all data: %.3f MB\n', AllDataMem) + + +end + diff --git a/LoadAllHours.m b/LoadAllHours.m new file mode 100644 index 0000000..9b645d0 --- /dev/null +++ b/LoadAllHours.m @@ -0,0 +1,29 @@ +function [HourMem] = LoadAllHours(FileName) +%LOADALLHOURS Summary of this function goes here +Contents = ncinfo(FileName); +HourMem = 0; % storage variable for the maximum memory in use by our data variable +StartLat = 1; % starting latitude +NumLat = 400; % number of latitude positions +StartLon = 1; % starying longitude +NumLon = 700; % number of lingitude positions +% StartHour = 1; % starting time for analyises +NumHour = 1; % Number of hours of data to load + +% loop through the hours loading one at a time +for StartHour = 1:25 + Models2Load = [1, 2, 4, 5, 6, 7, 8]; % list of models to load + idxModel = 0; % current model + for idx = 1:7 + idxModel = idxModel + 1; % move to next model index + LoadModel = Models2Load(idx);% which model to load + HourlyData(idxModel,:,:,:) = ncread(FileName, Contents.Variables(LoadModel).Name,... + [StartLon, StartLat, StartHour], [NumLon, NumLat, NumHour]); + fprintf('Loading %s\n', Contents.Variables(LoadModel).Name); % display loading information + end + + % Record the maximum memory used by the data variable so far + HourMem = max( [ HourMem, whos('HourlyData').bytes/1000000 ] ); + fprintf('Loaded Hour %i, memory used: %.3f MB\n', StartHour, HourMem); % display loading information +end +end + diff --git a/LoadHours.m b/LoadHours.m new file mode 100644 index 0000000..f7b419e --- /dev/null +++ b/LoadHours.m @@ -0,0 +1,30 @@ +function [HourDataMem] = LoadHours(FileName) +%LOADHOURS Summary of this function goes here +% Detailed explanation goes here +% We combine the aboce code to cycle through the names and load each model. +% We load the data into successive 'layers' using 'idx', and let the other +% two dimensions take care of themselves by using ':' +Contents = ncinfo(FileName); +StartLat = 1; % starting latitude +NumLat = 400; % number of latitude positions +StartLon = 1; % starying longitude +NumLon = 700; % number of lingitude positions +StartHour = 1; % starting time for analyises +NumHour = 1; % Number of hours of data to load + +% loop through the models loading *ALL* the data into an array +Models2Load = [1, 2, 4, 5, 6, 7, 8]; % list of models to load +idxModel = 0; % current model +for idx = 1:7 + idxModel = idxModel + 1; % move to next model index + LoadModel = Models2Load(idx); % which model to load + ModelData(idxModel,:,:,:) = ncread(FileName, Contents.Variables(LoadModel).Name,... + [StartLon, StartLat, StartHour], [NumLon, NumLat, NumHour]); + fprintf('Loading %s\n', Contents.Variables(LoadModel).Name); % display loading information +end + +HourDataMem = whos('ModelData').bytes/1000000; +fprintf('Memory used for 1 hour of data: %.3f MB\n', HourDataMem) + +end + diff --git a/MemorySaving.m b/MemorySaving.m new file mode 100644 index 0000000..6a94158 --- /dev/null +++ b/MemorySaving.m @@ -0,0 +1,79 @@ +%% This script allows you to open and explore the data in a *.nc file +clear all % clear all variables +close all % close all windows + +FileName = '/Users/juhildungrani/Desktop/Sem 4/Big Data/work/5011CEM2021-dungranij/2a Data Exploration/o3_surface_20180701000000.nc'; % define the name of the file to be used, the path is included + +Contents = ncinfo(FileName); % Store the file content information in a variable. + + +%% Section 2: Load all the model data together +% for idx = 1: 8 +% AllData(idx,:,:,:) = ncread(FileName, Contents.Variables(idx).Name); +% fprintf('Loading %s\n', Contents.Variables(idx).Name); % display loading information +% end +% +% AllDataMem = whos('AllData').bytes/1000000; +% fprintf('Memory used for all data: %.3f MB\n', AllDataMem) +[AllDataMem] = LoadAllData(FileName); + +%% Section 3: Loading all the data for a single hour from all the models +% We combine the aboce code to cycle through the names and load each model. +% We load the data into successive 'layers' using 'idx', and let the other +% two dimensions take care of themselves by using ':' +% StartLat = 1; % starting latitude +% NumLat = 400; % number of latitude positions +% StartLon = 1; % starying longitude +% NumLon = 700; % number of lingitude positions +% StartHour = 1; % starting time for analyises +% NumHour = 1; % Number of hours of data to load +% +% % loop through the models loading *ALL* the data into an array +% Models2Load = [1, 2, 4, 5, 6, 7, 8]; % list of models to load +% idxModel = 0; % current model +% for idx = 1:7 +% idxModel = idxModel + 1; % move to next model index +% LoadModel = Models2Load(idx); % which model to load +% ModelData(idxModel,:,:,:) = ncread(FileName, Contents.Variables(LoadModel).Name,... +% [StartLon, StartLat, StartHour], [NumLon, NumLat, NumHour]); +% fprintf('Loading %s\n', Contents.Variables(LoadModel).Name); % display loading information +% end +% +% HourDataMem = whos('ModelData').bytes/1000000; +% fprintf('Memory used for 1 hour of data: %.3f MB\n', HourDataMem) +[HourDataMem] = LoadHours(FileName); + +%% Section 4: Cycle through the hours and load all the models for each hour and record memory use +% We use an index named 'StartHour' in our loop +% HourMem = 0; % storage variable for the maximum memory in use by our data variable +% StartLat = 1; % starting latitude +% NumLat = 400; % number of latitude positions +% StartLon = 1; % starying longitude +% NumLon = 700; % number of lingitude positions +% % StartHour = 1; % starting time for analyises +% NumHour = 1; % Number of hours of data to load +% +% % loop through the hours loading one at a time +% for StartHour = 1:25 +% Models2Load = [1, 2, 4, 5, 6, 7, 8]; % list of models to load +% idxModel = 0; % current model +% for idx = 1:7 +% idxModel = idxModel + 1; % move to next model index +% LoadModel = Models2Load(idx);% which model to load +% HourlyData(idxModel,:,:,:) = ncread(FileName, Contents.Variables(LoadModel).Name,... +% [StartLon, StartLat, StartHour], [NumLon, NumLat, NumHour]); +% fprintf('Loading %s\n', Contents.Variables(LoadModel).Name); % display loading information +% end +% +% % Record the maximum memory used by the data variable so far +% HourMem = max( [ HourMem, whos('HourlyData').bytes/1000000 ] ); +% fprintf('Loaded Hour %i, memory used: %.3f MB\n', StartHour, HourMem); % display loading information +% end +[HourMem] = LoadAllHours(FileName); + +%% Section 5: Print our results +fprintf('\nResults:\n') +fprintf('Memory used for all data: %.2f MB\n', AllDataMem) +fprintf('Memory used for hourly data: %.2f MB\n', HourDataMem) +fprintf('Maximum memory used hourly = %.2f MB\n', HourMem) +fprintf('Hourly memory as fraction of all data = %.2f\n\n', HourMem / AllDataMem) \ No newline at end of file diff --git a/ReportResults.m b/ReportResults.m new file mode 100644 index 0000000..c856843 --- /dev/null +++ b/ReportResults.m @@ -0,0 +1,10 @@ +function [AllDataMem, HourDataMem, HourData] = ReportResults(FileName) +%REPORTRESULTS Summary of this function goes here +% Detailed explanation goes here +fprintf('\nResults:\n') +fprintf('Memory used for all data: %.2f MB\n', AllDataMem) +fprintf('Memory used for hourly data: %.2f MB\n', HourDataMem) +fprintf('Maximum memory used hourly = %.2f MB\n', HourMem) +fprintf('Hourly memory as fraction of all data = %.2f\n\n', HourMem / AllDataMem) +end +