Memory results

LoadAllData.m

@@ -0,0 +1,17 @@
+function [AllDataMem] = LoadAllData(FileName)
+%LOADALLDATA Summary of this function goes here
+
+Contents = ncinfo(FileName); 
+%% store the file content info into variable
+
+for idx = 1: 8
+    AllData(idx,:,:,:) = ncread(FileName, Contents.Variables(idx).Name);
+    fprintf('Loading %s\n', Contents.Variables(idx).Name); % display loading information
+end
+
+AllDataMem = whos('AllData').bytes/1000000;
+fprintf('Memory used for all data: %.3f MB\n', AllDataMem)
+
+
+end
+

LoadAllHours.m

@@ -0,0 +1,29 @@
+function [HourMem] = LoadAllHours(FileName)
+%LOADALLHOURS Summary of this function goes here
+Contents = ncinfo(FileName);
+HourMem = 0; % storage variable for the maximum memory in use by our data variable
+StartLat = 1; % starting latitude
+NumLat = 400; % number of latitude positions
+StartLon = 1; % starying longitude
+NumLon = 700; % number of lingitude positions
+% StartHour = 1; % starting time for analyises
+NumHour = 1; % Number of hours of data to load
+
+% loop through the hours loading one at a time
+for StartHour = 1:25
+    Models2Load = [1, 2, 4, 5, 6, 7, 8]; % list of models to load
+    idxModel = 0; % current model
+    for idx = 1:7
+        idxModel = idxModel + 1; % move to next model index
+        LoadModel = Models2Load(idx);% which model to load
+        HourlyData(idxModel,:,:,:) = ncread(FileName, Contents.Variables(LoadModel).Name,...
+            [StartLon, StartLat, StartHour], [NumLon, NumLat, NumHour]);
+        fprintf('Loading %s\n', Contents.Variables(LoadModel).Name); % display loading information
+    end
+
+    % Record the maximum memory used by the data variable so far
+    HourMem = max( [ HourMem, whos('HourlyData').bytes/1000000 ] );
+    fprintf('Loaded Hour %i, memory used: %.3f MB\n', StartHour, HourMem); % display loading information
+end
+end
+

LoadHours.m

@@ -0,0 +1,30 @@
+function [HourDataMem] = LoadHours(FileName)
+%LOADHOURS Summary of this function goes here
+%   Detailed explanation goes here
+% We combine the aboce code to cycle through the names and load each model.
+% We load the data into successive 'layers' using 'idx', and let the other
+% two dimensions take care of themselves by using ':'
+Contents = ncinfo(FileName);
+StartLat = 1; % starting latitude
+NumLat = 400; % number of latitude positions
+StartLon = 1; % starying longitude
+NumLon = 700; % number of lingitude positions
+StartHour = 1; % starting time for analyises
+NumHour = 1; % Number of hours of data to load
+
+% loop through the models loading *ALL* the data into an array
+Models2Load = [1, 2, 4, 5, 6, 7, 8]; % list of models to load
+idxModel = 0; % current model
+for idx = 1:7
+    idxModel = idxModel + 1; % move to next model index
+    LoadModel = Models2Load(idx); % which model to load
+    ModelData(idxModel,:,:,:) = ncread(FileName, Contents.Variables(LoadModel).Name,...
+        [StartLon, StartLat, StartHour], [NumLon, NumLat, NumHour]);
+    fprintf('Loading %s\n', Contents.Variables(LoadModel).Name); % display loading information
+end
+
+HourDataMem = whos('ModelData').bytes/1000000;
+fprintf('Memory used for 1 hour of data: %.3f MB\n', HourDataMem)
+
+end
+

MemorySaving.m

@@ -0,0 +1,79 @@
+%% This script allows you to open and explore the data in a *.nc file
+clear all % clear all variables
+close all % close all windows
+
+FileName = '/Users/juhildungrani/Desktop/Sem 4/Big Data/work/5011CEM2021-dungranij/2a Data Exploration/o3_surface_20180701000000.nc'; % define the name of the file to be used, the path is included
+
+Contents = ncinfo(FileName); % Store the file content information in a variable.
+
+
+%% Section 2: Load all the model data together
+% for idx = 1: 8
+%     AllData(idx,:,:,:) = ncread(FileName, Contents.Variables(idx).Name);
+%     fprintf('Loading %s\n', Contents.Variables(idx).Name); % display loading information
+% end
+% 
+% AllDataMem = whos('AllData').bytes/1000000;
+% fprintf('Memory used for all data: %.3f MB\n', AllDataMem)
+[AllDataMem] = LoadAllData(FileName);
+
+%% Section 3: Loading all the data for a single hour from all the models
+% We combine the aboce code to cycle through the names and load each model.
+% We load the data into successive 'layers' using 'idx', and let the other
+% two dimensions take care of themselves by using ':'
+% StartLat = 1; % starting latitude
+% NumLat = 400; % number of latitude positions
+% StartLon = 1; % starying longitude
+% NumLon = 700; % number of lingitude positions
+% StartHour = 1; % starting time for analyises
+% NumHour = 1; % Number of hours of data to load
+% 
+% % loop through the models loading *ALL* the data into an array
+% Models2Load = [1, 2, 4, 5, 6, 7, 8]; % list of models to load
+% idxModel = 0; % current model
+% for idx = 1:7
+%     idxModel = idxModel + 1; % move to next model index
+%     LoadModel = Models2Load(idx); % which model to load
+%     ModelData(idxModel,:,:,:) = ncread(FileName, Contents.Variables(LoadModel).Name,...
+%         [StartLon, StartLat, StartHour], [NumLon, NumLat, NumHour]);
+%     fprintf('Loading %s\n', Contents.Variables(LoadModel).Name); % display loading information
+% end
+% 
+% HourDataMem = whos('ModelData').bytes/1000000;
+% fprintf('Memory used for 1 hour of data: %.3f MB\n', HourDataMem)
+[HourDataMem] = LoadHours(FileName);
+
+%% Section 4: Cycle through the hours and load all the models for each hour and record memory use
+% We use an index named 'StartHour' in our loop
+% HourMem = 0; % storage variable for the maximum memory in use by our data variable
+% StartLat = 1; % starting latitude
+% NumLat = 400; % number of latitude positions
+% StartLon = 1; % starying longitude
+% NumLon = 700; % number of lingitude positions
+% % StartHour = 1; % starting time for analyises
+% NumHour = 1; % Number of hours of data to load
+% 
+% % loop through the hours loading one at a time
+% for StartHour = 1:25
+%     Models2Load = [1, 2, 4, 5, 6, 7, 8]; % list of models to load
+%     idxModel = 0; % current model
+%     for idx = 1:7
+%         idxModel = idxModel + 1; % move to next model index
+%         LoadModel = Models2Load(idx);% which model to load
+%         HourlyData(idxModel,:,:,:) = ncread(FileName, Contents.Variables(LoadModel).Name,...
+%             [StartLon, StartLat, StartHour], [NumLon, NumLat, NumHour]);
+%         fprintf('Loading %s\n', Contents.Variables(LoadModel).Name); % display loading information
+%     end
+%     
+%     % Record the maximum memory used by the data variable so far
+%     HourMem = max( [ HourMem, whos('HourlyData').bytes/1000000 ] );
+%     fprintf('Loaded Hour %i, memory used: %.3f MB\n', StartHour, HourMem); % display loading information
+% end
+[HourMem] = LoadAllHours(FileName);
+
+%% Section 5: Print our results
+fprintf('\nResults:\n')
+fprintf('Memory used for all data: %.2f MB\n', AllDataMem)
+fprintf('Memory used for hourly data: %.2f MB\n', HourDataMem)
+fprintf('Maximum memory used hourly = %.2f MB\n', HourMem)
+fprintf('Hourly memory as fraction of all data = %.2f\n\n', HourMem / AllDataMem)

ReportResults.m

@@ -0,0 +1,10 @@
+function [AllDataMem, HourDataMem, HourData] = ReportResults(FileName)
+%REPORTRESULTS Summary of this function goes here
+%   Detailed explanation goes here
+fprintf('\nResults:\n')
+fprintf('Memory used for all data: %.2f MB\n', AllDataMem)
+fprintf('Memory used for hourly data: %.2f MB\n', HourDataMem)
+fprintf('Maximum memory used hourly = %.2f MB\n', HourMem)
+fprintf('Hourly memory as fraction of all data = %.2f\n\n', HourMem / AllDataMem)
+end
+