test solution

CreateTestData_NaN.m

@@ -0,0 +1,20 @@
+%% Replaces one hours worth of data with NaN
+clear all
+close all
+
+OriginalFileName = '/Users/juhildungrani/Desktop/Sem 4/Big Data/work/5011CEM2021-dungranij/2a Data Exploration';
+NewFileName = './Model/TestFileNaN.nc';
+copyfile(OriginalFileName, NewFileName);
+
+C = ncinfo(NewFileName);
+ModelNames = {C.Variables(1:6).Name};
+
+
+%% Change data to NaN
+BadData = NaN(700,400,1);
+
+%% Write to *.nc file
+Hour2Replace = 12;
+for idx = 1:8
+    ncwrite(NewFileName, ModelNames{idx}, BadData, [1, 1, Hour2Replace]);
+end

TestNan.m

@@ -0,0 +1,72 @@
+%% Script to examine NetCDF data formats and check for NaN
+% Note, you would carry out this test each time you load data.
+% You should NOT test the whole file at the start
+
+clear all
+close all
+
+
+%% Test a good file
+NaNErrors = 0;
+%% Set file to test
+FileName = '/Users/juhildungrani/Desktop/Sem 4/Big Data/work/5011CEM2021-dungranij/2a Data Exploration/Model 2/TestFileNaN.nc'; % define our test file
+
+Contents = ncinfo(FileName); % Store the file content information in a variable.
+
+StartLat = 1;
+StartLon = 1;
+
+for idxHour = 1:25
+
+    for idxModel = 1:8
+        Data(idxModel,:,:) = ncread(FileName, Contents.Variables(idxModel).Name,...
+            [StartLat, StartLon, idxHour], [inf, inf, 1]);
+    end
+
+    % check for NaNs
+    if any(isnan(Data), 'All')
+        fprintf('NaNs present\n')
+        NaNErrors = 1;
+    end
+end
+
+fprintf('Testing files: %s\n', FileName)
+if NaNErrors
+    fprintf('NaN errors present!\n')
+else
+    fprintf('No errors!\n')
+end
+
+
+
+
+%% Test File with Errors
+NaNErrors = 0;
+%% Set file to test
+FileName = '../Model/TestFileNaN.nc'; % define our test file
+
+Contents = ncinfo(FileName); % Store the file content information in a variable.
+
+StartLat = 1;
+StartLon = 1;
+
+fprintf('Testing files: %s\n', FileName)
+for idxHour = 1:25
+
+    for idxModel = 1:8
+        Data(idxModel,:,:) = ncread(FileName, Contents.Variables(idxModel).Name,...
+            [StartLat, StartLon, idxHour], [inf, inf, 1]);
+    end
+
+    % check for NaNs
+    if any(isnan(Data), 'All')
+        fprintf('NaNs present during hour %i\n', idxHour)
+        NaNErrors = 1;
+    end
+end
+
+if NaNErrors
+    fprintf('NaN errors present!\n')
+else
+    fprintf('No errors!\n')
+end

TestSolutionsWithLogFile.m

@@ -0,0 +1,57 @@
+%% Script to examine NetCDF data formats and check for NaN
+% Note, you would carry out this test each time you load data.
+% You should NOT test the whole file at the start
+
+clear all
+close all
+
+
+%% Test File with Errors
+NaNErrors = 0;
+%% Set file to test
+DataFileName = '../Model/TestFileNaN.nc'; % define our test file
+% FileName = '../Model/o3_surface_20180701000000.nc'; % un rem this line to see what happens with good data
+Contents = ncinfo(DataFileName); % Store the file content information in a variable.
+
+%% Create and open log file
+LogFileName = 'AnalysisLog.txt';
+
+% create new log file, 'w' replaces the file if present. To continually
+% append, use 'a'
+LogID = fopen('AnalysisLog.txt', 'w');
+fprintf(LogID, '%s: Starting analysis of %s\n', datestr(now, 0), DataFileName);
+
+StartLat = 1;
+StartLon = 1;
+
+fprintf('Testing files: %s\n', DataFileName)
+for idxHour = 1:25
+
+    for idxModel = 1:8
+        Data(idxModel,:,:) = ncread(DataFileName, Contents.Variables(idxModel).Name,...
+            [StartLat, StartLon, idxHour], [inf, inf, 1]); % 'inf' reads all the data
+    end
+
+    % check for NaNs
+    if any(isnan(Data), 'All')
+        NaNErrors = 1;
+        %% display warning
+        fprintf('NaNs present\n')
+        ErrorModel = find(isnan(Data), 1, 'first');
+        %% find first error:
+        fprintf('Analysis for hour %i is invalid, NaN errors recorded in model %s\n',...
+            idxHour, Contents.Variables(ErrorModel).Name)
+
+        % Write to log file
+        fprintf(LogID, '%s: %s processing data hour %i\n', datestr(now, 0), 'NaN Error', idxHour);
+    else
+        % write to log file
+        fprintf(LogID, '%s: %s processing data hour %i\n', datestr(now, 0), 'Success', idxHour);
+    end
+
+end
+ fclose(LogID);
+
+if ~NaNErrors
+    fprintf('No errors!\n')
+end