diff --git a/CreateTestData_NaN.m b/CreateTestData_NaN.m new file mode 100644 index 0000000..16e05dd --- /dev/null +++ b/CreateTestData_NaN.m @@ -0,0 +1,20 @@ +%% Replaces one hours worth of data with NaN +clear all +close all + +OriginalFileName = '/Users/juhildungrani/Desktop/Sem 4/Big Data/work/5011CEM2021-dungranij/2a Data Exploration'; +NewFileName = './Model/TestFileNaN.nc'; +copyfile(OriginalFileName, NewFileName); + +C = ncinfo(NewFileName); +ModelNames = {C.Variables(1:6).Name}; + + +%% Change data to NaN +BadData = NaN(700,400,1); + +%% Write to *.nc file +Hour2Replace = 12; +for idx = 1:8 + ncwrite(NewFileName, ModelNames{idx}, BadData, [1, 1, Hour2Replace]); +end diff --git a/TestNan.m b/TestNan.m new file mode 100644 index 0000000..199cb84 --- /dev/null +++ b/TestNan.m @@ -0,0 +1,72 @@ +%% Script to examine NetCDF data formats and check for NaN +% Note, you would carry out this test each time you load data. +% You should NOT test the whole file at the start + +clear all +close all + + +%% Test a good file +NaNErrors = 0; +%% Set file to test +FileName = '/Users/juhildungrani/Desktop/Sem 4/Big Data/work/5011CEM2021-dungranij/2a Data Exploration/Model 2/TestFileNaN.nc'; % define our test file + +Contents = ncinfo(FileName); % Store the file content information in a variable. + +StartLat = 1; +StartLon = 1; + +for idxHour = 1:25 + + for idxModel = 1:8 + Data(idxModel,:,:) = ncread(FileName, Contents.Variables(idxModel).Name,... + [StartLat, StartLon, idxHour], [inf, inf, 1]); + end + + % check for NaNs + if any(isnan(Data), 'All') + fprintf('NaNs present\n') + NaNErrors = 1; + end +end + +fprintf('Testing files: %s\n', FileName) +if NaNErrors + fprintf('NaN errors present!\n') +else + fprintf('No errors!\n') +end + + + + +%% Test File with Errors +NaNErrors = 0; +%% Set file to test +FileName = '../Model/TestFileNaN.nc'; % define our test file + +Contents = ncinfo(FileName); % Store the file content information in a variable. + +StartLat = 1; +StartLon = 1; + +fprintf('Testing files: %s\n', FileName) +for idxHour = 1:25 + + for idxModel = 1:8 + Data(idxModel,:,:) = ncread(FileName, Contents.Variables(idxModel).Name,... + [StartLat, StartLon, idxHour], [inf, inf, 1]); + end + + % check for NaNs + if any(isnan(Data), 'All') + fprintf('NaNs present during hour %i\n', idxHour) + NaNErrors = 1; + end +end + +if NaNErrors + fprintf('NaN errors present!\n') +else + fprintf('No errors!\n') +end \ No newline at end of file diff --git a/TestSolutionsWithLogFile.m b/TestSolutionsWithLogFile.m new file mode 100644 index 0000000..43d3a9b --- /dev/null +++ b/TestSolutionsWithLogFile.m @@ -0,0 +1,57 @@ +%% Script to examine NetCDF data formats and check for NaN +% Note, you would carry out this test each time you load data. +% You should NOT test the whole file at the start + +clear all +close all + + +%% Test File with Errors +NaNErrors = 0; +%% Set file to test +DataFileName = '../Model/TestFileNaN.nc'; % define our test file +% FileName = '../Model/o3_surface_20180701000000.nc'; % un rem this line to see what happens with good data +Contents = ncinfo(DataFileName); % Store the file content information in a variable. + +%% Create and open log file +LogFileName = 'AnalysisLog.txt'; + +% create new log file, 'w' replaces the file if present. To continually +% append, use 'a' +LogID = fopen('AnalysisLog.txt', 'w'); +fprintf(LogID, '%s: Starting analysis of %s\n', datestr(now, 0), DataFileName); + +StartLat = 1; +StartLon = 1; + +fprintf('Testing files: %s\n', DataFileName) +for idxHour = 1:25 + + for idxModel = 1:8 + Data(idxModel,:,:) = ncread(DataFileName, Contents.Variables(idxModel).Name,... + [StartLat, StartLon, idxHour], [inf, inf, 1]); % 'inf' reads all the data + end + + % check for NaNs + if any(isnan(Data), 'All') + NaNErrors = 1; + %% display warning + fprintf('NaNs present\n') + ErrorModel = find(isnan(Data), 1, 'first'); + %% find first error: + fprintf('Analysis for hour %i is invalid, NaN errors recorded in model %s\n',... + idxHour, Contents.Variables(ErrorModel).Name) + + % Write to log file + fprintf(LogID, '%s: %s processing data hour %i\n', datestr(now, 0), 'NaN Error', idxHour); + else + % write to log file + fprintf(LogID, '%s: %s processing data hour %i\n', datestr(now, 0), 'Success', idxHour); + end + +end + fclose(LogID); + +if ~NaNErrors + fprintf('No errors!\n') +end \ No newline at end of file