From b820b35a8b84dd5ba7fde973912f92082ad682c3 Mon Sep 17 00:00:00 2001 From: "Madhav Murali (madhavmurm)" Date: Mon, 22 Nov 2021 17:39:20 +0000 Subject: [PATCH] Initial commit --- 2a Data Exploration/ExploreData.m | 52 +++++ 2b Memory/MemorySaving.m | 76 +++++++ 3a Sequential/SequentialProcessing.m | 69 ++++++ 3b Parallel/ParallelProcessing.m | 125 ++++++++++ 3c Plotting/Graphs.m | 43 ++++ 6a Testing Text/CreateTestData_Text.m | 39 ++++ 6a Testing Text/TestText.m | 62 +++++ 6b Testing NaN/CreateTestData_NaN.m | 20 ++ 6b Testing NaN/TestNan.m | 72 ++++++ Common Files/DDC_ver01_1_CAMS.m | 214 ++++++++++++++++++ Common Files/EnsembleValue.m | 13 ++ .../Parallel Progress Bar/ParforProgMon.m | 157 +++++++++++++ Common Files/Parallel Progress Bar/Readme.md | 31 +++ .../java/ParforProgressMonitor$1.class | Bin 0 -> 214 bytes .../ParforProgressMonitor$ProgServer$1.class | Bin 0 -> 1507 bytes .../ParforProgressMonitor$ProgServer.class | Bin 0 -> 3542 bytes .../ParforProgressMonitor$ProgThing.class | Bin 0 -> 244 bytes .../ParforProgressMonitor$ProgWorker.class | Bin 0 -> 913 bytes .../java/ParforProgressMonitor.class | Bin 0 -> 921 bytes .../java/ParforProgressMonitor.java | 198 ++++++++++++++++ .../Parallel Progress Bar/license.txt | 29 +++ .../Parallel Progress Bar/progress_bar.png | Bin 0 -> 15976 bytes Common Files/PrepareData.m | 34 +++ 23 files changed, 1234 insertions(+) create mode 100644 2a Data Exploration/ExploreData.m create mode 100644 2b Memory/MemorySaving.m create mode 100644 3a Sequential/SequentialProcessing.m create mode 100644 3b Parallel/ParallelProcessing.m create mode 100644 3c Plotting/Graphs.m create mode 100644 6a Testing Text/CreateTestData_Text.m create mode 100644 6a Testing Text/TestText.m create mode 100644 6b Testing NaN/CreateTestData_NaN.m create mode 100644 6b Testing NaN/TestNan.m create mode 100644 Common Files/DDC_ver01_1_CAMS.m create mode 100644 Common Files/EnsembleValue.m create mode 100644 Common Files/Parallel Progress Bar/ParforProgMon.m create mode 100644 Common Files/Parallel Progress Bar/Readme.md create mode 100644 Common Files/Parallel Progress Bar/java/ParforProgressMonitor$1.class create mode 100644 Common Files/Parallel Progress Bar/java/ParforProgressMonitor$ProgServer$1.class create mode 100644 Common Files/Parallel Progress Bar/java/ParforProgressMonitor$ProgServer.class create mode 100644 Common Files/Parallel Progress Bar/java/ParforProgressMonitor$ProgThing.class create mode 100644 Common Files/Parallel Progress Bar/java/ParforProgressMonitor$ProgWorker.class create mode 100644 Common Files/Parallel Progress Bar/java/ParforProgressMonitor.class create mode 100644 Common Files/Parallel Progress Bar/java/ParforProgressMonitor.java create mode 100644 Common Files/Parallel Progress Bar/license.txt create mode 100644 Common Files/Parallel Progress Bar/progress_bar.png create mode 100644 Common Files/PrepareData.m diff --git a/2a Data Exploration/ExploreData.m b/2a Data Exploration/ExploreData.m new file mode 100644 index 0000000..2d8fe80 --- /dev/null +++ b/2a Data Exploration/ExploreData.m @@ -0,0 +1,52 @@ +%% This script allows you to open and explore the data in a *.nc file +clear all +close all + +FileName = '..\Model\o3_surface_20180701000000.nc'; + +Contents = ncinfo(FileName); + +%% List dimensions names +% if you select dimension 1, then tyou are selecting along the 'longitude' +% for our project we will select along the 'ntim' or 'time' dimension +fprintf('Data Dimension Names: %s, %s, %s\n',... + Contents.Dimensions(1).Name,... + Contents.Dimensions(2).Name,... + Contents.Dimensions(3).Name) + +%% List variable names +% note that variable 3 is an ensemble, we will NOT use this in our project! +% Not each model is 700 x 400 x 12 and we know that lat, lon and time match +% these numbers. +% To visulaise this 3D array think of it as a 700 x 400 grid for each model +% for one hour. These are stacked up 25 high. + +NumVariables = size(Contents.Variables,2); +fprintf('Variable names and sizes:\n') +for idx = 1: NumVariables + fprintf('%i %s %i, %i, %i',... + idx, Contents.Variables(idx).Name, Contents.Variables(idx).Size); + fprintf('\n'); +end + + +%% Selecting data +% We want to load models only, i.e. variables 1, 2, 4, 5, 6, 7, 8 +% and we only want a single hour. We use indexing into our *.nc file: +% To load the variable 'chimere_ozone, starting at lat = 1, lon = 1 and +% hour = 1 we use: +StartLat = 1; +NumLat = 400; +StartLon = 1; +NumLon = 700; +StartHour = 1; +NumHour = 1; + +Data = ncread(FileName, 'chimere_ozone', [StartLon, StartLat, StartHour], [NumLon, NumLat, NumHour]); + +%% Cycling through the variable names +% We only want the models to load + +for idx = [1, 2, 4, 5, 6, 7, 8] + fprintf('Model %i : %s\n', idx, Contents.Variables(idx).Name); +end diff --git a/2b Memory/MemorySaving.m b/2b Memory/MemorySaving.m new file mode 100644 index 0000000..b9c962b --- /dev/null +++ b/2b Memory/MemorySaving.m @@ -0,0 +1,76 @@ +%% This script allows you to open and explore the data in a *.nc file +clear all % clear all variables +close all % close all windows + +FileName = '..\Model\o3_surface_20180701000000.nc'; % define the name of the file to be used, the path is included + +Contents = ncinfo(FileName); % Store the file content information in a variable. + + +%% Section 2: Load all the model data together +for idx = 1: 8 + AllData(idx,:,:,:) = ncread(FileName, Contents.Variables(idx).Name); + fprintf('Loading %s\n', Contents.Variables(idx).Name); % display loading information +end + +AllDataMem = whos('AllData').bytes/1000000; +fprintf('Memory used for all data: %.3f MB\n', AllDataMem) + +%% Section 3: Loading all the data for a single hour from all the models +% We combine the aboce code to cycle through the names and load each model. +% We load the data into successive 'layers' using 'idx', and let the other +% two dimensions take care of themselves by using ':' +StartLat = 1; % starting latitude +NumLat = 400; % number of latitude positions +StartLon = 1; % starying longitude +NumLon = 700; % number of lingitude positions +StartHour = 1; % starting time for analyises +NumHour = 1; % Number of hours of data to load + +% loop through the models loading *ALL* the data into an array +Models2Load = [1, 2, 4, 5, 6, 7, 8]; % list of models to load +idxModel = 0; % current model +for idx = 1:7 + idxModel = idxModel + 1; % move to next model index + LoadModel = Models2Load(idx); % which model to load + ModelData(idxModel,:,:,:) = ncread(FileName, Contents.Variables(LoadModel).Name,... + [StartLon, StartLat, StartHour], [NumLon, NumLat, NumHour]); + fprintf('Loading %s\n', Contents.Variables(LoadModel).Name); % display loading information +end + +HourDataMem = whos('ModelData').bytes/1000000; +fprintf('Memory used for 1 hour of data: %.3f MB\n', HourDataMem) + +%% Section 4: Cycle through the hours and load all the models for each hour and record memory use +% We use an index named 'StartHour' in our loop +HourMem = 0; % storage variable for the maximum memory in use by our data variable +StartLat = 1; % starting latitude +NumLat = 400; % number of latitude positions +StartLon = 1; % starying longitude +NumLon = 700; % number of lingitude positions +% StartHour = 1; % starting time for analyises +NumHour = 1; % Number of hours of data to load + +% loop through the hours loading one at a time +for StartHour = 1:25 + Models2Load = [1, 2, 4, 5, 6, 7, 8]; % list of models to load + idxModel = 0; % current model + for idx = 1:7 + idxModel = idxModel + 1; % move to next model index + LoadModel = Models2Load(idx);% which model to load + HourlyData(idxModel,:,:,:) = ncread(FileName, Contents.Variables(LoadModel).Name,... + [StartLon, StartLat, StartHour], [NumLon, NumLat, NumHour]); + fprintf('Loading %s\n', Contents.Variables(LoadModel).Name); % display loading information + end + + % Record the maximum memory used by the data variable so far + HourMem = max( [ HourMem, whos('HourlyData').bytes/1000000 ] ); + fprintf('Loaded Hour %i, memory used: %.3f MB\n', StartHour, HourMem); % display loading information +end + +%% Section 5: Print our results +fprintf('\nResults:\n') +fprintf('Memory used for all data: %.2f MB\n', AllDataMem) +fprintf('Memory used for hourly data: %.2f MB\n', HourDataMem) +fprintf('Maximum memory used hourly = %.2f MB\n', HourMem) +fprintf('Hourly memory as fraction of all data = %.2f\n\n', HourMem / AllDataMem) \ No newline at end of file diff --git a/3a Sequential/SequentialProcessing.m b/3a Sequential/SequentialProcessing.m new file mode 100644 index 0000000..a936c92 --- /dev/null +++ b/3a Sequential/SequentialProcessing.m @@ -0,0 +1,69 @@ +%% This script allows you to open and explore the data in a *.nc file +clear all +close all + +FileName = '..\Model\o3_surface_20180701000000.nc'; + +Contents = ncinfo(FileName); + +Lat = ncread(FileName, 'lat'); % load the latitude locations +Lon = ncread(FileName, 'lon'); % loadthe longitude locations + +%% Processing parameters provided by customer +RadLat = 30.2016; % cluster radius value for latitude +RadLon = 24.8032; % cluster radius value for longitude +RadO3 = 4.2653986e-08; % cluster radius value for the ozone data + +%% Cycle through the hours and load all the models for each hour and record memory use +% We use an index named 'NumHour' in our loop +% The section 'sequential processing' will process the data location one +% after the other, reporting on the time involved. + +StartLat = 1; % latitude location to start laoding +NumLat = 400; % number of latitude locations ot load +StartLon = 1; % longitude location to start loading +NumLon = 700; % number of longitude locations ot load +tic +for NumHour = 1:25 % loop through each hour + fprintf('Processing hour %i\n', NumHour) + DataLayer = 1; % which 'layer' of the array to load the model data into + for idx = [1, 2, 4, 5, 6, 7, 8] % model data to load + % load the model data + HourlyData(DataLayer,:,:) = ncread(FileName, Contents.Variables(idx).Name,... + [StartLon, StartLat, NumHour], [NumLon, NumLat, 1]); + DataLayer = DataLayer + 1; % step to the next 'layer' + end + + % We need to prepare our data for processing. This method is defined by + % our customer. You are not required to understand this method, but you + % can ask your module leader for more information if you wish. + [Data2Process, LatLon] = PrepareData(HourlyData, Lat, Lon); + + %% Sequential analysis + t1 = toc; + t2 = t1; + for idx = 1: size(Data2Process,1) % step through each data location to process the data + + % The analysis of the data creates an 'ensemble value' for each + % location. This method is defined by + % our customer. You are not required to understand this method, but you + % can ask your module leader for more information if you wish. + [EnsembleVector(idx, NumHour)] = EnsembleValue(Data2Process(idx,:,:,:), LatLon, RadLat, RadLon, RadO3); + + % To monitor the progress we will print out the status after every + % 50 processes. + if idx/50 == ceil( idx/50) + tt = toc-t2; + fprintf('Total %i of %i, last 50 in %.2f s predicted time for all data %.1f s\n',... + idx, size(Data2Process,1), tt, size(Data2Process,1)/50*25*tt) + t2 = toc; + end + end + T2(NumHour) = toc - t1; % record the total processing time for this hour + fprintf('Processing hour %i - %.2f s\n\n', NumHour, sum(T2)); + + +end +tSeq = toc; + +fprintf('Total time for sequential processing = %.2f s\n\n', tSeq) \ No newline at end of file diff --git a/3b Parallel/ParallelProcessing.m b/3b Parallel/ParallelProcessing.m new file mode 100644 index 0000000..65ddae6 --- /dev/null +++ b/3b Parallel/ParallelProcessing.m @@ -0,0 +1,125 @@ +function ParallelProcessing +%% 1: Load Data +clear all +close all + +FileName = '..\Model\o3_surface_20180701000000.nc'; + +Contents = ncinfo(FileName); + +Lat = ncread(FileName, 'lat'); +Lon = ncread(FileName, 'lon'); +NumHours = 25; + +%% 2: Processing parameters +% ## provided by customer ## +RadLat = 30.2016; +RadLon = 24.8032; +RadO3 = 4.2653986e-08; + +StartLat = 1; +NumLat = 400; +StartLon = 1; +NumLon = 700; + +%% 3: Pre-allocate output array memory +% the '-4' value is due to the analysis method resulting in fewer output +% values than the input array. +NumLocations = (NumLon - 4) * (NumLat - 4); +EnsembleVectorPar = zeros(NumLocations, NumHours); % pre-allocate memory + +%% 4: Cycle through the hours and load all the models for each hour and record memory use +% We use an index named 'NumHour' in our loop +% The section 'parallel processing' will process the data location one +% after the other, reporting on the time involved. +tic +for idxTime = 1:NumHours + + %% 5: Load the data for each hour + % Each hour we read the data from the required models, defined by the + % index variable. Each model data are placed on a 'layer' of the 3D + % array resulting in a 7 x 700 x 400 array. + % We do this by indexing through the model names, then defining the + % start position as the beginnning of the Lat, beginning of the Lon and + % beginning of the new hour. We then define the number of elements + % along each data dimension, so the total number of Lat, the total + % number of Lon, but only 1 hour. + % You can use these values to select a smaller sub-set of the data if + % required to speed up testing o fthe functionality. + + DataLayer = 1; + for idx = [1, 2, 4, 5, 6, 7, 8] + HourlyData(DataLayer,:,:) = ncread(FileName, Contents.Variables(idx).Name,... + [StartLon, StartLat, idxTime], [NumLon, NumLat, 1]); + DataLayer = DataLayer + 1; + end + + %% 6: Pre-process the data for parallel processing + % This takes the 3D array of data [model, lat, lon] and generates the + % data required to be processed at each location. + % ## This process is defined by the customer ## + % If you want to know the details, please ask, but this is not required + % for the module or assessment. + [Data2Process, LatLon] = PrepareData(HourlyData, Lat, Lon); + + +%% Parallel Analysis + %% 7: Create the parallel pool and attache files for use + PoolSize = 2 ; % define the number of processors to use in parallel + if isempty(gcp('nocreate')) + parpool('local',PoolSize); + end + poolobj = gcp; + % attaching a file allows it to be available at each processor without + % passing the file each time. This speeds up the process. For more + % information, ask your tutor. + addAttachedFiles(poolobj,{'EnsembleValue'}); + +% %% 8: Parallel processing is difficult to monitor progress so we define a +% % special function to create a wait bar which is updated after each +% % process completes an analysis. The update function is defined at the +% % end of this script. Each time a parallel process competes it runs the +% % function to update the waitbar. + DataQ = parallel.pool.DataQueue; % Create a variable in the parallel pool +% +% % Create a waitbar and handle top it: + hWaitBar = waitbar(0, sprintf('Time period %i, Please wait ...', idxTime)); +% % Define the function to call when new data is received in the data queue +% % 'DataQ'. See end of script for the function definition. + afterEach(DataQ, @nUpdateWaitbar); + N = size(Data2Process,1); % the total number of data to process + p = 20; % offset so the waitbar shows some colour quickly. + + %% 9: The actual parallel processing! + % Ensemble value is a function defined by the customer to calculate the + % ensemble value at each location. Understanding this function is not + % required for the module or the assessment, but it is the reason for + % this being a 'big data' project due to the processing time (not the + % pure volume of raw data alone). + T4 = toc; + parfor idx = 1: 100 % size(Data2Process,1) + [EnsembleVectorPar(idx, idxTime)] = EnsembleValue(Data2Process(idx,:,:,:), LatLon, RadLat, RadLon, RadO3); + send(DataQ, idx); + end + + close(hWaitBar); % close the wait bar + + T3(idxTime) = toc - T4; % record the parallel processing time for this hour of data + fprintf('Parallel processing time for hour %i : %.1f s\n', idxTime, T3(idxTime)) + +end % end time loop +T2 = toc; +delete(gcp); + +%% 10: Reshape ensemble values to Lat, lon, hour format +EnsembleVectorPar = reshape(EnsembleVectorPar, 696, 396, []); +fprintf('Total processing time for %i workers = %.2f s\n', PoolSize, sum(T3)); + +%% 11: ### PROCESSING COMPLETE DATA NEEDS TO BE SAVED ### + +function nUpdateWaitbar(~) % nested function + waitbar(p/N, hWaitBar, sprintf('Hour %i, %.3f complete, %i out of %i', idxTime, p/N*100, p, N)); + p = p + 1; +end + +end % end function \ No newline at end of file diff --git a/3c Plotting/Graphs.m b/3c Plotting/Graphs.m new file mode 100644 index 0000000..93c14fc --- /dev/null +++ b/3c Plotting/Graphs.m @@ -0,0 +1,43 @@ +%% Plotting graphs in Matlab +clear all +close all + + +%% Show two plots on different y-axes +%% 250 data processed +x1Vals = [2, 3, 4, 5, 6, 7]; +y1Vals = [65, 56, 47, 44, 40, 39]; +figure(1) +yyaxis left +plot(x1Vals, y1Vals, '-bd') +xlabel('Number of Processors') +ylabel('Processing time (s)') +title('Processing time vs number of processors') + + +%% 5,000 data processed +x2Vals = [2, 3, 4, 5, 6, 7, 8]; +y2Vals = [1560, 1077, 945, 838, 852, 725, 707]; +figure(1) +yyaxis right +plot(x2Vals, y2Vals, '-rx') +xlabel('Number of Processors') +ylabel('Processing time (s)') +title('Processing time vs number of processors') + +legend('250 Data', '5,000 Data') + + +%% Show two plots on same y-axis +%% Mean processing time +y1MeanVals = y1Vals / 250; +y2MeanVals = y2Vals / 5000; + +figure(2) +plot(x1Vals, y1MeanVals, '-bd') +hold on +plot(x2Vals, y2MeanVals, '-rx') +xlabel('Number of Processors') +ylabel('Processing time (s)') +title('Mean Processing time vs number of processors') +legend('250 Data', '5,000 Data') \ No newline at end of file diff --git a/6a Testing Text/CreateTestData_Text.m b/6a Testing Text/CreateTestData_Text.m new file mode 100644 index 0000000..475311a --- /dev/null +++ b/6a Testing Text/CreateTestData_Text.m @@ -0,0 +1,39 @@ +%% Replaces one hours worth of data with empty strings +clear all +close all + +FileIn = '.\Model\o3_surface_20180701000000.nc'; +C = ncinfo(FileIn); +VarNames = {C.Variables.Name}; + + +%% Move to new *.nc file +FileOut = 'TestyTest.nc'; +nccreate(FileOut, 'lat', 'Dimensions', {'lat', 400}, 'DataType', 'single'); +ncwrite(FileOut, 'lat', ncread(FileIn, 'lat')); +nccreate(FileOut, 'lon', 'Dimensions', {'lon', 700}, 'DataType', 'single'); +ncwrite(FileOut, 'lon', ncread(FileIn, 'lon')); +nccreate(FileOut, 'hour', 'Dimensions', {'hour', 25}, 'DataType', 'single'); +ncwrite(FileOut, 'hour', ncread(FileIn, 'hour')); + +Model2Change = 6; % Select the model that will be overwritten with errors + +for idx = 1:7 + if idx ~= Model2Change + Var = ncread(FileIn, VarNames{idx}); + nccreate('TestyTest.nc', VarNames{idx},... + 'Dimensions', { 'lon', 700, 'lat', 400, 'hour', 25},... + 'DataType', 'single'); + ncwrite('TestyTest.nc', VarNames{idx}, Var); + else + Var = ncread(FileIn, VarNames{idx}); + nccreate('TestyTest.nc', VarNames{idx},... + 'Dimensions', { 'lon', 700, 'lat', 400, 'hour', 25},... + 'DataType', 'char'); + var = char(Var); + ncwrite('TestyTest.nc', VarNames{idx}, var); + end + + +end + diff --git a/6a Testing Text/TestText.m b/6a Testing Text/TestText.m new file mode 100644 index 0000000..ae44193 --- /dev/null +++ b/6a Testing Text/TestText.m @@ -0,0 +1,62 @@ +%% Script to examine NetCDF data formats and check for non-numeric values (chars only) + +clear all +close all + +%% Define plain text variable types +DataTypes = {'NC_Byte', 'NC_Char', 'NC_Short', 'NC_Int', 'NC_Float', 'NC_Double'}; + +%% Test a good file +%% Set file to test +FileName = '../Model/o3_surface_20180701000000.nc'; % define our test file + +Contents = ncinfo(FileName); % Store the file content information in a variable. +FileID = netcdf.open(FileName,'NC_NOWRITE'); % open file read only and create handle + +for idx = 0:size(Contents.Variables,2)-1 % loop through each variable + % read data type for each variable and store + [~, datatype(idx+1), ~, ~] = netcdf.inqVar(FileID,idx); +end + +%% display data types +DataInFile = DataTypes(datatype)' + +%% find character data types +FindText = strcmp('NC_Char', DataInFile); + +%% print results +fprintf('Testing file: %s\n', FileName) +if any(FindText) + fprintf('Error, text variables present:\n') +else + fprintf('All data is numeric, continue analysis.\n') +end + +%% ##### + +%% Test File with Errors +%% Set file to test + FileName = '../Model/TestFileText.nc'; % define our test file + + Contents = ncinfo(FileName); % Store the file content information in a variable. + FileID = netcdf.open(FileName,'NC_NOWRITE'); % open file read only and create handle + + for idx = 0:size(Contents.Variables,2)-1 % loop through each variable + % read data type for each variable and store + [~, datatype(idx+1), ~, ~] = netcdf.inqVar(FileID,idx); + end + + %% display data types + DataInFile = DataTypes(datatype)' + + %% find character data types + FindText = strcmp('NC_Char', DataInFile); + + %% print results + fprintf('Testing file: %s\n', FileName) + if any(FindText) + fprintf('Error, text variables present:\n') + else + fprintf('All data is numeric, continue analysis.\n') + end + diff --git a/6b Testing NaN/CreateTestData_NaN.m b/6b Testing NaN/CreateTestData_NaN.m new file mode 100644 index 0000000..ab754f3 --- /dev/null +++ b/6b Testing NaN/CreateTestData_NaN.m @@ -0,0 +1,20 @@ +%% Replaces one hours worth of data with NaN +clear all +close all + +OriginalFileName = './Model/o3_surface_20180701000000.nc'; +NewFileName = './Model/TestFileNaN.nc'; +copyfile(OriginalFileName, NewFileName); + +C = ncinfo(NewFileName); +ModelNames = {C.Variables(1:8).Name}; + + +%% Change data to NaN +BadData = NaN(700,400,1); + +%% Write to *.nc file +Hour2Replace = 12; +for idx = 1:8 + ncwrite(NewFileName, ModelNames{idx}, BadData, [1, 1, Hour2Replace]); +end diff --git a/6b Testing NaN/TestNan.m b/6b Testing NaN/TestNan.m new file mode 100644 index 0000000..8448c34 --- /dev/null +++ b/6b Testing NaN/TestNan.m @@ -0,0 +1,72 @@ +%% Script to examine NetCDF data formats and check for NaN +% Note, you would carry out this test each time you load data. +% You should NOT test the whole file at the start + +clear all +close all + + +%% Test a good file +NaNErrors = 0; +%% Set file to test +FileName = '../Model/o3_surface_20180701000000.nc'; % define our test file + +Contents = ncinfo(FileName); % Store the file content information in a variable. + +StartLat = 1; +StartLon = 1; + +for idxHour = 1:25 + + for idxModel = 1:8 + Data(idxModel,:,:) = ncread(FileName, Contents.Variables(idxModel).Name,... + [StartLat, StartLon, idxHour], [inf, inf, 1]); + end + + % check for NaNs + if any(isnan(Data), 'All') + fprintf('NaNs present\n') + NaNErrors = 1; + end +end + +fprintf('Testing files: %s\n', FileName) +if NaNErrors + fprintf('NaN errors present!\n') +else + fprintf('No errors!\n') +end + + + + +%% Test File with Errors +NaNErrors = 0; +%% Set file to test +FileName = '../Model/TestFileNaN.nc'; % define our test file + +Contents = ncinfo(FileName); % Store the file content information in a variable. + +StartLat = 1; +StartLon = 1; + +fprintf('Testing files: %s\n', FileName) +for idxHour = 1:25 + + for idxModel = 1:8 + Data(idxModel,:,:) = ncread(FileName, Contents.Variables(idxModel).Name,... + [StartLat, StartLon, idxHour], [inf, inf, 1]); + end + + % check for NaNs + if any(isnan(Data), 'All') + fprintf('NaNs present during hour %i\n', idxHour) + NaNErrors = 1; + end +end + +if NaNErrors + fprintf('NaN errors present!\n') +else + fprintf('No errors!\n') +end \ No newline at end of file diff --git a/Common Files/DDC_ver01_1_CAMS.m b/Common Files/DDC_ver01_1_CAMS.m new file mode 100644 index 0000000..4369583 --- /dev/null +++ b/Common Files/DDC_ver01_1_CAMS.m @@ -0,0 +1,214 @@ +function [ Clusters, Results ] = DDC_ver01_1_CAMS( varargin ) +%DDC_VER01.1 Data Density Based Clustering +% Copyright R Hyde 2017 +% Released under the GNU GPLver3.0 +% You should have received a copy of the GNU General Public License +% along with this program. If not, see 0 + % size(DataIn,1) % uncomment to trace remaining data + NumClusters=NumClusters+1; + Clusters.Rad(NumClusters,:)=InitR; + %% Find Cluster Centre + Glob_Mean=mean(DataIn,1); % array of means of data dim + Glob_Scalar=sum(sum((DataIn.*DataIn),2),1)/size(DataIn,1); % array of scalar products for each data dim + % full calculations +% GDensity=1./(1+(pdist2(DataIn,Glob_Mean,'euclidean').^2)+Glob_Scalar-(sum(Glob_Mean.^2))); % calculate global densities +% [~, CentreIndex]=max(GDensity); % find index of max densest point + % slim calculations + GDensity=pdist2(DataIn,Glob_Mean,'euclidean').^2 + Glob_Scalar - sum(Glob_Mean.^2); % calculate global densities + [~, CentreIndex]=min(GDensity); % find index of max densest point + + %% Find points belonging to cluster + Include=bsxfun(@minus,DataIn,DataIn(CentreIndex,:)).^2; % sum square of distances from centre + RadSq=Clusters.Rad(NumClusters,:).^2; % square radii + Include=sum(bsxfun(@rdivide,Include,RadSq),2); % divide by radii and add terms + Include=find(Include<1); + + %% Remove outliers >3sigma + Dist=pdist2(DataIn(Include,:),DataIn(CentreIndex,:)); % distances to all potential members + Include=Include(abs(Dist - mean(Dist) <= 3*std(Dist))==1,:); % keep only indices of samples with 3 sigma + + %% Move cluster centre to local densest point + LocMean=mean(DataIn(Include,:),1); + LocScalar=sum((DataIn(Include,:).^2),2)/size(Include,1); % array of scalar products of data dims + % full calculations +% LocDens=1./(1+(pdist2(DataIn(Include,:),LocMean,'euclidean').^2)+LocScalar-(sum(LocMean.^2))); % calculate local densities +% [~,CentreIndex]=max(LocDens); + % slim calculations + LocDens=pdist2(DataIn(Include,:),LocMean,'euclidean').^2 + LocScalar - sum(LocMean.^2); % calculate local densities + [~,CentreIndex]=min(LocDens); + CentreIndex=Include(CentreIndex); + Clusters.Centre(NumClusters,:)=DataIn(CentreIndex,:); % assign cluster centre + + %% Assign data to new centre + Include=bsxfun(@minus,DataIn,Clusters.Centre(NumClusters,:)).^2; % sum square of distances from centre + RadSq=Clusters.Rad(NumClusters,:).^2; % square radii + Include=sum(bsxfun(@rdivide,Include,RadSq),2); % divide by radii and add terms + Include=find(Include<1); + + %% Remove outliers >3sigma + Dist=pdist2(Clusters.Centre(NumClusters,:),DataIn(Include,:)); % distances to all potential members + Include=Include(abs(Dist - mean(Dist) <= 3*std(Dist))==1,:); % keep only indices of samples with 3 sigma + + %% Update radii to maximum distances + for idx=1:size(DataIn,2) + value01=pdist2(DataIn(Include,idx),Clusters.Centre(NumClusters,idx),'Euclidean'); + if max(value01)>0 + Clusters.Rad(NumClusters,idx)=max(value01); + end + end + + %% Assign data to cluster based on new radii + Include=bsxfun(@minus,DataIn,Clusters.Centre(NumClusters,:)).^2; % sum square of distances from centre + RadSq=Clusters.Rad(NumClusters,:).^2; % square radii + Include=sum(bsxfun(@rdivide,Include,RadSq),2); % divide by radii and add terms + Include=find(Include<1); + + %% Remove outliers >3sigma + Dist=pdist2(Clusters.Centre(NumClusters,:),DataIn(Include,:)); % distances to all potential members + Include=Include(abs(Dist - mean(Dist) <= 3*std(Dist))==1,:); % keep only indices of samples with 3 sigma + + %% Update radii to maximum distances + + for idx=1:size(DataIn,2) + value01=pdist2(DataIn(Include,idx),Clusters.Centre(NumClusters,idx),'Euclidean'); + if max(value01)>0 + Clusters.Rad(NumClusters,idx)=max(value01); + else +% Clusters.Rad(NumClusters,idx)=DefaultRadii(idx); + end + end + + %% Plot + if Verbose==1 + hold off;scatter(DataIn(:,1),DataIn(:,2));hold on + scatter(DataIn(CentreIndex,1),DataIn(CentreIndex,2),'r') + scatter(DataIn(Include,1),DataIn(Include,2),'g'); + scatter(Clusters.Centre(NumClusters,1),Clusters.Centre(NumClusters,2),'*','r') + title(sprintf('Clustered: %i, Remaining: %i',size(Results,1)-size(DataIn,1), size(DataIn,1))) + axis([0 1 0 1]) + drawnow + for zz=1:size(Clusters.Centre,1) + rectangle('Position',[Clusters.Centre(zz,1)-Clusters.Rad(zz,1), Clusters.Centre(zz,2)-Clusters.Rad(zz,2), 2*Clusters.Rad(zz,1), 2*Clusters.Rad(zz,2)],'Curvature',[1,1]) + end + end + %% Assign data to final clusters + StartIdx=find(all(Results==0,2),1,'first'); + EndIdx=StartIdx+size(Include,1)-1; + Results(StartIdx:EndIdx,:)=[DataIn(Include,:),ones(size(Include,1),1)*NumClusters]; + DataIn(Include,:)=[]; % remove clustered data +end + +%% Merge clusters if centre is within another cluster +if Merge==1 +MergeAny=1; + while MergeAny==1 + if Verbose==1 + figure(2) + clf + for zz=1:size(Clusters.Centre,1) + rectangle('Position',[Clusters.Centre(zz,1)-Clusters.Rad(zz,1),... + Clusters.Centre(zz,2)-Clusters.Rad(zz,2), 2*Clusters.Rad(zz,1),... + 2*Clusters.Rad(zz,2)],'Curvature',[1,1]) + end + hold on + scatter(Clusters.Centre(:,1),Clusters.Centre(:,2),'*','r') + drawnow + end + + MergeAny=0; + Merges=[]; + % for each cluster & find if cluster centre is within other clusters + for idx1=1:size(Clusters.Centre,1); + InEll=bsxfun(@minus,Clusters.Centre,Clusters.Centre(idx1,1:end)).^2; + InEll=sum(bsxfun(@rdivide,InEll,Clusters.Rad(idx1,:).^2),2); % divide by rad^2 & add + InEll=(InEll<1); + Merges(idx1,:)=InEll.'; + end + Merges(logical(eye(size(Merges))))=0; + % Merge clusters + for idx=1:size(Clusters.Centre,1) + [~,idx1]=find(Merges(idx,:),1); + Results(ismember(Results(:,end),idx1),end)=idx; + if idx1 + MergeAny=1; + end + end + %% renumber clusters + [C,~,ic]=unique(Results(:,end)); + C=1:size(C,1); + Results(:,end)=C(ic); + %% Re-create cluster data + Clusters.Centre=[]; + Clusters.Rad=[]; + for idx1=1:max(Results(:,end)) + Clusters.Centre(idx1,:)=mean(Results(Results(:,3)==idx1,1:end-1),1); + for idx2=1:size(Results,2)-1 + value01=pdist2(Results(Results(:,3)==idx1,idx2),Clusters.Centre(idx1,idx2),'Euclidean'); + if max(value01)>0 + Clusters.Rad(idx1,idx2)=max(value01); + else + Clusters.Rad(idx1,idx2)=0; + end + end + end + + end +end + +end % end function \ No newline at end of file diff --git a/Common Files/EnsembleValue.m b/Common Files/EnsembleValue.m new file mode 100644 index 0000000..9fde6df --- /dev/null +++ b/Common Files/EnsembleValue.m @@ -0,0 +1,13 @@ +function EV = EnsembleValue(Data, LatLon, RadLat, RadLon, RadO3) + +%ENSEMBLEVALUE Summary of this function goes here +% Detailed explanation goes here + + +Data4Cluster = [Data(:),LatLon]; +[Clusters, Results] = DDC_ver01_1_CAMS(Data4Cluster, [RadLat, RadLon, RadO3], 0, 0); +MostCommonCluster = mode(Results(:,end)); +EV = Clusters.Centre(MostCommonCluster); + +end + diff --git a/Common Files/Parallel Progress Bar/ParforProgMon.m b/Common Files/Parallel Progress Bar/ParforProgMon.m new file mode 100644 index 0000000..4a0d0f6 --- /dev/null +++ b/Common Files/Parallel Progress Bar/ParforProgMon.m @@ -0,0 +1,157 @@ +% ParforProgMon - CLASS Progress monitor for `parfor` loops +% +% Usage +% Begin by creating a parallel pool. +% +% Then construct a ParforProgMon object: +% ppm = ParforProgMon(strWindowTitle, nNumIterations <, nProgressStepSize, nWidth, nHeight, nMinIterations>); +% +% 'strWindowTitle' is a string containing the title of the progress bar +% window. 'nNumIterations' is an integer with the total number of +% iterations in the loop. +% +% Optional arguments 'nProgressStepSize' specifies how +% many loop iterations should correspond to a single call to 'increment()'. +% 'nWidth' and 'nHeight' specify the size of the progress window. +% +% Within the parfor loop +% parfor (nIndex = 1:nNumIterations) +% if (mod(nIndex, nProgressStepSize) == 0) +% ppm.increment(); +% end +% end +% +% Modified from ParforProgMonv2. + +classdef ParforProgMon < handle + + properties ( GetAccess = private, SetAccess = private ) + Port + HostName + strAttachedFilesFolder + end + + properties (Transient, GetAccess = private, SetAccess = private) + JavaBit + end + + methods ( Static ) + function o = loadobj( X ) + % loadobj - METHOD REconstruct a ParforProgMon object + + % Once we've been loaded, we need to reconstruct ourselves correctly as a + % worker-side object. + % fprintf('Worker: Starting with {%s, %f, %s}\n', X.HostName, X.Port, X.strAttachedFilesFolder); + o = ParforProgMon( {X.HostName, X.Port, X.strAttachedFilesFolder} ); + end + end + + methods + function o = ParforProgMon(strWindowTitle, nNumIterations, nProgressStepSize, nWidth, nHeight, nMinIterations) + % ParforProgMon - CONSTRUCTOR Create a ParforProgMon object + % + % Usage: ppm = ParforProgMon(strWindowTitle, nNumIterations <, nProgressStepSize, nWidth, nHeight>) + % + % 'strWindowTitle' is a string containing the title of the + % progress bar window. 'nNumIterations' is an integer with the + % total number of iterations in the loop. 'nProgressStepSize' + % indicates that one update (call to 'increment') corresponds to + % this many iterations. 'nWidth' indicates the width of the + % progress window. 'nHeight' indicates the width of the progress + % window. + + if (~exist('nMinIterations', 'var') || isempty(nMinIterations)) + nMinIterations = 0; + end + + % - Are we a worker or a server? + if ((nargin == 1) && iscell(strWindowTitle)) + % - Worker constructor + % Get attached files + o.strAttachedFilesFolder = getAttachedFilesFolder(strWindowTitle{3}); + % fprintf('Worker: Attached files folder on worker is [%s]\n', o.strAttachedFilesFolder); + + % Add to java path + w = warning('off', 'MATLAB:Java:DuplicateClass'); + javaaddpath(o.strAttachedFilesFolder); + warning(w); + + % "Private" constructor used on the workers + o.JavaBit = ParforProgressMonitor.createWorker(strWindowTitle{1}, strWindowTitle{2}); + o.Port = []; + + elseif (nargin > 1) && (nNumIterations >= nMinIterations) + % - Server constructor + % Check arguments + if (~exist('nProgressStepSize', 'var') || isempty(nProgressStepSize)) + nProgressStepSize = 1; + end + + if (~exist('nWidth', 'var') || ~exist('nHeight', 'var') || isempty(nHeight) || isempty(nWidth)) + nWidth = 400; + nHeight = 80; + end + + % Check for an existing pool + pPool = gcp('nocreate'); + if (isempty(pPool)) + error('ParforProgMon:NeedPool', ... + '*** ParforProgMon: You must construct a pool before creating a ParforProgMon object.'); + end + + % Amend java path + strPath = fileparts(which('ParforProgMon')); + o.strAttachedFilesFolder = fullfile(strPath, 'java'); + % fprintf('Server: JAVA class folder is [%s]\n', o.strAttachedFilesFolder); + w = warning('off', 'MATLAB:Java:DuplicateClass'); + javaaddpath(o.strAttachedFilesFolder); + warning(w); + + % Distribute class to pool + if (ismember(pPool.AttachedFiles, o.strAttachedFilesFolder)) + pPool.updateAttachedFiles(); + else + pPool.addAttachedFiles(o.strAttachedFilesFolder); + end + + % Normal construction + o.JavaBit = ParforProgressMonitor.createServer( strWindowTitle, nNumIterations, nProgressStepSize, nWidth, nHeight ); + o.Port = double( o.JavaBit.getPort() ); + % Get the client host name from pctconfig + cfg = pctconfig; + o.HostName = cfg.hostname; + end + end + + function X = saveobj( o ) + % saveobj - METHOD Save a ParforProgMon object for serialisations + + % Only keep the Port, HostName and strAttachedFilesFolder + X.Port = o.Port; + X.HostName = o.HostName; + X.strAttachedFilesFolder = o.strAttachedFilesFolder; + end + + function increment( o ) + % increment - METHOD Indicate that a single loop execution has finished + + % Update the UI + if (~isempty(o.JavaBit)) + o.JavaBit.increment(); + end + end + + function delete( o ) + % delete - METHOD Delete a ParforProgMon object + + % - Make sure that any other threads that may have closed + % the UI down have a chance to do it first + pause(.01); + + % Close the UI + if (~isempty(o.JavaBit)) + o.JavaBit.done(); + end + end + end +end diff --git a/Common Files/Parallel Progress Bar/Readme.md b/Common Files/Parallel Progress Bar/Readme.md new file mode 100644 index 0000000..4c37ee0 --- /dev/null +++ b/Common Files/Parallel Progress Bar/Readme.md @@ -0,0 +1,31 @@ +# Parfor progress monitor + + +## A Java-based `Matlab` class for progress monitoring during a `parfor` loop + +### Usage +Begin by creating a parallel pool. + +Then construct a ParforProgMon object: + + ppm = ParforProgMon(strWindowTitle, nNumIterations <, nProgressStepSize, nWidth, nHeight>); + + `strWindowTitle` is a string containing the title of the progress bar + window. `nNumIterations` is an integer with the total number of + iterations in the loop. + +#### Optional arguments + `nProgressStepSize` specifies to update the progress bar every time this + number of steps passes. `nWidth` and `nHeight` specify the size of the + progress window. + +Within the `parfor` loop: + + parfor (nIndex = 1:nNumIterations) + ppm.increment(); + end + +### Credits +[Parfor Progress monitor](https://www.mathworks.com/matlabcentral/fileexchange/24594-parfor-progress-monitor) + +[Parfor Progress monitor v2](https://www.mathworks.com/matlabcentral/fileexchange/31673-parfor-progress-monitor-v2) diff --git a/Common Files/Parallel Progress Bar/java/ParforProgressMonitor$1.class b/Common Files/Parallel Progress Bar/java/ParforProgressMonitor$1.class new file mode 100644 index 0000000000000000000000000000000000000000..1dc7bbd28cfc798aa97bdebfb07c399a2e7493d2 GIT binary patch literal 214 zcmZvWK?=e^3`KvmI$8^MBZ4azt}WsbM8QQ-@C0KWYs)YLnd;$OcmNM2ru*iTgb)9P ze1Go`fGrXQr4SS1i+^%q4wW^8#aZ*+^O=3gCWPF(s`0$8^-B|`dskQ=Dp%&FdH9>c zgfQrN9<5dKbESy&c9~m~i*a+4n+!@s*SR^7QZ0*H4A*8CRP&W#3KNrCR@Bfn z+c5n{E+Ygss*(0CLwn$C^GN41gKxq}aE8iRP3Lpj^eQ)()YTMeyk=G>8W2VmFhBwwBJOo#CP?_!5RLJWlN*w9=zgNBvk2PgRu|8Gty=4?Ra7`di3?;VP{< zNcIx140i2<1)KNb2_C@vnN-yFL`#SRZ;4)qKkxJfKF8^G@GICi_*{05m?xP-Ne!gD zW_LsA97=X&yT-J;A^$-6$^jHts>tDd$D14~tE*_0!-X8G^8ms<>8^b_T-<{!gdA!_ zfF38nMhX#lNB4b?K762o{Rj(t*uf{Xxfl429WMpk?)Y>dK-@El)KuZm5NF-@+u~jC!vteK}vz?s{ zSZgn0)wZ_wroGo|joPZvlHCd|rT*w&;osonk3QD-JF^#JmdEV#?D@|5&UZiGdC&a) zKbNlp*p0s>FbqeA(=wcqp(w+g!NE1K3!aXWflfTE!#7}{EQ`QE1@kgHE5kV%z9YjU zqWoPQj~ZBu@5$mZnLV!K2?Oi#q;S43v-3KBU|tN|K23Wt1a&em{q=Cn0$olBR`I__-xu;Pwrh!qZ4 zRNT09S8V!rxy(Jv=!Hoycg7AhtXE=b*ACL5%W#7-9ffJDSfL+nR70e2R71Ql8Q3M# zzzK@n=m~X;7Tr$BS zEAZx=TzbDE2RyH6Tdo9jx8pj&J`EcNmOCEhZ*ZE~AN2AJZB52;?W2{s8QY(-W+YhS zzH>Rd6gZw+*6}MQQ4TDh1&9uaTGX$&8dgsRR_@H0RZ==UT_$1WL~XyDA9ZuSJ;(TX z&^mstAyFyit-w~*jtwa9`m7yHcy+TZTf%85F|xep+B$xtVb=}f&rDc;!SfqOHRds0 z&+i`|kq8Y5q+IUbwR5M2+XikR+iIPWLHchf?G~kdD^XIFZT3_nl(1_W*7k_C#^O#{ z>|L?x^wMr=SlKI9?pm>xW`K>RHzvJ`pR*4+a?~5!*6|LxEhg^5mrNYMK@(raJtn@4 z5fl4m_BD*kFwQeyey?Qbw;FC)vU6Q^WdB_gzr%u#-<$XY{%GPojGK60*nh&GP5cEH zOzguw8j@mV*_9I$$4%Ugub8-q;~Kh^n5tKJeMI5|6Zhgi6CdJ&hCwBZ?QUBaS?Ny&C?%T4o&n5c?S4IgRP(u!7L#*{N>dzGLm%e!FWuOfaz%x!C1{Qg#B z_76)imvvk*@v-Fd39j5F>Cy3K8HhQ}6q?}L-s&)0pbZ4b zEsqa`FxEvNIq10o;by{e+0*L=21C%Z&IW1Ou?TPiH_ET@EkVJ1Eql%*4o8{2Lgo{!YKad^pn!SGw{i@uBZOpjZlfmZ^-|5r zg2Z!0uS~J+D?k$k+N$*$h%zaU=>vR_oWPL>ca`JLduQwnfu4_Ql7UH0ic1=l-Mc)J zdh;@%C^KVW^G>;>0!y~-3T)+v!7fDjwX+8aQUVpfeIkNTiV_$UC4?wS_)?VMr6@5? zQNn|wyz8W0v_NFzc$lkyA&w5x*HSg;sVnHr4%LuYgvPm%jrP{CY7w#2C1_QcB(W-1 z9}kyZLXxp$yoxn4r9`(-irkcnE}|#ZyNGD2ZxQj-&>|wVxHcQD;pRmoDVOZdSFw)F z^;!D9Wf2?5+(6}x#i``>NRTI(nJM)`idinQ3zLp(?g5eEyG%lr>oSoA_-QQQ2(cS03;v zxsQ>5o;!~-zbDX(CmF|i+={30bxI@{=Y)=9T@fl0o$JSTk_MyPUPo8SDsIgzAU<}H z-fl}(v0dCT4Xi?c-Q8yBG zr9~;xAj6TQS6{)7Y-&>t10P|qRVPDjzDe`%Qh^{{SKCf1AxX>FVoU0?lm97qKH-P?RetGT!)AP@ilrlRjK>n| z1~1b?^dr^j=T(wAo2jARKDik+bhKg(lN@VRw!4!p`|WzVeWa3ZoMFZPw=!cCJ;~k< z58-;z*ahlbUqlg|X=^pw&T5XgWVzKyJF7MdZA7(YqAbQNgNoo3_L6R6%M9a9+{r`Q S!L|GzN>S&eN*x1-uu+fD_kVq^<6JkGVRGXSfX3E1^cmNM2ZmleA?stDq z&ONX9;|XAazCb7t6T(7gR$Y~@39aeuK=4mSD?+q0H=C(dQSmTJr9B&)T9aGl+{Wml zHg;ahyAeZ1CM*<~f2Q;slbY?4aqJlP&gm2r~cU%!Ar@Hju< Z`5YHMz-%|r;)}_$TySaG+uU7@|Hx+& zu!$!A0DqM6+)ZOmsM%y@cIM32x%2Dy_n!cEu;ai%$-xvJq-i}g;*pKAgCrgs@x;bc z2a{N{QL(XM`3WIKs1g0R)w_ZS{LqTx7Krm^IAL7esJCpNbUN8AMFX`*Y1X! z>+b~;`|4B*WLth953epyr0%#UJ$ll9;Awd&1J<5cGtlXFb>!+ZrH`~a*D?$b6vtD#TyK~LPFr1RPrmU@|H4@0rdcwB zSWLV9Fq_5US0pZFH!(^MGRi{sICzNXhxp?Qx!UoeUZ?`YSfXJKp( zO1zgTLq05iB!X9E8CIwhw8#{r^YN`&n4r#92CypCTTE1nw@6&mC2)^&jtM}Gatmpa zGlOkL7LXqn_!t)iR(>G)p>o6g8zcruwW{A>f5oKH>1%?)a*I4!D=u3=ik|13=mq~@ zGSa{jc2UG0ZNP?gY-1_G#O1%emKiXG#Av)!?0Ch+X0bxs?6k^_%+PkFS{&R3Z*YYB a3|j;isSAV`uny!VN!vQAlo{&HRQeAfoViB; literal 0 HcmV?d00001 diff --git a/Common Files/Parallel Progress Bar/java/ParforProgressMonitor.class b/Common Files/Parallel Progress Bar/java/ParforProgressMonitor.class new file mode 100644 index 0000000000000000000000000000000000000000..210fe5bf77652ac9f8e0120b091d2dc21aea209a GIT binary patch literal 921 zcmaJ=&rcIU6#k~&-Jwf`0#<0D@~djwgAIDnP{W02(xeJWiN@o$4t25YnAwH+XX!-} zNxXUSk1~GaBJ{9y4)bQ-`@Z+Rd2jdkpC3N~JVi@l&4)nEZuceDeK^?gF@sHsEs1T3 zx&ULvZWJlAKMvwp#RC4J(WAaH=gM4N-f8nm8Hoo11sgp27)B%VhDHU6`VZZ0^iG2F zpgj(vQM;cQVme)}X74aC$J$t?QE~iQM`5B({YIuE;Oy%UN?^VhM(WMk=|GvIU@)e~ zfB9voJ}04$Vu^>>qG?G2&$2|PDW*g!dM`rOu_uskG~ZIEug}a-y$Y>pHM8Q!_IUyu znZb$Y6tRLwMKn;E3QjAr;HKROE@x&IDBbk+U~r;_iNJCuw7w&-o*kK(+tR;up>B5% zuA3|HB&(Ag-!0SI>DaEhaS|Ao)6ODKVPX@KAEI4(6i~tfa#-Z;IlUh3$|c16*6LcV z?1&5GcHN5m9r=p;l`(-O?sH^B9)&;z8JfylQ=@+wB>C0giv$R{(F7Qn#%@8M1g#Ftj?$FNB&U=Nw D&Jp1* literal 0 HcmV?d00001 diff --git a/Common Files/Parallel Progress Bar/java/ParforProgressMonitor.java b/Common Files/Parallel Progress Bar/java/ParforProgressMonitor.java new file mode 100644 index 0000000..2087af6 --- /dev/null +++ b/Common Files/Parallel Progress Bar/java/ParforProgressMonitor.java @@ -0,0 +1,198 @@ +import javax.swing.*; +import java.io.*; +import java.net.*; +import java.util.concurrent.atomic.AtomicBoolean; + +// Copyright 2009 The MathWorks, Inc. + +public class ParforProgressMonitor { + + /** + * Create a "server" progress monitor - this runs on the desktop client and + * pops up the progress monitor UI. + */ + public static ProgServer createServer( String s, int N, int progressStepSize, int width, int height ) + throws IOException { + ProgServer ret = new ProgServer( s, N, progressStepSize, width, height ); + ret.start(); + return ret; + } + + /** + * Create a "worker" progress monitor - runs on the remote lab and sends updates + */ + public static ProgWorker createWorker( String host, int port ) + throws IOException { + return new ProgWorker( host, port ); + } + + /** + * Common interface exposed by both objects + */ + public interface ProgThing { + public void increment(); + public void done(); + } + + /** + * The worker-side object. Simply connects to the server to indicate that a + * quantum of progress has been made. This is a very basic implementation - + * a more sophisticated implementation would use a persistent connection, + * and a SocketChannel on the client with a thread doing a select loop and + * accepting connections etc. + */ + private static class ProgWorker implements ProgThing { + private int fPort; + private String fHost; + private ProgWorker( String host, int port ) { + fHost = host; + fPort = port; + } + + /** + * Connect and disconnect immediately to indicate progress + */ + public void increment() { + try { + Socket s = new Socket( fHost, fPort ); + s.close(); + } catch( Exception e ) { + e.printStackTrace(); + } + } + + /** + * Nothing for us to do here + */ + public void done() { + } + } + + /** + * The client-side object which pops up a window with a + * JProgressBar. Accepts connections from the workers, and then disconnects + * them immediately. Beware, the connection backlog of the ServerSocket + * might be insufficient. + */ + private static class ProgServer implements Runnable, ProgThing { + private JFrame fFrame; + private JProgressBar fBar; + private ServerSocket fSocket; + private int fValue, fN, fStep; + private String title; + private Thread fThread; + private AtomicBoolean fKeepGoing; + + private ProgServer( String s, int N, int progressStepSize, int width, int height ) throws IOException { + // The UI + fFrame = new JFrame( s ); + fBar = new JProgressBar( 0, N ); + fFrame.getContentPane().add( fBar ); + fFrame.pack(); + fFrame.setSize(width,height); + fFrame.setLocationRelativeTo( null ); + fFrame.setVisible( true ); + + // How far we are through - requires synchronized access + fValue = 0; + fN = N; + fStep = progressStepSize; + title = s; + + // Get an anonymous port + fSocket = new ServerSocket( 0 ); + // Set SO_TIMEOUT so that we don't block forever + fSocket.setSoTimeout( 100 ); + + // Our background thread + fThread = new Thread( this ); + fThread.setDaemon( true ); + + // Used to indicate to fThread when it's time to go + fKeepGoing = new AtomicBoolean( true ); + } + + /** + * Don't start the Thread in the constructor + */ + public void start() { fThread.start(); } + + /** + * Loop over accepting connections and updating + */ + public void run() { + while( fKeepGoing.get() ) { + try { + acceptAndIncrement(); + } catch( Exception e ) { + if( fKeepGoing.get() ) { + e.printStackTrace(); + } + } + } + } + + /** + * If there's a connection - accept and then disconnect; increment our count. + */ + private void acceptAndIncrement() throws IOException { + Socket worker; + try { + worker = fSocket.accept(); + } catch( SocketTimeoutException timeout ) { + // don't care about timeouts + return; + } + worker.close(); + increment(); + } + + + /** + * On the EDT, update the progress bar + */ + private void updateBar( final int newVal ) { + SwingUtilities.invokeLater( new Runnable() { + public void run() { + fBar.setValue( fStep*newVal ); + double percentage = 100.0*fStep*newVal/fN; + fFrame.setTitle(title + (int)percentage + "% completed."); + if ( fStep*newVal >= fBar.getMaximum() ) { + done(); + } + } + } ); + } + + /** + * M-code needs to know which port we got + */ + public int getPort() { + return ((InetSocketAddress)fSocket.getLocalSocketAddress()).getPort(); + } + + /** + * Provide public access to this for pool-close PARFORs + */ + public synchronized void increment() { + fValue++; + updateBar( fValue ); + } + + /** + * Shut it all down + */ + public void done() { + fKeepGoing.set( false ); + try { + fSocket.close(); + } catch( Exception e ) { + e.printStackTrace(); + } + fFrame.dispose(); + } + } + + /** This class isn't useful - use the static methods */ + private ParforProgressMonitor() {} +} \ No newline at end of file diff --git a/Common Files/Parallel Progress Bar/license.txt b/Common Files/Parallel Progress Bar/license.txt new file mode 100644 index 0000000..68eb73a --- /dev/null +++ b/Common Files/Parallel Progress Bar/license.txt @@ -0,0 +1,29 @@ +Copyright (c) 2016, Dylan Muir +Copyright (c) 2011, Willem-Jan de Goeij +Copyright (c) 2009, The MathWorks, Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution + * Neither the name of the The MathWorks, Inc. nor the names + of its contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/Common Files/Parallel Progress Bar/progress_bar.png b/Common Files/Parallel Progress Bar/progress_bar.png new file mode 100644 index 0000000000000000000000000000000000000000..8fc476c492b4ef203aabdd0c63372b700b98f661 GIT binary patch literal 15976 zcmZX*bzGDG*FTPL6bxjF0;3TXB*#Dy#s(+?N=tW*(IGL(fyi4CN$EzUrMpH+=jbkJ z7(GJzccJ&^bKl>`&p&*O>$+ZbUMHXD96u!m>DxDHZxRs^-G2S*r7{uGl?dQ>=5-R_ z|0ivZr@#Xm3keCO*Af!UN)C3W7S<@>#aE=Eq4F!PCv8R)hK6n3oa{FpoRvR+4p27q zs;}>^?_q9ZZZb@W*43Q_-`?>#2w9tEB! z97cwQT|`9@ow7H@*BP009FN^3+W@cu%gNZ3j`K8qV3-g~RB>$P`CCU-$VDZ+;!p@fYa$X~2JG2vmh2?Ug z|Ni^CPn3(r|E^@~_&>)24hXq?1A+4JLjF59Fcg0I6sBb1g0j|pX<>u1bp+NBeab5U z|MUI-@6G?N_bc0Y7v_(j^Lk$%{QrcJ=%-y#yU_#7cPoK!} z-+0RR<9hP7=P#eXwDz@qlgeOTUuL6i;sUo$Ct(UHdBPtf!OYqE5>?o-X7*IJ&QbEZkI+E~Z~#stk5Y?h(v3d2kE4}8204b8!tR=K4xL3=h! z=kKPRS92l++8!WO#|1FuLl%qaFKv16kU2dg-mqMlQqN$>0gWa12T6R3pv_$ve(d>V zDW`h4?H0)=5#WPSH5YSKkGex;%ez6t+$x7xA!eCN`@)SD5>}1vGH+l|qoP7-T zUGw{TzS19erZ(NwmV!nNnZYYK@oGBjF*nrQIOTv>t9-Vc4J4muct;{l0m&Vva)025 z!98tN+15{tWLJr;sEE~K9o|@Hd-Ce2HX`d3f;2M7h6^RqY#;J;bP`7Pzh+`RTFkWs z^q2p9Kh-Qg)S|;G3Q5lc2|&u4VbYQr0-H|D1{^2;%R}3=ss4 zq^3Kd<@9R8O7VH@yC1x5QQuMm3!B7-LSyT0ZZ^AtH=o~JK4ahI4=S>R6TX;CkVpKu z4TXp<61MrGsfmq$%st|dkoFQ;H0&AbK}+WveoG*8`f$SS0x-W7X)N*8YIQg5_W+rDCmMqVHMSWk$nJ&1|Y2jPJApq_6uPf-gW%N zn%4xW3?AE?TP-e<$?29Kvk_MP@pKTYi8CvN(~0VHYUJJ54agXL8QvT+-@K+h*x3RV zH33+YOq3WEXTj0aNZ|b~6-YZ&@r^LoV`KX@5Kh;qEh);NAuzn;_O8=cCW1Bg%l=#c zh(i0%dwY9r+4;2`osryDE1c>%nwZ`YBAr_dI zKxwo+Pm6FX31ARWZwaJhn{N%aTM;s`_Ee*DFBG0MxS3(l$x#Ty4d}ptZMUiB`08-s z(m|v*$-_S0qP|HUS~{Yy!>SO!vLRdmu;6tPk_7L^vXO0b&Hi+xT~+wJ2_fCL1%@l> zd1yRg{eGlxqnU5Q$Z_Ymk3(j`&V;0EMRQ3)|Itm z1whOOaXK1tpQQ=C1Il*Jd~CmqnaSDyh{4&Hk1F_<#QwY47=4~X)tm#BL(6^*$)GHX z9Au~9vTA%YVLd!ph&&)oHcC)G8M`Eqzly1rE%Gsuoag4nQnqN;_zLLRjey!?dsNo# z-h#ndxaZi#kL9%htrHkXScK&CJ$9OuebWRtCliOJ=9G~m!TR_KlnpIoTmkKjg4q4G|2Dg{cpJNiT~2gg zXdq<@XG`)Zx?44*uG8Klvc+YG!RuVJOID+q($DYEXAuN07}iSUevh1LkBgBbi}+(r zkmpuNgXLZ6Br}b%ueoZEsS6`SGnYyHS$F2RBFzS*{f;u-DIgD%yv|)?;I0nydu9RU zJ*`Y$y0$g@!!1TdJuv*!T5oTmLCq5Lud{E3SHAg}#MzYX^phF_YgW#p_?Ep-P(8|b z oMQzLC<6Lt^7q(GznwAZ?l2=6D~EpIJ0YqWNjB+RgGxt-9NFw``nAe~#b7a!rj zn{M%W5)u~jLE|d=BFWE)Y7I_bFzS?eV{MXZ7H@~O=Jt5}>YHiz;*Mw8^5|6W&%NJM zxzXt`wcEH5Ph6&5v#G%DBdn2ez^jSwJKfpHr`C6QE;#KnF9n+O4hhwa+qVd?GL!2# zZ~$AS<)Y5m;}EQi`2ouUPE?ng;jVN-f8*=`wSL;{g5quL_j>Q$Ww8q_!7-CCPa|YY zz+HO2?FUPr9|k?~Xzt(GKd#W>2|iy?t~^Ng)aeO`aX4}od!$lT$~nHMUP6sHcNAXB zvGZTADDR3t&(U>W9dnqPPWp-$I})^rr{heF@Pf}hsua$4pFJ__P?@=i^;kLqb-2#J zpOF!|L^jdIssbHL{@y>$+M8>r)P=V^Qs01NkAPMM>p*in`6$ z%7w9JW5!y&1E)(!+f|fe$Kzs!Cm!-NR?k)8E!Rg%aOu2d=R8I%xnOzZid1!@`b_Wnt8_PUI(=vN-UVxG61wzsduE0`$?b57_(1T~ zJI!I_u!wk(fvpXlG@fGrdA)Mu_U`IfU3jJRDP#;LqP8@-ldvv6_Coa82|Nb+ScLoB z?USx+mzKk@;$ip}W@UeOa-PPl)suIAeQLM9&O?VPw;lq8w237+#k5_B%8OPw65>8h zb*Pn%TU&G+U>!zAPs%CA*jloa+$(po6F7_Pbc1@%H(xyJ95)E2dcg%;n71p^IFjy^gD=_CdLIknYKmL5JX00*i)$eOMcnbVsSd zqyd$vQ?z>xbW$e&a-LJHP>Hnj+X6`CYD8Z3A^WK&fz!4;LaBd ztTAVeGlHS-JX(J_EJ)8h9N50K|K(`6w`k;KZ|(`8~NKXjoYB6o_RgaPV~M7$Vll4!ZbmJs2p@J<1usGK{h!%NpS2)_GC z!X%sRLMyKZqd-+7`7T&N9}~Ro&YZ-=qB*cX78?t<-3fZQxS+V6R3p>96uE6~b4K-tWv!CVH%8Wten?Pp7Y3 z+cW-ccK=RirE>D;)g?C!9#?A=YGXW<30>Q0NGlR@-)HG`G<;DlFN!%YHwgwMJu>h-Q9S~bUu3@SsfCtP3*~Mt16#X~n2@N5QdvlFlAAZy zzVXOmVuUA>$&|r#_)6vo|F`L;M6u4m)}+T`GxLoO2hG(TE6ZC^jxlycoW}^L7U>}nfA~`zoHGOdt1yhKq_~sjvA|M7;h46hm-9pDY zV_@y?xT3?78{J7W>y_ST&qhk}=p9>{i>6Ye7R@p+Z$!`fIVzXBYD_NpP1`r9r%27be*bFZL#jLSBt_KXt}35@ z{ZwrBe0RH-{toBEudOik!}a?l*r7LTN~=~1`xyfQ@PueKf8O7&Lk7R<<$6SVRX{ZB zZWuytD0Iv&)a!JMba6)#-}YtEe_^NE>&Zp5z)&z#xkbwqA@DY4<#c`_K}XPw60_b~ z?q_{rFTD1x{HAB!^!m4yg_VnQ(9sbR*U@F8|JCa(tL>aEiN$@R#OUGSL$k!k7w9h6 zvrV78a+6JlT>{tcg2X}Oh+TNikLarMcf4RgR_e5nliEV@)UDYLY z%;MS!!*!+v^R$tFTOH#}UCIs~8ZgZlv_0#iAHz3|6>Y__J(~r^A3YvBE{sB{sh?(@ z5L_5$K3~KUKv4GELgNK!e!H|yU1J5EEUvAld zk9X;fqsu=KJgI-pcX2`g@ZzX936-CvlUGaa^7xCu#@AfTAwzmw2c^nlO$HN(v#wJx zyx7EG%&mF@C8ch;J2=vA8iyiKHRV@aD15bGiV8-F(fziFIra`Kb`1`7cTe)wR1YvF z87+Pi(;)G$;G;sgpU~iofFU~ZKvKPYYqwv?AHGXA`tR@#_6hA@Y7&ch!NNhe48KCOyc)377tEx-R3xsHlEe7C@RLWIxc@%=S4~%O+ex~>@XWIwPG;Q z)k5=5atAk&iyiC5AJ=n(*|riNPHa`8=F5kdAPXsrXuD~(C6D>nS_Ly@fxqPq_-nVy zPuI&QgexF1P1rNMfC2Y=Tz>qi-}&HHfo5c0{1dyAg_#bf{3BA~qZK;ZbHwz?Yhml9 z>V;CfixB#qh4Kae4Em9-2_rAz#kQCt!je{)o>jc{an%UQYqkD>qKtQa3KOQpXGGDkTFkW7lXz!+U=YX`aChB(xPsA zBPQeIT2sCm?d%tC6W6iHZg}pK6rFWU2D{avJ-@xm7%ED>_y$$mlq;5TbOBAd)wpY# z>xUd!%Pm!w%8%pD+ganAAk3*1ohhgcMqMkiqqM=wP~})o`>bCxFN>1HZ9*z8Hb#mO z9yOJ?rFk-%j~G)zeI196`~0~XA48pZ;>uK`(1PI2z_5`GQMFCo;dl+)H}vUOg8)$y z+IXMC>1-VT_8o7?miY6)?~N~8-|~P`f{WzHkPise*+JINTfeU}N3tQ9cmoI0O9=WZ?O} z5?!5!Viw*g zY_3MP7vb|*W-8|E7V2%B@l(jM4EGBG&O-m3B#h@zrU|k0HRa09TD`>&_-UR_ySi5k z2(ylBTQl05LIkA4$wRNN3GY-4oWT6*+a!2?9AtAoeHrDtrYLJNZxOdlu-t02tUh|C zSrM!`U>U@J_A4V1x1Loq(Q{r;4p}d=ZON}KN4B)5l6m5md7=)tCOuGoCzVdCGBhHh z*_BT_J5N)j_dUzPHy$vEsYil-kF9*oIN#FMawZw}71;`gCnXh{reF>W+vz~JH)91K zzg=?;Jqn2?F7y%BJ|Aj0dEOVDVI^l5DU>?oxA^)rGRAqMsFSCtnM$Ya^oX!ua_02= zj?XAodmHUuv9>XY6}!*vYJKiNBWCnn zZd}MQN94H_iolh07}Ax9PCuov;`i$R=#>Z*VZ$=7RM#B$?nkk zcqZy(85^IwdaV*)ZPwW*?IoSqJsqke;E5R%y;XBkE7o@FRxyP2ev{qP+$}TY^v|eb z*Sd!zVxcuZ@Om0+wf7N8Qn3az%PckaZ0<*C$+F-;NyW-DsZjExnTPoQL1dVt|`5r61$9kp1>@eO`*KFpu@E*M|rXzT< zRYQ;KBAkI*I>9M`tM+tlT9{-w%wDIB4FZ3YCw%50yx)VTrOenkx_dO|KEpgF^2VX9 z-M4c+-$PAdCBgdSEGV?mIgyLb`owkY(2uZidT+}*&v-o1p?`<}s$I<v;q03IS5Ulr8htcIs&I4o#LV5Jw6JyhaeJb4;sfcf z=*gV&J&U8^ql+h7q6Z#hDTg-2+|%zeu_rBrg$hNJ{q!0|p7Ej!(zCIx(B1ThJYk9f zCsn%WO*1#PDLsWYQMQ;+Oa;Uhdw?Ci5H-o46KoSKu%z6QJ&;u~RZQglQCN!_n=r_J zC!$Gpx~?m{^J7W1LV9bO#a)(vo|ts&$NF@huCTT#y}e(Twtx6ZzZqDQf>$gC&9 z9wfVyjyQqy!ZPfB|6#u@q@5T9X9kI#zMX%2C8*KFm*Jf`_IT7hB@nDF^Uez4plNw8 z>)X17y+>^b<^vB>n&12*9J9r+yAZvB!s)rqxM^*F_E>L%w3D8n-xMZd;#dw~JoWES zDF#iS+&qvnbL8%xX6^Jq2HHvDlvmr&S&e^QB{aD7z|-`!HY7XMO$Z7hXx5x*69w2b zZ0&2|SlvuGbq&jj^>;Q-eBRiifLX_tytG|Bgptj;ZUZii#%0mijfq8tEGFITS`0RLH9p>vEI{GK0MX^7}RIK=ia|m0oC%!Lua~J&=Q=CwU<`6cYe-gS!IDJ zPm``ekHJKW33Qwb%^4RujF?@;P|V8*c$=y~>`vf-EXHNJQXnv=193|~{D@{gUn~De zVB2;E6Z2_s!`*!OSUF#{{vM=DT2D669R}qlIQ{13m7adG`^?fLuyq4|!)4!NIH|~Q z`SpfR9z-tFCL1sDSU)SL7I8Z*EMN5)cVC@XsY@KIyo!6%#~CsBAoS;(1o^sG4vzz5 z$C}t};P%`T=g$#eQg)VaAG?b4N>6mLsXyuyzk4qw*qcv`2-BB*jW_|toS1ZWgu)ry z;kP#IR+o#G?h4tg!OFZM_IwWo^C+u_lYh#)+_PdrbAFW$s4^qYc0laDI;d#@12`}M zG82=Ob)3iDo|@_4iSZ?IPcO?20vm=$OW@!2z>@;y$4bd5b;xe9i#X*Be+5-{G>0WR zT$~+3ssldXBKZL1CtlIG?Da|0&FD+wBtY562h5;5*8d2YTh*BRs2TfP_RZ`vi4}!J zXmWjupiKz^w_GXq@l@OuZ|^MM|4II5hD;hmu>nLtbYvFV4O%ahsq@+^!=y7`wPKd< z)?i_8X`qL(dQn+(Zg)k0Hw+_q}^~ z(Cl(-=%|$ONMB=Vh-OaNu!EooWI5{EHsvQQfT;lJ)Ie`Z8a{l-FEz{Q+8f6SU z)Wn&1$$x+552|JHgD;uaJjrjH=}(@cTqenV$;CSdd-f4pn=>dN@t*|?Ft@Exkkmo2 z6XQR<&^yaU+jq?ZB0jG+nV%%jaVy`uDrSwLG)gpBmI%hF)++}9NIaN_Y(;~8#R#lE zXULq6-h_Va3)W@iX@YT|VA4aM(L9T!4qNW*EEgSZf+v7eYoV?LUkCv20{Ii;_ZYcD zUgHpON$&L9gX1inc3LWk)krr4lR5fvVmBC69&5XA5*2GdG18ltPBVXrne0X{FkH! zXJl57rSxQLhtg(xUSJu57$O%b4N=JvP$@}qGb7Ahq9pqGThO`e^~Du9nH2| zbaR2Np*mmQuazR-GA52lqR{8@Cv5EDt`$Mao3DqjU#4R&Nk6;KYphuf_++h`x+mX| z%u=17`n-K3K@l9c7F%Lc4keO&p^4B$hMDvJu8)`Lv8kT-XYU#D-FSO9v4DZ-D7&-q zVT|g0?iwHvJn81UMfO$X?Z$a!{!V&qdw&uiRWaV*J=4!deM_^~Z&#wE#wxJd0E&q;PY-H#<|+;D*P)lG*4 zO7g760|&}>N)TE2KYMBrfA;4fy5(ibcq5zN)>!HktNW`}`fs?-KKkAy&Tbo&`tr8= zij7J+1)fD66?m8K^7!>ieBVx}(42*$#=6;M$?ho}n?srYrB1+QTIt}knjd_!5C1$= zQRe5)Cm=ohSH?Jr^&!RJ5e40eUT%fDh|lxAFkUPEyddjUr7Zpjj#zo30U?_}XwqfS zB0&O>n75akEPPc@pA1?Jnw{bucgS4nPIao$Vj!L@RsnAhrY&Db{5`fQ^HA&4yb;Ao zV^={;+v2O$y>gBTAfd+nfvzA_m@4D8n3Fy%F*+jP#(yr)@Fc3N^58y+n1{9?8L1il zSwdt4Ii+pE+V=`}nWYSCyTtpEKP2IoNi*Q$+TL$^X}4;toCVp3Pil0O3C$B8Ijk)d znx;v7e4Ut6b)6v}tw$#WnH^;3PIvsTh#4~ZH1(GGy1EVmA z|4a?uc8lDw4|x4+V-mD%S!$|tAx6o=YG*&5fnL&X2g@8A;r(3{6@$AH5jsFpL_Ox2 z9pf(nF$!hv{qegtDa>e^=j7$3!MHh!R95yau=Ik@) z;a(yILG&D&(b7tm4T6G{tyTK)!cIt??k1L?FlJl5p*~ZAb!!~+ZREeyg=8QSDu_Op z*vS}fflchlkKS!1nb_^oJ8#hzeRWff&x=OJ^SQWj!CJj{8|8Hs?|)oP-PI&jHuyXu znvg#a_K6d;87~ny7k>cylWce4EjSpr>U>Vh@Tl&n+m7~D*_l6rOkquX&4pp$NxB1^ z2?=i(h8r;CEJ{4K>*weuZ+M1CFAJ^4@L`-i{UB?7P@=B zWPi*89bgu2;Rghr#rdVR1ncc+yo-oMrQ>l0iDq|cUE~FjVNhbXxFM(tQ zW~-u~&=kwDr8Y>K0r{d0-!H876Pe40ypU&qlc^uPMM<#Jt40_aVi5O+mGAU{MB3)} zFQ3Kr7*h7D9HsRCvOK_z7TSur`Eqg`BRGwgO}>w&@hvxpr(7=6`xHJr;(vH69OV zvH@wyVCML%Sxy#OC1gIM<5{hc^Y4k%&K=7&NLs5VTqz%K7hkr+?x@m7n2sBn``7{iuB02Vy!Zs{4X^Xc;9dhN10+$+O~kF5TRt^iYq!lllIr?TjnP2oeU-pWo`l}f8P=7~u9`N(T3O%-e6=|TALFbhXMkn=o|M~+pw;L1wXb#!&hy8Eosn|y(fDddt7HS9}#zMrRP0%Pi)y+ z`i%G3P+WyUC{?a8sXVTb;iCF+K>%CfwOvISNEr$%gv^6!_@#z*><~u-N#~DlQE`P> z45YV9(b<$7r~j>~05b92^&JtMbapsgf{4ZvE(sFGDo?S;bpXk09dA-mK72uk#0;11 zY!rz&WE$=qi}(2tiC(IsuqneoZ_@VKRm4r$e1T_A_TfJPqCixE^q)h=Yh*BJZb)47w{pH`BL7N1y0Nj+ zwdMI7YY^SDya)D#NQgo0F>jx*Zw8WauGjTSN)Naz7rH{5GEG!Wdd$5!)|EV1+4O>h z_+j7`cpc%>zSYEl_uUKyKi6Jaa|t&=XxsN&q}TOhl=z|?-`bTtfR>~x&If)IM7+&Y z?;H|U)s6xCn-!-5CeZE~nU29{-@v%$sEA%yCGlO#n`s9|h>~Q&KT-+MXjx>Uyqs)y zVqUv~L?$oEs$USDU^Tyj zTvuuXjP0mgL`&v;Y2It1o?E3^D(S|*TIEZG>)*aIZ>M$J=(UGNmQhP^QTiMQ6vCF% z2kkG9)$kril(?Lgj2UGAN}Ld7mLlD&m<3%a$qjNjj*k1ecZ}w8veSbldKz~n>pjYD zp&m-9>rehEscPfdCPDs~O>wLqKw1%C4e7b``CdbwwHIvUI`{(U`?P~#kK;X0Z{!^- za1AKHx4k0VU>;E35B;ueRVE*99SKMw6n{olZDTVO@qcPU+6Oi~?_`rYdb^6!;hR^Z z!)<}e5M`G%iN|V>K&}6b$(5QYBr=KWTi2ElQu7kNNCX`8eQE-clGCq6$C2K#U-)Oa zD)U}BMd^EcUR)!XIO~iSgaCtA$y{)OmvM{=%#G+1;z>^6X%Y37MvMzY9of4-zDju8 zvKp%Tzo3%< zenfJnedy6d`+DcnZ%U&nz%c-+N$9KWdjk|Wcih2S|5f-3m`ZjmmZ(J&DEbK|{8U&C zuu&^pRLUy8ZY}prXK>ckuW?B-$DF4nG_MdG+fnMFSr& z37)Fk5!HCEC7S5rsDbh~W$>)ylTrg*J~Vq^m;z>VK=NuXys0F$Cz>T@+_xFcaQwM1 z{p$4Xlr)KJalP`26WWS4;g1S-im2WuC|3Vy3M}*S{l`O{jl{2X_5~RUR3qZ zH5G!hgHVVyH{GY72qt8X3Aev5!{u_@ryTrF@xxzbC&Flb603UIR#Fsh=;8xX{)OOh zLfnz?1|%cU+%>>$ryh}U8~~hsYLG7OIbA4h&LZQHEiH%)B6X6bazy?#RotT|89i%Y0jeRHIFo_1O%H_gMKvV$;+QEA)Vj>zIcXnkT zEFBOkI_h$MFJlmlezn#$P!=W2rMOngoVWHQPDPN%?p)jV3(sfCZ#!=+lZN&`*c1bL ztVxqDlzKV!wqLT%q#Pq_J+O7h=i|hZOsv~@&2iesNp?C>vg0^)LSJKv*kv5|#1v4| z3>MIFR|--6Ov!|P#ychK{&OpbsR_7ZZzJA1i+u!zFBfeMPtkfN9-cCA}w!>hhcALC-3-y1SA()Ygc7kmu;{RUK48@+6_XJLWHk+gVMK`*fc zdgApf`X&1oaaC;IJ1_Ls)Yl~btzz?gV1Q4|UG8=^;QV@*PaygL2OW*eQ2XBFhE23K} zCxlbllti{)1ArWScXko9muX6MN=WK2RQ#P|;?dZL0@(nr_M4{?G89I1k6!odu7H>L zd%|c3V7A{7O7D%fDQ`Sp{&(S{(3-k7lZ~Qf*II!vukkQjA`WD2%Xo@t93M4oHFvIYDlHVSX@nCAR zzYGx_Ruy>=(kr;nl7-jO!&#w6!ELVt;RYPi^Js!B6Ng*T4;WQOT#6o-himi!f8MJ( z(_NxMiv7=T%n+l;KlK6j(@Y$!m3nbduRE?EaPTLrUj)w*G^BOpCI$(QJg`bpu*Ys> zBp~wHM1_FRBh;fceg(WF*cf`}*=Wj!?@dAPC5yD_Z#H=WLvbS}E->ii6UIB*-y@eB zCj)$XO2t_cqgzzKOTG35(9c>mzt%CmP}*|0F`+o>cf19x(E4TfzC>kv+CwMf5OrMR zY_adhvMiM~7vu1FHpTI?hr{Gc_W)>VuUx0h2@qdvH*HAuD_aVIOuVjMoY4FeZzlA(hsOjTz0q6pRN5*SduFv&pxGR9!1A>JPe{8j3I8n!nFKAD$9P7 zjPcR-04cq|%7Fb|1MXJkB|>1MNczd!gFJWH2<7n?k~pK8CDe+Im%x)+;0$1U#d6V> zZ{)BwQKvRQIl4`gG7piHq}$%+8nk`gVB4Me+ zs}J)qv?-em^eE`?EdBIkfOl}CF)RtnuJu?@O=6y#iK8ldxKQb-D(78^02o2}wos{s zKi=nG0l#EH(7nJbBhW*>Nv9`HZ&JZQ~;??LZk& z1l#MbxZ2%TE6p(C`-uwhEpmB7MkGq|U->3M6Kyd8uiL--q_%Ah4Z-aZ^(A>oc#TX> z)Rqx|tZ_XWx?dVx6`^?@x)zdi!(hxiE*V4~r{NT0sG#4L)>GP>jtiqLhFOyhTWOyo zW58nky@rMhP^EN57w8{=Uwf+9d#LgU;}h&ayzGSf2);ygT$@iHskW~T1Zs~oHT8W` zljy{`(1XL3w2Iw$2Kp?YPmV}(cYE8plFnBQ#0(N!DSms1rK98TFD`LL4zh*)%dmc* zj{*t{DvQ)CF3ZwDh(~7#Gm~U3C+8t^d+7bk2W%&jKQy}qhPucY^8;Q2or!#tD$VqB zAMKph%Fl1sO7BQ;5po7afvkw3!hnoeWT33Zbwwb61wWa%hZPER1ri9Q3u zpq^?AM*&-Old)#Ikj$|~x$Lm0YQZ-w>&g*<*EU4J!ZD?uTgrf@z4DE;0hKg&p0_>K zyhp%*Cr-M{f8ei^^DiIoK5E^$4-LGJ^v0@Oo(>ShOBxm#Lxf6!rz-~BAa&@$C7b0Of& z=Tj}(ag|LL6KGAH);H3(Ti&Ge21>N{3d@_!B4eVxwY)FGcSd3ta_ZZ3VMPK8Q(^3K z^SwHSg={*F6B&sw&?mCezKUC0>ZMiofh+eYt za~Wm#)7`KFj}bcp0ttYJ5RJbOqcO?vNvZI;vW4QXz0ZwGp;+@Brx z>=?9&)(>hfkx!tAS2_GUpeFDBe*d(5bg3-a zfV!1S9fpo-27~+>u<)s4L*&XNtMaKPTSkTDOQ7+Dl~^r~L37)ID}93{!IQhRN7%oB zMGU|2dZW0?EQ5EJUYB}9GddbVX3(g!r#k%-gRKLoo`;tT9 zFJd|G&wGBKH@K{4AUeanQa`>$hKb`9#{iKEly;;lFB=GS_31Go+K?|FnHrZ;FPXs5 zcED29E&9L;P;foBF7fYvGF6KY`Z>+YWi44M#^}M$w(sD4{hI?7Lq?8o6~+^}?#(Zu zx^uUE+*l)T@BgQQK1(A41+=&Ty0ZXUk)AHFUDzhZ`!M9v`JyJ4?Cv7~&VwZEOx;+F ziqp!ylGZ1v!A;)k+eS0*u#8UUvC6wN4UQ+uWzXI0sXB{SQwALYH}s90YocPSR!u-4 zm|QPO^AdSkAt%|5P1|1_8(&^+dQ#}V#=hEo-)I|IU^+kxuv|cC|8UVcym~kJgE1=B zl%r+Nwd~+c#vtCDI^9Z}LGV*zw%=1Gpf#e;ZOA2o&J6%Pev_%%e9CeGjQ}=@Vmv&d z$;oB~?52;iy((W#C~++;gEpg?V7)rQawoDg-;rf+xq5N8IFt(SM9qB@{+}kyd)Kk45W)5 z^BojbRr+p?lxx21OPTs&JCbdsmGp-x%UE|V&2cXV*oYF&t`J_Gqu3hJ*N__Yu$VFO z3o0lw%Y#8hiYZ*;+O5|Dx}b8+#9WMax#QB2HJ3G6T-ZMG>%Zbd-TfU+70eDA4Q#Aj&&^KiZ!(xosbe0h7U!lwqI zWO4xrJ81UJHcW0YqS+=Mm>{9l(p{h}I#+%&4Cp8G2rTu;{H7UU8bggg;u^LUu{(Rm zBd-hGHs$KZ50$5impV1Z`<*~D7RH|BtCKS&xS5ulwPknt#ZX}(YQ2n}BRW&YvCDto z16A;@Ipk?RBS!5ooA71tyQ}|hBNJJ`Rgxc1yE#1W56K@PCXI(%s<=#TAgmv+v~F8I zMG!M(H<5Q8e0=}_65-gTm|a&FnlAr#9SM%a`&36XBjm!-j-|+&HxI!pSBVqMJ+shM z-4VM{%{3K8;NkUV2Z*ca-^R1_JpdDm{$@$c^+oXrf#r%USIGbLI8QF1o88K}=-rsW zOI@Is4!rVGVqv;-P@!41p+*zbSf%U|A;4cIhhWf=w8JNV@WVfC!N&inRrRo+N*`HfZrvQrPu8 z9P%+r3#o5=w>@`(YKYtmxPy`wan9SNPo4oiYH4rR`EVL7TTnICfzGTCKvP`9KWl=+ z-?4BnJ*FF-on`@%YV&B@OHS_917xBMHAgUBJFeH(w`I&m6Lu}N4O+=56;v~JBHP)& zSPbFzC8&n_3olJmeQdLMgp}_6i62Z|R72aN`C;=z>^|sIr3RKY!9_mTjV~6c1orQp zIouI+mz+@&<^As2aM6B8Qz>T)GUrbVpx_hVn^Yq*KHPB=n?kadW5Ye8I~kcI;cAlFaxR6z z8*3WFwR|EJZ{TqEUBea3G*A#*jihHID-z*Q!)OsBG{be;c#%E~}b->UNu zTT3R?P999PVx>XoDhZN&lkIYe05TGils-@keePs#9$a+VA*_>p2O`|A69_w@3l46zlFbg zdVwr~>N?50(epMg<~74ElLO5_{va#**9TCigk+eNofJJ^Y{H+omX4+XlTA0G=`7!P yu2e{OXsb^DG&1c^8)yNLMC>>0M%*WydMVE&6~H&UX)b%8UrQ>y%zI(@;r|14-+&|l literal 0 HcmV?d00001 diff --git a/Common Files/PrepareData.m b/Common Files/PrepareData.m new file mode 100644 index 0000000..31adfad --- /dev/null +++ b/Common Files/PrepareData.m @@ -0,0 +1,34 @@ +function [ SegVector, LatLon ] = PrepareData(O3Data, Lat, Lon) +%UNTITLED2 Summary of this function goes here +% Detailed explanation goes here + +fprintf('Creating segments....') + +GeogSlice = 2; +DimSize = 2*GeogSlice+1; + +% tic +SegLatLon = zeros(400-GeogSlice, 700-GeogSlice,7,2*GeogSlice+1,2*GeogSlice+1); +idxSeg = 0; + + +for idxLat = GeogSlice+1:400-GeogSlice +% idxLat + for idxLon = GeogSlice+1:700-GeogSlice + SegLatLon(idxLat, idxLon, :, :, :) =... + O3Data(:, idxLon-GeogSlice:idxLon+GeogSlice, idxLat-GeogSlice:idxLat+GeogSlice); + end +end + +fprintf('Segments created\n') + +SegVector = reshape(SegLatLon,[],7,DimSize,DimSize); +LatSpace = abs(Lat(2)-Lat(1)); +LatList = [1:DimSize]*LatSpace; +LonSpace = abs(Lon(2)-Lon(1)); +LonList = [1:DimSize]*LonSpace; +[X, Y] = meshgrid(LonList,LatList); +LatLon = repmat([X(:),Y(:)], 7, 1); + +end +