% function Read_SOCATv2()
%------------------------------------------------------------------%
% This is the routine to get the SOCAT v2 data into matlab.        %
% The data files are the ?SOCATv2? global and regional zip files   %
% which can be downloaded from www.socat.info                      %
% It can also read individual cruise files from Pangaea.           %
% Several files can be selected, all with the same extension.      %
% For allowed extensions, see the 'exts' variable.                 %
% It reads the file(s) until the column headers are found.         %
% Determines # of columns from # of headers.                       %
% Creates GUI to ask user which columns to import                  %
% If Lat or Lon are selected for import, it creates another GUI    %
%      to ask the user which geographical region to import         % 
% If cruise flags are present and selected for import,             %
%          it creates a GUI to select which flags to import.       % 
% Likewise with data flags.                                        %
% Data is stored in variables                                      %
%        named after (sometimes modified) column headers           %
% Anything before column headers is stored in 'StartText' variable %
% For questions, contact:                                          %
%       Denis Pierrot (Denis.Pierrot@noaa.gov)                     %
%   or  Peter Landschutzer (p.landschutzer@uea.ac.uk)              %
%------------------------------------------------------------------%

clear all
%dbstop at 135;
%--------------------------------------------------------------------------
%  TO USER:
%  Data file(s) should prefarably be in same directory as this .m file 
%  User Input: 
%              readlines = number of lines read per loop
%              dflt      = column number initially selected to be imported 
%                          when interface is created
%--------------------------------------------------------------------------

exts={'txt';'tab.tsv';'tsv'};
axts='';for i=1 : size(exts,1),axts=[axts '*.' exts{i} ';'];end
delim='\t';
readlines=100000;
%adds thousands Separators to a 1x1 array.
import java.text.*
v = DecimalFormat;
readlinesstr = char(v.format(readlines));

%----Gets mfile directory ----------------------------------------------------
if exist(mfilename,'file')==2
    projDir = strrep([mfilename('fullpath') '.m'],[filesep mfilename '.m'],'');
else
    projDir =pwd;
end
%--------------------------------------------------------------------------

%----Select files to be imported (only 1 kind at a time)-------------------
[fname,fpath] = uigetfile(axts,'Select Data File(s)',[projDir filesep],'MultiSelect','on');
if (fpath==0),return;end
if strcmp(fpath(end),filesep)>0,fpath=fpath(1:end-1);end

lname=1;

%--------------- Detects extensions in files selected  -------------------
% 1 or more files selected
if ischar(fname), lname=1; else lname=length(fname);  fname=sort(fname);end
cond=0;
if iscell(fname)
    for i=1:size(exts,1)
        for j=1:lname
            cond=cond + ~isempty(regexpi(fname{j},['.' exts{i} '$'], 'match'));
        end
        if cond>0 ,ftype=i; break;end %breaks on first extension detected
    end

else %one file only selected
    for i=1:size(exts,1)
        cond=cond + ~isempty(regexpi(fname,['.' exts{i} '$'], 'match'));
        if cond,ftype=i;break;end
    end
end
if ~cond 
    mmm=msgbox(sprintf('No files with correct extension selected!\n\nAllowed extensions are: %s\n\nProgram aborted.\n\n',exts),'Input File Error');
    return;
end
if cond<lname
    mmm=msgbox(sprintf('Files selected have different extensions!\n\nProgram aborted.\n\n'),'Input File Error');
    return;
end

% ************************************** txt or tsv files*****************************************************************
switch ftype
    case 1 %txt
        mots={'cruise_ID';'cruise_name'};
        dflt=[8 9 19 22 23];% lon lat fCO2 JDay WOCE_flag
        tcol=[1 2 24 25];%text columns
    case 2 %tab.tsv   individual cruise files on PANGAEA (http://doi.pangaea.de)
        dflt=[1 2 3 6 12];% Date_Time lon lat temp fCO2 ? columns vary depending on the file
        tcol=[1];%text columns
        mots={'Date/Time';'Flag [#]'};
    case 3%tsv
        dflt=[1 3 4 5 6 7 8 9 10 11 13 14 22 25];% Expocode QC_ID  yr mon day hh mm ss lon lat sal SST GVCO2 fCO2rec fCO2rec_flag
        tcol=[1 2];%text columns  (expocode and doi)
        mots={'Expocode';'fCO2rec_flag'};
end
ln=0;%lines read
for fnum=1:lname
    if lname>1 ff= fopen([fpath filesep fname{fnum}],'r');
    else  ff=fopen([fpath filesep fname],'r'); end
    
    hdr=0;SText='';kk=0;
%--------------------------------------------------------------------------
% searches for header line
%--------------------------------------------------------------------------
    while (~hdr & ~feof(ff)) %
        tline = fgets(ff);
        for k=1:size(mots,1)
            if ~isempty(findstr(tline, mots{k})), continue; else SText=char(SText,tline);kk=kk+1; end
            k=size(mots,1)+1;break;
        end
        if k~=size(mots,1)+1,hdr=1;end
    end
    StartText{fnum}=SText;skip{fnum}=kk;
    %--------------------------------------------------------------------------
    % Reads headers and number of columns
    %--------------------------------------------------------------------------
    hdr2=textscan(tline,'%s','delimiter',delim)
    fclose(ff);
    if fnum==1
        if hdr
            hdrs=hdr2{1,1};
            colmax=length(hdrs);
        else
            break;
        end
            
        %--------------------------------------------------------------------------
        % Creates interface
        %--------------------------------------------------------------------------
        [scrsz]=get(0,'ScreenSize');   ht=700;wd=600;
        fig = figure('position', [(scrsz(3)-wd)/2 (scrsz(4)-ht)/2 wd ht]);cols=[50 300];spc=50;ctrln=13;
        uicontrol ('Style','text','position', [(wd-200)/2 ht-20 200 20],'String','Select Variable to Import');
        
        for i=1:colmax
            b(i) = uicontrol ('Style','radiobutton','position', [cols((i>ctrln)+1) ht-50-spc*i+(ctrln*spc*(i>ctrln)) 50+length(hdrs{i})*10 50],'String',hdrs{i});
            set(b(i),'Value',ismember(i,dflt));
        end
        okb=uicontrol ('Style','togglebutton','position', [(wd-50) 20 30 20],'String','OK','Callback','uiresume(gcbf)');
        ccb=uicontrol ('Style','togglebutton','position', [(wd-100) 20 40 20],'String','Cancel','Callback','uiresume(gcbf)');
        uiwait(fig);
        set(fig,'visible','off');
        if ~get(okb,'value'), return;end
        for i=1:colmax,bv(i)=get(b(i),'Value');end
        rcol=find(bv);% columns to read
        
        
        for i=1:colmax
            if(regexpi(hdrs{i},'longitude')==1)
                ind2=i;
            elseif(regexpi(hdrs{i},'latitude')==1)
                ind3=i;
            elseif(regexpi(hdrs{i},'QC_ID')==1)
                ind4=i;
            else
                if(regexpi(hdrs{i},'fco2rec_flag')==1), ind5=i; end
                if(regexpi(hdrs{i},'Flag \[\#\]')==1), ind5=i; end                
            end
        end
        
        %--------------------------------------------------------------------------
        % open GUI to set lon-lat borders (PL 17102011)
        %--------------------------------------------------------------------------
        
        if(bv(ind2)~=0 || bv(ind3)~=0)
            prompt={'Min. Longitude','Max. Longitude', 'Min. Latitude', 'Max. Latitude'};
            if ftype==3, defans={'0','360', '-90', '90'}; else defans={'-180','180', '-90', '90'}; end
            fields = {'lonmin','lonmax', 'latmin', 'latmax'};
            info = inputdlg(prompt, 'Enter Limits', 1, defans,'on');
            if isempty(info),return;end              %see if user hit cancel
            info = cell2struct(info,fields);
            lonmin = str2num(info.lonmin);%convert string to number
            lonmax = str2num(info.lonmax);
            latmin = str2num(info.latmin);
            latmax = str2num(info.latmax);
            clear prompt defans fields info
        end
        
        
        if exist('ind4','var')
            if bv(ind4)~=0
                %--------------------------------------------------------------------------
                % Creates Cruise Flag Dialog
                %--------------------------------------------------------------------------
                ht=200;wd=400;hdr2s={'A(11)','B(12)','C(13)','D(14)'};cols=[25 100 200 300];ctrln=4;htt=100;wdt=400;
                cfdlg = figure('Name','Cruise Flags Selection','NumberTitle','off','position', [(scrsz(3)-wd)/2 (scrsz(4)-ht)/2 wd ht]);
                msg=sprintf('%s\n\n\n%s\n%s','Select Cruise Flags to Import','(see the QC Cookbook on the SOCAT website)','(http://www.socat.info/upload/Images_Header/SOCAT_QC_Cookbook_v2-8.doc)');
                uicontrol ('Style','text','position', [(wd-wdt)/2 ht-htt wdt htt],'String',msg);
                for i=1:ctrln
                    b(i) = uicontrol ('Style','radiobutton','position', [cols(i) 50 50+length(hdrs{i})*10 50],'String',hdr2s{i});
                    set(b(i),'Value',1);
                end
                okb=uicontrol ('Style','togglebutton','position', [(wd-50) 20 30 20],'String','OK','Callback','uiresume(gcbf)');
                ccb=uicontrol ('Style','togglebutton','position', [(wd-100) 20 40 20],'String','Cancel','Callback','uiresume(gcbf)');
                uiwait(cfdlg);
                set(cfdlg,'visible','off');
                if ~get(okb,'value'), return;end
                for i=1:ctrln,bc(i)=get(b(i),'Value');end
                cf=find(bc(1:ctrln))+10;% cruise flags to read
            end
        end
        
        if bv(ind5)~=0  %if ind5 exists and has been selected
            %--------------------------------------------------------------------------
            % Creates WOCE Flag Dialog
            %--------------------------------------------------------------------------
            ht=150;wd=400;hdr3s={'2 (good)','3 (questionable)','4 (bad)'};cols=[50 175 300];ctrln=size(hdr3s,2);htt=20;wdt=400;
            dfdlg = figure('Name','Data Flag Selection','NumberTitle','off','position', [(scrsz(3)-wd)/2 (scrsz(4)-ht)/2 wd ht]);
            msg=sprintf('%s','Select Data WOCE Flags to Import');
            uicontrol ('Style','text','position', [(wd-wdt)/2 ht-htt wdt htt],'String',msg);
            for i=1:ctrln
                b(i) = uicontrol ('Style','radiobutton','position', [cols(i) 50 50+length(hdrs{i})*10 50],'String',hdr3s{i});
                set(b(i),'Value',1);
            end
            okb=uicontrol ('Style','togglebutton','position', [(wd-50) 20 30 20],'String','OK','Callback','uiresume(gcbf)');
            ccb=uicontrol ('Style','togglebutton','position', [(wd-100) 20 40 20],'String','Cancel','Callback','uiresume(gcbf)');
            uiwait(dfdlg);
            set(dfdlg,'visible','off');
            if ~get(okb,'value'), return;end
            for i=1:ctrln,bd(i)=get(b(i),'Value');end
            df=find(bd(1:ctrln))+1;% cruise flags to read
        end
        
        
        h=waitbar(0,sprintf('Reading first set of data...(%s lines)',readlinesstr));
    end %of if fnum==1
    clear hdr2;

%--------------------------------------------------------------------------
% start reading the data according to the format string 'fstr'
%--------------------------------------------------------------------------
%tic;    
fstr='';% Creates the format string to read data
for i=1:colmax 
    if ~ismember(i,rcol)
      fstr=[fstr '%*s '];
    elseif ~ismember(i,tcol)
      fstr=[fstr '%f '];
    else
      fstr=[fstr '%s '];
    end
end
fstr=[fstr '\n'];

if lname>1
    ff= fopen([fpath filesep fname{fnum}],'r');
    fs=dir([fpath filesep fname{fnum}]);
else
    ff=fopen([fpath filesep fname],'r'); 
    fs=dir([fpath filesep fname]);
end 
    
%Reads data by chunks of lines. Number of lines read at once = 'readlines'
clc;ii=0;
data_in1=textscan(ff,fstr,readlines,'delimiter',delim,'HeaderLines',skip{fnum}+1);ln=ln+readlines;
if fnum==1
    data_in=data_in1;ncol=size(data_in1,2);
else
    for j=1:ncol,data_in{1,j}=cat(1,data_in{1,j},data_in1{1,j});end
end
%estimates numlines from size of file
numlines=fs.bytes * 0.0072;
numlinesi=int64(numlines);
if numlines>0
    waitbar(ln/numlines,h,sprintf('Loading file %d/%d...\n\n%s  of  ~%s lines loaded',fnum,lname,char(v.format(ln)),char(v.format(numlinesi))));
else
    waitbar(fnum/lname,h,sprintf('Loading file %d/%d...\n\n%s  lines loaded',fnum,lname,char(v.format(ln))));
end
while ~feof(ff)
    data_in1=textscan(ff,fstr,readlines,'delimiter',delim);
    for j=1:ncol,data_in{1,j}=cat(1,data_in{1,j},data_in1{1,j});end    
    ln=ln+readlines;ii=ii+1;
    if numlines>0
        waitbar(ln/numlines,h,sprintf('Loading file %d/%d...\n\n%s  of  ~%s lines loaded',fnum,lname,char(v.format(ln)),char(v.format(numlinesi))));
    else
        waitbar(fnum/lname,h,sprintf('Loading file %d/%d...\n\n%s  lines loaded',fnum,lname,char(v.format(ln))));
    end
end
fprintf('\n');
fclose(ff);
%toc
end
%--------------------------------------------------------------------------
% remove lines outside lat and lon border (PL 17102011)
%--------------------------------------------------------------------------

if(bv(ind2)~=0)
    ind=find(data_in{1,sum(bv(1:ind2))}<lonmin | data_in{1,sum(bv(1:ind2))}>lonmax);
    for i=1:size(data_in,2)
        data_in{1,i}(ind)=[];
    end
    clear ind ind2
end
if(bv(ind3)~=0)
    ind=find(data_in{1,sum(bv(1:ind3))}<latmin | data_in{1,sum(bv(1:ind3))}>latmax);
    for i=1:size(data_in,2)
        data_in{1,i}(ind)=[];
    end
    clear ind ind3
end
%--------------------------------------------------------------------------
% remove unwanted flags
%--------------------------------------------------------------------------

if exist('ind4','var')
    if(bv(ind4)~=0)  %Cruise Flags
        ind=find(~ismember(data_in{1,sum(bv(1:ind4))},cf));
        for i=1:size(data_in,2)
            data_in{1,i}(ind)=[];
        end
        clear ind ind4
    end
end
if(bv(ind5)~=0)  %Data Flags
    ind=find(~ismember(data_in{1,sum(bv(1:ind5))},df));
    for i=1:size(data_in,2)
        data_in{1,i}(ind)=[];
    end
    clear ind ind5
end

if size(data_in{1},1)==0 msgbox(sprintf('No data within the criteria specified.\n\nTry again with different selections\n\n')); end
%--------------------------------------------------------------------------
% Puts each column read in a variable named after column header.
%--------------------------------------------------------------------------
%Cleans hdrs for use as variable name
%gets rid of units
hdrs=regexprep(hdrs,'\s\[\S*\]','');
%gets rid of what is in parentheses
hdrs=regexprep(hdrs,'\s(\S*)','');
%replace (spaces, /) by underscore
hdrs=regexprep(hdrs,'[\s/]','_');
for i = rcol
    eval([hdrs{i} '=data_in{1,find(rcol==i)};']);
end
delete (h);

%--------------------------------------------------------------------------
% clears unnecessary variables
%--------------------------------------------------------------------------
close all; 
clear data_in1 readlines fs numlines numlinesi dflt
clear ht wd scrsz cols spc ctrln b okb ccb colmax bv rcol fig h data_in
clear ff fstr hdr i ii j k kk mots remain str ncol tline filen fnum lname
clear lonmin latmin lonmax latmax 
clear SText axts cond exts ftype ln skip tcol
clear htt wdt hdr2s cfdlg
clear hdr3s dfdlg bd df bc cf msg v



% function out = ThousandSep(in)
% %THOUSANDSEP adds thousands Separators to a 1x1 array.
% % Example:
% % ThousandSep(1234567)
% import java.text.*
% v = DecimalFormat;
% out = char(v.format(in));
%  
