%------------------------------------------------------------------%
% This is the routine to get the SOCAT v1.5 data into matlab.      %
% These are the 'Database Files from CDIAC v1.5' and               %
%     and the 'Individual Cruise Files from Pangea v1.4'           %
% Several files can be selected, all with the same extension.      %
% For allowed extensions, see the 'exts' variable.                 %
% It reads the file(s) until the column headers are found.         %
% Determines # of columns from # of headers.                       %
% Creates GUI to ask user which columns to import                  %
% Creates another GUI to ask user which                            %
%       geographical region to import (see figure below)           % 
% Data is stored in variables                                      %
%        named after (sometimes modified) column headers           %
% Anything before column headers is stored in 'StartText' variable %
% For questions, contact:                                          %
%       Denis Pierrot (Denis.Pierrot@noaa.gov)                     %
%   or  Peter Landschutzer (p.landschutzer@uea.ac.uk)              %
%------------------------------------------------------------------%

clear all
%dbstop at 135;
%--------------------------------------------------------------------------
%  TO USER:
%  Data file(s) should prefarably be in same directory as this .m file 
%  User Input: 
%              readlines = number of lines read per loop
%              dflt      = column number initially selected to be imported 
%                          when interface is created
%--------------------------------------------------------------------------

exts={'txt';'tsv';'tab'};
axts='';for i=1 : size(exts,1),axts=[axts '*.' exts{i} ';'];end
delim='\t';
% numlines is only used to show progress. Set to 0 if # of lines unknown
readlines=100000;numlines=0;
% filen='SOCATv1.5_2011_09_22_test2.txt';numlines=7814908;
% dflt=[8 9 19 22 23];% lon lat fCO2 JDay WOCE_flag
% tcol=[1 2 24 25];%text columns

%----Gets mfile directory ----------------------------------------------------
if exist(mfilename,'file')==2
    projDir = strrep([mfilename('fullpath') '.m'],[filesep mfilename '.m'],'');
else
    projDir =pwd;
end
%--------------------------------------------------------------------------

%----Select files to be imported (only 1 kind at a time)-------------------
[fname,fpath] = uigetfile(axts,'Select Data File(s)',[projDir filesep],'MultiSelect','on');
if (fpath==0),return;end
if strcmp(fpath(end),filesep)>0,fpath=fpath(1:end-1);end

lname=1;

%--------------- Detects extensions in files selected  -------------------
% 1 or more files selected
if ischar(fname), lname=1; else lname=length(fname);  fname=sort(fname);end
cond=0;
if iscell(fname)
    for i=1:size(exts,1)
        for j=1:lname
            cond=cond + ~isempty(regexpi(fname{j},['.' exts{i} '$'], 'match'));
        end
        if cond>0 ,ftype=i; break;end %breaks on first extension detected
   end

else %one file only selected
    for i=1:size(exts,1)
        cond=cond + ~isempty(regexpi(fname,['.' exts{i} '$'], 'match'));
        if cond,ftype=i;break;end
    end
end
if ~cond 
    mmm=msgbox(sprintf('No files with correct extension selected!\n\nAllowed extensions are: %s\n\nProgram aborted.\n\n',exts),'Input File Error');
    return;
end
if cond<lname
    mmm=msgbox(sprintf('Files selected have different extensions!\n\nProgram aborted.\n\n'),'Input File Error');
    return;
end

% ************************************** txt or tsv files*****************************************************************
switch ftype
    case 1 %txt
        mots={'cruise_ID';'cruise_name'};
        dflt=[8 9 19 22 23];% lon lat fCO2 JDay WOCE_flag
        tcol=[1 2 24 25];%text columns
    case 2 %tab
        dflt=[1 2 3 5 13];% Date_Time lon lat temp fCO2
        tcol=[1];%text columns
        mots={'Date/Time';'Latitude'};
    case 3%tsv
        dflt=[1 2 3 5 15];% Date_Time lon lat temp fCO2
        tcol=[1];%text columns
        mots={'Date/Time';'Latitude'};
end
ln=0;%lines read
for fnum=1:lname
    if lname>1 ff= fopen([fpath filesep fname{fnum}],'r');
    else  ff=fopen([fpath filesep fname],'r'); end
    
    hdr=0;SText='';kk=0;
%--------------------------------------------------------------------------
% searches for header line
%--------------------------------------------------------------------------
    while (~hdr & ~feof(ff)) %
        tline = fgets(ff);
        for k=1:size(mots,1)
            if ~isempty(findstr(tline, mots{k})), continue; else SText=char(SText,tline);kk=kk+1; end
            k=size(mots,1)+1;break;
        end
        if k~=size(mots,1)+1,hdr=1;end
    end
    StartText{fnum}=SText;skip{fnum}=kk;
    %--------------------------------------------------------------------------
    % Reads headers and number of columns
    %--------------------------------------------------------------------------
    hdr2=textscan(tline,'%s','delimiter',delim)
    fclose(ff);
    if fnum==1
        if hdr
            hdrs=hdr2{1,1};
            colmax=length(hdrs);
        else
            break;
        end
            
        %--------------------------------------------------------------------------
        % Creates interface
        %--------------------------------------------------------------------------
        [scrsz]=get(0,'ScreenSize');   ht=700;wd=600;
        fig = figure('position', [(scrsz(3)-wd)/2 (scrsz(4)-ht)/2 wd ht]);cols=[50 300];spc=50;ctrln=13;
        uicontrol ('Style','text','position', [(wd-200)/2 ht-20 200 20],'String','Select Variable to Import');
        
        for i=1:colmax
            b(i) = uicontrol ('Style','radiobutton','position', [cols((i>ctrln)+1) ht-50-spc*i+(ctrln*spc*(i>ctrln)) 50+length(hdrs{i})*10 50],'String',hdrs{i});
            set(b(i),'Value',ismember(i,dflt));
        end
        okb=uicontrol ('Style','togglebutton','position', [(wd-50) 20 30 20],'String','OK','Callback','uiresume(gcbf)');
        ccb=uicontrol ('Style','togglebutton','position', [(wd-100) 20 40 20],'String','Cancel','Callback','uiresume(gcbf)');
        uiwait(fig);
        set(fig,'visible','off');
        if ~get(okb,'value'), return;end
        for i=1:colmax,bv(i)=get(b(i),'Value');end
        rcol=find(bv);% columns to read
        
        %--------------------------------------------------------------------------
        % open GUI to set lon-lat borders (PL 17102011)
        %--------------------------------------------------------------------------
        
        for i=1:colmax
            if(strcmpi(hdrs{i},'longitude')==1)
                ind2=i;
            elseif(strcmpi(hdrs{i},'latitude')==1)
                ind3=i;
            end
        end
        
        if(bv(ind2)~=0 && bv(ind3)~=0)
            prompt={'Min. Longitude','Max. Longitude', 'Min. Latitude', 'Max. Latitude'};
            defans={'-180','180', '-90', '90'};
            fields = {'lonmin','lonmax', 'latmin', 'latmax'};
            info = inputdlg(prompt, 'Enter Limits', 1, defans,'on');
            if isempty(info),return;end              %see if user hit cancel
            info = cell2struct(info,fields);
            lonmin = str2num(info.lonmin);%convert string to number
            lonmax = str2num(info.lonmax);
            latmin = str2num(info.latmin);
            latmax = str2num(info.latmax);
            clear prompt defans fields info
        end
        h=waitbar(0);
    end %of if fnum==1
    clear hdr2;

%--------------------------------------------------------------------------
% start reading the data according to the format string 'fstr'
%--------------------------------------------------------------------------
%tic;    
fstr='';% Creates the format string to read data
for i=1:colmax 
    if ~ismember(i,rcol)
      fstr=[fstr '%*s '];
    elseif ~ismember(i,tcol)
      fstr=[fstr '%f '];
    else
      fstr=[fstr '%s '];
    end
end
fstr=[fstr '\n'];

if lname>1 ff= fopen([fpath filesep fname{fnum}],'r');
else  ff=fopen([fpath filesep fname],'r'); end

%Reads data by chunks of lines. Number of lines read at once = 'readlines'
clc;ii=0;
data_in1=textscan(ff,fstr,readlines,'delimiter',delim,'HeaderLines',skip{fnum}+1);ln=ln+readlines;
if fnum==1 data_in=data_in1;ncol=size(data_in1,2);end
for j=1:ncol,data_in{1,j}=cat(1,data_in{1,j},data_in1{1,j});end

if numlines>0
    waitbar(ln/numlines,h,sprintf('Loading file %d/%d...\n\n%d  of  %d lines loaded',fnum,lname,ln,numlines));
else
    waitbar(fnum/lname,h,sprintf('Loading file %d/%d...\n\n%d  lines loaded',fnum,lname,ln));
end
while ~feof(ff)
    data_in1=textscan(ff,fstr,readlines,'delimiter',delim);
    for j=1:ncol,data_in{1,j}=cat(1,data_in{1,j},data_in1{1,j});end    
    ln=ln+readlines;ii=ii+1;
    if numlines>0 
        waitbar(ln/numlines,h,sprintf('Loading file %d/%d...\n\n%d  of  %d lines loaded',fnum,lname,ln,numlines));
    else
        waitbar(fnum/lname,h,sprintf('Loading file %d/%d...\n\n%d  lines loaded',fnum,lname,ln));
    end
end
fprintf('\n');
fclose(ff);
%toc
end
%--------------------------------------------------------------------------
% remove lines outside lat and lon boarder (PL 17102011)
%--------------------------------------------------------------------------

if(bv(ind2)~=0 && bv(ind3)~=0)
    ind=find(data_in{1,sum(bv(1:ind2))}<lonmin | data_in{1,sum(bv(1:ind2))}>lonmax | data_in{1,sum(bv(1:ind3))}<latmin | data_in{1,sum(bv(1:ind3))}>latmax);
    for i=1:size(data_in,2)
        data_in{1,i}(ind)=[];
    end
    clear ind ind2 ind3
end

%--------------------------------------------------------------------------
% Puts each column read in a variable named after column header.
%--------------------------------------------------------------------------
%Cleans hdrs for use as variable name
%gets rid of units
hdrs=regexprep(hdrs,'\s\[\S*\]','');
%gets rid of what is in parentheses
hdrs=regexprep(hdrs,'\s(\S*)','');
%replace (spaces, /) by underscore
hdrs=regexprep(hdrs,'[\s/]','_');
for i = rcol
    eval([hdrs{i} '=data_in{1,find(rcol==i)};']);
end
delete (h);

%--------------------------------------------------------------------------
% clears unnecessary variables
%--------------------------------------------------------------------------
close all; 
clear data_in1 readlines numlines dflt
clear ht wd scrsz cols spc ctrln b okb ccb colmax bv rcol fig h data_in
clear ff fstr hdr i ii j k kk mots remain str ncol tline filen fnum lname
clear lonmin latmin lonmax latmax 
clear SText axts cond exts ftype ln skip tcol



