/*------------------------------------------------------------------------------
* download.c : gnss data downloader
*
*          Copyright (C) 2012-2020 by T.TAKASU, All rights reserved.
*
* version : $Revision:$ $Date:$
* history : 2012/12/28  1.0  new
*           2013/06/02  1.1  replace S_IREAD by S_IRUSR
*           2020/11/30  1.2  support protocol https:// and ftps://
*                            support compressed RINEX (CRX) files
*                            support wild-card (*) in URL for FTP and FTPS
*                            use "=" to separate file name from URL
*                            fix bug on double-free of download paths
*                            limit max number of download paths
*                            use integer types in stdint.h
*-----------------------------------------------------------------------------*/
#include <errno.h>
#include <sys/stat.h>
#include <sys/types.h>

#include "rtklib.h"

#define FTP_CMD     "wget"          /* FTP/HTTP command */
#define FTP_TIMEOUT 60              /* FTP/HTTP timeout (s) */
#define FTP_LISTING ".listing"      /* FTP listing file */
#define FTP_NOFILE  2048            /* FTP error no file */
#define HTTP_NOFILE 1               /* HTTP error no file */
#define FTP_RETRY   3               /* FTP number of retry */
#define MAX_PATHS   131072          /* max number of download paths */

/* type definitions ----------------------------------------------------------*/

typedef struct {                    /* download path type */
    char *remot;                    /* remote path */
    char *local;                    /* local path */
} path_t;

typedef struct {                    /* download paths type */
    path_t *path;                   /* download paths */
    int n,nmax;                     /* number and max number of paths */
} paths_t;

/* execute command with test timeout -----------------------------------------*/
extern int execcmd_to(const char *cmd)
{
#ifdef WIN32
    PROCESS_INFORMATION info;
    STARTUPINFO si={0};
    DWORD stat;
    char cmds[4096];
    
    si.cb=sizeof(si);
    sprintf(cmds,"cmd /c %s",cmd);
    if (!CreateProcess(NULL,(LPTSTR)cmds,NULL,NULL,FALSE,CREATE_NO_WINDOW,NULL,
                       NULL,&si,&info)) return -1;
    
    while (WaitForSingleObject(info.hProcess,10)==WAIT_TIMEOUT) {
        showmsg("");
    }
    if (!GetExitCodeProcess(info.hProcess,&stat)) stat=-1;
    CloseHandle(info.hProcess);
    CloseHandle(info.hThread);
    return (int)stat;
#else
    return system(cmd);
#endif
}
/* generate path by replacing keywords ---------------------------------------*/
static void genpath(const char *file, const char *name, gtime_t time, int seqno,
                    char *path)
{
    char buff[1024],*p,*q,*r,*env,var[1024]="";
    char l_name[1024]="",u_name[1024]="";
    
    for (p=l_name,q=(char *)name;(*p=(char)tolower(*q));p++,q++) ;
    for (p=u_name,q=(char *)name;(*p=(char)toupper(*q));p++,q++) ;
    
    for (p=buff,q=(char *)file;(*p=*q);p++,q++) {
        if (*q=='%') q++; else continue;
        if      (*q=='s'||*q=='r') p+=sprintf(p,"%s",l_name)-1;
        else if (*q=='S'||*q=='R') p+=sprintf(p,"%s",u_name)-1;
        else if (*q=='N') p+=sprintf(p,"%d",seqno)-1;
        else if (*q=='{'&&(r=strchr(q+1,'}'))) {
            strncpy(var,q+1,r-q-1);
            var[r-q-1]='\0';
            if ((env=getenv(var))) p+=sprintf(p,"%s",env)-1;
            q=r;
        }
        else q--;
    }
    reppath(buff,path,time,"","");
}
/* parse field strings separated by spaces -----------------------------------*/
static char *parse_str(char *buff, char *str, int nmax)
{
    char *p,*q,sep[]=" \r\n";
    
    for (p=buff;*p&&*p==' ';p++) ;
    
    if (*p=='"') sep[0]=*p++; /* enclosed within quotation marks */
    
    for (q=str;*p&&!strchr(sep,*p);p++) {
        if (q<str+nmax-1) *q++=*p;
    }
    *q='\0';
    return *p?p+1:p;
}
/* compare str1 and str2 with wildcards (*) ----------------------------------*/
static int cmp_str(const char *str1, const char *str2)
{
    char s1[1026],s2[1026],*p,*q;
    
    sprintf(s1,"^%s$",str1);
    sprintf(s2,"^%s$",str2);
    
    for (p=s1,q=strtok(s2,"*");q;q=strtok(NULL,"*")) {
        if ((p=strstr(p,q))) p+=strlen(q); else break;
    }
    return p!=NULL;
}
/* remote to local file path -------------------------------------------------*/
static void remot2local(const char *remot, const char *dir, char *local)
{
    char *p;
    
    if      ((p=strrchr(remot,'='))) p++;
    else if ((p=strrchr(remot,'/'))) p++;
    else p=(char *)remot;
    
    sprintf(local,"%s%c%s",dir,FILEPATHSEP,p);
}
/* test file existence -------------------------------------------------------*/
static int exist_file(const char *local)
{
#ifdef WIN32
    DWORD stat=GetFileAttributes(local);
    return stat!=0xFFFFFFFF;
#else
    struct stat buff;
    if (stat(local,&buff)) return 0;
    return buff.st_mode&S_IRUSR;
#endif
}
/* test local file existence -------------------------------------------------*/
static int test_file(const char *local)
{
    char buff[1024],*p;
    int comp=0;
    
    if (strchr(local,'*')) { /* test wild-card (*) in path */
        return 0;
    }
    strcpy(buff,local);
    if ((p=strrchr(buff,'.'))&&
        (!strcmp(p,".z")||!strcmp(p,".gz")||!strcmp(p,".zip")||
         !strcmp(p,".Z")||!strcmp(p,".GZ")||!strcmp(p,".ZIP"))) {
        *p='\0';
        if (exist_file(buff)) return 1;
        comp=1;
    }
    if ((p=strrchr(buff,'.'))&&strlen(p)==4&&(*(p+3)=='d'||*(p+3)=='D')) {
        *(p+3)=*(p+3)=='d'?'o':'O';
        if (exist_file(buff)) return 1;
        comp=1;
    }
    if ((p=strrchr(buff,'.'))&&(!strcmp(p,".crx")||!strcmp(p,".CRX"))) {
        if (!strcmp(p,".crx")) strcpy(p,".cro"); else strcpy(p,".CRO");
        if (exist_file(buff)) return 1;
        comp=1;
    }
    if (!exist_file(buff)) return 0;
    return comp?2:1;
}
/* free download paths -------------------------------------------------------*/
static void free_path(paths_t *paths)
{
    int i;
    if (!paths) return;
    for (i=0;i<paths->n;i++) {
        free(paths->path[i].remot);
        free(paths->path[i].local);
    }
    free(paths->path);
}
/* add download paths --------------------------------------------------------*/
static int add_path(paths_t *paths, const char *remot, const char *dir)
{
    path_t *paths_path;
    char local[1024];
    
    if (paths->n>=paths->nmax) {
        if ((paths->nmax=paths->nmax<=0?1024:paths->nmax*2)>MAX_PATHS) {
            return 0;
        }
        paths_path=(path_t *)realloc(paths->path,sizeof(path_t)*paths->nmax);
        if (!paths_path) {
            return 0;
        }
        paths->path=paths_path;
    }
    remot2local(remot,dir,local);
    
    paths->path[paths->n].remot=paths->path[paths->n].local=NULL;
    
    if (!(paths->path[paths->n].remot=(char *)malloc(strlen(remot)+1))||
        !(paths->path[paths->n].local=(char *)malloc(strlen(local)+1))) {
        return 0;
    }
    strcpy(paths->path[paths->n].remot,remot);
    strcpy(paths->path[paths->n].local,local);
    paths->n++;
    return 1;
}
/* generate download path ----------------------------------------------------*/
static int gen_path(gtime_t time, gtime_t time_p, int seqnos, int seqnoe,
                    const url_t *url, const char *sta, const char *dir,
                    paths_t *paths)
{
    char remot[1024],remot_p[1024],dir_t[1024];
    int i;
    
    if (!*dir) dir=url->dir;
    if (!*dir) dir=".";
    
    if (strstr(url->path,"%N")) {
        for (i=seqnos;i<=seqnoe;i++) {
            genpath(url->path,sta,time,i,remot);
            genpath(dir      ,sta,time,i,dir_t);
            if (time_p.time) {
                genpath(url->path,sta,time_p,i,remot_p);
                if (!strcmp(remot_p,remot)) continue;
            }
            if (!add_path(paths,remot,dir_t)) return 0;
        }
    }
    else {
        genpath(url->path,sta,time,0,remot);
        genpath(dir      ,sta,time,0,dir_t);
        if (time_p.time) {
            genpath(url->path,sta,time_p,0,remot_p);
            if (!strcmp(remot_p,remot)) return 1;
        }
        if (!add_path(paths,remot,dir_t)) return 0;
    }
    return 1;
}
/* generate download paths ---------------------------------------------------*/
static int gen_paths(gtime_t time, gtime_t time_p, int seqnos, int seqnoe,
                     const url_t *url, char **stas, int nsta, const char *dir,
                     paths_t *paths)
{
    int i;
    
    if (strstr(url->path,"%s")||strstr(url->path,"%S")) {
        for (i=0;i<nsta;i++) {
            if (!gen_path(time,time_p,seqnos,seqnoe,url,stas[i],dir,paths)) {
                return 0;
            }
        }
    }
    else {
        if (!gen_path(time,time_p,seqnos,seqnoe,url,"",dir,paths)) {
            return 0;
        }
    }
    return 1;
}
/* compact download paths ----------------------------------------------------*/
static void compact_paths(paths_t *paths)
{
    int i,j,k;
    
    for (i=0;i<paths->n;i++) {
        for (j=i+1;j<paths->n;j++) {
            if (strcmp(paths->path[i].remot,paths->path[j].remot)) continue;
            free(paths->path[j].remot);
            free(paths->path[j].local);
            for (k=j;k<paths->n-1;k++) paths->path[k]=paths->path[k+1];
            paths->n--; j--;
        }
    }
}
/* generate local directory recursively --------------------------------------*/
static int mkdir_r(const char *dir)
{
    char pdir[1024],*p;
    
#ifdef WIN32
    HANDLE h;
    WIN32_FIND_DATA data;
    
    if (!*dir||!strcmp(dir+1,":\\")) return 1;
    
    strcpy(pdir,dir);
    if ((p=strrchr(pdir,FILEPATHSEP))) {
        *p='\0';
        h=FindFirstFile(pdir,&data);
        if (h==INVALID_HANDLE_VALUE) {
            if (!mkdir_r(pdir)) return 0;
        }
        else FindClose(h);
    }
    if (CreateDirectory(dir,NULL)||
        GetLastError()==ERROR_ALREADY_EXISTS) return 1;
    
    trace(2,"directory generation error: dir=%s\n",dir);
    return 0;
#else
    FILE *fp;
    
    if (!*dir) return 1;
    
    strcpy(pdir,dir);
    if ((p=strrchr(pdir,FILEPATHSEP))) {
        *p='\0';
        if (!(fp=fopen(pdir,"r"))) {
            if (!mkdir_r(pdir)) return 0;
        }
        else fclose(fp);
    }
    if (!mkdir(dir,0777)||errno==EEXIST) return 1;
    
    trace(2,"directory generation error: dir=%s\n",dir);
    return 0;
#endif
}
/* get remote file list for FTP or FTPS --------------------------------------*/
static int get_list(const path_t *path, const char *usr, const char *pwd,
                    const char *proxy)
{
    FILE *fp;
    char cmd[4096],env[1024]="",remot[1024],*opt="",*opt2="",*p;
    int stat;
    
#ifndef WIN32
    opt2=" -o /dev/null";
#endif
    remove(FTP_LISTING);
    
    strcpy(remot,path->remot);
    
    if ((p=strrchr(remot,'/'))) strcpy(p+1,"__REQUEST_LIST__"); else return 0;
    
    if (*proxy) {
        sprintf(env,"set ftp_proxy=http://%s & ",proxy);
        opt="--proxy=on ";
    }
    sprintf(cmd,"%s%s %s --ftp-user=%s --ftp-password=%s --glob=off "
            "--passive-ftp --no-remove-listing -N %s-t 1 -T %d%s\n",
            env,FTP_CMD,remot,usr,pwd,opt,FTP_TIMEOUT,opt2);
    execcmd_to(cmd);
    
    if (!(fp=fopen(FTP_LISTING,"r"))) return 0;
    fclose(fp);
    return 1;
}
/* replace wild-card (*) in the paths ----------------------------------------*/
static int rep_paths(path_t *path, const char *file)
{
    char buff1[1024],buff2[1024],*p,*q,*remot,*local;
    
    strcpy(buff1,path->remot);
    strcpy(buff2,path->local);
    if ((p=strrchr(buff1,'/'))) p++; else p=buff1;
    if ((q=strrchr(buff2,FILEPATHSEP))) q++; else q=buff2;
    strcpy(p,file);
    strcpy(q,file);
    
    if (!(remot=(char *)malloc(strlen(buff1)+1)) ||
        !(local=(char *)malloc(strlen(buff2)+1))) {
        free(remot);
        return 0;
    }
    strcpy(remot,buff1);
    strcpy(local,buff2);
    free(path->remot);
    free(path->local);
    path->remot=remot;
    path->local=local;
    return 1;
}
/* test file in remote file list ---------------------------------------------*/
static int test_list(path_t *path)
{
    FILE *fp;
    char buff[1024],*file,*list,*p;
    int i;
    
    if (!(fp=fopen(FTP_LISTING,"r"))) return 1;
    
    if ((p=strrchr(path->remot,'/'))) file=p+1; else return 1;
    
    /* search file in remote file list */
    while (fgets(buff,sizeof(buff),fp)) {
        
        /* remove symbolic link */
        if ((p=strstr(buff,"->"))) *p='\0';
        
        for (i=strlen(buff)-1;i>=0;i--) {
            if (strchr(" \r\n",buff[i])) buff[i]='\0'; else break;
        }
        /* file as last field */
        if ((p=strrchr(buff,' '))) list=p+1; else list=buff;
        
        if (!strcmp(file,list)) {
            fclose(fp);
            return 1;
        }
        /* compare with wild-card (*) */
        if (cmp_str(list,file)) {
            
            /* replace wild-card (*) in the paths */
            if (!rep_paths(path,list)) {
                fclose(fp);
                return 0;
            }
            fclose(fp);
            return 1;
        }
    }
    fclose(fp);
    return 0;
}
/* execute download ----------------------------------------------------------*/
static int exec_down(path_t *path, char *remot_p, const char *usr,
                     const char *pwd, const char *proxy, int opts, int *n,
                     FILE *fp)
{
    char dir[1024],errfile[1024],tmpfile[1024],cmd[4096],env[1024]="";
    char opt[1024]="",*opt2="",*p;
    int ret,proto;
    
#ifndef WIN32
    opt2=" 2> /dev/null";
#endif
    strcpy(dir,path->local);
    if ((p=strrchr(dir,FILEPATHSEP))) *p='\0';
    
    if      (!strncmp(path->remot,"ftp://"  ,6)) proto=0;
    else if (!strncmp(path->remot,"ftps://" ,7)) proto=2;
    else if (!strncmp(path->remot,"http://" ,7)) proto=1;
    else if (!strncmp(path->remot,"https://",8)) proto=1;
    else {
        trace(2,"exec_down: invalid path %s\n",path->remot);
        showmsg("STAT=X");
        if (fp) fprintf(fp,"%s ERROR (INVALID PATH)\n",path->remot);
        n[1]++;
        return 0;
    }
    /* test local file existence */
    if (!(opts&DLOPT_FORCE)&&test_file(path->local)) {
        showmsg("STAT=.");
        if (fp) fprintf(fp,"%s in %s\n",path->remot,dir);
        n[2]++;
        return 0;
    }
    showmsg("STAT=_");
    
    /* get remote file list for FTP or FTPS */
    if ((proto==0||proto==2)&&(p=strrchr(path->remot,'/'))&&
        strncmp(path->remot,remot_p,p-path->remot)) {
        
        if (get_list(path,usr,pwd,proxy)) {
            strcpy(remot_p,path->remot);
        }
    }
    /* test file in listing for FTP or FTPS or extend wild-card in file path */
    if ((proto==0||proto==2)&&!test_list(path)) {
        showmsg("STAT=x");
        if (fp) fprintf(fp,"%s NO_FILE\n",path->remot);
        n[1]++;
        return 0;
    }
    /* generate local directory recursively */
    if (!mkdir_r(dir)) {
        showmsg("STAT=X");
        if (fp) fprintf(fp,"%s -> %s ERROR (LOCAL DIR)\n",path->remot,dir);
        n[3]++;
        return 0;
    }
    /* re-test local file existence for file with wild-card */
    if (!(opts&DLOPT_FORCE)&&test_file(path->local)) {
        showmsg("STAT=.");
        if (fp) fprintf(fp,"%s in %s\n",path->remot,dir);
        n[2]++;
        return 0;
    }
    /* proxy option */
    if (*proxy) {
        sprintf(env,"set %s_proxy=http://%s & ",proto==0||proto==2?"ftp":"http",
                proxy);
        sprintf(opt," --proxy=on ");
    }
    /* download command */
    sprintf(errfile,"%s.err",path->local);
    if (proto==0||proto==2) {
        sprintf(cmd,"%s%s %s --ftp-user=%s --ftp-password=%s --glob=off "
                "--passive-ftp %s-t %d -T %d -O \"%s\" -o \"%s\"%s\n",
                env,FTP_CMD,path->remot,usr,pwd,opt,FTP_RETRY,FTP_TIMEOUT,
                path->local,errfile,opt2);
    }
    else {
        if (*pwd) {
            sprintf(opt+strlen(opt)," --http-user=%s --http-password=%s ",usr,
                    pwd);
        }
        sprintf(cmd,"%s%s %s %s-t %d -T %d -O \"%s\" -o \"%s\"%s\n",env,FTP_CMD,
                path->remot,opt,FTP_RETRY,FTP_TIMEOUT,path->local,errfile,opt2);
    }
    if (fp) fprintf(fp,"%s -> %s",path->remot,dir);
    
    /* execute download command */
    if ((ret=execcmd_to(cmd))) {
        if ((proto==0&&ret==FTP_NOFILE)||
            (proto==1&&ret==HTTP_NOFILE)) {
            showmsg("STAT=x");
            if (fp) fprintf(fp," NO_FILE\n");
            n[1]++;
        }
        else {
            trace(2,"exec_down: error proto=%d %d\n",proto,ret);
            showmsg("STAT=X");
            if (fp) fprintf(fp," ERROR (%d)\n",ret);
            n[3]++;
        }
        remove(path->local);
        if (!(opts&DLOPT_HOLDERR)) {
            remove(errfile);
        }
        return ret==2;
    }
    remove(errfile);
    
    /* uncompress download file */
    if (!(opts&DLOPT_KEEPCMP)&&(p=strrchr(path->local,'.'))&&
        (!strcmp(p,".z")||!strcmp(p,".gz")||!strcmp(p,".zip")||
         !strcmp(p,".Z")||!strcmp(p,".GZ")||!strcmp(p,".ZIP"))) {
        
        if (rtk_uncompress(path->local,tmpfile)) {
            remove(path->local);
        }
        else {
            trace(2,"exec_down: uncompress error\n");
            showmsg("STAT=C");
            if (fp) fprintf(fp," ERROR (UNCOMP)\n");
            n[3]++;
            return 0;
        }
    }
    showmsg("STAT=o");
    if (fp) fprintf(fp," OK\n");
    n[0]++;
    return 0;
}
/* test local file -----------------------------------------------------------*/
static int test_local(gtime_t ts, gtime_t te, double ti, const char *path,
                      const char *sta, const char *dir, int *nc, int *nt,
                      FILE *fp)
{
    gtime_t time;
    char remot[1024],remot_p[1024],dir_t[1024],local[1024],str[1024];
    int stat,abort=0;
    
    for (time=ts;timediff(time,te)<=1E-3;time=timeadd(time,ti)) {
        
        sprintf(str,"%s->%s",path,local);
        
        if (showmsg(str)) {
            abort=1;
            break;
        }
        genpath(path,sta,time,0,remot);
        genpath(dir ,sta,time,0,dir_t);
        remot2local(remot,dir_t,local);
        
        stat=test_file(local);
        
        fprintf(fp," %s",stat==0?"-":(stat==1?"o":"z"));
        
        showmsg("STAT=%s",stat==0?"x":(stat==1?"o":"z"));
        
        (*nt)++; if (stat) (*nc)++;
    }
    fprintf(fp,"\n");
    return abort;
}
/* test local files ----------------------------------------------------------*/
static int test_locals(gtime_t ts, gtime_t te, double ti, const url_t *url,
                       char **stas, int nsta, const char *dir, int *nc, int *nt,
                       FILE *fp)
{
    int i;
    
    if (strstr(url->path,"%s")||strstr(url->path,"%S")) {
        fprintf(fp,"%s\n",url->type);
        for (i=0;i<nsta;i++) {
            fprintf(fp,"%-12s:",stas[i]);
            if (test_local(ts,te,ti,url->path,stas[i],*dir?dir:url->dir,nc+i,
                           nt+i,fp)) {
                return 1;
            }
        }
    }
    else {
        fprintf(fp,"%-12s:",url->type);
        if (test_local(ts,te,ti,url->path,"",*dir?dir:url->dir,nc,nt,fp)) {
            return 1;
        }
    }
    return 0;
}
/* print total count of local files ------------------------------------------*/
static int print_total(const url_t *url, char **stas, int nsta, int *nc,
                       int *nt, FILE *fp)
{
    int i;
    
    if (strstr(url->path,"%s")||strstr(url->path,"%S")) {
        fprintf(fp,"%s\n",url->type);
        for (i=0;i<nsta;i++) {
            fprintf(fp,"%-12s: %5d/%5d\n",stas[i],nc[i],nt[i]);
        }
        return nsta;
    }
    fprintf(fp,"%-12s: %5d/%5d\n",url->type,nc[0],nt[0]);
    return 1;
}
/* read URL list file ----------------------------------------------------------
* read URL list file for GNSS data
* args   : char   *file     I   URL list file
*          char   **types   I   selected types ("*":wildcard)
*          int    ntype     I   number of selected types
*          urls_t *urls     O   URL list
*          int    nmax      I   max number of URL list
* return : number of URL addresses (0:error)
* notes  :
*    (1) URL list file contains records containing the following fields
*        separated by spaces. if a field contains spaces, enclose it within "".
*
*        data_type  url_address       default_local_directory
*
*    (2) strings after # in a line are treated as comments
*    (3) url_address should be:
*
*        ftp://host_address/file_path or
*        ftps://host_address/file_path or
*        http://host_address/file_path or
*        https://host_address/file_path
*
*    (4) the field url_address or default_local_directory can include the
*        follwing keywords replaced by date, time, station names and environment
*        variables.
*
*        %Y -> yyyy    : year (4 digits) (2000-2099)
*        %y -> yy      : year (2 digits) (00-99)
*        %m -> mm      : month           (01-12)
*        %d -> dd      : day of month    (01-31)
*        %h -> hh      : hours           (00-23)
*        %H -> a       : hour code       (a-x)
*        %M -> mm      : minutes         (00-59)
*        %n -> ddd     : day of year     (001-366)
*        %W -> wwww    : gps week        (0001-9999)
*        %D -> d       : day of gps week (0-6)
*        %N -> nnn     : general number
*        %s -> ssss    : station name    (lower-case)
*        %S -> SSSS    : station name    (upper-case)
*        %r -> rrrr    : station name
*        %{env} -> env : environment variable
*-----------------------------------------------------------------------------*/
extern int dl_readurls(const char *file, char **types, int ntype, url_t *urls,
                       int nmax)
{
    FILE *fp;
    char buff[2048],type[32],path[1024],dir[1024],*p;
    int i,n=0;
    
    if (!(fp=fopen(file,"r"))) {
        fprintf(stderr,"options file read error %s\n",file);
        return 0;
    }
    for (i=0;i<ntype;i++) {
        rewind(fp);
        while (fgets(buff,sizeof(buff),fp)&&n<nmax) {
            if ((p=strchr(buff,'#'))) *p='\0';
            p=buff;
            p=parse_str(p,type,sizeof(type));
            p=parse_str(p,path,sizeof(path));
            p=parse_str(p,dir ,sizeof(dir ));
            if (!*type||!*path) continue;
            if (!cmp_str(type,types[i])) continue;
            strcpy(urls[n  ].type,type);
            strcpy(urls[n  ].path,path);
            strcpy(urls[n++].dir ,dir );
        }
    }
    fclose(fp);
    
    if (n<=0) {
        fprintf(stderr,"no url in options file %s\n",file);
        return 0;
    }
    return n;
}
/* read station list file ------------------------------------------------------
* read station list file
* args   : char   *file     I   station list file
*          char   **stas    O   stations
*          int    nmax      I   max number of stations
* return : number of stations (0:error)
* notes  :
*    (1) station list file contains station names separated by spaces.
*    (2) strings after # in a line are treated as comments
*-----------------------------------------------------------------------------*/
extern int dl_readstas(const char *file, char **stas, int nmax)
{
    FILE *fp;
    char buff[4096],*p;
    int n=0;
    
    if (!(fp=fopen(file,"r"))) {
        fprintf(stderr,"station list file read error %s\n",file);
        return 0;
    }
    while (fgets(buff,sizeof(buff),fp)&&n<nmax) {
        if ((p=strchr(buff,'#'))) *p='\0';
        for (p=strtok(buff," \r\n");p&&n<nmax;p=strtok(NULL," \r\n")) {
            strcpy(stas[n++],p);
        }
    }
    fclose(fp);
    
    if (n<=0) {
        fprintf(stderr,"no station in station file %s\n",file);
        return 0;
    }
    return n;
}
/* execute download ------------------------------------------------------------
* execute download
* args   : gtime_t ts,te    I   time start and end
*          double tint      I   time interval (s)
*          int    seqnos    I   sequence number start
*          int    seqnoe    I   sequence number end
*          url_t  *urls     I   URL list
*          int    nurl      I   number of URL list
*          char   **stas    I   station list
*          int    nsta      I   number of station list
*          char   *dir      I   local directory
*          char   *remote_p I   previous remote file path
*          char   *usr      I   login user for FTP or FTPS
*          char   *pwd      I   login password for FTP or FTPS
*          char   *proxy    I   proxy server address
*          int    opts      I   download options (or of the followings)
*                                 DLOPT_FORCE = force download existing file
*                                 DLOPT_KEEPCMP=keep compressed file
*                                 DLOPT_HOLDERR=hold on error file
*                                 DLOPT_HOLDLST=hold on listing file
*          char   *msg      O   output messages
*          FILE   *fp       IO  log file pointer (NULL: no output log)
* return : status (1:ok,0:error,-1:aborted)
* notes  : The URL list should be read by using dl_readurl()
*          In the FTP or FTPS cases, the file name in a URL can contain wild-
*          cards (*). The directory in a URL can not contain any wild-cards.
*          If the file name contains wild-cards, dl_exec() gets a file-list in
*          the remote directory and downloads the firstly matched file in the
*          remote file-list. The secondary matched or the following files are
*          not downloaded.
*-----------------------------------------------------------------------------*/
extern int dl_exec(gtime_t ts, gtime_t te, double ti, int seqnos, int seqnoe,
                   const url_t *urls, int nurl, char **stas, int nsta,
                   const char *dir, const char *usr, const char *pwd,
                   const char *proxy, int opts, char *msg, FILE *fp)
{
    paths_t paths={0};
    gtime_t ts_p={0};
    char str[2048],remot_p[1024]="";
    int i,n[4]={0};
    uint32_t tick=tickget();
    
    showmsg("STAT=_");
    
    /* generate download paths  */
    while (timediff(ts,te)<1E-3) {
        
        for (i=0;i<nurl;i++) {
            if (!gen_paths(ts,ts_p,seqnos,seqnoe,urls+i,stas,nsta,dir,&paths)) {
                free_path(&paths);
                sprintf(msg,"too many download files");
                return 0;
            }
        }
        ts_p=ts; ts=timeadd(ts,ti);
    }
    /* compact download paths */
    compact_paths(&paths);
    
    if (paths.n<=0) {
        sprintf(msg,"no download data");
        return 0;
    }
    for (i=0;i<paths.n;i++) {
        
        sprintf(str,"%s->%s (%d/%d)",paths.path[i].remot,paths.path[i].local,i+1,
                paths.n);
        if (showmsg(str)) break;
        
        /* execute download */
        if (exec_down(paths.path+i,remot_p,usr,pwd,proxy,opts,n,fp)) {
            break;
        }
    }
    if (!(opts&DLOPT_HOLDLST)) {
        remove(FTP_LISTING);
    }
    sprintf(msg,"OK=%d No_File=%d Skip=%d Error=%d (Time=%.1f s)",n[0],n[1],n[2],
            n[3],(tickget()-tick)*0.001);
    
    free_path(&paths);
    
    return 1;
}
/* execute local file test -----------------------------------------------------
* execute local file test
* args   : gtime_t ts,te    I   time start and end
*          double tint      I   time interval (s)
*          url_t  *urls     I   remote URL addresses
*          int    nurl      I   number of remote URL addresses
*          char   **stas    I   stations
*          int    nsta      I   number of stations
*          char   *dir      I   local directory
*          int    ncol      I   number of column
*          int    datefmt   I   date format (0:year-dow,1:year-dd/mm,2:week)
*          FILE   *fp       IO  log test result file pointer
* return : status (1:ok,0:error,-1:aborted)
*-----------------------------------------------------------------------------*/
extern void dl_test(gtime_t ts, gtime_t te, double ti, const url_t *urls,
                    int nurl, char **stas, int nsta, const char *dir,
                    int ncol, int datefmt, FILE *fp)
{
    gtime_t time;
    double tow;
    char year[32],date[32],date_p[32];
    int i,j,n,m,*nc,*nt,week,flag,abort=0;
    
    if (ncol<1) ncol=1; else if (ncol>200) ncol=200;
     
    fprintf(fp,"** LOCAL DATA AVAILABILITY (%s, %s) **\n\n",
            time_str(timeget(),0),*dir?dir:"*");
    
    for (i=n=0;i<nurl;i++) {
        n+=strstr(urls[i].path,"%s")||strstr(urls[i].path,"%S")?nsta:1;
    }
    nc=imat(n,1);
    nt=imat(n,1);
    for (i=0;i<n;i++) nc[i]=nt[i]=0;
    
    for (;timediff(ts,te)<1E-3&&!abort;ts=timeadd(ts,ti*ncol)) {
        
        genpath(datefmt==0?"   %Y-":"%Y/%m/","",ts,0,year);
        if      (datefmt<=1) fprintf(fp,"%s %s",datefmt==0?"DOY ":"DATE",year);
        else                 fprintf(fp,"WEEK          ");
        *date_p='\0'; flag=0;
        
        m=datefmt==2?1:2;
        
        for (i=0;i<(ncol+m-1)/m;i++) {
            time=timeadd(ts,ti*i*m);
            if (timediff(time,te)>=1E-3) break;
            
            if (datefmt<=1) {
                genpath(datefmt==0?"%n":"%d","",time,0,date);
                fprintf(fp,"%-4s",strcmp(date,date_p)?date:"");
            }
            else {
                if (fabs(time2gpst(time,&week))<1.0) {
                    fprintf(fp,"%04d",week); flag=1;
                }
                else {
                    fprintf(fp,"%s",flag?"":"  "); flag=0;
                }
            }
            strcpy(date_p,date);
        }
        fprintf(fp,"\n");
        
        for (i=j=0;i<nurl&&!abort;i++) {
            time=timeadd(ts,ti*ncol-1.0);
            if (timediff(time,te)>=0.0) time=te;
            
            /* test local files */
            abort=test_locals(ts,time,ti,urls+i,stas,nsta,dir,nc+j,nt+j,fp);
            
            j+=strstr(urls[i].path,"%s")||strstr(urls[i].path,"%S")?nsta:1;
        }
        fprintf(fp,"\n");
    }
    fprintf(fp,"# COUNT     : FILES/TOTAL\n");
    
    for (i=j=0;i<nurl;i++) {
        j+=print_total(urls+i,stas,nsta,nc+j,nt+j,fp);
    }
    free(nc); free(nt);
}