00001 /*****************************************************************************
00002    Major portions of this software are copyrighted by the Medical College
00003    of Wisconsin, 1994-2000, and are released under the Gnu General Public
00004    License, Version 2.  See the file README.Copyright for details.
00005 ******************************************************************************/
00006 
00007 #include "thd_iochan.h"
00008 #include "Amalloc.h"
00009 #include <sys/stat.h>
00010 
00011 static int debug = 0 ;
00012 #define FAILED     if(debug)fprintf(stderr," **FAILED\n")
00013 #define DMESS(s,t) if(debug)fprintf(stderr,s,t)
00014 
00015 extern unsigned long THD_filesize( char * pathname ) ;
00016 
00017 /*---------------------------------------------------------------------*/
00018 static int   use_http_10     = 0          ;
00019 static char *http_user_agent = "read_URL" ;
00020 
00021 void set_HTTP_10( int n ){ use_http_10 = n; }  /* 24 Mar 2005 */
00022 
00023 extern void set_HTTP_user_agent( char *ua )
00024 {
00025    if( ua == NULL || *ua == '\0' ) http_user_agent = "read_URL" ;
00026    else                            http_user_agent = strdup(ua) ;
00027 }
00028 
00029 /*---------------------------------------------------------------------*/
00030 static char tmpdir[256] = "\0" ;
00031 
00032 static void setup_tmpdir(void)  /* 02 Apr 1999 */
00033 {
00034    char * td ;
00035 
00036    if( tmpdir[0] != '\0' ) return ;
00037 
00038                     td = getenv("TMPDIR") ;   /* try two possibilities */
00039    if( td == NULL ) td = getenv("TEMPDIR") ;
00040 
00041    if( td == NULL || td[0] == '\0' || strlen(td) > 222 ){
00042       strcpy(tmpdir,"/tmp/") ;
00043    } else {
00044       int ltd = strlen(td) ;
00045       strcpy(tmpdir,td) ;
00046       if( tmpdir[ltd-1] != '/' ) strcat(tmpdir,"/") ;
00047    }
00048 
00049    return ;
00050 }
00051 
00052 /*---------------------------------------------------------------------
00053   Open an "http://" URL in host, port, and filename pieces.
00054   Wait up to msec milliseconds for network functions to occur.
00055   If an error occurs, return NULL, otherwise the caller can read
00056   from this IOCHAN (using iochan_recv, etc.).
00057 -----------------------------------------------------------------------*/
00058 
00059 IOCHAN * open_URL_hpf( char * host , int port , char * file , int msec )
00060 {
00061    IOCHAN * ioc ;
00062    char str[512] ;
00063    int ii ;
00064 
00065    if( host == NULL || port <= 0 || file == NULL ) return NULL ;
00066 
00067    sprintf(str,"tcp:%s:%d",host,port) ;
00068    DMESS(" ++Opening %s",str);
00069    ioc = iochan_init( str , "create" ) ;
00070    if( ioc == NULL ){ FAILED; return NULL; }
00071    fprintf(stderr,".");
00072    iochan_set_cutoff( ioc ) ;
00073    fprintf(stderr,".");
00074    ii = iochan_writecheck( ioc , msec ) ;
00075    if( ii <= 0 ){ FAILED; IOCHAN_CLOSE(ioc) ; return NULL ; }
00076 
00077    DMESS(" ++GET %s",file);
00078    if( use_http_10 )
00079      sprintf(str,"GET %s HTTP/1.0\r\n"
00080                  "User-Agent: %s\r\n"
00081                  "\r\n", file , http_user_agent ) ;       /* HTTP 1.0 */
00082    else
00083      sprintf(str,"GET %s\r\n",file) ;                     /* HTTP 0.9 */
00084    ii = iochan_sendall( ioc , str , strlen(str) ) ;
00085    if( ii <= 0 ){ FAILED; IOCHAN_CLOSE(ioc); return NULL; }
00086 
00087    ii = iochan_readcheck( ioc , msec ) ;
00088    if( ii <= 0 ){ FAILED; IOCHAN_CLOSE(ioc) ; return NULL ; }
00089    DMESS("%s"," **OPENED");
00090    return ioc ;
00091 }
00092 
00093 /*----------------------------------------------------------------------
00094   Open an "http://" URL and prepare to read it (but the caller must
00095   actually do the reading).  If NULL is returned, an error occurred.
00096 ------------------------------------------------------------------------*/
00097 
00098 #define HTTP     "http://"
00099 #define HTTPLEN  7
00100 
00101 #define FTP      "ftp://"
00102 #define FTPLEN   6
00103 
00104 IOCHAN * open_URL_http( char * url , int msec )
00105 {
00106   char * s, * h , * file ;
00107   char hostname[256] ;
00108   int port;
00109   IOCHAN * ioc ;
00110 
00111   /* check inputs */
00112 
00113   if( url == NULL || strstr(url,HTTP) != url ) return NULL ;
00114 
00115   /* parse hostname */
00116 
00117   for( s=url+HTTPLEN , h=hostname ;
00118        (*s != '\0') && (*s != ':') && (*s != '/') ; s++ , h++ ) *h = *s ;
00119 
00120   *h = '\0' ; if( hostname[0] == '\0' ) return NULL ;
00121 
00122   /* parse port number if present */
00123 
00124   port = 0 ;
00125   if( *s == ':' ){ port = strtol( ++s , &h , 10 ) ; s = h ; }
00126   if( port <= 0 ) port = 80 ;
00127 
00128   /* get the file name (keep leading "/") */
00129 
00130   file = (*s == '/') ? s : "/" ;
00131 
00132   /* do the actual work */
00133 
00134   ioc = open_URL_hpf( hostname , port , file , msec ) ;
00135   return ioc ;
00136 }
00137 
00138 /*--------------------------------------------------------------*/
00139 
00140 static int prog=0 ;
00141 void set_URL_progress( int p ){ prog=p; }  /* 20 Mar 2003 */
00142 
00143 /*---------------------------------------------------------------
00144   Read an "http://" URL, with network waits of up to msec
00145   milliseconds allowed.  Returns number of bytes read -- if this
00146   is > 0, then *data will be a pointer to malloc-ed bytes holding
00147   the contents of the file.
00148 
00149   If the file is gzip-ed, then it will be un-gzip-ed before being
00150   loaded into memory.  This uses temporary files in $TMPDIR or
00151   /tmp, which must have space to hold the compressed and
00152   uncompressed file.  If the file is not compressed, then input
00153   is directly to memory and no temporary files are used.
00154 -----------------------------------------------------------------*/
00155 
00156 #define QBUF 4096
00157 
00158 int read_URL_http( char * url , int msec , char ** data )
00159 {
00160    IOCHAN * ioc ;
00161    char * buf=NULL , * cpt , qbuf[QBUF] , qname[256] ;
00162    int ii,jj , nall , nuse , nget=0, nmeg=0 ;
00163    int cflag , first ;
00164    FILE * cfile ;
00165 
00166    /* sanity check */
00167 
00168    if( url == NULL || data == NULL || msec < 0 ) return( -1 );
00169 
00170    /* open http channel to get url */
00171 
00172    ioc = open_URL_http( url , msec ) ;
00173    if( ioc == NULL ){ DMESS("%s","\n"); return( -1 ); }
00174 
00175    /* check if url will be returned gzip-ed */
00176 
00177    ii = strlen(url) ;
00178    if( ii > 3 ){
00179       cpt = url + (ii-3) ; cflag = (strcmp(cpt,".gz") == 0) ;
00180    } else {
00181       cflag = 0 ;
00182    }
00183 
00184    if( cflag ){
00185       setup_tmpdir() ;
00186       strcpy(qname,tmpdir) ; strcat(qname,"gosiaXXXXXX") ;
00187       mktemp(qname) ;
00188       if( qname[0] != '\0' ){
00189          strcat(qname,".gz") ; cfile = fopen( qname , "wb" ) ;
00190          if( cfile == NULL ) cflag == 0 ;
00191       } else {
00192          cflag = 0 ;
00193       }
00194 
00195       if( cflag == 0 ){
00196          DMESS(" **Temp file %s FAILS\n",qname); IOCHAN_CLOSE(ioc); return(-1);
00197       }
00198       DMESS(" ++Temp file=%s",qname);
00199    }
00200 
00201    /* read all of url */
00202 
00203    if( !cflag ){ buf = AFMALL(char, QBUF ) ; nall = QBUF ; }
00204    nuse = 0 ; first = 1 ;
00205 
00206    do{
00207       if(debug)fprintf(stderr,".");
00208       ii = iochan_readcheck( ioc , msec ) ;  /* wait for data to be ready */
00209       if( ii <= 0 ) break ;                  /* quit if no data */
00210       ii = iochan_recv( ioc , qbuf , QBUF ) ;
00211       if( ii <= 0 ) break ;                  /* quit if no data */
00212 
00213       if( prog ){
00214         nget += ii ; jj = nget/(1024*1024) ;
00215         if( jj > nmeg ){ nmeg=jj; fprintf(stderr,"."); }
00216       }
00217 
00218       if( first ){                           /* check for "not found" */
00219          if( buf == NULL ){ buf = AFMALL(char, ii) ; }
00220          memcpy( buf , qbuf , ii ) ;
00221          for( jj=0 ; jj < ii ; jj++ ) buf[jj] = toupper(buf[jj]) ;
00222          buf[ii-1] = '\0' ;
00223          cpt = strstr(buf,"NOT FOUND") ;
00224          if( cpt != NULL ){
00225             if( cflag ){ fclose(cfile) ; unlink(qname) ; }
00226             DMESS("%s"," **NOT FOUND\n");
00227             free(buf) ; IOCHAN_CLOSE(ioc) ; return( -1 );
00228          }
00229          first = 0 ;
00230          if( cflag ){ free(buf) ; buf = NULL ; }
00231       }
00232 
00233       if( cflag ){                           /* write to temp file */
00234          nall = fwrite( qbuf , 1 , ii , cfile ) ;
00235          if( nall != ii ){                   /* write failed? */
00236             DMESS("\n** Write to temp file %s FAILED!\n",qname);
00237             fclose(cfile) ; unlink(qname) ;
00238             IOCHAN_CLOSE(ioc) ; return( -1 );
00239          }
00240       } else {                               /* save to buffer */
00241          if( nuse+ii > nall ){               /* enlarge buffer? */
00242             nall += QBUF ;
00243             buf   = AFREALL(buf, char, nall) ;
00244          }
00245          memcpy( buf+nuse , qbuf , ii ) ;    /* copy data into buffer */
00246       }
00247       nuse += ii ;                           /* how many bytes so far */
00248    } while(1) ;
00249    IOCHAN_CLOSE(ioc) ;
00250 
00251    if( prog && nmeg > 0 ) fprintf(stderr,"!\n") ;
00252 
00253    /* didn't get anything? */
00254 
00255    if( nuse <= 0 ){
00256       if( cflag ){ fclose(cfile) ; unlink(qname) ; }
00257       else       { free(buf) ; }
00258       FAILED; return(-1);
00259    }
00260    if(debug)fprintf(stderr,"!\n");
00261 
00262    /* uncompression time? */
00263 
00264    if( cflag ){
00265       fclose(cfile) ;
00266       sprintf( qbuf , "gzip -dq %s" , qname ) ;     /* execute gzip */
00267       ii = system(qbuf) ;
00268       if( ii != 0 ){ DMESS("%s"," **gzip failed!\n");
00269                      unlink(qname) ; return( -1 );   }  /* gzip failed  */
00270       ii = strlen(qname) ; qname[ii-3] = '\0' ;     /* fix filename */
00271       nuse = THD_filesize( qname ) ;                /* find how big */
00272       if( nuse <= 0 ){ DMESS("%s"," **gzip failed!\n");
00273                        unlink(qname) ; return( -1 );   }
00274 
00275       cfile = fopen( qname , "rb" ) ;
00276       if( cfile == NULL ){ DMESS("%s"," **gzip failed!\n");
00277                            unlink(qname) ; return( -1 );   }
00278       buf = AFMALL(char, nuse) ;
00279       fread( buf , 1 , nuse , cfile ) ;             /* read file in */
00280       fclose(cfile) ; unlink(qname) ;
00281    }
00282 
00283    /* data is in buf, nuse bytes of it */
00284 
00285    DMESS("%s","\n"); *data = buf ; return( nuse );
00286 }
00287 
00288 /*---------------------------------------------------------------------*/
00289 
00290 static char ftp_name[128] = "anonymous" ;
00291 static char ftp_pwd[128]  = "AFNI@nowhere.org" ;
00292 
00293 void set_URL_ftp_ident( char * name , char * pwd )  /* 05 Apr 1999 */
00294 {
00295    int ll ;
00296 
00297    if( name == NULL || pwd == NULL ) return ;
00298 
00299    ll = strlen(name) ; if( ll < 1 || ll > 127 ) return ;
00300    ll = strlen(pwd)  ; if( ll < 1 || ll > 127 ) return ;
00301 
00302    strcpy(ftp_name,name) ; strcpy(ftp_pwd,pwd) ; return ;
00303 }
00304 
00305 /*---------------------------------------------------------------------
00306   Reads an "ftp://" URL, similarly to read_URL_http above;
00307   however, staging is always done through a temporary file.
00308 -----------------------------------------------------------------------*/
00309 
00310 int read_URL_ftp( char * url , char ** data )
00311 {
00312    char * s, * h , * file , qname[256] , sname[256] , * cpt , * buf ;
00313    char hostname[256] ;
00314    int port , ii , cflag , nuse ;
00315    FILE * sp ;
00316 
00317    /* sanity check */
00318 
00319    if( url == NULL || data == NULL || strstr(url,FTP) != url ) return( -1 );
00320 
00321    /* parse hostname */
00322 
00323    for( s=url+FTPLEN , h=hostname ;
00324         (*s != '\0') && (*s != ':') && (*s != '/') ; s++ , h++ ) *h = *s ;
00325 
00326    *h = '\0' ; if( hostname[0] == '\0' ) return( -1 );
00327 
00328    /* parse port number, if present */
00329 
00330    port = 0 ;
00331    if( *s == ':' ){ port = strtol( ++s , &h , 10 ) ; s = h ; }
00332 
00333    /* get the file name (strip off leading "/") */
00334 
00335    if( *s == '/' ){
00336       file = s+1 ; if( file[0] == '\0' ) return( -1 );
00337    } else {
00338                                          return( -1 );
00339    }
00340 
00341    /* check if file will be returned gzip-ed */
00342 
00343    ii = strlen(file) ;
00344    if( ii > 3 ){
00345       cpt = file + (ii-3) ; cflag = (strcmp(cpt,".gz") == 0) ;
00346    } else {
00347       cflag = 0 ;
00348    }
00349 
00350    /* make name for output file */
00351 
00352    setup_tmpdir() ;
00353    strcpy(qname,tmpdir) ; strcat(qname,"elvisXXXXXX") ;
00354    mktemp(qname) ;
00355    if( qname[0] == '\0' ) return( -1 );
00356    if( cflag ) strcat(qname,".gz") ;
00357 
00358    /* write the script file that will be used to run ftp */
00359 
00360    strcpy(sname,tmpdir) ; strcat(sname,"dahmerXXXXXX") ;
00361    mktemp(sname) ;             if( sname[0] == '\0' ) return( -1 );
00362    sp = fopen( sname , "w" ) ; if( sp == NULL )       return( -1 );
00363 
00364    fprintf( sp , "#!/bin/sh\n" ) ;
00365    fprintf( sp , "ftp -n << EEEEE &> /dev/null\n") ;
00366    if( port > 0 )
00367       fprintf( sp , "open %s %d\n" , hostname , port ) ;
00368    else
00369       fprintf( sp , "open %s\n" , hostname ) ;
00370    fprintf( sp , "user %s %s\n" , ftp_name, ftp_pwd ) ;
00371    fprintf( sp , "binary\n" ) ;
00372    fprintf( sp , "get %s %s\n" , file , qname ) ;
00373    fprintf( sp , "bye\n" ) ;
00374    fprintf( sp , "EEEEE\n" ) ;
00375    fprintf( sp , "exit\n" ) ;
00376    fclose( sp ) ;
00377    chmod( sname , S_IRUSR | S_IWUSR | S_IXUSR ) ;
00378 
00379    /* execute the script, then delete it */
00380 
00381    system( sname ) ; unlink( sname ) ;
00382 
00383    /* check the size of the output file */
00384 
00385    nuse = THD_filesize( qname ) ;
00386    if( nuse <= 0 ){ unlink(qname) ; return( -1 ); }
00387 
00388    /* uncompress the file, if needed */
00389 
00390    if( cflag ){
00391       sprintf( sname , "gzip -dq %s" , qname ) ;    /* execute gzip */
00392       ii = system(sname) ;
00393       if( ii != 0 ){ unlink(qname) ; return( -1 ); }  /* gzip failed  */
00394       ii = strlen(qname) ; qname[ii-3] = '\0' ;     /* fix filename */
00395       nuse = THD_filesize( qname ) ;                /* find how big */
00396       if( nuse <= 0 ){ unlink(qname) ; return( -1 ); }
00397    }
00398 
00399    /* suck the file into memory */
00400 
00401    sp = fopen( qname , "rb" ) ;
00402    if( sp == NULL ){ unlink(qname) ; return( -1 ); }
00403    buf = AFMALL(char,nuse) ; if( buf == NULL ){ unlink(qname) ; return( -1 ); }
00404 
00405    fread( buf , 1 , nuse , sp ) ;  /* AT LAST! */
00406    fclose(sp) ; unlink(qname) ;
00407 
00408    /* data is in buf, nuse bytes of it */
00409 
00410    *data = buf ; return( nuse );
00411 }
00412 
00413 /*-------------------------------------------------------------------
00414    Read a URL (ftp:// or http://) into memory.  The return value
00415    is the number of bytes read, and *data points to the data.
00416    If the return value is negative, then something bad happened.
00417 ---------------------------------------------------------------------*/
00418 
00419 int read_URL( char * url , char ** data )
00420 {
00421    int nn ;
00422    if( url == NULL || data == NULL ) return( -1 );
00423 
00424    if( getenv("AFNI_WWW_DEBUG") != NULL ) debug = 1 ;
00425 
00426    if( strstr(url,HTTP) == url ){
00427       nn = read_URL_http( url , 4000 , data ) ; return(nn) ;
00428    }
00429 
00430    else if( strstr(url,FTP) == url ){
00431       nn = read_URL_ftp( url , data ) ; return(nn) ;
00432    }
00433 
00434    return( -1 );
00435 }
00436 
00437 /*------------------------------------------------------------------
00438   Read a URL and save it to disk in tmpdir.  The filename
00439   it is saved in is returned in the malloc-ed space *tname.
00440   The byte count is the return value of the function;
00441   if <= 0, then an error transpired (and *tname is not set).
00442 --------------------------------------------------------------------*/
00443 
00444 extern char * THD_trailname( char * fname , int lev ) ;
00445 
00446 int read_URL_tmpdir( char * url , char ** tname )
00447 {
00448    int nn , ll ;
00449    char * data , * fname , * tt ;
00450    FILE * fp ;
00451 
00452    if( url == NULL || tname == NULL ) return( -1 );
00453 
00454    nn = read_URL( url , &data ) ;  /* get the data into memory */
00455    if( nn <= 0 ) return( -1 );       /* bad */
00456 
00457    /* make the output filename */
00458 
00459    setup_tmpdir() ;
00460    fname = AFMALL(char, strlen(url)+strlen(tmpdir)+1) ;
00461    tt    = THD_trailname(url,0) ;
00462    strcpy(fname,tmpdir) ; strcat(fname,tt) ; ll = strlen(fname) ;
00463    if( ll > 3 && strcmp(fname+(ll-3),".gz") == 0 ) fname[ll-3] = '\0' ;
00464 
00465    /* open and write output */
00466 
00467    fp = fopen( fname , "wb" ) ;
00468    if( fp == NULL ){
00469       fprintf(stderr,"** Can't open temporary file %s\n",fname);
00470       free(data) ; return( -1 );
00471    }
00472    ll = fwrite(data,1,nn,fp) ; fclose(fp) ; free(data) ;
00473    if( ll != nn ){ unlink(fname); return( -1 ); } /* write failed */
00474 
00475    *tname = fname ; return( nn );
00476 }