static char rcsid[] = "ftpget.c,v 1.18.6.22 1996/01/20 07:50:01 duane Exp";
/* 
 *  ftpget.c - An FTP client used by liburl and cached.
 *
 *  Duane Wessels, University of Colorado, September 1995
 *
 *  DEBUG: section  26, level 1           ftpget - standalone liburl program.
 *
 *  ----------------------------------------------------------------------
 *  Copyright (c) 1994, 1995.  All rights reserved.
 *  
 *    The Harvest software was developed by the Internet Research Task
 *    Force Research Group on Resource Discovery (IRTF-RD):
 *  
 *          Mic Bowman of Transarc Corporation.
 *          Peter Danzig of the University of Southern California.
 *          Darren R. Hardy of the University of Colorado at Boulder.
 *          Udi Manber of the University of Arizona.
 *          Michael F. Schwartz of the University of Colorado at Boulder.
 *          Duane Wessels of the University of Colorado at Boulder.
 *  
 *    This copyright notice applies to software in the Harvest
 *    ``src/'' directory only.  Users should consult the individual
 *    copyright notices in the ``components/'' subdirectories for
 *    copyright information about other software bundled with the
 *    Harvest source code distribution.
 *  
 *  TERMS OF USE
 *    
 *    The Harvest software may be used and re-distributed without
 *    charge, provided that the software origin and research team are
 *    cited in any use of the system.  Most commonly this is
 *    accomplished by including a link to the Harvest Home Page
 *    (http://harvest.cs.colorado.edu/) from the query page of any
 *    Broker you deploy, as well as in the query result pages.  These
 *    links are generated automatically by the standard Broker
 *    software distribution.
 *    
 *    The Harvest software is provided ``as is'', without express or
 *    implied warranty, and with no support nor obligation to assist
 *    in its use, correction, modification or enhancement.  We assume
 *    no liability with respect to the infringement of copyrights,
 *    trade secrets, or any patents, and are not responsible for
 *    consequential damages.  Proper use of the Harvest software is
 *    entirely the responsibility of the user.
 *  
 *  DERIVATIVE WORKS
 *  
 *    Users may make derivative works from the Harvest software, subject 
 *    to the following constraints:
 *  
 *      - You must include the above copyright notice and these 
 *        accompanying paragraphs in all forms of derivative works, 
 *        and any documentation and other materials related to such 
 *        distribution and use acknowledge that the software was 
 *        developed at the above institutions.
 *  
 *      - You must notify IRTF-RD regarding your distribution of 
 *        the derivative work.
 *  
 *      - You must clearly notify users that your are distributing 
 *        a modified version and not the original Harvest software.
 *  
 *      - Any derivative product is also subject to these copyright 
 *        and use restrictions.
 *  
 *    Note that the Harvest software is NOT in the public domain.  We
 *    retain copyright, as specified above.
 *  
 *  HISTORY OF FREE SOFTWARE STATUS
 *  
 *    Originally we required sites to license the software in cases
 *    where they were going to build commercial products/services
 *    around Harvest.  In June 1995 we changed this policy.  We now
 *    allow people to use the core Harvest software (the code found in
 *    the Harvest ``src/'' directory) for free.  We made this change
 *    in the interest of encouraging the widest possible deployment of
 *    the technology.  The Harvest software is really a reference
 *    implementation of a set of protocols and formats, some of which
 *    we intend to standardize.  We encourage commercial
 *    re-implementations of code complying to this set of standards.  
 *  
 */

/*
 *    NOTES
 *      
 *      Directories vs. Files
 *      ---------------------
 *      
 *      Given 'ftp://foo/dir/bar' how do we know if 'bar' is a file or
 *      directory?  We can't assume anything from the URL, because people
 *      often leave trailing slashes off of directory URLs.  
 *      What we do here is try it one way first, and then try it the
 *      other way.
 *      
 *      Initially we tried CWD first, and RETR second.  This seemed to
 *      work pretty well.  If the last pathname component didn't exist
 *      at all (ie both CWD and RETR fail) then we got back a "No such
 *      file or directory" message from the ftpd.
 *      
 *      Then we tried ftp.microsoft.com (a Windows/NT) server.  On this
 *      "ftpd" you could successfully CWD to a file.  eg:
 *      
 *              ftp -d ftp.microsoft.com
 *              ...
 *              ftp> cd index.txt
 *              ---> CWD index.txt
 *              250 CWD command successful.
 *      
 *      So then we tried RETR before CWD.  Gets around the Microsoft bug,
 *      but this can cause incorrect error messages when a file is not
 *      readable.  Something like this happens:
 *      
 *              --> RETR passwd
 *              <-- 550 passwd: Permission denied.
 *              --> CWD passwd
 *              <-- 550 passwd: Not a directory.
 *      
 *      And the user ends up viewing the "Not a directory." message
 *      which is misleading and confusing.  Sigh.  The CERN httpd
 *      does this too.
 *      
 *      
 *      PASV vs. PORT
 *      -------------
 *      
 *      As of v1.4 pl1 we try to use PASV to make the data connection
 *      before using PORT.  PASV is more firewall-friendly.  At least
 *      one ftpd has been found which does not support PASV:
 *      ftp.isoc.org.  Because we fall back to PORT, users behind 
 *      firewalls may be confused as to why that site would fail.
 *      
 *      Use of HTML BASE tag
 *      --------------------
 *      
 *      Sometimes we have to use the HTML BASE tag to "force" the
 *      current URL to be a directory.  [[ we could do like httpd's
 *      do.  When they get requests for directories w/o the trailing
 *      slash, they return redirects with the slash so the browser
 *      can properly grok relative URLs. ]]  This is especially a
 *      problem for us with symbolic links.  We assume symbolic
 *      links point to files, not directories, so when a symlink
 *      really points to a directory, the URL we generate
 *      will be wrong.
 *      
 *      We insert the BASE tag when we know its a directory, but
 *      the given url-path didn't end with a slash.
 *      
 */


#include <stdlib.h>
#include <fcntl.h>
#include <errno.h>
#include <string.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>

#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>

#include "config.h"
#include "util.h"

char *rfc1738_escape();
char *http_time();

typedef struct _ext_table_entry {
    char *name;
    char *mime_type;
    char *mime_encoding;
    char *icon;
} ext_table_entry;

#include "mime_table.h"

#define FIELDSIZE 32

#define FTP_PORT 21

#define DEFAULT_MIME_TYPE "text/plain"

#define F_HTTPIFY	0x01
#define F_HDRSENT	0x02
#define F_ISDIR		0x04
#define F_NOERRS	0x08
#define F_TRYDIR	0x10
#define F_NEEDACCEPT	0x20
#define F_USEBASE	0x40

typedef enum {
    BEGIN,
    PARSE_OK,
    CONNECTED,
    SERVICE_READY,
    NEED_PASSWD,
    LOGGED_IN,
    TYPE_OK,
    MDTM_OK,
    SIZE_OK,
    PORT_OK,
    PASV_OK,
    PASV_FAIL,
    CWD_OK,
#ifdef TRY_CWD_FIRST
    CWD_FAIL,
#else
    RETR_FAIL,
#endif
    TRANSFER_BEGIN,
    DATA_TRANSFER,
    TRANSFER_DONE,
    DONE,
    FAIL_SOFT,			/* don't cache these */
    FAIL_HARD			/* do cache these */
} state_t;

typedef struct _request {
    char *host;
    char *path;
    char *type;
    char *user;
    char *pass;
    char *url_escaped;
    char *title_url;
    char *url;
    int cfd;
    int sfd;
    int dfd;
    int connect_attempts;
    int login_attempts;
    state_t state;
    int rc;
    char *errmsg;
    time_t mdtm;
    int size;
    int flags;
    char *mime_type;
    char *mime_enc;
    char *html_icon;
    FILE *readme_fp;
    struct _list_t *cmd_msg;
} request_t;

typedef struct _parts {
    char type;
    int size;
    char *date;
    char *name;
    char *showname;
    char *link;
} parts_t;

typedef struct _list_t {
    char *ptr;
    struct _list_t *next;
} list_t;

/*
 *  GLOBALS
 */
int connect_retries = 1;
int login_retries = 1;
char *progname = NULL;
char cbuf[BUFSIZ];		/* send command buffer */
char *htmlbuf = NULL;
char *server_reply_msg = NULL;
struct sockaddr_in ifc_addr;
int timeout = XFER_TIMEOUT;	/* 120, from config.h */
static time_t last_alarm_set = 0;

/* This linked list holds the "continuation" lines before the final
 * reply code line is sent for a FTP command */
list_t *cmd_msg = NULL;

static int process_request _PARAMS((request_t *));

static char *state_str[] =
{
    "BEGIN",
    "PARSE_OK",
    "CONNECTED",
    "SERVICE_READY",
    "NEED_PASSWD",
    "LOGGED_IN",
    "TYPE_OK",
    "MDTM_OK",
    "SIZE_OK",
    "PORT_OK",
    "PASV_OK",
    "PASV_FAIL",
    "CWD_OK",
#ifdef TRY_CWD_FIRST
    "CWD_FAIL",
#else
    "RETR_FAIL",
#endif
    "TRANSFER_BEGIN",
    "DATA_TRANSFER",
    "TRANSFER_DONE",
    "DONE",
    "FAIL_SOFT",
    "FAIL_HARD",
};

/* 
 *  CACHED_RETRIEVE_ERROR_MSG args: 
 *      $1 is URL, 
 *      $2 is URL, 
 *      $3 is protocol type string
 *      $4 is error code, 
 *      $5 is error msg, 
 *      $6 is message to user
 *      $7 is time string
 *      $8 is cached version
 *      $9 is cached hostname
 */

#define CACHED_RETRIEVE_ERROR_MSG "\
<HTML><HEAD>\n\
<TITLE>ERROR: The requested URL could not be retrieved</TITLE>\n\
</HEAD><BODY>\n\
<H2>ERROR: The requested URL could not be retrieved</H2>\n\
<HR>\n\
<P>\n\
While trying to retrieve the URL:\n\
<A HREF=\"%s\">%s</A>\n\
<P>\n\
The following %s error was encountered:\n\
<UL>\n\
<LI><STRONG>ERROR %d -- %s</STRONG>\n\
</UL>\n\
<P>This means that:\n\
<PRE>\n\
    %s\n\
</PRE>\n\
<P>\n\
\n"

char *html_trailer()
{
    static char buf[1024];

    sprintf(buf, "<HR><ADDRESS>\nGenerated %s, by ftpget/%s@%s\n</ADDRESS>\n </BODY></HTML>\n", http_time(NULL), HARVEST_VERSION, getfullhostname());
    return buf;
}

void fail(r)
     request_t *r;
{
    FILE *fp = NULL;
    char *longmsg = NULL;
    time_t expire_time;
    list_t *l = NULL;

    if (r->flags & F_NOERRS)
	return;

    switch (r->rc) {
    case 0:
	longmsg = "Success!  Huh?";
	break;
    case 2:
	longmsg = "A local socket error occured.  Please try again.";
	break;
    case 3:
	longmsg = "A network socket error occured.  Please try again.";
	break;
    case 4:
	longmsg = "A network read or write error occured.  Please try again.";
	break;
    case 5:
	longmsg = "An FTP protocol error occured.  Please try again.";
	break;
    case 10:
	longmsg = "The given URL does not exist, or is not readable.";
	break;
    default:
	break;
    }

    if ((r->flags & F_HTTPIFY) && !(r->flags & F_HDRSENT)) {
	Debug(26, 1, ("Preparing HTML error message\n"));
	expire_time = time(NULL) + 300;		/* XXX hardcoded 5 min */
	htmlbuf = (char *) xmalloc(8192);
	sprintf(htmlbuf, CACHED_RETRIEVE_ERROR_MSG,
	    r->title_url,
	    r->title_url,
	    "FTP",
	    304,
	    r->errmsg,
	    longmsg);
	if ((fp = fdopen(dup(r->cfd), "w")) == NULL) {
	    log_errno2(__FILE__, __LINE__, "fdopen");
	    exit(1);
	}
	setbuf(fp, NULL);
	fprintf(fp, "HTTP/1.0 500 Proxy Error\r\n");
	fprintf(fp, "Expires: %s\r\n", mkrfc850(&expire_time));
	fprintf(fp, "MIME-Version: 1.0\r\n");
	fprintf(fp, "Server: Harvest %s\r\n", HARVEST_VERSION);
	fprintf(fp, "Content-Type: text/html\r\n");
	fprintf(fp, "Content-Length: %d\r\n", (int) strlen(htmlbuf));
	fprintf(fp, "\r\n");
	fputs(htmlbuf, fp);
	if (cmd_msg) {
	    fprintf(fp, "<HR><H4>Remote server replied with:</H4>\n");
	    fprintf(fp, "<PRE>\n");
	    for (l = cmd_msg; l; l = l->next)
		fputs(l->ptr, fp);
	    fprintf(fp, "</PRE>\n");
	}
	fputs(html_trailer(), fp);
	fclose(fp);
    } else if (r->flags & F_HTTPIFY) {
	if ((fp = fdopen(dup(r->cfd), "w")) == NULL) {
	    log_errno2(__FILE__, __LINE__, "fdopen");
	    exit(1);
	}
	setbuf(fp, NULL);
	htmlbuf = (char *) xmalloc(8192);
	sprintf(htmlbuf, CACHED_RETRIEVE_ERROR_MSG,
	    r->title_url,
	    r->title_url,
	    "FTP",
	    304,
	    r->errmsg,
	    longmsg);
	fputs(htmlbuf, fp);
	xfree(r->errmsg);
	if (cmd_msg) {
	    fprintf(fp, "<HR><H4>Remote server replied with:</H4>\n");
	    fprintf(fp, "<PRE>\n");
	    for (l = cmd_msg; l; l = l->next)
		fputs(l->ptr, fp);
	    fprintf(fp, "</PRE>\n");
	}
	fputs(html_trailer(), fp);
	fclose(fp);
    } else if (r->errmsg) {
	errorlog("%s\n\t<URL:%s>\n", r->errmsg, r->url);
	xfree(r->errmsg);
    }
}

void generic_sig_handler(sig)
     int sig;
{
    errorlog("Received signal %d, exiting.\n", sig);
    exit(1);
}

void timeout_handler()
{
    time_t now;

    now = time(NULL);
    errorlog("Timeout after %d seconds, exiting.\n", now - last_alarm_set);
    exit(1);
}

void reset_timeout()
{
    last_alarm_set = time(NULL);
    alarm(timeout);
}


/*
 *  If there are two extensions and both are listed in the types table
 *  then return the leftmost extention type.  The rightmost extention
 *  type becomes the content encoding (eg .gz)
 */
void mime_get_type(r)
     request_t *r;
{
    char *filename = NULL;
    char *ext = NULL;
    char *t = NULL;
    char *type = NULL;
    char *enc = NULL;
    int i;

    if (r->flags & F_ISDIR) {
	r->mime_type = xstrdup("text/html");
	return;
    }
    type = DEFAULT_MIME_TYPE;

    if ((t = strrchr(r->path, '/')))
	filename = xstrdup(t + 1);
    else
	filename = xstrdup(r->path);

    if (!(t = strrchr(filename, '.')))
	goto mime_get_type_done;
    ext = xstrdup(t + 1);
    for (i = 0; i < EXT_TABLE_LEN; i++) {
	if (!strcmp(ext, ext_mime_table[i].name)) {
	    type = ext_mime_table[i].mime_type;
	    enc = ext_mime_table[i].mime_encoding;
	    break;
	}
    }
    if (i == EXT_TABLE_LEN) {
	for (i = 0; i < EXT_TABLE_LEN; i++) {
	    if (!strcasecmp(ext, ext_mime_table[i].name)) {
		type = ext_mime_table[i].mime_type;
		enc = ext_mime_table[i].mime_encoding;
		break;
	    }
	}
    }
    /* now check for another extension */

    *t = '\0';
    if (!(t = strrchr(filename, '.')))
	goto mime_get_type_done;
    xfree(ext);
    ext = xstrdup(t + 1);
    for (i = 0; i < EXT_TABLE_LEN; i++) {
	if (!strcmp(ext, ext_mime_table[i].name)) {
	    type = ext_mime_table[i].mime_type;
	    break;
	}
    }
    if (i == EXT_TABLE_LEN) {
	for (i = 0; i < EXT_TABLE_LEN; i++) {
	    if (!strcasecmp(ext, ext_mime_table[i].name)) {
		type = ext_mime_table[i].mime_type;
		break;
	    }
	}
    }
  mime_get_type_done:
    xfree(filename);
    xfree(ext);
    r->mime_type = xstrdup(type);
    if (enc)
	r->mime_enc = xstrdup(enc);
}

char *mime_get_icon(name)
     char *name;
{
    char *ext = NULL;
    char *t = NULL;
    int i = 0;

    if (!(t = strrchr(name, '.')))
	return xstrdup("unknown");
    ext = xstrdup(t + 1);
    Debug(26, 1, ("mime_get_icon: ext = '%s'\n", ext));
    for (i = 0; i < EXT_TABLE_LEN; i++) {
	if (!strcmp(ext, ext_mime_table[i].name)) {
	    Debug(26, 1, ("mime_get_icon: matched entry #%d\n", i));
	    Debug(26, 1, ("mime_get_icon: returning '%s'\n",
		    ext_mime_table[i].icon));
	    xfree(ext);
	    return xstrdup(ext_mime_table[i].icon);
	    /* NOTREACHED */
	}
    }
    if (i == EXT_TABLE_LEN) {
	for (i = 0; i < EXT_TABLE_LEN; i++) {
	    if (!strcasecmp(ext, ext_mime_table[i].name)) {
		Debug(26, 1, ("mime_get_icon: matched entry #%d\n", i));
		Debug(26, 1, ("mime_get_icon: returning '%s'\n",
			ext_mime_table[i].icon));
		xfree(ext);
		return xstrdup(ext_mime_table[i].icon);
		/* NOTREACHED */
	    }
	}
    }
    return xstrdup("unknown");
}

char *http_time(t)
     time_t t;
{
    struct tm *gmt;
    time_t when;
    static char tbuf[128];

    when = t ? t : time(NULL);
    gmt = gmtime(&when);
    strftime(tbuf, 128, "%A, %d-%b-%y %H:%M:%S GMT", gmt);
    return tbuf;
}

void send_success_hdr(r)
     request_t *r;
{
    FILE *fp = NULL;

    if (r->flags & F_HDRSENT)
	return;

    r->flags |= F_HDRSENT;

    mime_get_type(r);

    if ((fp = fdopen(dup(r->cfd), "w")) == NULL) {
	log_errno2(__FILE__, __LINE__, "fdopen");
	exit(1);
    }
    setbuf(fp, NULL);
    fprintf(fp, "HTTP/1.0 200 Gatewaying\r\n");
    fprintf(fp, "MIME-Version: 1.0\r\n");
    fprintf(fp, "Server: Harvest %s\r\n", HARVEST_VERSION);
    if (r->mime_type)
	fprintf(fp, "Content-Type: %s\r\n", r->mime_type);
    if (r->size > 0)
	fprintf(fp, "Content-Length: %d\r\n", r->size);
    if (r->mime_enc)
	fprintf(fp, "Content-Encoding: %s\r\n", r->mime_enc);
    if (r->mdtm > 0)
	fprintf(fp, "Last-Modified: %s\r\n", http_time(r->mdtm));
    fprintf(fp, "\r\n");
    fclose(fp);
}

/*
 *  read_reply()
 *  Read reply strings from an FTP server.
 * 
 *  Returns the reply code.
 */
int read_reply(fd)
     int fd;
{
    FILE *fp = NULL;
    static char buf[BUFSIZ];
    static char xbuf[BUFSIZ];
    int quit = 0;
    char *t = NULL;
    int code;
    list_t **Tail = NULL;
    list_t *l = NULL;
    list_t *next = NULL;

    for (l = cmd_msg; l; l = next) {
	next = l->next;
	xfree(l->ptr);
	xfree(l);
    }
    cmd_msg = NULL;
    Tail = &cmd_msg;

    if (server_reply_msg) {
	xfree(server_reply_msg);
	server_reply_msg = NULL;
    }
    if ((fp = fdopen(dup(fd), "r")) == (FILE *) NULL) {
	log_errno2(__FILE__, __LINE__, "fdopen");
	exit(1);
    }
    setbuf(fp, NULL);

    while (!quit) {
	if (fgets(buf, BUFSIZ, fp) == (char *) NULL) {
	    reset_timeout();
	    sprintf(xbuf, "read failed: %s", xstrerror());
	    server_reply_msg = xstrdup(xbuf);
	    fclose(fp);
	    return -1;
	}
	quit = (buf[2] >= '0' && buf[2] <= '9' && buf[3] == ' ');
	if (!quit) {
	    l = (list_t *) xmalloc(sizeof(list_t));
	    l->ptr = xstrdup(&buf[4]);
	    l->next = NULL;
	    *Tail = l;
	    Tail = &(l->next);
	}
	if ((t = strchr(buf, '\r')))
	    *t = 0;
	if ((t = strchr(buf, '\n')))
	    *t = 0;
	Debug(26, 1, ("read_reply: %s\n", buf));
    }
    fclose(fp);
    code = atoi(buf);
    server_reply_msg = xstrdup(&buf[4]);
    return code;
}

/*
 *  send_cmd()
 *  Write a command string
 * 
 *  Returns # bytes written
 */
int send_cmd(fd, buf)
     int fd;
     char *buf;
{
    char *xbuf = NULL;
    int len;
    int x;

    len = strlen(buf) + 2;
    xbuf = (char *) xmalloc(len + 1);
    sprintf(xbuf, "%s\r\n", buf);
    Debug(26, 1, ("send_cmd: %s\n", buf));
    x = write(fd, xbuf, len);
    if (x < 0)
	log_errno2(__FILE__, __LINE__, "write");
    reset_timeout();
    xfree(xbuf);
    return x;
}


#define ASCII_DIGIT(c) ((c)-48)
time_t parse_iso3307_time(buf)
     char *buf;
{
/* buf is an ISO 3307 style time: YYYYMMDDHHMMSS or YYYYMMDDHHMMSS.xxx */
    struct tm tms;
    time_t t;

    while (*buf == ' ' || *buf == '\t')
	buf++;
    if ((int) strlen(buf) < 14)
	return 0;
    memset(&tms, '\0', sizeof(struct tm));
    tms.tm_year = (ASCII_DIGIT(buf[2]) * 10) + ASCII_DIGIT(buf[3]);
    tms.tm_mon = (ASCII_DIGIT(buf[4]) * 10) + ASCII_DIGIT(buf[5]) - 1;
    tms.tm_mday = (ASCII_DIGIT(buf[6]) * 10) + ASCII_DIGIT(buf[7]);
    tms.tm_hour = (ASCII_DIGIT(buf[8]) * 10) + ASCII_DIGIT(buf[9]);
    tms.tm_min = (ASCII_DIGIT(buf[10]) * 10) + ASCII_DIGIT(buf[11]);
    tms.tm_sec = (ASCII_DIGIT(buf[12]) * 10) + ASCII_DIGIT(buf[13]);

#ifdef HAVE_TIMEGM
    t = timegm(&tms);
#elif defined(_HARVEST_SYSV_) || defined(_HARVEST_LINUX_) || defined(_HARVEST_HPUX_) || defined(_HARVEST_AIX_)
    t = mktime(&tms);
#else
    t = (time_t) 0;
#endif

    Debug(26, 1, ("parse_iso3307_time: %d\n", t));
    return t;
}
#undef ASCII_DIGIT

#define SEND_CBUF \
        if (send_cmd(r->sfd, cbuf) < 0) { \
                r->errmsg = (char *) xmalloc (BUFSIZ); \
                sprintf(r->errmsg, "Failed to send '%s'", cbuf); \
                r->rc = 4; \
                return FAIL_SOFT; \
        }

/*
 *  parse_request()
 *  Perform validity checks on request parameters.
 *    - lookup hostname
 * 
 *  Returns states:
 *    FAIL_HARD
 *    PARSE_OK
 */
state_t parse_request(r)
     request_t *r;
{
    Debug(26, 1, ("parse_request: looking up '%s'\n", r->host));
    if (get_host(r->host) == (Host *) NULL) {
	r->errmsg = (char *) xmalloc(BUFSIZ);
	sprintf(r->errmsg, "Unknown host: %s", r->host);
	r->rc = 10;
	return FAIL_HARD;
    }
    return PARSE_OK;
}

/*
 *  do_connect()
 *  Connect to the FTP server r->host on port 21.
 * 
 *  Returns states:
 *    FAIL_SOFT
 *    CONNECTED
 *    r->state        ( retry )
 */
state_t do_connect(r)
     request_t *r;
{
    Host *h = NULL;
    int sock;
    struct sockaddr_in S;
    int len;

    ++r->connect_attempts;
    Debug(26, 1, ("do_connect: connect attempt #%d to '%s'\n",
	    r->connect_attempts, r->host));
    if (r->connect_attempts > connect_retries) {
	r->errmsg = (char *) xmalloc(BUFSIZ);
	sprintf(r->errmsg, "%s: Giving up after %d connect attempts",
	    r->host, connect_retries);
	r->rc = 3;
	return FAIL_SOFT;
    }
    if ((sock = socket(PF_INET, SOCK_STREAM, 0)) < 0) {
	r->errmsg = (char *) xmalloc(BUFSIZ);
	sprintf(r->errmsg, "socket: %s", xstrerror());
	r->rc = 2;
	return FAIL_SOFT;
    }
    h = get_host(r->host);
    memcpy(&(S.sin_addr.s_addr), h->ipaddr, h->addrlen);
    S.sin_family = AF_INET;
    S.sin_port = htons(FTP_PORT);

    if (connect(sock, (struct sockaddr *) &S, sizeof(S)) < 0) {
	r->errmsg = (char *) xmalloc(BUFSIZ);
	sprintf(r->errmsg, "%s (port %d): %s",
	    r->host, FTP_PORT, xstrerror());
	r->rc = 3;
	return r->state;
    }
    r->sfd = sock;
    reset_timeout();

    /* get the address of whatever interface we're using so we know */
    /* what to use in the PORT command.                             */
    len = sizeof(ifc_addr);
    if (getsockname(sock, (struct sockaddr *) &ifc_addr, &len) < 0) {
	log_errno2(__FILE__, __LINE__, "getsockname");
	exit(1);
    }
    return CONNECTED;
}

/*
 *  read_welcome()
 *  Parse the ``welcome'' message from the FTP server
 * 
 *  Returns states:
 *    FAIL_SOFT
 *    SERVICE_READY
 *    PARSE_OK        ( retry )
 */
state_t read_welcome(r)
     request_t *r;
{
    int code;

    if ((code = read_reply(r->sfd)) > 0) {
	if (code == 220)
	    return SERVICE_READY;
    }
    close(r->sfd);
    r->sfd = -1;
    if (code < 0) {
	r->errmsg = xstrdup(server_reply_msg);
	r->rc = 4;
	return FAIL_SOFT;
    }
    if (r->connect_attempts >= connect_retries) {
	r->errmsg = xstrdup(server_reply_msg);
	r->rc = 5;
	return FAIL_SOFT;
    }
    return PARSE_OK;		/* retry */
}

/*
 *  do_user()
 *  Send the USER command to the FTP server
 * 
 *  Returns states:
 *    FAIL_SOFT
 *    LOGGED_IN
 *    NEED_PASSWD
 *    PARSE_OK        ( retry )
 */
state_t do_user(r)
     request_t *r;
{
    int code;

    r->login_attempts++;

    sprintf(cbuf, "USER %s", r->user);
    SEND_CBUF;

    if ((code = read_reply(r->sfd)) > 0) {
	if (code == 230)
	    return LOGGED_IN;
	if (code == 331)
	    return NEED_PASSWD;
    }
    close(r->sfd);
    r->sfd = -1;
    if (code < 0) {
	r->errmsg = xstrdup(server_reply_msg);
	r->rc = 4;
	return FAIL_SOFT;
    }
    if (r->login_attempts >= login_retries) {
	r->errmsg = xstrdup(server_reply_msg);
	r->rc = 5;
	return FAIL_SOFT;
    }
    return PARSE_OK;		/* retry */
}

/*
 *  do_passwd()
 *  Send the USER command to the FTP server
 * 
 *  Returns states:
 *    FAIL_SOFT
 *    LOGGED_IN
 *    PARSE_OK        ( retry )
 */
state_t do_passwd(r)
     request_t *r;
{
    int code;

    sprintf(cbuf, "PASS %s", r->pass);
    SEND_CBUF;

    if ((code = read_reply(r->sfd)) > 0) {
	if (code == 230)
	    return LOGGED_IN;
    }
    close(r->sfd);
    r->sfd = -1;
    if (code < 0) {
	r->errmsg = xstrdup(server_reply_msg);
	r->rc = 4;
	return FAIL_SOFT;
    }
    if (r->login_attempts >= login_retries) {
	r->errmsg = xstrdup(server_reply_msg);
	r->rc = 5;
	return FAIL_SOFT;
    }
    return PARSE_OK;		/* retry */
}

state_t do_type(r)
     request_t *r;
{
    int code;

    sprintf(cbuf, "TYPE %c", *(r->type));
    SEND_CBUF;

    if ((code = read_reply(r->sfd)) > 0) {
	if (code == 200)
	    return TYPE_OK;
    }
    r->errmsg = xstrdup(server_reply_msg);
    r->rc = code < 0 ? 4 : 5;
    return FAIL_SOFT;
}

state_t do_mdtm(r)
     request_t *r;
{
    int code;

    sprintf(cbuf, "MDTM %s", r->path);
    SEND_CBUF;

    if ((code = read_reply(r->sfd)) > 0) {
	if (code == 213)
	    r->mdtm = parse_iso3307_time(server_reply_msg);
    }
    if (code < 0) {
	r->errmsg = xstrdup(server_reply_msg);
	r->rc = 4;
	return FAIL_SOFT;
    }
    return MDTM_OK;
}

state_t do_size(r)
     request_t *r;
{
    int code;

    sprintf(cbuf, "SIZE %s", r->path);
    SEND_CBUF;

    if ((code = read_reply(r->sfd)) > 0) {
	if (code == 213)
	    r->size = atoi(server_reply_msg);
    }
    if (code < 0) {
	r->errmsg = xstrdup(server_reply_msg);
	r->rc = 4;
	return FAIL_SOFT;
    }
    return SIZE_OK;
}

state_t do_port(r)
     request_t *r;
{
    int code;
    int sock;
    struct sockaddr_in S;
    unsigned int naddr;
    int tries = 0;
    int port = 0;
    static int init = 0;

    if ((sock = socket(PF_INET, SOCK_STREAM, 0)) < 0) {
	r->errmsg = (char *) xmalloc(BUFSIZ);
	sprintf(r->errmsg, "socket: %s", xstrerror());
	r->rc = 2;
	return FAIL_SOFT;
    }
    S = ifc_addr;
    S.sin_family = AF_INET;

    if (!init) {
	init = 1;
#if defined(HAVE_SRAND48)
	srand48(time(NULL));
#else
	srand(time(NULL));
#endif
    }
    while (1) {
#if defined(HAVE_LRAND48)
	port = (lrand48() & 0x3FFF) | 0x4000;
#else
	port = (rand() & 0x3FFF) | 0x4000;
#endif
	S.sin_port = htons(port);
	if (bind(sock, (struct sockaddr *) &S, sizeof(S)) >= 0)
	    break;
	if (++tries < 10)
	    continue;
	r->errmsg = (char *) xmalloc(BUFSIZ);
	sprintf(r->errmsg, "bind: %s", xstrerror());
	r->rc = 2;
	return FAIL_SOFT;
    }

    if (listen(sock, 1) < 0) {
	r->errmsg = (char *) xmalloc(BUFSIZ);
	sprintf(r->errmsg, "listen: %s", xstrerror());
	r->rc = 2;
	return FAIL_SOFT;
    }
    naddr = ntohl(ifc_addr.sin_addr.s_addr);
    sprintf(cbuf, "PORT %d,%d,%d,%d,%d,%d",
	(naddr >> 24) & 0xFF,
	(naddr >> 16) & 0xFF,
	(naddr >> 8) & 0xFF,
	naddr & 0xFF,
	(port >> 8) & 0xFF,
	port & 0xFF);
    SEND_CBUF;

    if ((code = read_reply(r->sfd)) > 0) {
	if (code == 200) {
	    r->dfd = sock;
	    r->flags |= F_NEEDACCEPT;
	    return PORT_OK;
	}
    }
    r->errmsg = xstrdup(server_reply_msg);
    r->rc = code < 0 ? 4 : 5;
    return FAIL_SOFT;
}

state_t do_pasv(r)
     request_t *r;
{
    int code;
    int sock;
    struct sockaddr_in S;
    int port = 0;
    int n;
    int h1, h2, h3, h4;
    int p1, p2;
    static char junk[BUFSIZ];
    static int pasv_supported = 1;

    /* if PASV previously failed, don't even try it again.  Just return
     * PASV_FAIL and let the state machine fall back to using PORT */
    if (!pasv_supported)
	return PASV_FAIL;

    if ((sock = socket(PF_INET, SOCK_STREAM, 0)) < 0) {
	r->errmsg = (char *) xmalloc(BUFSIZ);
	sprintf(r->errmsg, "socket: %s", xstrerror());
	r->rc = 2;
	return FAIL_SOFT;
    }
    S = ifc_addr;
    S.sin_family = AF_INET;

    sprintf(cbuf, "PASV");
    SEND_CBUF;

    if ((code = read_reply(r->sfd)) != 227) {
	r->errmsg = xstrdup(server_reply_msg);
	r->rc = code < 0 ? 4 : 5;
	pasv_supported = 0;
	return PASV_FAIL;
    }
    /*  227 Entering Passive Mode (h1,h2,h3,h4,p1,p2).  */
    n = sscanf(server_reply_msg, "%[^(](%d,%d,%d,%d,%d,%d)",
	junk, &h1, &h2, &h3, &h4, &p1, &p2);
    if (n != 7) {
	r->errmsg = xstrdup(server_reply_msg);
	r->rc = 5;
	pasv_supported = 0;
	return PASV_FAIL;
    }
    sprintf(junk, "%d.%d.%d.%d", h1, h2, h3, h4);
    S.sin_addr.s_addr = inet_addr(junk);
    S.sin_port = htons((p1 << 8) + p2);

    if (connect(sock, (struct sockaddr *) &S, sizeof(S)) < 0) {
	r->errmsg = (char *) xmalloc(BUFSIZ);
	sprintf(r->errmsg, "%s, port %d: %s", junk, port, xstrerror());
	r->rc = 2;
	return FAIL_SOFT;
    }
    r->dfd = sock;
    return PORT_OK;
}

state_t do_cwd(r)
     request_t *r;
{
    int code;

    if (!strcmp(r->path, "."))
	return CWD_OK;

    sprintf(cbuf, "CWD %s", r->path);
    SEND_CBUF;

    if ((code = read_reply(r->sfd)) > 0) {
	if (code >= 200 && code < 300)
	    return CWD_OK;
#ifdef TRY_CWD_FIRST
	return CWD_FAIL;
#else
	r->errmsg = xstrdup(server_reply_msg);
	r->rc = 10;
	return FAIL_HARD;
#endif
    }
    r->errmsg = xstrdup(server_reply_msg);
    r->rc = code < 0 ? 4 : 5;
    return FAIL_SOFT;
}

state_t do_retr(r)
     request_t *r;
{
    int code;

    sprintf(cbuf, "RETR %s", r->path);
    SEND_CBUF;

    if ((code = read_reply(r->sfd)) > 0) {
	if (code >= 100 && code < 200)
	    return TRANSFER_BEGIN;
#ifdef TRY_CWD_FIRST
	if (code == 550) {
	    r->errmsg = xstrdup(server_reply_msg);
	    r->rc = 10;
	    return FAIL_HARD;
	}
#else
	if (r->dfd > 0)
	    close(r->dfd);
	if (r->flags & F_TRYDIR)
	    return RETR_FAIL;
	r->errmsg = xstrdup(server_reply_msg);
	r->rc = 10;
	return FAIL_HARD;
#endif
    }
    r->errmsg = xstrdup(server_reply_msg);
    r->rc = code < 0 ? 4 : 5;
    return FAIL_SOFT;
}

state_t do_list(r)
     request_t *r;
{
    int code;

    sprintf(cbuf, "LIST");
    SEND_CBUF;

    if ((code = read_reply(r->sfd)) > 0) {
	if (code == 125)
	    return TRANSFER_BEGIN;
	if (code == 150)
	    return TRANSFER_BEGIN;
	if (code == 450) {
	    r->errmsg = xstrdup(server_reply_msg);
	    r->rc = 10;
	    return FAIL_HARD;
	}
    }
    sprintf(cbuf, "NLST");
    SEND_CBUF;

    if ((code = read_reply(r->sfd)) > 0) {
	if (code == 125)
	    return TRANSFER_BEGIN;
	if (code == 150)
	    return TRANSFER_BEGIN;
	if (code == 450) {
	    r->errmsg = xstrdup(server_reply_msg);
	    r->rc = 10;
	    return FAIL_HARD;
	}
    }
    r->errmsg = xstrdup(server_reply_msg);
    r->rc = code < 0 ? 4 : 5;
    return FAIL_SOFT;
}

state_t do_accept(r)
     request_t *r;
{
    int sock;
    struct sockaddr S;
    int len;

    len = sizeof(S);
    memset(&S, '\0', len);
    if ((sock = accept(r->dfd, &S, &len)) < 0) {
	r->errmsg = (char *) xmalloc(BUFSIZ);
	sprintf(r->errmsg, "accept: %s", xstrerror());
	r->rc = 3;
	return FAIL_SOFT;
    }
    reset_timeout();
    close(r->dfd);
    r->dfd = sock;
    return DATA_TRANSFER;
}

state_t read_data(r)
     request_t *r;
{
    int code;
    int n;
    static char buf[8192];

    n = read(r->dfd, buf, 8192);
    reset_timeout();
    if (n < 0) {
	r->errmsg = (char *) xmalloc(BUFSIZ);
	sprintf(r->errmsg, "read: %s", xstrerror());
	r->rc = 4;
	return FAIL_SOFT;
    }
    if (n == 0) {
	if ((code = read_reply(r->sfd)) > 0) {
	    if (code == 226)
		return TRANSFER_DONE;
	}
	r->errmsg = xstrdup(server_reply_msg);
	r->rc = code < 0 ? 4 : 5;
	return FAIL_SOFT;
    }
    write(r->cfd, buf, n);
    return r->state;
}

static char *Month[] =
{
    "Jan", "Feb", "Mar", "Apr", "May", "Jun",
    "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
};

int is_month(buf)
     char *buf;
{
    int i;

    for (i = 0; i < 12; i++)
	if (!strcasecmp(buf, Month[i]))
	    return 1;
    return 0;
}

#define MAX_TOKENS 64

parts_t *parse_entry(buf)
     char *buf;
{
    parts_t *p = NULL;
    char *t = NULL;
    char *tokens[MAX_TOKENS];
    int i;
    int n;
    static char *WS = " \t\n";
    static char sbuf[128];
    char *xbuf = NULL;

    if (buf == NULL)
	return NULL;

    p = (parts_t *) xmalloc(sizeof(parts_t));
    memset(p, '\0', sizeof(parts_t));

    n = 0;
    xbuf = xstrdup(buf);
    for (t = strtok(xbuf, WS); t && n < MAX_TOKENS; t = strtok(NULL, WS))
	tokens[n++] = xstrdup(t);
    xfree(xbuf);

    /* locate the Month field */
    for (i = 3; i < n - 3; i++) {
	if (!is_month(tokens[i]))	/* Month */
	    continue;
	if (!sscanf(tokens[i - 1], "%[0-9]", sbuf))	/* Size */
	    continue;
	if (!sscanf(tokens[i + 1], "%[0-9]", sbuf))	/* Day */
	    continue;
	if (!sscanf(tokens[i + 2], "%[0-9:]", sbuf))	/* Yr | hh:mm */
	    continue;
	p->type = *tokens[0];
	p->size = atoi(tokens[i - 1]);
	sprintf(sbuf, "%s %2s %5s",
	    tokens[i], tokens[i + 1], tokens[i + 2]);
	if (!strstr(buf, sbuf))
	    sprintf(sbuf, "%s %2s %-5s",
		tokens[i], tokens[i + 1], tokens[i + 2]);
	if ((t = strstr(buf, sbuf))) {
	    p->date = xstrdup(sbuf);
	    t += strlen(sbuf);
	    while (strchr(WS, *t))
		t++;
	    p->name = xstrdup(t);
	    if ((t = strstr(p->name, " -> "))) {
		*t = '\0';
		p->link = xstrdup(t + 4);
	    }
	}
	break;
    }

    /* try it as a DOS listing */
    if (tokens[3] && p->name == NULL &&
	!sscanf(tokens[0], "%[-0-9]", sbuf) &&	/* 04-05-70 */
	!sscanf(tokens[1], "%[0-9:apm]", sbuf)) {	/* 09:33pm */
	if (!strcasecmp(tokens[2], "<dir>")) {
	    p->type = 'd';
	    sprintf(sbuf, "%s %s", tokens[0], tokens[1]);
	    p->date = xstrdup(sbuf);
	    p->name = xstrdup(tokens[3]);
	}
	p->type = '-';
	sprintf(sbuf, "%s %s", tokens[0], tokens[1]);
	p->date = xstrdup(sbuf);
	p->size = atoi(tokens[2]);
	p->name = xstrdup(tokens[3]);
    }
    for (i = 0; i < n; i++)
	xfree(tokens[i]);
    if (p->name == NULL) {
	xfree(p->date);
	xfree(p);
	p = NULL;
    }
    return p;
}

char *dots_fill(len)
     size_t len;
{
    static char buf[256];
    int i = 0;


    for (i = len; i < FIELDSIZE; i++)
	buf[i - len] = (i % 2) ? '.' : ' ';
    buf[i - len] = '\0';
    return buf;
}

char *htmlize_list_entry(line, r)
     char *line;
     request_t *r;
{
    char *link = NULL;
    char *icon = NULL;
    char *html = NULL;
    char *ename = NULL;
    parts_t *parts = NULL;

    link = (char *) xmalloc(2048);
    icon = (char *) xmalloc(2048);
    html = (char *) xmalloc(8192);

    /* check .. as special case */
    if (!strcmp(line, "..")) {
	sprintf(icon, "<IMG BORDER=0 SRC=\"%s\" ALT=\"%-6s\">",
	    "internal-gopher-menu", "[DIR]");
	sprintf(link, "<A HREF=\"%s\">%s</A>",
	    "../",
	    "Parent Directory");
	sprintf(html, "%s %s\n", icon, link);
	xfree(icon);
	xfree(link);
	return (html);
    }
    if ((parts = parse_entry(line)) == NULL) {
	sprintf(html, "%s\n", line);
	return html;
    }
    if (!strcmp(parts->name, ".") || !strcmp(parts->name, "..")) {
	/* sprintf(html, "<!-- %s -->\n", line); */
	*html = '\0';
	return html;
    }
    parts->size += 1023;
    parts->size >>= 10;
    parts->showname = xstrdup(parts->name);
    if ((int) strlen(parts->showname) > FIELDSIZE - 1) {
	*(parts->showname + FIELDSIZE - 1) = '>';
	*(parts->showname + FIELDSIZE - 0) = '\0';
    }
    ename = xstrdup(rfc1738_escape(parts->name));
    switch (parts->type) {
    case 'd':
	sprintf(icon, "<IMG SRC=\"internal-gopher-%s\" ALT=\"%-6s\">",
	    "menu", "[DIR]");
	sprintf(link, "<A HREF=\"%s/\">%s</A>%s",
	    ename,
	    parts->showname,
	    dots_fill(strlen(parts->showname)));
	sprintf(html, "%s %s  [%s]\n",
	    icon,
	    link,
	    parts->date);
	break;
    case 'l':
	sprintf(icon, "<IMG SRC=\"internal-gopher-%s\" ALT=\"%-6s\">",
	    mime_get_icon(parts->link), "[LINK]");
	sprintf(link, "<A HREF=\"%s\">%s</A>%s",
	    ename,
	    parts->showname,
	    dots_fill(strlen(parts->showname)));
	sprintf(html, "%s %s  [%s]\n",
	    icon,
	    link,
	    parts->date);
	break;
    case '-':
    default:
	sprintf(icon, "<IMG SRC=\"internal-gopher-%s\" ALT=\"%-6s\">",
	    mime_get_icon(parts->name), "[FILE]");
	sprintf(link, "<A HREF=\"%s\">%s</A>%s",
	    ename,
	    parts->showname,
	    dots_fill(strlen(parts->showname)));
	sprintf(html, "%s %s  [%s] %6dk\n",
	    icon,
	    link,
	    parts->date,
	    parts->size);
	break;
    }

    xfree(parts->name);
    xfree(parts->showname);
    xfree(parts->date);
    xfree(parts->link);
    xfree(parts);
    xfree(ename);
    xfree(icon);
    xfree(link);
    return html;		/* html should be freed by caller */
}

void try_readme(r)
     request_t *r;
{
    char *t = NULL;
    char *tfname = NULL;
    request_t *readme = NULL;
    int fd = -1;
    struct stat sb;
    FILE *fp = NULL;

    if ((t = tempnam(NULL, progname)) == NULL)
	return;

    tfname = xstrdup(t);

    if ((fd = open(tfname, O_WRONLY | O_CREAT, 0600)) < 0) {
	xfree(tfname);
	return;
    }
    readme = (request_t *) xmalloc(sizeof(request_t));
    memset(readme, '\0', sizeof(request_t));

    readme->path = xstrdup("README");
    readme->cfd = fd;
    readme->sfd = r->sfd;
    readme->dfd = -1;
#ifdef TRY_CWD_FIRST
    readme->state = CWD_FAIL;
#else
    readme->state = SIZE_OK;
#endif
    readme->flags = F_NOERRS;

    process_request(readme);
    if (readme->cfd > 0)
	close(readme->cfd);
    if (readme->dfd > 0)
	close(readme->dfd);

    fp = fopen(tfname, "r");
    unlink(tfname);

    if (fp) {
	if (fstat(fileno(fp), &sb) < 0 || sb.st_size == 0) {
	    fclose(fp);
	    fp = NULL;
	}
    }
    r->readme_fp = fp;
    xfree(tfname);
    xfree(readme->path);
    xfree(readme);
}



state_t htmlify_listing(r)
     request_t *r;
{
    int code;
    static char buf[8192];
    char *t = NULL;
    FILE *rfp = NULL;
    FILE *wfp = NULL;
    time_t stamp;

    rfp = fdopen(dup(r->dfd), "r");
    wfp = fdopen(dup(r->cfd), "w");
    setbuf(rfp, NULL);
    setbuf(wfp, NULL);

    stamp = time(NULL);
    fprintf(wfp, "<!-- HTML listing generated by Harvest %s -->\n",
	HARVEST_VERSION);
    fprintf(wfp, "<!-- %s -->\n", http_time(stamp));
    fprintf(wfp, "<TITLE>\n");
    fprintf(wfp, "FTP Directory: %s\n", r->title_url);
    fprintf(wfp, "</TITLE>\n");
    if (r->flags & F_USEBASE)
	fprintf(wfp, "<BASE HREF=\"%s\">\n", r->url_escaped);

    if (r->cmd_msg) {		/* There was a message sent with the CWD cmd */
	list_t *l;
	fprintf(wfp, "<PRE>\n");
	for (l = r->cmd_msg; l; l = l->next)
	    write(r->cfd, l->ptr, strlen(l->ptr));
	fprintf(wfp, "</PRE>\n");
	fprintf(wfp, "<HR>\n");
    } else if (r->readme_fp) {
	fprintf(wfp, "<H4>README file from %s</H4>\n", r->title_url);
	fprintf(wfp, "<PRE>\n");
	while (fgets(buf, 1024, r->readme_fp))
	    fputs(buf, wfp);
	fclose(r->readme_fp);
	fprintf(wfp, "</PRE>\n");
	fprintf(wfp, "<HR>\n");
    }
    fprintf(wfp, "<H2>\n");
    fprintf(wfp, "FTP Directory: %s\n", r->title_url);
    fprintf(wfp, "</H2>\n");
    fprintf(wfp, "<PRE>\n");
    if (strcmp(r->path, ".")) {
	if ((t = htmlize_list_entry("..", r))) {
	    fputs(t, wfp);
	    xfree(t);
	}
    }
    while (fgets(buf, 8192, rfp)) {
	reset_timeout();
	Debug(26, 1, ("Input: %s", buf));
	if ((t = strchr(buf, '\r')))
	    *t = '\0';
	if ((t = strchr(buf, '\n')))
	    *t = '\0';
	if (!strncmp(buf, "total", 5))
	    continue;
	if ((t = htmlize_list_entry(buf, r))) {
	    fputs(t, wfp);
	    xfree(t);
	}
    }
    fprintf(wfp, "</PRE>\n");
    fprintf(wfp, "<HR>\n");
    fprintf(wfp, "<ADDRESS>\n");
    fprintf(wfp, "Generated %s, by %s/%s@%s\n",
	http_time(stamp), progname, HARVEST_VERSION, getfullhostname());
    fprintf(wfp, "</ADDRESS>\n");
    fclose(wfp);
    fclose(rfp);

    if ((code = read_reply(r->sfd)) > 0) {
	if (code == 226)
	    return TRANSFER_DONE;
    }
    r->errmsg = xstrdup(server_reply_msg);
    r->rc = code < 0 ? 4 : 5;
    return FAIL_SOFT;
}

static int process_request(r)
     request_t *r;
{
    if (r == (request_t *) NULL)
	return 1;

    while (1) {
	Debug(26, 1, ("process_request: in state %s\n",
		state_str[r->state]));
	switch (r->state) {
	case BEGIN:
	    r->state = parse_request(r);
	    break;
	case PARSE_OK:
	    r->state = do_connect(r);
	    break;
	case CONNECTED:
	    r->state = read_welcome(r);
	    break;
	case SERVICE_READY:
	    r->state = do_user(r);
	    break;
	case NEED_PASSWD:
	    r->state = do_passwd(r);
	    break;
	case LOGGED_IN:
	    r->state = do_type(r);
	    break;
	case TYPE_OK:
	    r->state = do_mdtm(r);
	    break;
	case MDTM_OK:
	    r->state = do_size(r);
	    break;
	case SIZE_OK:
#ifdef TRY_CWD_FIRST
	    r->state = do_cwd(r);
#else
	    r->state = do_pasv(r);
#endif
	    break;
	case CWD_OK:
	    r->flags |= F_ISDIR;
	    if (strcmp(r->path, ".")) {
		/* tack on the trailing slash now that we know its a dir */
		strcat(r->url, "/");
		strcat(r->title_url, "/");
		strcat(r->url_escaped, "/");
		if (*(r->path + strlen(r->path) - 1) != '/')
		    r->flags |= F_USEBASE;
	    }
	    if (r->flags & F_HTTPIFY) {
		if (cmd_msg) {
		    r->cmd_msg = cmd_msg;
		    cmd_msg = NULL;
		} else {
		    try_readme(r);
		}
	    }
	    r->state = do_pasv(r);
	    break;
#ifdef TRY_CWD_FIRST
	case CWD_FAIL:
	    r->flags &= ~F_ISDIR;
	    r->state = do_pasv(r);
	    break;
#else
	case RETR_FAIL:
	    r->flags |= F_ISDIR;
	    r->state = do_cwd(r);
	    break;
#endif
	case PASV_FAIL:
	    /* fallback to PORT */
	    r->state = do_port(r);
	    break;
	case PORT_OK:
	case PASV_OK:
	    r->state = r->flags & F_ISDIR ? do_list(r) : do_retr(r);
	    break;
	case TRANSFER_BEGIN:
	    if (r->flags & F_HTTPIFY)
		send_success_hdr(r);
	    if (r->flags & F_NEEDACCEPT)
		r->state = do_accept(r);
	    else
		r->state = DATA_TRANSFER;
	    break;
	case DATA_TRANSFER:
	    if ((r->flags & F_HTTPIFY) && (r->flags & F_ISDIR))
		r->state = htmlify_listing(r);
	    else
		r->state = read_data(r);
	    break;
	case TRANSFER_DONE:
	    r->state = DONE;
	    break;
	case DONE:
	    return 0;
	case FAIL_HARD:
	case FAIL_SOFT:
	    fail(r);
	    return (r->rc);
	    /* NOTREACHED */
	default:
	    errorlog("Nothing to do with state %s\n",
		state_str[r->state]);
	    return (1);
	    /* NOTREACHED */
	}
    }
}

void cleanup_path(r)
     request_t *r;
{
    int again;
    int l;
    char *t = NULL;
    char *s = NULL;

    do {
	again = 0;
	l = strlen(r->path);
	/* check for null path */
	if (*r->path == '\0') {
	    t = r->path;
	    r->path = xstrdup(".");
	    xfree(t);
	    again = 1;
	} else if (*r->path == '/') {
	    /* remove any leading slashes from path */
	    t = r->path;
	    r->path = xstrdup(t + 1);
	    xfree(t);
	    again = 1;
	} else if (!strncmp(r->path, "./", 2)) {
	    /* remove leading ./ */
	    t = r->path;
	    r->path = xstrdup(t + 2);
	    xfree(t);
	    again = 1;
	} else if (*(r->path + l - 1) == '/') {
	    /* remove any trailing slashes from path */
	    *(r->path + l - 1) = '\0';
	    again = 1;
	} else if (!strcmp(r->path + l - 2, "/.")) {
	    /* remove trailing /. */
	    *(r->path + l - 2) = '\0';
	    again = 1;
	} else if ((t = strstr(r->path, "/./"))) {
	    /* remove /./ */
	    s = xstrdup(t + 2);
	    strcpy(t, s);
	    xfree(s);
	    again = 1;
	} else if ((t = strstr(r->path, "//"))) {
	    /* remove // */
	    s = xstrdup(t + 1);
	    strcpy(t, s);
	    xfree(s);
	    again = 1;
	}
    } while (again);
}

void usage()
{
    fprintf(stderr, "usage: %s [-l num] [-c num] [-t timeout] [-htmlify] filename host path A,I user pass\n", progname);
    exit(1);
}


int main(argc, argv)
     int argc;
     char *argv[];
{
    request_t *r = NULL;
    FILE *logfp = NULL;
    char *t = NULL;
    int httpify_flag = 0;
    int rc;
    int i;
    int len;

    if ((t = strrchr(argv[0], '/'))) {
	progname = xstrdup(t + 1);
    } else
	progname = xstrdup(argv[0]);
    if (getenv("HARVEST_GATHERER_LOGFILE") != (char *) NULL)
	logfp = fopen(getenv("HARVEST_GATHERER_LOGFILE"), "a+");
    if (logfp == (FILE *) NULL)
	logfp = stderr;
    init_log3(progname, logfp, stderr);
    debug_init();
    for (argc--, argv++; argc > 0 && **argv == '-'; argc--, argv++) {
	if (!strcmp(*argv, "-"))
	    break;
	if (!strncmp(*argv, "-D", 2)) {
	    debug_flag(*argv);
	    continue;
	}
	if (!strcmp(*argv, "-htmlify") || !strcmp(*argv, "-httpify")) {
	    httpify_flag = 1;
	    continue;
	}
	if (!strcmp(*argv, "-t")) {
	    if (--argc < 1)
		usage();
	    argv++;
	    timeout = atoi(*argv);
	    if (timeout < 1)
		timeout = XFER_TIMEOUT;
	    continue;
	}
	if (!strcmp(*argv, "-c")) {
	    if (--argc < 1)
		usage();
	    argv++;
	    connect_retries = atoi(*argv);
	    continue;
	}
	if (!strcmp(*argv, "-l")) {
	    if (--argc < 1)
		usage();
	    argv++;
	    login_retries = atoi(*argv);
	    continue;
	}
	if (!strcmp(*argv, "-v")) {
	    printf("%s version %s\n", progname, HARVEST_VERSION);
	    exit(0);
	}
    }

    if (argc != 6)
	usage();

    r = (request_t *) xmalloc(sizeof(request_t));
    memset(r, '\0', sizeof(request_t));

    if (strcmp(argv[0], "-") == 0) {
	r->cfd = 1;
    } else if ((r->cfd = open(argv[0], O_WRONLY | O_CREAT, 0666)) < 0) {
	perror(argv[0]);
	exit(1);
    }
    r->host = xstrdup(argv[1]);
    r->path = xstrdup(argv[2]);
    r->type = xstrdup(argv[3]);
    r->user = xstrdup(argv[4]);
    r->pass = xstrdup(argv[5]);
    r->sfd = -1;
    r->dfd = -1;
    r->size = -1;
    r->state = BEGIN;
    r->flags |= httpify_flag ? F_HTTPIFY : 0;
    r->flags |= F_TRYDIR;

    if (*(r->type) != 'A' && *(r->type) != 'I') {
	errorlog("Invalid transfer type: %s\n", r->type);
	usage();
    }
    cleanup_path(r);

    len = 15 + strlen(r->user) + strlen(r->pass) + strlen(r->host)
	+ strlen(r->path);
    r->url = (char *) xmalloc(len);
    r->title_url = (char *) xmalloc(len);

    *r->url = '\0';
    strcat(r->url, "ftp://");
    if (strcmp(r->user, "anonymous")) {
	strcat(r->url, r->user);
	strcat(r->url, ":");
	strcat(r->url, r->pass);
	strcat(r->url, "@");
    }
    strcat(r->url, r->host);
    strcat(r->url, "/");
    if (strcmp(r->path, "."))
	strcat(r->url, r->path);

    *r->title_url = '\0';
    strcat(r->title_url, "ftp://");
    if (strcmp(r->user, "anonymous")) {
	strcat(r->title_url, r->user);
	strcat(r->title_url, "@");
    }
    strcat(r->title_url, r->host);
    strcat(r->title_url, "/");
    if (strcmp(r->path, "."))
	strcat(r->title_url, r->path);

    /* Make a copy of the escaped URL with some room to grow at the end */
    t = rfc1738_escape(r->url);
    r->url_escaped = (char *) xmalloc(strlen(t) + 10);
    strcpy(r->url_escaped, t);

#ifdef NSIG
    for (i = 0; i < NSIG; i++) {
#else
    for (i = 0; i < _sys_nsig; i++) {
#endif
	switch (i) {
	case SIGALRM:
	    signal(SIGALRM, timeout_handler);
	    break;
	case SIGINT:
	case SIGHUP:
	case SIGTERM:
	case SIGQUIT:
	    signal(i, generic_sig_handler);
	    break;
	case SIGSEGV:
	case SIGBUS:
	    break;
	default:
	    signal(i, SIG_IGN);
	    break;
	}
    }

    reset_timeout();
    rc = process_request(r);
    if (r->sfd > 0)
	send_cmd(r->sfd, "QUIT");
    if (r->sfd > 0)
	close(r->sfd);
    if (r->cfd > 0)
	close(r->cfd);
    if (r->dfd > 0)
	close(r->dfd);
    close(0);
    close(1);
    exit(rc);
    /* NOTREACHED */
}
