Harvest v1.3 Patchlevel 2 fixes To apply this patch: cd to the top of the source tree (to the directory containing the "src" and "components" subdirectories) and do: patch -p -s < ThisFile Patch will work silently unless an error occurs. If you want to watch patch do its thing, leave out the "-s" argument to patch. Finally, to rebuild after applying this patch, do: make make install Brief notes on what this patch fixes: CACHE: * Make cached.conf a symlink to cached.conf.default the first time. Now won't clobber cached.conf after a re-install. * Add 'bind_address' config option (for multihomed hosts). * Remove leading slash from ftp URLs so user directories work. * Include non-anonymous user:passwd in FTP HREF links. * Fix broken HREF links to long FTP filenames. * Add date timestamp to FTP directory listings. * Fix FTP errors not being returned to the browser. * Fix bug where an error is returned when two clients request the same object before any data has been received. * Fix memory leak introduced by pl1. * Increase bitfields to 3 bits because enums are signed. * Fix InvokeHandler() coredump bug. Lock objects inside InvokeHandlers() so they don't get freed in storeGetMemSpace(). * Change timestamps in access.log to local timezone. * Misc cosmetic fixes. * Removed old, unused code. * Changed HOT-VM count to cache_mem/80,000. GATHERER * Fixes a Makefile to actually build and install HTML-lax.sum. * Remove leading slash from ftp URLs so user directories work. * Added much debugging and cleanup to newsget.pl. * Change Essence unnesting to occur in individual directories. BROKER * Remove getpeername() call in the broker. * Log the FQDN in admin/LOG. * Redirect 'sync' output to null in Glimpse. * Remove glimpse version ambiguities in Glimpse/index.c. Index: README =================================================================== RCS file: /projects/agile/cvs/harvest/README,v retrieving revision 1.43 retrieving revision 1.44 diff -c -r1.43 -r1.44 *** 1.43 1995/02/14 20:13:54 --- 1.44 1995/09/13 19:38:47 *************** *** 9,13 **** --- 9,18 ---- http://harvest.cs.colorado.edu/ + Source code fixes will occasionally be made available in the Fixes/ + directory. Binary distributions will be rebuilt approximately every one + to two months. We urge you to compile the source code if you want to + incorporate fixes immediately. + The Harvest documentation is on-line, and is available via the Home Page. For further assistance, see http://harvest.cs.colorado.edu/support.html. Index: components/broker/standard/glimpse/index/build_in.c =================================================================== RCS file: /projects/agile/cvs/harvest/components/broker/standard/glimpse/index/build_in.c,v retrieving revision 1.19 retrieving revision 1.20 diff -c -r1.19 -r1.20 *** 1.19 1995/08/01 02:29:06 --- 1.20 1995/09/13 22:54:42 *************** *** 435,441 **** #endif sprintf(s, "exec mv %s/%s %s/%s\n", INDEX_DIR, O2, INDEX_DIR, I2); system(s); ! system("sync"); /* sync() has a bug */ #if 0 printf("traversed\n"); sprintf(s, "exec head -10 %s/%s\n", INDEX_DIR, I2); --- 435,441 ---- #endif sprintf(s, "exec mv %s/%s %s/%s\n", INDEX_DIR, O2, INDEX_DIR, I2); system(s); ! system("sync 2>/dev/null"); /* sync() has a bug */ #if 0 printf("traversed\n"); sprintf(s, "exec head -10 %s/%s\n", INDEX_DIR, I2); *************** *** 482,488 **** system(s); #endif /*0*/ FirstTraverse1 = 0; ! system("sync"); /* sync() has a bug */ return; } /* else not first-traverse */ --- 482,488 ---- system(s); #endif /*0*/ FirstTraverse1 = 0; ! system("sync 2>/dev/null"); /* sync() has a bug */ return; } /* else not first-traverse */ Index: components/broker/standard/glimpse/index/partition.c =================================================================== RCS file: /projects/agile/cvs/harvest/components/broker/standard/glimpse/index/partition.c,v retrieving revision 1.13 retrieving revision 1.14 diff -c -r1.13 -r1.14 *** 1.13 1995/08/01 02:29:20 --- 1.14 1995/09/13 22:54:44 *************** *** 647,653 **** fclose(i_out); sprintf(s, "exec mv %s/.glimpse_split.%d %s/%s", INDEX_DIR, getpid(), INDEX_DIR, INDEX_FILE); system(s); ! system("sync"); /* sync() has a BUG */ sprintf(s, "%s/%s", INDEX_DIR, INDEX_FILE); if (BuildTurbo) dump_mini(s); } --- 647,653 ---- fclose(i_out); sprintf(s, "exec mv %s/.glimpse_split.%d %s/%s", INDEX_DIR, getpid(), INDEX_DIR, INDEX_FILE); system(s); ! system("sync 2>/dev/null"); /* sync() has a BUG */ sprintf(s, "%s/%s", INDEX_DIR, INDEX_FILE); if (BuildTurbo) dump_mini(s); } *************** *** 817,823 **** system(s); sprintf(s, "exec mv %s/%s.tmp %s/%s", INDEX_DIR, INDEX_FILE, INDEX_DIR, INDEX_FILE); system(s); ! system("sync"); /* sync() has a BUG */ fclose(i_in); } sprintf(s, "%s/%s", INDEX_DIR, INDEX_FILE); --- 817,823 ---- system(s); sprintf(s, "exec mv %s/%s.tmp %s/%s", INDEX_DIR, INDEX_FILE, INDEX_DIR, INDEX_FILE); system(s); ! system("sync 2>/dev/null"); /* sync() has a BUG */ fclose(i_in); } sprintf(s, "%s/%s", INDEX_DIR, INDEX_FILE); Index: components/gatherer/standard/summarizers/Makefile.in =================================================================== RCS file: /projects/agile/cvs/harvest/components/gatherer/standard/summarizers/Makefile.in,v retrieving revision 1.31 retrieving revision 1.32 diff -c -r1.31 -r1.32 *** 1.31 1995/09/06 18:07:44 --- 1.32 1995/09/14 23:23:07 *************** *** 1,7 **** # # Makefile for the standard Harvest Gatherer summarizers # ! # $Id: Makefile.in,v 1.31 1995/09/06 18:07:44 duane Exp $ ####################################################################### # Copyright (c) 1994, 1995. All rights reserved. # --- 1,7 ---- # # Makefile for the standard Harvest Gatherer summarizers # ! # $Id: Makefile.in,v 1.32 1995/09/14 23:23:07 duane Exp $ ####################################################################### # Copyright (c) 1994, 1995. All rights reserved. # *************** *** 136,142 **** HTML-lax.sum: HTML-lax.sum.o $(CC) -o $@ HTML-lax.sum.o $(HTML_OBJS) $(LDFLAGS) $(LIBS) ! all: $(SCRIPTS) install: all @for f in $(SCRIPTS); do \ --- 136,142 ---- HTML-lax.sum: HTML-lax.sum.o $(CC) -o $@ HTML-lax.sum.o $(HTML_OBJS) $(LDFLAGS) $(LIBS) ! all: $(SCRIPTS) $(PROGS) install: all @for f in $(SCRIPTS); do \ Index: src/Makefile =================================================================== RCS file: /projects/agile/cvs/harvest/src/Makefile,v retrieving revision 1.60 retrieving revision 1.61 diff -c -r1.60 -r1.61 *** 1.60 1995/09/01 21:04:49 --- 1.61 1995/09/13 17:50:26 *************** *** 1,7 **** # # Makefile for the Harvest source tree # ! # $Id: Makefile,v 1.60 1995/09/01 21:04:49 duane Exp $ # # -------------------------------------------------------------------- # --- 1,7 ---- # # Makefile for the Harvest source tree # ! # $Id: Makefile,v 1.61 1995/09/13 17:50:26 duane Exp $ # # -------------------------------------------------------------------- # *************** *** 146,153 **** chmod 755 $(prefix)/bin/Harvest cp RunHarvest $(prefix)/RunHarvest chmod 755 $(prefix)/RunHarvest ! cp cache/server/cached.conf $(prefix)/lib/cache/cached.conf ! chmod 644 $(prefix)/lib/cache/cached.conf # ckconfigure runs the configure command ckconfigure: --- 146,157 ---- chmod 755 $(prefix)/bin/Harvest cp RunHarvest $(prefix)/RunHarvest chmod 755 $(prefix)/RunHarvest ! cp cache/server/cached.conf $(prefix)/lib/cache/cached.conf.default ! chmod 644 $(prefix)/lib/cache/cached.conf.default ! @if test ! -f $(prefix)/lib/cache/cached.conf ; then \ ! echo "ln -s cached.conf.default $(prefix)/lib/cache/cached.conf"; \ ! ln -s cached.conf.default $(prefix)/lib/cache/cached.conf; \ ! fi # ckconfigure runs the configure command ckconfigure: Index: src/broker/Glimpse/index.c =================================================================== RCS file: /projects/agile/cvs/harvest/src/broker/Glimpse/index.c,v retrieving revision 1.128 retrieving revision 1.129 diff -c -r1.128 -r1.129 *** 1.128 1995/09/05 19:07:56 --- 1.129 1995/09/15 23:32:54 *************** *** 1,4 **** ! static char rcsid[] = "$Id: index.c,v 1.128 1995/09/05 19:07:56 hardy Exp $"; /* * index.c -- Broker indexing/search support using Glimpse * --- 1,4 ---- ! static char rcsid[] = "$Id: index.c,v 1.129 1995/09/15 23:32:54 duane Exp $"; /* * index.c -- Broker indexing/search support using Glimpse * *************** *** 872,882 **** */ int Glimpse_IND_initialize() { - #ifdef GLIMPSE_2 GL_IndexOption = xstrdup("-b -s -T -B -M 10"); - #else - GL_IndexOption = xstrdup("-o -s -T -B -M 10"); - #endif IndexType = I_FULL; GL_IndexFlags = xstrdup(""); GL_Glimpse = xstrdup("glimpse"); --- 872,878 ---- *************** *** 923,933 **** fatal("sscanf GL_GlimpseSrvRestart failed"); } else if (strcasecmp(tag, S_GLIMPSEINDOPT) == 0) { if (strcasecmp(value, "Fast-Search") == 0) { - #ifdef GLIMPSE_2 GL_IndexOption = xstrdup("-b -s -T -B -M 10"); - #else - GL_IndexOption = xstrdup("-o -s -T -B -M 10"); - #endif } else if (strcasecmp(value, "Medium") == 0) { GL_IndexOption = xstrdup("-o -s -T -B -M 10"); } else if (strcasecmp(value, "Small-Index") == 0) { --- 919,925 ---- *************** *** 984,998 **** xbuf, GL_errflag); } - #ifdef GLIMPSE_2 - { /* Glimpse 2.0 can limit the result set; always use -L 1000 if undef */ ! char tbuf[BUFSIZ]; ! tbuf[0] = '\0'; ! sprintf(tbuf, " -L %d ", GL_maxresults < 1 ? 1000 : GL_maxresults); ! strcat(commandstr, tbuf); ! } ! #endif /* Add Glimpse flags */ if (GL_caseflag == 1) { --- 976,984 ---- xbuf, GL_errflag); } /* Glimpse 2.0 can limit the result set; always use -L 1000 if undef */ ! sprintf(xbuf, " -L %d ", GL_maxresults < 1 ? 1000 : GL_maxresults); ! strcat(commandstr, xbuf); /* Add Glimpse flags */ if (GL_caseflag == 1) { *************** *** 1025,1043 **** return ERROR; } - - #ifndef GLIMPSE_2 /* optimization for Glimpse 1.1 */ - /* - * Try to optimize the search by using -N when the query meets the - * following criteria: -i is used, -l is used, no structured query - * is used, the error is 0 or "Best Match". - */ - if ((patstr != NULL) && (QM_opaqueflag != 1) && (GL_caseflag == 1) && - ((GL_errflag == 0) || (GL_errflag == -1)) && (QM_gotphrase != 1) && - (strchr(patstr, '=') == NULL)) { - sprintf(commandstr, "%s -a -H %s -y -N ", GL_Glimpse, DIRpath); - } - #endif if (patstr != NULL) { sprintf(commandstr + strlen(commandstr), " \'%s\' ", patstr); --- 1011,1016 ---- Index: src/broker/Glimpse/index.h =================================================================== RCS file: /projects/agile/cvs/harvest/src/broker/Glimpse/index.h,v retrieving revision 1.24 retrieving revision 1.25 diff -c -r1.24 -r1.25 *** 1.24 1995/09/05 19:07:58 --- 1.25 1995/09/15 23:32:55 *************** *** 1,7 **** /* * index.h - Broker support for Glimpse 1.1. * ! * $Id: index.h,v 1.24 1995/09/05 19:07:58 hardy Exp $ * * ---------------------------------------------------------------------- * Copyright (c) 1994, 1995. All rights reserved. --- 1,7 ---- /* * index.h - Broker support for Glimpse 1.1. * ! * $Id: index.h,v 1.25 1995/09/15 23:32:55 duane Exp $ * * ---------------------------------------------------------------------- * Copyright (c) 1994, 1995. All rights reserved. *************** *** 90,96 **** #define AD_GLIMPSE 5 #define AD_GLIND 6 ! #define GLIMPSE_2 /* Glimpse version 2.0, otherwise 1.1 */ #define SEL_SIZE BUFSIZ --- 90,98 ---- #define AD_GLIMPSE 5 #define AD_GLIND 6 ! #undef GLIMPSE_1 ! #undef GLIMPSE_2 ! #define GLIMPSE_3 #define SEL_SIZE BUFSIZ Index: src/broker/select_loop.c =================================================================== RCS file: /projects/agile/cvs/harvest/src/broker/select_loop.c,v retrieving revision 1.23 retrieving revision 1.25 diff -c -r1.23 -r1.25 *** 1.23 1995/09/05 18:25:54 --- 1.25 1995/09/15 05:54:41 *************** *** 1,4 **** ! static char rcsid[] = "$Id: select_loop.c,v 1.23 1995/09/05 18:25:54 hardy Exp $"; /* * select_loop.c - Input socket processing for the Broker * --- 1,4 ---- ! static char rcsid[] = "$Id: select_loop.c,v 1.25 1995/09/15 05:54:41 duane Exp $"; /* * select_loop.c - Input socket processing for the Broker * *************** *** 110,115 **** --- 110,116 ---- fd_set qready; int err, clen, msock; struct sockaddr_in cin; + Host *H = NULL; if (qsock < 0) /* we may call this before qsock is init */ return 0; *************** *** 139,144 **** --- 140,150 ---- log_errno("accept"); return -1; } + if (H = get_host(inet_ntoa(cin.sin_addr))) { + LOGCONNECT(H->fqdn); + } else { + LOGCONNECT(inet_ntoa(cin.sin_addr)); + } setsocket_linger(msock, 30); /* 30 sec linger time */ /* Immediately send the protocol version */ *************** *** 153,169 **** (void)write(msock, offlinemsg, strlen(offlinemsg)); (void)close(msock); } else if (Accept) { - struct sockaddr_in peersock; - int peersocksz = sizeof(struct sockaddr_in); - if (EV_add_sevent(msock) == ERROR) { (void)close(msock); ndenied_connections++; - } - if (getpeername(msock, (struct sockaddr *) &peersock, &peersocksz) < 0) { - log_errno("getpeername"); - } else { - LOGCONNECT(inet_ntoa(peersock.sin_addr)); } /* msock lives on... */ } else { --- 159,167 ---- Index: src/cache/CachedLynx =================================================================== RCS file: /projects/agile/cvs/harvest/src/cache/CachedLynx,v retrieving revision 1.5 retrieving revision 1.6 diff -c -r1.5 -r1.6 *** 1.5 1995/04/12 23:21:31 --- 1.6 1995/09/15 16:51:24 *************** *** 6,12 **** # # Darren Hardy, University of Colorado - Boulder, August 1994 # ! # $Id: CachedLynx,v 1.5 1995/04/12 23:21:31 wessels Exp $ # # set cachehost to the machine running the Harvest Object cache --- 6,12 ---- # # Darren Hardy, University of Colorado - Boulder, August 1994 # ! # $Id: CachedLynx,v 1.6 1995/09/15 16:51:24 duane Exp $ # # set cachehost to the machine running the Harvest Object cache *************** *** 17,20 **** ftp_proxy="http://${cachehost}:${cacheport}/"; export ftp_proxy gopher_proxy="http://${cachehost}:${cacheport}/"; export gopher_proxy ! exec lynx $* --- 17,20 ---- ftp_proxy="http://${cachehost}:${cacheport}/"; export ftp_proxy gopher_proxy="http://${cachehost}:${cacheport}/"; export gopher_proxy ! exec lynx ${1+"$@"} Index: src/cache/CachedMosaic =================================================================== RCS file: /projects/agile/cvs/harvest/src/cache/CachedMosaic,v retrieving revision 1.5 retrieving revision 1.6 diff -c -r1.5 -r1.6 *** 1.5 1995/04/12 23:21:29 --- 1.6 1995/09/15 16:51:24 *************** *** 6,12 **** # # Darren Hardy, University of Colorado - Boulder, August 1994 # ! # $Id: CachedMosaic,v 1.5 1995/04/12 23:21:29 wessels Exp $ # # set cachehost to the machine running the Harvest Object cache --- 6,12 ---- # # Darren Hardy, University of Colorado - Boulder, August 1994 # ! # $Id: CachedMosaic,v 1.6 1995/09/15 16:51:24 duane Exp $ # # set cachehost to the machine running the Harvest Object cache *************** *** 17,20 **** ftp_proxy="http://${cachehost}:${cacheport}/"; export ftp_proxy gopher_proxy="http://${cachehost}:${cacheport}/"; export gopher_proxy ! exec Mosaic $* --- 17,20 ---- ftp_proxy="http://${cachehost}:${cacheport}/"; export ftp_proxy gopher_proxy="http://${cachehost}:${cacheport}/"; export gopher_proxy ! exec mosaic ${1+"$@"} Index: src/cache/server/Makefile.in =================================================================== RCS file: /projects/agile/cvs/harvest/src/cache/server/Makefile.in,v retrieving revision 1.68 retrieving revision 1.69 diff -c -r1.68 -r1.69 *** 1.68 1995/09/05 20:23:11 --- 1.69 1995/09/15 16:51:26 *************** *** 1,7 **** # # Makefile for the Harvest Object Cache server # ! # $Id: Makefile.in,v 1.68 1995/09/05 20:23:11 duane Exp $ # # If you don't want cache traces (the hostname and port on which your # cache is running) to be sent to the University of Colorado, --- 1,7 ---- # # Makefile for the Harvest Object Cache server # ! # $Id: Makefile.in,v 1.69 1995/09/15 16:51:26 duane Exp $ # # If you don't want cache traces (the hostname and port on which your # cache is running) to be sent to the University of Colorado, *************** *** 43,51 **** # MUST use -I. first INCLUDE = -I. -I../../common/include CFLAGS = $(DEBUG) $(INCLUDE) $(DEFINES) $(XTRA_CFLAGS) ! LDFLAGS = -L../../common/lib ! LIBS = -lregex $(XTRA_LIBS) $(CRYPT_LIB) ! CLIENT_LIBS = $(XTRA_LIBS) PROGS = cached UTILS = client dnsserver --- 43,51 ---- # MUST use -I. first INCLUDE = -I. -I../../common/include CFLAGS = $(DEBUG) $(INCLUDE) $(DEFINES) $(XTRA_CFLAGS) ! LDFLAGS = $(CFLAGS) -L../../common/lib ! LIBS = -lregex $(XTRA_LIBS) $(CRYPT_LIB) ! CLIENT_LIBS = $(XTRA_LIBS) PROGS = cached UTILS = client dnsserver *************** *** 61,76 **** all: $(PROGS) $(UTILS) $(CGIPROGS) cached: mime_table.h main.o $(OBJS) ! $(CC) -o $@ $(OBJS) main.o $(LDFLAGS) $(LIBS) -lm client: client.o util.o ! $(CC) -o $@ $@.o util.o $(CLIENT_LIBS) dnsserver: dnsserver.o util.o ! $(CC) -o $@ $@.o util.o $(LDFLAGS) $(LIBS) cachemgr.cgi: cachemgr.o util.o ! $(CC) -o $@ cachemgr.o util.o $(CLIENT_LIBS) cachemgr.o: ../manager/cache_cgi/cachemgr.c $(CC) $(CFLAGS) -c ../manager/cache_cgi/cachemgr.c -o $@ --- 61,76 ---- all: $(PROGS) $(UTILS) $(CGIPROGS) cached: mime_table.h main.o $(OBJS) ! $(CC) -o $@ $(OBJS) $(LDFLAGS) main.o $(LIBS) -lm client: client.o util.o ! $(CC) -o $@ $(LDFLAGS) $@.o util.o $(CLIENT_LIBS) dnsserver: dnsserver.o util.o ! $(CC) -o $@ $(LDFLAGS) $@.o util.o $(LIBS) cachemgr.cgi: cachemgr.o util.o ! $(CC) -o $@ $(LDFLAGS) cachemgr.o util.o $(CLIENT_LIBS) cachemgr.o: ../manager/cache_cgi/cachemgr.c $(CC) $(CFLAGS) -c ../manager/cache_cgi/cachemgr.c -o $@ Index: src/cache/server/cache_cf.c =================================================================== RCS file: /projects/agile/cvs/harvest/src/cache/server/cache_cf.c,v retrieving revision 1.41 retrieving revision 1.43 diff -c -r1.41 -r1.43 *** 1.41 1995/09/06 23:25:35 --- 1.43 1995/09/12 22:48:17 *************** *** 1,4 **** ! static char rcsid[] = "$Id: cache_cf.c,v 1.41 1995/09/06 23:25:35 duane Exp $"; /* * File: cache_cf.c * Description: cache initialization functions --- 1,4 ---- ! static char rcsid[] = "$Id: cache_cf.c,v 1.43 1995/09/12 22:48:17 duane Exp $"; /* * File: cache_cf.c * Description: cache initialization functions *************** *** 157,167 **** --- 157,171 ---- stoplist *http_stoplist = NULL; stoplist *gopher_stoplist = NULL; stoplist *ftp_stoplist = NULL; + stoplist *bind_addr_list = NULL; + ip_acl *ip_access_control_list = NULL; ip_acl *manager_ip_access_control_list = NULL; + #ifdef CHECK_LOCAL_NETS ip_acl *local_ip_list = NULL; #endif + int quick_abort = 0; /* off by default */ void self_destruct(in_string) *************** *** 803,808 **** --- 807,827 ---- } #endif /* APPEND_DOMAIN */ + void + parseBindAddressLine(line_in) + char *line_in; + { + char *token; + int a[4]; + struct in_addr *addr = NULL; + struct in_addr *getAddress(); + + token = strtok(NULL, w_space); + if (token == (char *) NULL) + self_destruct(line_in); + debug(1, "parseBindAddressLine: adding %s\n", token); + addToStopList(&bind_addr_list, token); + } /* Maybe a bit heavy handed, but parser is immune to virtually every sane *************** *** 920,931 **** else if (!strncmp(token, "manager_access_deny", 19)) parseManagerAccessDenyLine(line_in); - #ifdef CHECK_LOCAL_NETS - /* Parse a local_ip line */ - else if (!strncmp(token, "local_ip", 19)) - parseLocalIPLine(line_in); - #endif - /* Parse a http_stop line */ else if (!strncmp(token, "http_stop", 9)) parseHttpStopLine(line_in); --- 939,944 ---- *************** *** 1013,1018 **** --- 1026,1041 ---- else if (!strcmp(token, "wais_relay")) parseWAISRelayLine(line_in); #endif + + #ifdef CHECK_LOCAL_NETS + /* Parse a local_ip line */ + else if (!strcmp(token, "local_ip")) + parseLocalIPLine(line_in); + #endif + + /* Parse a bind_address line */ + else if (!strcmp(token, "bind_address")) + parseBindAddressLine(line_in); /* If unknown, treat as a comment line */ else { Index: src/cache/server/cache_cf.h =================================================================== RCS file: /projects/agile/cvs/harvest/src/cache/server/cache_cf.h,v retrieving revision 1.17 retrieving revision 1.18 diff -c -r1.17 -r1.18 *** 1.17 1995/09/06 07:47:09 --- 1.18 1995/09/12 18:32:55 *************** *** 1,5 **** /* ! * $Id: cache_cf.h,v 1.17 1995/09/06 07:47:09 duane Exp $ * * File: config.h * Description: Declarations of parsing and config functions --- 1,5 ---- /* ! * $Id: cache_cf.h,v 1.18 1995/09/12 18:32:55 duane Exp $ * * File: config.h * Description: Declarations of parsing and config functions *************** *** 122,127 **** --- 122,128 ---- extern stoplist *http_stoplist; extern stoplist *gopher_stoplist; extern stoplist *ftp_stoplist; + extern stoplist *bind_addr_list; extern ip_acl *ip_access_control_list; extern ip_acl *manager_ip_access_control_list; Index: src/cache/server/cached.conf =================================================================== RCS file: /projects/agile/cvs/harvest/src/cache/server/cached.conf,v retrieving revision 1.45 retrieving revision 1.47 diff -c -r1.45 -r1.47 *** 1.45 1995/09/05 20:26:50 --- 1.47 1995/09/13 17:54:56 *************** *** 1,4 **** --- 1,11 ---- + # $Id: cached.conf,v 1.47 1995/09/13 17:54:56 duane Exp $ # + # NOTE: Upon installation, cached.conf is a symbolic link to this file, + # cached.conf.default. If you want to change this file, please + # remove the symbolic link and copy cached.conf.default to + # cached.conf. Then make your changes to cached.conf. + # + # # cached.conf - Harvest Object Cache configuration file # # Host: ??? *************** *** 177,182 **** --- 184,199 ---- # #local_ip 10.0.0.0 #local_ip 172.16.0.0 + + # + # TAG: bind_address + # Use this to force the cache to listen on a specific IP address + # (interface) instead of listening on all addresses. Either + # an IP address or hostname can be given. For example: + # + # bind_address 172.16.1.1 + # bind_address localhost + # bind_address www-slip.my.domain ################################################################### Index: src/cache/server/comm.c =================================================================== RCS file: /projects/agile/cvs/harvest/src/cache/server/comm.c,v retrieving revision 1.74 retrieving revision 1.77 diff -c -r1.74 -r1.77 *** 1.74 1995/09/07 18:20:29 --- 1.77 1995/09/13 00:12:48 *************** *** 1,4 **** ! static char rcsid[] = "$Id: comm.c,v 1.74 1995/09/07 18:20:29 duane Exp $"; /* * File: comm.c * Description: socket-based communication facility. Adapted from DHT --- 1,4 ---- ! static char rcsid[] = "$Id: comm.c,v 1.77 1995/09/13 00:12:48 duane Exp $"; /* * File: comm.c * Description: socket-based communication facility. Adapted from DHT *************** *** 191,196 **** --- 191,224 ---- } } + static int do_bind (s, host, port) + int s; + char *host; + int port; + { + struct sockaddr_in S; + struct in_addr *addr = NULL; + struct in_addr *getAddress(); + + addr = getAddress(host); + if (addr == (struct in_addr *) NULL) { + debug(0, "do_bind: Unknown host: %s\n", host); + return COMM_ERROR; + } + + memset(&S, '\0', sizeof(S)); + S.sin_family = AF_INET; + S.sin_port = htons(port); + S.sin_addr = *addr; + + if (bind(s, (struct sockaddr *) &S, sizeof(S)) == 0) + return COMM_OK; + + debug(0, "do_bind: Cannot bind %s port %d to socket %d: %s\n", + inet_ntoa(S.sin_addr), port, s, sys_errlist[errno]); + return COMM_ERROR; + } + /* Create a socket. Default is blocking, stream (TCP) socket. IO_TYPE is OR of flags specified in comm.h. */ int comm_open(io_type, port, handler) *************** *** 198,209 **** int port; int (*handler) (); /* Interrupt handler. */ { - struct sockaddr_in addr; - struct in_addr *getAddress(); int io_handler(); int new_socket; ! FD_ENTRY *conn; int sock_type = io_type & COMM_DGRAM ? SOCK_DGRAM : SOCK_STREAM; /* Create socket for accepting new connections. */ if ((new_socket = socket(AF_INET, sock_type, 0)) < 0) { --- 226,236 ---- int port; int (*handler) (); /* Interrupt handler. */ { int io_handler(); int new_socket; ! FD_ENTRY *conn = NULL; int sock_type = io_type & COMM_DGRAM ? SOCK_DGRAM : SOCK_STREAM; + stoplist *p = NULL; /* Create socket for accepting new connections. */ if ((new_socket = socket(AF_INET, sock_type, 0)) < 0) { *************** *** 217,238 **** memset(conn, '\0', sizeof(FD_ENTRY)); conn->openned = 1; ! /* Create name for the socket */ ! memset(&addr, '\0', sizeof(addr)); ! addr.sin_family = AF_INET; - /* - * If port is nonzero, it is the port number specified by user. - */ - - if (getAddress(0)) { - memcpy(&addr.sin_addr, getAddress(0), sizeof(addr.sin_addr)); - } else { - addr.sin_addr.s_addr = INADDR_ANY; - } - - addr.sin_port = htons(port); - if (port > 0) { setsocketNolinger(new_socket); if (do_reuse) { --- 244,252 ---- memset(conn, '\0', sizeof(FD_ENTRY)); conn->openned = 1; ! /* Add INADDR_ANY to end of bind_addr_list as last chance */ ! addToStopList(&bind_addr_list, "0.0.0.0"); if (port > 0) { setsocketNolinger(new_socket); if (do_reuse) { *************** *** 239,249 **** setsocketReuse(new_socket); } } ! if (port && bind(new_socket, (struct sockaddr *) &addr, sizeof(addr))) { ! debug(0, "comm_open: bind failure on addr %s port %d: %s\n", ! inet_ntoa(addr.sin_addr), port, sys_errlist[errno]); ! return (COMM_ERROR); } conn->port = port; if (io_type & COMM_INTERRUPT) { --- 253,266 ---- setsocketReuse(new_socket); } } ! ! if (port) for (p=bind_addr_list; p; p=p->next) { ! if (do_bind(new_socket, p->key, port) == COMM_OK) ! break; ! if (p->next == (stoplist *) NULL) ! return COMM_ERROR; } + conn->port = port; if (io_type & COMM_INTERRUPT) { *************** *** 322,328 **** return new_socket; } ! /* NOTE: set the listen queue to 50 and rely on the kernel to ** ** impose an upper limit. Solaris' listen(3n) page says it has ** ** no limit on this parameter, but sys/socket.h sets SOMAXCONN ** ** to 5. HP-UX currently has a limit of 20. SunOS is 5 and ** --- 339,345 ---- return new_socket; } ! /* NOTE: set the listen queue to 50 and rely on the kernel to ** ** impose an upper limit. Solaris' listen(3n) page says it has ** ** no limit on this parameter, but sys/socket.h sets SOMAXCONN ** ** to 5. HP-UX currently has a limit of 20. SunOS is 5 and ** *************** *** 891,902 **** struct in_addr * ! getAddress(name) char *name; { char hostname[256]; ! struct hostent *hp; ! static struct in_addr *our_first_addr; if (name == NULL) { return NULL; --- 908,919 ---- struct in_addr * ! getAddress(name) char *name; { char hostname[256]; ! struct hostent *hp = NULL; ! static struct in_addr *our_first_addr = NULL; if (name == NULL) { return NULL; Index: src/cache/server/ftp.c =================================================================== RCS file: /projects/agile/cvs/harvest/src/cache/server/ftp.c,v retrieving revision 1.67 retrieving revision 1.69 diff -c -r1.67 -r1.69 *** 1.67 1995/09/05 19:08:34 --- 1.69 1995/09/15 21:48:09 *************** *** 1,4 **** ! static char rcsid[] = "$Id: ftp.c,v 1.67 1995/09/05 19:08:34 hardy Exp $"; /* * File: ftp.c * Description: state machine for ftp retrieval protocol. Based on John's --- 1,4 ---- ! static char rcsid[] = "$Id: ftp.c,v 1.69 1995/09/15 21:48:09 duane Exp $"; /* * File: ftp.c * Description: state machine for ftp retrieval protocol. Based on John's *************** *** 355,361 **** data->pfp = ftp_open_pipe(ftp_program, data->host, data->request, user, password, "r", &(data->cpid)); ! if (data->pfp == NULL || fileno(data->pfp) < 0) { sprintf(tmp_error_buf, CACHED_RETRIEVE_ERROR_MSG, entry->url, entry->url, --- 355,365 ---- data->pfp = ftp_open_pipe(ftp_program, data->host, data->request, user, password, "r", &(data->cpid)); ! if (data->pfp == NULL ! #if !defined(_HARVEST_SYSV_) ! || fileno(data->pfp) < 0 ! #endif ! ) { sprintf(tmp_error_buf, CACHED_RETRIEVE_ERROR_MSG, entry->url, entry->url, *************** *** 443,448 **** --- 447,463 ---- for (fd = 3; fd < getMaxFD(); fd++) (void) close(fd); (void) close(pfd[0]); + + + /* + * Remove leading slash from FTP url-path so that we can + * handle ftp://user:pw@host/path objects where path and /path + * are quite different. -DW + */ + if (!strcmp(p3, "/")) + *p3 = '.'; + if (*p3 == '/') + p3++; /* * Run the ftpget.pl command: Index: src/cache/server/icp.c =================================================================== RCS file: /projects/agile/cvs/harvest/src/cache/server/icp.c,v retrieving revision 1.107 retrieving revision 1.110 diff -c -r1.107 -r1.110 *** 1.107 1995/09/08 18:19:36 --- 1.110 1995/09/15 05:57:29 *************** *** 1,4 **** ! static char rcsid[] = "$Id: icp.c,v 1.107 1995/09/08 18:19:36 hardy Exp $"; /* * File: icp.c * Description: Implementation of binary cache protocol. --- 1,4 ---- ! static char rcsid[] = "$Id: icp.c,v 1.110 1995/09/15 05:57:29 duane Exp $"; /* * File: icp.c * Description: Implementation of binary cache protocol. *************** *** 443,449 **** if ((state->offset == 0) && (header->opcode != ICP_OP_DATABEG)) { header->opcode = ICP_OP_DATABEG; } else if ((store_mem_obj(entry, e_current_len) == entry->object_len) && ! ((entry->object_len - state->offset) == len)) { /* No more data; this is the last message. */ header->opcode = ICP_OP_DATAEND; } else { --- 443,450 ---- if ((state->offset == 0) && (header->opcode != ICP_OP_DATABEG)) { header->opcode = ICP_OP_DATABEG; } else if ((store_mem_obj(entry, e_current_len) == entry->object_len) && ! ((entry->object_len - state->offset) == len) && ! (entry->status != STORE_PENDING)) { /* No more data; this is the last message. */ header->opcode = ICP_OP_DATAEND; } else { *************** *** 529,534 **** --- 530,538 ---- fetching module to abort the fetching */ if (getQuickAbort()) { BIT_SET(state->entry->flag, CLIENT_ABORT_REQUEST); + /* uncomment this to have aborted requests be immediately */ + /* released from the cache */ + /*BIT_SET(state->entry->flag, RELEASE_REQUEST); */ peer.sin_addr.s_addr = (long) 0; CacheInfo->log_append(CacheInfo, state->url, *************** *** 549,556 **** /* More data available locally; write it now */ icpSendMoreData(fd, state); } else ! /* We're finished case */ if ((binaryMode(state) && (header->opcode == ICP_OP_DATAEND)) || ! (asciiMode(state) && (state->offset == state->entry->object_len))) { storeUnregister(state->entry, fd); storeUnlockObject(state->entry); comm_close(fd); --- 553,562 ---- /* More data available locally; write it now */ icpSendMoreData(fd, state); } else ! /* We're finished case */ ! if ((binaryMode(state) && (header->opcode == ICP_OP_DATAEND)) || ! (asciiMode(state) && (state->offset == state->entry->object_len) ! && state->entry->status != STORE_PENDING)) { storeUnregister(state->entry, fd); storeUnlockObject(state->entry); comm_close(fd); *************** *** 1275,1288 **** } if ((handler != NULL) && (client_data != NULL)) { ! rw_state = (icpReadWriteData *) client_data; ! if (rw_state->buf) ! safe_free(rw_state->buf); ! safe_free(rw_state); } - /* astm->url is already freed by safe_free(rw_state->buf) */ - /* safe_free(astm->url); */ safe_free(astm->mime_hdr); safe_free(astm->type); safe_free(astm); --- 1281,1294 ---- } if ((handler != NULL) && (client_data != NULL)) { ! rw_state = (icpReadWriteData *) client_data; ! if (rw_state->buf) ! safe_free(rw_state->buf); ! safe_free(rw_state); ! } else { ! safe_free(astm->url); } safe_free(astm->mime_hdr); safe_free(astm->type); safe_free(astm); Index: src/cache/server/ipcache.h =================================================================== RCS file: /projects/agile/cvs/harvest/src/cache/server/ipcache.h,v retrieving revision 1.17 retrieving revision 1.18 diff -c -r1.17 -r1.18 *** 1.17 1995/09/05 19:08:48 --- 1.18 1995/09/15 05:56:07 *************** *** 1,6 **** /* ! * $Id: ipcache.h,v 1.17 1995/09/05 19:08:48 hardy Exp $ * * File: ipcache.h * Description: ip address cache. speed up gethostbyname() --- 1,6 ---- /* ! * $Id: ipcache.h,v 1.18 1995/09/15 05:56:07 duane Exp $ * * File: ipcache.h * Description: ip address cache. speed up gethostbyname() *************** *** 107,113 **** unsigned char alias_count; enum { CACHED, PENDING, NEGATIVE_CACHED ! } status:2; struct hostent entry; struct _ip_pending *pending_head; struct _ip_pending *pending_tail; --- 107,113 ---- unsigned char alias_count; enum { CACHED, PENDING, NEGATIVE_CACHED ! } status:3; struct hostent entry; struct _ip_pending *pending_head; struct _ip_pending *pending_tail; Index: src/cache/server/stat.c =================================================================== RCS file: /projects/agile/cvs/harvest/src/cache/server/stat.c,v retrieving revision 1.85 retrieving revision 1.86 diff -c -r1.85 -r1.86 *** 1.85 1995/09/07 20:24:32 --- 1.86 1995/09/15 23:52:27 *************** *** 1,4 **** ! static char rcsid[] = "$Id: stat.c,v 1.85 1995/09/07 20:24:32 duane Exp $"; /* * File: stat.c * Description: stat module for object cache --- 1,4 ---- ! static char rcsid[] = "$Id: stat.c,v 1.86 1995/09/15 23:52:27 duane Exp $"; /* * File: stat.c * Description: stat module for object cache *************** *** 694,704 **** sprintf(line, "{\t\tStoreEntry %d x %d}\n", sizeof(StoreEntry), meta_data.store_entries); strncat(buffer, line, MAX_LINELEN); - #if !OLD_STORE_STRU sprintf(line, "{\t\tStoreMemObject %d x %d}\n", sizeof(MemObject), meta_data.store_in_mem_objects); strncat(buffer, line, MAX_LINELEN); - #endif sprintf(line, "{\t\tIPCacheEntry %d x %d}\n", sizeof(ipcache_entry), meta_data.ipcache_count); strncat(buffer, line, MAX_LINELEN); --- 694,702 ---- *************** *** 711,719 **** strncat(buffer, line, MAX_LINELEN); sprintf(line, "{\tTotal Accounted %d}\n", meta_data.store_entries * sizeof(StoreEntry) + - #if !OLD_STORE_STRU meta_data.store_in_mem_objects * sizeof(MemObject) + - #endif meta_data.ipcache_count * sizeof(ipcache_entry) + meta_data.hash_links * sizeof(hash_link) + meta_data.url_strings); --- 709,715 ---- Index: src/cache/server/stat.h =================================================================== RCS file: /projects/agile/cvs/harvest/src/cache/server/stat.h,v retrieving revision 1.16 retrieving revision 1.17 diff -c -r1.16 -r1.17 *** 1.16 1995/09/05 19:09:07 --- 1.17 1995/09/15 23:52:28 *************** *** 1,5 **** /* ! * $Id: stat.h,v 1.16 1995/09/05 19:09:07 hardy Exp $ * * File: stat.h * Description: prototype for stat module for object cache --- 1,5 ---- /* ! * $Id: stat.h,v 1.17 1995/09/15 23:52:28 duane Exp $ * * File: stat.h * Description: prototype for stat module for object cache *************** *** 128,136 **** typedef struct _meta_data_stat { int hot_vm; int store_entries; - #if !OLD_STORE_STRU int store_in_mem_objects; - #endif int ipcache_count; int hash_links; int url_strings; --- 128,134 ---- Index: src/cache/server/store.c =================================================================== RCS file: /projects/agile/cvs/harvest/src/cache/server/store.c,v retrieving revision 1.135 retrieving revision 1.140 diff -c -r1.135 -r1.140 *** 1.135 1995/09/10 04:17:12 --- 1.140 1995/09/15 23:53:51 *************** *** 1,4 **** ! static char rcsid[] = "$Id: store.c,v 1.135 1995/09/10 04:17:12 duane Exp $"; /* * File: store.c * Description: Storage manager for the Cache --- 1,4 ---- ! static char rcsid[] = "$Id: store.c,v 1.140 1995/09/15 23:53:51 duane Exp $"; /* * File: store.c * Description: Storage manager for the Cache *************** *** 109,116 **** #define SWAP_BUF (64*1024) #define FATAL_BUF_SIZE 1024 #define SWAP_DIRECTORIES 100 ! #define HOT_OBJECT_HIGH (store_mem_high / 20000) ! #define HOT_OBJECT_LOW (store_mem_low / 20000) /* rate of checking expired objects in main loop */ #define STORE_MAINTAIN_RATE (20) --- 109,116 ---- #define SWAP_BUF (64*1024) #define FATAL_BUF_SIZE 1024 #define SWAP_DIRECTORIES 100 ! #define HOT_OBJECT_HIGH (store_mem_high / 80000) ! #define HOT_OBJECT_LOW (store_mem_low / 80000) /* rate of checking expired objects in main loop */ #define STORE_MAINTAIN_RATE (20) *************** *** 227,246 **** } - /* ------- FOR SPLITTING STORE STRUCTURE ---------------- */ - /* A switch 'OLD_STORE_STRU' has been used for switching - back to old implementation. If there is any doubt about new one, - try set 'OLD_STORE_STRU' to 1 in store.h, then it shall - be functioning as same as old implementation. - */ - /* Allocate memory for a new store structure */ - #if OLD_STORE_STRU - StoreEntry *create_StoreEntry() - { - return ((StoreEntry *)xcalloc (1,sizeof(StoreEntry))); - } - #else StoreEntry *create_StoreEntry() { StoreEntry *e = NULL; --- 227,233 ---- *************** *** 251,257 **** return (e); } - #endif StoreEntry *create_StoreEntry_only() { --- 238,243 ---- *************** *** 259,267 **** } /* Free memory of a store structure */ - #if OLD_STORE_STRU - #define destroy_StoreEntry(e) safe_free(e) - #else /* free a StoreEntry */ void destroy_StoreEntry(e) StoreEntry *e; --- 245,250 ---- *************** *** 274,286 **** safe_free (e); } } - #endif /* free unused memory while object is not in memory */ - #if OLD_STORE_STRU - #define destroy_store_mem_obj(e) - #else void destroy_store_mem_obj(e) StoreEntry *e; { --- 257,265 ---- *************** *** 290,305 **** meta_data.store_in_mem_objects--; } } - #endif /* Check if there is memory allocated for object in memory */ - #if OLD_STORE_STRU - /* always TRUE for old implementation */ - int has_mem_obj(e) - { - return (TRUE); - } - #else int has_mem_obj(e) StoreEntry *e; { --- 269,276 ---- *************** *** 306,317 **** if (e && e->mem_obj) return (TRUE); return (FALSE); } - #endif /* allocate memory for swapping object in memory */ - #if OLD_STORE_STRU - #define create_store_mem_obj(e) - #else void create_store_mem_obj(e) StoreEntry *e; { --- 277,284 ---- *************** *** 321,327 **** meta_data.store_in_mem_objects++; } } - #endif /* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */ --- 288,293 ---- *************** *** 414,422 **** safe_free(e->url); if (!(e->flag & KEY_URL)) safe_free(e->key); if (has_mem_obj(e)) { ! safe_free(store_mem_obj(e,mime_hdr)); ! safe_free(store_mem_obj(e,e_abort_msg)); ! safe_free(store_mem_obj(e,pending)); } destroy_StoreEntry(e); --- 380,389 ---- safe_free(e->url); if (!(e->flag & KEY_URL)) safe_free(e->key); if (has_mem_obj(e)) { ! safe_free(store_mem_obj(e,mime_hdr)); ! safe_free(store_mem_obj(e,e_abort_msg)); ! safe_free(store_mem_obj(e,pending)); ! store_mem_obj(e,pending_list_size) = 0; } destroy_StoreEntry(e); *************** *** 479,485 **** (e->swap_status != SWAP_OK) && (e->status != STORE_PENDING)) { /* where is it ???? */ ! debug(1, "storeLockObject: Conflicting status: NOT_IN_MEMORY && NO_SWAP: .", e->url); return -1; } e->lastref = time(0); --- 446,452 ---- (e->swap_status != SWAP_OK) && (e->status != STORE_PENDING)) { /* where is it ???? */ ! debug(1, "storeLockObject: Conflicting status: NOT_IN_MEMORY && NO_SWAP: .\n", e->url); return -1; } e->lastref = time(0); *************** *** 682,697 **** e->key = e->url; BIT_SET(e->flag, KEY_URL); - #if OLD_STORE_STRU - store_mem_obj(e,mime_hdr) = NULL; - store_mem_obj(e,e_current_len) = 0; - store_mem_obj(e,data) = NULL; - - /* allocate pending list */ - store_mem_obj(e,pending_list_size) = MIN_PENDING; - store_mem_obj(e,pending) = (struct pentry **) - xcalloc(store_mem_obj(e,pending_list_size), sizeof(struct pentry *)); - #endif if (!table) { storeCreateHashTable(urlcmp); } --- 649,654 ---- *************** *** 812,827 **** } debug(5, "storeAppend: appending %d bytes for %s\n", len, e->url); - #ifdef DW_QUESTIONABLE_CODE - if (store_mem_obj(e,e_current_len) + len > store_mem_high) { - debug(0, "storeAppend: object length about to exceed high water mark!\n"); - /* NOTE: this just gets called repeatedly. Need to do */ - /* something smarter, like call storeAbort()? -DW */ - InvokeHandlers(e); - return -1; - } - #endif - /* get some extra storage if needed */ if (storeGetMemSpace(len, 0) < 0) { debug(0, "storeAppend: Unable to allocate %d bytes, aborting...\n", --- 769,774 ---- *************** *** 942,948 **** #ifdef METADATA_IN_SWAPFILE /* read the swapfile header */ ! file_read (e->swap_fd, store_mem_obj(e,e_swap_buf), sizeof(swap_hdr), 0, (FILE_READ_HD) storeSwapInHandleHdr, (caddr_t) e); return 0; } --- 889,895 ---- #ifdef METADATA_IN_SWAPFILE /* read the swapfile header */ ! file_read (store_mem_obj(e,swap_fd), store_mem_obj(e,e_swap_buf), sizeof(swap_hdr), 0, (FILE_READ_HD) storeSwapInHandleHdr, (caddr_t) e); return 0; } *************** *** 1342,1348 **** debug(1, "Finished rebuilding storage from disk image.\n"); debug(1, " %7d Lines read from previous logfile.\n", linecount); debug(1, " %7d Objects loaded.\n", objcount); ! debug(1, " %7d Ojbects expired.\n", expcount); debug(1, " %7d Duplicate URLs purged.\n", dupcount); debug(1, " %7d Swapfile clashes avoided.\n", clashcount); debug(1, " Took %d seconds (%6.1lf objects/sec).\n", --- 1289,1295 ---- debug(1, "Finished rebuilding storage from disk image.\n"); debug(1, " %7d Lines read from previous logfile.\n", linecount); debug(1, " %7d Objects loaded.\n", objcount); ! debug(1, " %7d Objects expired.\n", expcount); debug(1, " %7d Duplicate URLs purged.\n", dupcount); debug(1, " %7d Swapfile clashes avoided.\n", clashcount); debug(1, " Took %d seconds (%6.1lf objects/sec).\n", *************** *** 1412,1417 **** --- 1359,1365 ---- e->swap_status = NO_SWAP; e->lastref = e->timestamp = time(0); + storeLockObject(e); mk_mime_hdr(mime_hdr, (time_t) e->ttl, 6 + strlen(msg), (time_t) time(NULL), "text/html"); if (msg) { /* This can run off the end here. Be careful */ *************** *** 1429,1438 **** /* Call handlers so they can report error. */ InvokeHandlers(e); ! if (e->flag & RELEASE_REQUEST) { ! storeRelease(e); ! return 0; ! } return 0; } --- 1377,1383 ---- /* Call handlers so they can report error. */ InvokeHandlers(e); ! storeUnlockObject(e); return 0; } *************** *** 1724,1858 **** /* The maximum objects to scan for maintain storage space */ #define SWAP_LRUSCAN_COUNT (256) /* Removes at most 30 LRU objects for one loop */ #define SWAP_LRU_REMOVE_COUNT (24) /* Clear Swap storage to accommodate the given object len */ - #ifdef USE_OLD_GET_SWAP_CODE - int storeGetSwapSpace(size) - int size; - { - static int swap_help = 0; - StoreEntry *LRU = NULL, *e = NULL; - StoreEntry **LRU_list = NULL; - int LRU_list_count = 0; - int scanned = 0; - int removed = 0; - int expired = 0; - int pending = 0; - int locked = 0; - int locked_size = 0; - int i; - int LRU_cur_size = meta_data.store_entries; - - - debug(2, "storeGetSwapSpace: Need %d bytes...\n", size); - - if (store_swap_size + ((size + 1023) >> 10) <= store_swap_high) - return 0; - - LRU_list = (StoreEntry **) xcalloc(LRU_cur_size, sizeof(StoreEntry *)); - - - /* begin to scan for LRU and kick out any expired object if found */ - for (LRU_list_count = 0, e = storeGetFirst(); e; e = storeGetNext()) { - - scanned++; - - if (LRU_list_count >= LRU_cur_size) { - /* have to realloc */ - LRU_cur_size += SWAP_LRUSCAN_BLOCK; - LRU_list = (StoreEntry **) xrealloc((char *) LRU_list, - LRU_cur_size * sizeof(StoreEntry *)); - } - if (e->status == STORE_PENDING) { - pending++; - continue; - } - if ((cached_curtime - e->timestamp > e->ttl) && - (e->swap_status != SWAP_IN_PROGRESS)) { - debug(2, "storeGetSwapSpace: Expired: \n", e->url); - /* just call release. don't have to check for lock status. - storeRelease will take care of that and set a pending flag - if it's still locked. */ - expired++; - storeRelease(e); - continue; - } - /* now see if it's ok to kick out */ - if (!(e->flag & OBJ_LOCK) || (e->swap_status != SWAP_IN_PROGRESS)) { - /* ok add it to a candidate list */ - LRU_list[LRU_list_count++] = e; - } else { - debug(2, "storeGetSwapSpace: Can't purge %7d bytes: \n", - store_mem_obj(e,e_current_len), e->url); - if (e->flag & OBJ_LOCK) { - debug(2, "\t\te->flag & OBJ_LOCK\n"); - } - if (e->swap_status == SWAP_IN_PROGRESS) { - debug(2, "\t\te->swap_status == SWAP_IN_PROGRESS\n"); - } - locked++; - locked_size += store_mem_obj(e,e_current_len); - } - - } - - /* end of candidate selection */ - debug(2, "storeGetSwapSpace: Current Size: %7d kbytes\n", store_swap_size); - debug(2, "storeGetSwapSpace: High W Mark: %7d kbytes\n", store_swap_high); - debug(2, "storeGetSwapSpace: Low W Mark: %7d kbytes\n", store_swap_low); - debug(2, "storeGetSwapSpace: Entry count: %7d items\n", meta_data.store_entries); - debug(2, "storeGetSwapSpace: Scanned: %7d items\n", scanned); - debug(2, "storeGetSwapSpace: Expired: %7d items\n", expired); - debug(2, "storeGetSwapSpace: Locked: %7d items\n", locked); - debug(2, "storeGetSwapSpace: Locked Space: %7d bytes\n", locked_size); - debug(2, "storeGetSwapSpace: LRU candidate: %7d items\n", LRU_list_count); - - /* sort LRU candidate list */ - qsort((char *) LRU_list, LRU_list_count, sizeof(e), compareLastRef); - - - /* Kick LRU out until we have enough swap space */ - for (i = 0; - (i < LRU_list_count) && - (store_swap_size + ((size + 1023) >> 10) > store_swap_low); - ++i) { - - if ((LRU = LRU_list[i]) != NULL) { - if (storeRelease(LRU) == 0) { - removed++; - } else { - debug(2, "storeGetSwapSpace: Help! Can't remove objects. <%s>\n", - LRU->url); - } - } else { - break; - } - - } - debug(2, "storeGetSwapSpace: After Freeing Size: %7d kbytes\n", store_swap_size); - - /* free the list */ - safe_free(LRU_list); - - if ((store_swap_size + ((size + 1023) >> 10) > store_swap_high)) { - debug(0, "storeGetSwapSpace: Help! Nothing to free with %d Kbyte allocated\n", - store_swap_size); - debug(0, " Ask for %d bytes\n", size); - ++swap_help; - if (swap_help > SWAP_MAX_HELP) { - /* dump core for tracing back */ - fatal_dump("storeGetSwapSpace: Repeat Errors. Exiting."); - return -1; - } - } - /* else */ - debug(1, "storeGetSwapSpace: Finished, %d objects removed.\n", removed); - return 0; - } - #else - int storeGetSwapSpace(size) int size; { --- 1669,1679 ---- /* The maximum objects to scan for maintain storage space */ #define SWAP_LRUSCAN_COUNT (256) + /* Removes at most 30 LRU objects for one loop */ #define SWAP_LRU_REMOVE_COUNT (24) /* Clear Swap storage to accommodate the given object len */ int storeGetSwapSpace(size) int size; { *************** *** 1996,2002 **** return 0; } - #endif /* USE_OLD_GET_SWAP_CODE */ /* release an object from a cache */ --- 1817,1822 ---- Index: src/cache/server/store.h =================================================================== RCS file: /projects/agile/cvs/harvest/src/cache/server/store.h,v retrieving revision 1.33 retrieving revision 1.35 diff -c -r1.33 -r1.35 *** 1.33 1995/09/05 19:09:14 --- 1.35 1995/09/15 23:52:30 *************** *** 1,5 **** /* ! * $Id: store.h,v 1.33 1995/09/05 19:09:14 hardy Exp $ * * File: store.h * Description: Interface to cache storage manager. --- 1,5 ---- /* ! * $Id: store.h,v 1.35 1995/09/15 23:52:30 duane Exp $ * * File: store.h * Description: Interface to cache storage manager. *************** *** 119,217 **** extern char *HTTP_OPS[]; - /* The data structure StoreEntry has been split into two structures - for cutting down the memory usage. - You can disable this implementation by setting OLD_STORE_STRU to 1, - */ - #define OLD_STORE_STRU 0 - - #if OLD_STORE_STRU - /* This sentry data structure is optimized to reduce space */ - typedef struct sentry { - char *url; - char *key; - char *mime_hdr; /* Mime header info */ - u_num32 flag; - enum { - NOT_IN_MEMORY, SWAPPING_IN, IN_MEMORY - } mem_status:2; - enum { - WAITING, TIMEOUT, DONE, NOPING - } ping_status:2; - enum { - STORE_OK, STORE_PENDING, STORE_ABORTED - } status:2; - enum { - NO_SWAP, SWAP_IN_PROGRESS, SWAP_OK - } swap_status:2; - enum { - REQ_GET = 0, REQ_POST = 1, REQ_HEAD = 2 - } type_id:2; - unsigned short swap_fd; - mem_ptr data; - - /* These items are mutually exclusive */ - #define e_swap_buf crunch.swap_buf - #define e_pings_first_miss crunch.first_miss - - union { - edge *first_miss; - char *swap_buf; - } crunch; - - /* These items are also mutually exclusive */ - #define e_swap_buf_len crunch2.swap_buf_len - #define e_pings_n_pings crunch2.pings.npings - #define e_pings_n_acks crunch2.pings.nacks - union { - struct { - unsigned char npings; - unsigned char nacks; - } pings; - int swap_buf_len; - } crunch2; - - #ifdef USE_DANZIG_OLD_CODE - #define e_swap_access crunch3.swap_access - #define e_abort_msg crunch3.abort_msg - union { - int swap_access; - char *abort_msg; - } crunch3; - #else - #define e_swap_access no_union_swap_access - #define e_abort_msg no_union_abort_msg - int no_union_swap_access; - char *no_union_abort_msg; - #endif - - int e_current_len; - int object_len; - u_num32 timestamp; - u_num32 lastref; - u_num32 refcount; - u_num32 ttl; - - /* use another field to avoid changing the existing code */ - struct pentry **pending; - unsigned char pending_list_size; - - #ifdef VERY_RISKY_CODE - u_num32 *pending_ip; - unsigned char ipnum_pending; - #endif - - /* WARNING: Explicit assummption that fewer than 256 - WARNING: clients all hop onto the same object. The code - WARNING: doesn't deal with this case. - */ - unsigned char lock_count; - - int swap_file_number; - u_num32 swap_offset; - - } StoreEntry; - #else /* --------------- SPLIT STORE STRUCTURE ----------------- */ /* Split 'StoreEntry' into two structure, when object is purged out from memory, one structure can be freed for saving memory --- 119,124 ---- *************** *** 288,306 **** enum { NOT_IN_MEMORY, SWAPPING_IN, IN_MEMORY ! } mem_status:2; enum { WAITING, TIMEOUT, DONE, NOPING ! } ping_status:2; enum { STORE_OK, STORE_PENDING, STORE_ABORTED ! } status:2; enum { NO_SWAP, SWAP_IN_PROGRESS, SWAP_OK ! } swap_status:2; enum { REQ_GET = 0, REQ_POST = 1, REQ_HEAD = 2 ! } type_id:2; /* WARNING: Explicit assummption that fewer than 256 --- 195,213 ---- enum { NOT_IN_MEMORY, SWAPPING_IN, IN_MEMORY ! } mem_status:3; enum { WAITING, TIMEOUT, DONE, NOPING ! } ping_status:3; enum { STORE_OK, STORE_PENDING, STORE_ABORTED ! } status:3; enum { NO_SWAP, SWAP_IN_PROGRESS, SWAP_OK ! } swap_status:3; enum { REQ_GET = 0, REQ_POST = 1, REQ_HEAD = 2 ! } type_id:3; /* WARNING: Explicit assummption that fewer than 256 *************** *** 310,322 **** unsigned char lock_count; } StoreEntry; - #endif - #if OLD_STORE_STRU - #define store_mem_obj(a,b) (a->b) - #else #define store_mem_obj(a,b) ((a)->mem_obj->b) ! #endif /* ----------------------------------------------------------------- */ typedef int (*PIF) _PARAMS((int, StoreEntry *, caddr_t)); --- 217,225 ---- unsigned char lock_count; } StoreEntry; #define store_mem_obj(a,b) ((a)->mem_obj->b) ! /* ----------------------------------------------------------------- */ typedef int (*PIF) _PARAMS((int, StoreEntry *, caddr_t)); Index: src/common/include/version.h =================================================================== RCS file: /projects/agile/cvs/harvest/src/common/include/version.h,v retrieving revision 1.4 retrieving revision 1.5 diff -c -r1.4 -r1.5 *** 1.4 1995/09/10 05:51:07 --- 1.5 1995/09/13 21:04:56 *************** *** 2,6 **** * HARVEST_VERSION - String for version id of this distribution */ #ifndef HARVEST_VERSION ! #define HARVEST_VERSION "1.3 pl 1" #endif --- 2,6 ---- * HARVEST_VERSION - String for version id of this distribution */ #ifndef HARVEST_VERSION ! #define HARVEST_VERSION "1.3 pl 2" #endif Index: src/common/url/ftp.c =================================================================== RCS file: /projects/agile/cvs/harvest/src/common/url/ftp.c,v retrieving revision 1.15 retrieving revision 1.16 diff -c -r1.15 -r1.16 *** 1.15 1995/09/05 19:09:59 --- 1.16 1995/09/11 22:29:32 *************** *** 1,4 **** ! static char rcsid[] = "$Id: ftp.c,v 1.15 1995/09/05 19:09:59 hardy Exp $"; /* * ftp.c - URL processing for ftp-specific URLs * --- 1,4 ---- ! static char rcsid[] = "$Id: ftp.c,v 1.16 1995/09/11 22:29:32 duane Exp $"; /* * ftp.c - URL processing for ftp-specific URLs * *************** *** 110,118 **** URL *up; { char cmd[BUFSIZ]; sprintf(cmd, "ftpget.pl \"%s\" \"%s\" \"%s\" %c \"%s\" \"%s\"", ! up->filename, up->host, up->pathname, 'I', up->user, up->password); return (run_cmd(cmd)); --- 110,129 ---- URL *up; { char cmd[BUFSIZ]; + char *pathname = NULL; + /* Remove leading slash from FTP url-path so that we can */ + /* handle ftp://user:pw@host/path objects where path and /path */ + /* are quite different. -DW */ + + pathname = up->pathname; + if (!strcmp(pathname, "/")) + *pathname = '.'; + if (*pathname == '/'); + pathname++; + sprintf(cmd, "ftpget.pl \"%s\" \"%s\" \"%s\" %c \"%s\" \"%s\"", ! up->filename, up->host, pathname, 'I', up->user, up->password); return (run_cmd(cmd)); Index: src/common/url/ftpget.pl.in =================================================================== RCS file: /projects/agile/cvs/harvest/src/common/url/ftpget.pl.in,v retrieving revision 1.12 retrieving revision 1.14 diff -c -r1.12 -r1.14 *** 1.12 1995/09/07 23:48:23 --- 1.14 1995/09/13 00:55:44 *************** *** 2,8 **** eval 'exec perl -S $0 "$@"' if $running_under_some_shell; ! # $Id: ftpget.pl.in,v 1.12 1995/09/07 23:48:23 duane Exp $ # ftpget.pl beaten (not hacked) to stuff HTTP and MIME on top of an FTP # retrieval. Works like this: --- 2,8 ---- eval 'exec perl -S $0 "$@"' if $running_under_some_shell; ! # $Id: ftpget.pl.in,v 1.14 1995/09/13 00:55:44 duane Exp $ # ftpget.pl beaten (not hacked) to stuff HTTP and MIME on top of an FTP # retrieval. Works like this: *************** *** 134,140 **** if ($#ARGV > $[-1 && $ARGV[0] eq '-v') { @F = split ('/', $0); print pop @F, ': '; ! print '$Id: ftpget.pl.in,v 1.12 1995/09/07 23:48:23 duane Exp $'; print "\n"; exit (0); } --- 134,140 ---- if ($#ARGV > $[-1 && $ARGV[0] eq '-v') { @F = split ('/', $0); print pop @F, ': '; ! print '$Id: ftpget.pl.in,v 1.14 1995/09/13 00:55:44 duane Exp $'; print "\n"; exit (0); } *************** *** 163,168 **** --- 163,172 ---- $user = shift(@ARGV); # user name $password = shift(@ARGV); # password + $userinfo = ''; + $userinfo = $user . ':' . $password . '@' + unless ($user eq 'anonymous'); + # $do_tmpfile = 1 is only allowed if $htmlify also == 1. # $do_tmpfile = 0 unless ($htmlify); *************** *** 307,317 **** open (OUT, ">$lfile") || &fail (5, "$lfile: $!"); ! $dir = "/" . $dir if ($dir !~ /^\//); $dir_escaped = &url_escape($dir); ! $URL = "ftp://" . $host . $dir . "/"; $URL =~ s'//$'/'; ! $URLescaped = "ftp://" . $host . $dir_escaped . "/"; $URLescaped =~ s'//$'/'; print OUT "\n"; --- 311,322 ---- open (OUT, ">$lfile") || &fail (5, "$lfile: $!"); ! $dir = '/' if ($dir eq '.'); ! $dir = '/' . $dir if ($dir !~ /^\//); $dir_escaped = &url_escape($dir); ! $URL = 'ftp://' . $host . $dir . '/'; $URL =~ s'//$'/'; ! $URLescaped = 'ftp://' . $userinfo . $host . $dir_escaped . '/'; $URLescaped =~ s'//$'/'; print OUT "\n"; *************** *** 347,352 **** --- 352,365 ---- print OUT &htmlize_list_entry ($_); } print OUT "\n"; + print OUT "
\n"; + @T = gmtime(time); + printf OUT ("Listing generated %s, %02d-%s-%02d %02d:%02d:%02d GMT\n", + $weekday[$T[6]], + $T[3], + $month[$T[4]], + $T[5] % 100, + $T[2], $T[1], $T[0]); close (OUT); &ftp'dir_close(); 0; *************** *** 427,434 **** $pd = $dir_escaped; $pd =~ s'/[^/]+$'/'; $icon = '[DIR] '; ! $link = sprintf ('%s', ! $host, $pd, "Parent Directory"); $buf = sprintf ("%s %s\n", $icon, $link); return $buf; } --- 440,447 ---- $pd = $dir_escaped; $pd =~ s'/[^/]+$'/'; $icon = '[DIR] '; ! $link = sprintf ('%s', ! $userinfo, $host, $pd, "Parent Directory"); $buf = sprintf ("%s %s\n", $icon, $link); return $buf; } *************** *** 450,462 **** LE]">', &get_mime_icon($longf)); $link = sprintf ('%s', $URLescaped, ! &url_escape($f), &make_leader($fA)); $buf = sprintf ("%s %s [%s] %6dk\n", $icon, $link, $d, $s); } elsif ($p =~ /^d/) { $icon = '[DIR] ' ; $link = sprintf ('%s', $URLescaped, ! &url_escape($f), &make_leader($fA)); $buf = sprintf ("%s %s [%s]\n", $icon, $link, $d); } elsif ($p =~ /^l/) { $icon = sprintf('[LI --- 463,475 ---- LE]', &get_mime_icon($longf)); $link = sprintf ('%s', $URLescaped, ! &url_escape($longf), &make_leader($fA)); $buf = sprintf ("%s %s [%s] %6dk\n", $icon, $link, $d, $s); } elsif ($p =~ /^d/) { $icon = '[DIR] ' ; $link = sprintf ('%s', $URLescaped, ! &url_escape($longf), &make_leader($fA)); $buf = sprintf ("%s %s [%s]\n", $icon, $link, $d); } elsif ($p =~ /^l/) { $icon = sprintf('[LI *************** *** 464,470 **** ($_ = &get_mime_icon($longf)) eq 'unknown' && defined($fl) ? &get_mime_icon($fl) : $_); $link = sprintf ('<A HREF=%s', $URLescaped, ! &url_escape($f), &make_leader($fA)); $buf = sprintf ("%s %s [%s]\n", $icon, $link, $d); } $buf; --- 477,483 ---- ($_ = &get_mime_icon($longf)) eq 'unknown' && defined($fl) ? &get_mime_icon($fl) : $_); $link = sprintf ('%s', $URLescaped, ! &url_escape($longf), &make_leader($fA)); $buf = sprintf ("%s %s [%s]\n", $icon, $link, $d); } $buf; *************** *** 552,559 **** # If this is a 'raw' transfer, don't write a HTTP error header, # also if we're not putting the FTP data into a tmpfile. # ! if (! $htmlify || $sent_header) { print STDERR "ftpget.pl: ", $msg, "\n"; exit($exitstat); } --- 565,576 ---- # If this is a 'raw' transfer, don't write a HTTP error header, # also if we're not putting the FTP data into a tmpfile. # ! unless ($htmlify) { print STDERR "ftpget.pl: ", $msg, "\n"; + exit($exitstat); + } + if ($sent_header) { + print STDOUT "$msg\n"; exit($exitstat); } Index: src/common/url/newsget.pl =================================================================== RCS file: /projects/agile/cvs/harvest/src/common/url/newsget.pl,v retrieving revision 1.11 retrieving revision 1.12 diff -c -r1.11 -r1.12 *** 1.11 1995/09/05 20:18:19 --- 1.12 1995/09/11 17:57:45 *************** *** 2,8 **** eval 'exec perl -S $0 "$@"' if $running_under_some_shell; # ! # $Id: newsget.pl,v 1.11 1995/09/05 20:18:19 hardy Exp $ # ####################################################################### # Copyright (c) 1994, 1995. All rights reserved. --- 2,8 ---- eval 'exec perl -S $0 "$@"' if $running_under_some_shell; # ! # $Id: newsget.pl,v 1.12 1995/09/11 17:57:45 duane Exp $ # ####################################################################### # Copyright (c) 1994, 1995. All rights reserved. *************** *** 81,88 **** --- 81,92 ---- $ENV{'HARVEST_HOME'} = "/usr/local/harvest" if (!defined($ENV{'HARVEST_HOME'})); unshift(@INC, "$ENV{'HARVEST_HOME'}/lib"); # use local files + @F = split('/', $0); $prog = pop @F; undef @F; + require "socket.ph"; + $debug = 0; + alarm (300); # limit process to 5 minutes real time $nntp_sock = -1; *************** *** 97,140 **** $F = shift; $URL = shift; ! $host="news"; ! $host=$ENV{NNTPSERVER} if ($ENV{NNTPSERVER} ne ""); $port=119; ! open (F, ">$F") || die "$F: $!\n"; if ( $nntp_sock == -1 ) { ! $sockaddr = 'S n a4 x8'; ! chop($thishost = `hostname`); ! ($name, $aliases, $proto) = getprotobyname("tcp"); ! ($name, $aliases, $type, $len, $thisaddr) = gethostbyname($thishost); ! ($fqdn, $aliases, $type, $len, $thataddr) = gethostbyname($host); ! $thissock = pack($sockaddr, &AF_INET, 0, $thisaddr); ! $that = pack($sockaddr, &AF_INET, $port, $thataddr); ! ! socket(NNTP, &AF_INET, &SOCK_STREAM, $proto) || die "socket: $!\n"; ! bind(NNTP, $thissock) || die "bind: $!\n"; ! connect(NNTP, $that) || die "connect: $!\n"; ! $NNTPR = NNTP; ! $NNTPW = NNTP; } else { ! open (NNTPR, "<&$nntp_sock") || die "$nntp_sock: $!\n"; ! open (NNTPW, ">&$nntp_sock") || die "$nntp_sock: $!\n"; $NNTPR = NNTPR; $NNTPW = NNTPW; } select($NNTPW); $| = 1; select(F); if ( $nntp_sock == -1 ) { ! $welcome = <$NNTPR>; ! ($code, @stuff) = split (/[ \t\n]+/, $welcome); ! exit (1) if ($code != 200); } ($access, $path) = split (/:/, $URL); ! die "Specify a news: URL\n" if ($access ne "news"); ! if ($path =~ /@/) { &do_msgid ($path); } else { &do_group ($path); --- 101,167 ---- $F = shift; $URL = shift; ! $host='news'; ! $host=$ENV{'NNTPSERVER'} if ($ENV{'NNTPSERVER'} ne ''); $port=119; ! open (F, ">$F") || &mydie("$F: $!\n"); if ( $nntp_sock == -1 ) { ! local ($sockaddr) = 'S n a4 x8'; ! local ($name, $aliases, $proto) = getprotobyname('tcp'); ! local ($connected) = 0; ! ! # Lookup addresses for remote hostname ! # ! local($w,$x,$y,$z,@thataddrs) = gethostbyname($host); ! &mydie("Unknown Host: $host\n") ! unless (@thataddrs); ! ! # bind local socket to INADDR_ANY ! # ! local($thissock) = pack($sockaddr, &AF_INET, 0, "\0\0\0\0"); ! &mydie("socket: $!\n") unless ! socket (NNTP, &AF_INET, &SOCK_STREAM, $proto); ! &mydie("bind: $!\n") unless ! bind (NNTP, $thissock); ! ! # Try all addresses ! # ! foreach $thataddr (@thataddrs) { ! local ($that) = pack($sockaddr, &AF_INET, $port, $thataddr); ! if (connect (NNTP, $that)) { ! $connected = 1; ! last; ! } ! } ! &mydie("$host:$port: $!\n") ! unless ($connected); ! ! $NNTPR = NNTP; ! $NNTPW = NNTP; ! print STDERR "Connected to $host:$port\n" if ($debug); } else { ! open (NNTPR, "<&$nntp_sock") || &mydie("nntp_sock: $!\n"); ! open (NNTPW, ">&$nntp_sock") || &mydie("nntp_sock: $!\n"); $NNTPR = NNTPR; $NNTPW = NNTPW; + print STDERR "Connected to fd $nntp_sock\n" if ($debug); } select($NNTPW); $| = 1; select(F); if ( $nntp_sock == -1 ) { ! $reply = &read_reply($NNTPR); ! ($code, @stuff) = split (/[ \t\n]+/, $reply); ! &mydie("Bad welcome: $reply") ! if ($code != 200); } ($access, $path) = split (/:/, $URL); ! &mydie("Not a news URL") ! if ($access ne "news"); ! if ($path =~ /\@/) { &do_msgid ($path); } else { &do_group ($path); *************** *** 142,166 **** close F; if ( $nntp_sock == -1 ) { ! print NNTP "QUIT\r\n"; close NNTP; } ! exit 0; # ----- SUBROUTINES -------------------------------------------------- sub do_group { local ($group) = shift @_; ! print $NNTPW "GROUP $group\r\n"; ! $/="\n"; ! $reply = <$NNTPR>; ($code, $nmsgs, $min, $max, $realgroup) = split (/\s+/, $reply); ! exit (1) if ($code != 211); ! exit (0) if ($max == 0); ! print $NNTPW "XOVER $min-$max\r\n"; ! ($code, @stuff) = split (/\s+/, <$NNTPR>); ! exit (1) if ($code != 224); while (<$NNTPR>) { s/\r//g; # remove CR last if (/^\.$/); --- 169,199 ---- close F; if ( $nntp_sock == -1 ) { ! &send_cmd("QUIT"); close NNTP; } ! exit(0); # ----- SUBROUTINES -------------------------------------------------- sub do_group { local ($group) = shift @_; ! ! &send_cmd("GROUP $group"); ! $reply = &read_reply($NNTPR); ($code, $nmsgs, $min, $max, $realgroup) = split (/\s+/, $reply); ! &mydie($reply) ! if ($code != 211); ! &mydie("No messages") ! if ($max == 0); ! ! &send_cmd("XOVER $min-$max"); ! $reply = &read_reply($NNTPR); ! ($code, @stuff) = split (/\s+/, $reply); ! &mydie($reply) ! if ($code != 224); ! while (<$NNTPR>) { s/\r//g; # remove CR last if (/^\.$/); *************** *** 170,186 **** sub do_msgid { local ($msgid) = shift @_; ! print $NNTPW "ARTICLE <$msgid>\r\n"; ! $/="\n"; ! $reply = <$NNTPR>; ($code, $nmsgs, $min, $max, $realgroup) = split (/\s+/, $reply); ! if ($code != 220) { ! print STDERR "newsget.pl: <$msgid> $reply"; ! exit (1); ! } while (<$NNTPR>) { s/\r//g; # remove CR last if (/^\.$/); print F $_; } } --- 203,237 ---- sub do_msgid { local ($msgid) = shift @_; ! ! &send_cmd("ARTICLE <$msgid>"); ! $reply = &read_reply($NNTPR); ($code, $nmsgs, $min, $max, $realgroup) = split (/\s+/, $reply); ! &mydie($reply) ! if ($code != 220); ! while (<$NNTPR>) { s/\r//g; # remove CR last if (/^\.$/); print F $_; } + } + + sub mydie { + $_ = shift; + print STDERR "$prog: $URL: $_\n"; + exit(1); + } + + sub send_cmd { + local ($cmd) = @_; + print STDERR "--> $cmd\n" if ($debug); + print $NNTPW "$cmd\r\n"; + } + + sub read_reply { + $reply = <$NNTPR>; + $reply =~ s/[\r\n]//g; + print STDERR "<-- $reply\n" if ($debug); + return $reply; } Index: src/common/url/rfc850.c =================================================================== RCS file: /projects/agile/cvs/harvest/src/common/url/rfc850.c,v retrieving revision 1.13 retrieving revision 1.14 diff -c -r1.13 -r1.14 *** 1.13 1995/07/27 00:39:10 --- 1.14 1995/09/15 16:55:40 *************** *** 1,4 **** ! static char rcsid[] = "$Id: rfc850.c,v 1.13 1995/07/27 00:39:10 hardy Exp $"; /* * Adapted from HTSUtils.c in CERN httpd 3.0 (http://info.cern.ch/httpd/) * by Darren Hardy , November 1994. --- 1,4 ---- ! static char rcsid[] = "$Id: rfc850.c,v 1.14 1995/09/15 16:55:40 duane Exp $"; /* * Adapted from HTSUtils.c in CERN httpd 3.0 (http://info.cern.ch/httpd/) * by Darren Hardy , November 1994. *************** *** 143,150 **** --- 143,174 ---- struct tm *gmt = gmtime(t); + #ifndef USE_GMT + int gmt_min,gmt_hour,gmt_yday,day_offset; + size_t len; + struct tm *lt; + + /* localtime & gmtime may use the same static data */ + gmt_min = gmt->tm_min; + gmt_hour = gmt->tm_hour; + gmt_yday = gmt->tm_yday; + + lt = localtime(t), + day_offset = lt->tm_yday - gmt_yday; + + /* wrap round on end of year */ + if (day_offset > 1) day_offset = -1; + else if (day_offset < -1) day_offset = 1; + + len = strftime(buf, 127 - 5, "%d/%b/%Y:%H:%M:%S ", lt); + (void)sprintf(buf + len, "%+03d%02d", + lt->tm_hour - gmt_hour + 24 * day_offset, + lt->tm_min - gmt_min); + #else /* USE_GMT */ buf[0] = '\0'; (void)strftime(buf, 127, "%d/%b/%Y:%H:%M:%S -000", gmt); + #endif /* USE_GMT */ + return buf; } Index: src/gatherer/enumerate/ftpenum.pl =================================================================== RCS file: /projects/agile/cvs/harvest/src/gatherer/enumerate/ftpenum.pl,v retrieving revision 1.20 retrieving revision 1.21 diff -c -r1.20 -r1.21 *** 1.20 1995/09/06 06:44:55 --- 1.21 1995/09/11 22:29:34 *************** *** 16,22 **** # # Jim Guyton & Darren Hardy, hardy@cs.colorado.edu, April 1994 # ! # $Id: ftpenum.pl,v 1.20 1995/09/06 06:44:55 duane Exp $ # ####################################################################### # --- 16,22 ---- # # Jim Guyton & Darren Hardy, hardy@cs.colorado.edu, April 1994 # ! # $Id: ftpenum.pl,v 1.21 1995/09/11 22:29:34 duane Exp $ # ####################################################################### # *************** *** 215,221 **** while (!eof($rls)) { ( $path, $size, $time, $type, $mode ) = &lsparse'line($rls); - $path = '/' . $path unless ($path =~ /^\//); $path =~ s/\/\.\//\//g; # remove /./ components print "PATH=$path SIZE=$size TIME=$time TYPE=$type MODE=$mode\n" if ($debug); last if ($path eq ''); --- 215,220 ---- Index: src/gatherer/essence/unnest.c =================================================================== RCS file: /projects/agile/cvs/harvest/src/gatherer/essence/unnest.c,v retrieving revision 1.50 retrieving revision 1.51 diff -c -r1.50 -r1.51 *** 1.50 1995/09/06 07:15:09 --- 1.51 1995/09/12 21:53:32 *************** *** 1,4 **** ! static char rcsid[] = "$Id: unnest.c,v 1.50 1995/09/06 07:15:09 duane Exp $"; /* * unnest.c - Presentation Unnesting for the Essence system. * --- 1,4 ---- ! static char rcsid[] = "$Id: unnest.c,v 1.51 1995/09/12 21:53:32 duane Exp $"; /* * unnest.c - Presentation Unnesting for the Essence system. * *************** *** 184,189 **** --- 184,191 ---- struct dirent *dp; struct stat sb; char buf[MAXPATHLEN+1], *s; + static char unnestsubdir[MAXPATHLEN+1]; + static int count = 0; Debug(65,1,("Unnesting: %s\n", object->url->url)); *************** *** 226,236 **** if (nested_types[cur].do_timestamp && mk_timestamp()) return(NULL); /* We'll need to create new file(s) in the unnestdir, so first cd */ ! if (chdir(unnestdir) < 0) { ! log_errno2(__FILE__,__LINE__,unnestdir); return(NULL); } /* * Run the Extraction process, then gather the extracted * files and return as DataObjectList --- 228,247 ---- if (nested_types[cur].do_timestamp && mk_timestamp()) return(NULL); + + sprintf(unnestsubdir, "%s/%05d", unnestdir, ++count); + if (mkdir(unnestsubdir, 0775) < 0) { + log_errno2(__FILE__,__LINE__,unnestsubdir); + return(NULL); + } + /* We'll need to create new file(s) in the unnestdir, so first cd */ ! Debug(65,1,("presentation_unnest: chdir %s\n", unnestsubdir)); ! if (chdir(unnestsubdir) < 0) { ! log_errno2(__FILE__,__LINE__,unnestsubdir); return(NULL); } + /* * Run the Extraction process, then gather the extracted * files and return as DataObjectList *************** *** 243,248 **** --- 254,260 ---- } /* Go back to previous directory */ + Debug(65,1,("presentation_unnest: chdir %s\n", topdir)); if (chdir(topdir) < 0) { log_errno2(__FILE__,__LINE__,topdir); } *************** *** 266,281 **** * files generated. */ dol = walker = NULL; ! if ((dirp = opendir(unnestdir)) == NULL) { ! log_errno2(__FILE__,__LINE__,unnestdir); errorlog("presentation_unnest: Cannot open directory: %s\n", ! unnestdir); return(NULL); } while ((dp = readdir(dirp)) != NULL) { if (!strcmp(dp->d_name, ".") || !strcmp(dp->d_name, "..")) continue; ! sprintf(buf, "%s/%s", unnestdir, dp->d_name); if (lstat(buf, &sb) < 0) { log_errno2(__FILE__,__LINE__,buf); errorlog("presentation_unnest: Cannot stat: %s\n", buf); --- 278,293 ---- * files generated. */ dol = walker = NULL; ! if ((dirp = opendir(unnestsubdir)) == NULL) { ! log_errno2(__FILE__,__LINE__,unnestsubdir); errorlog("presentation_unnest: Cannot open directory: %s\n", ! unnestsubdir); return(NULL); } while ((dp = readdir(dirp)) != NULL) { if (!strcmp(dp->d_name, ".") || !strcmp(dp->d_name, "..")) continue; ! sprintf(buf, "%s/%s", unnestsubdir, dp->d_name); if (lstat(buf, &sb) < 0) { log_errno2(__FILE__,__LINE__,buf); errorlog("presentation_unnest: Cannot stat: %s\n", buf); *************** *** 340,359 **** fatal("init_presentation_unnest: Cannot create %s\n", unnestdir); } } void finish_presentation_unnest() { char buf[BUFSIZ]; ! if (rmdir(unnestdir) < 0) { ! if (errno == EEXIST) { ! sprintf(buf, "/bin/rm -rf '%s'", unnestdir); ! run_cmd(buf); ! } else { ! log_errno2(__FILE__,__LINE__,unnestdir); ! } ! } ! (void) unlink(extracted_filename); } static int mk_timestamp() --- 352,369 ---- fatal("init_presentation_unnest: Cannot create %s\n", unnestdir); } + memset(extracted_filename, '\0', MAXPATHLEN+1); } void finish_presentation_unnest() { char buf[BUFSIZ]; ! sprintf(buf, "/bin/rm -rf '%s'", unnestdir); ! Debug(65,1,("finish_presentation_unnest: %s\n", buf)); ! run_cmd(buf); ! if (extracted_filename[0]) ! if (unlink(extracted_filename) < 0) ! log_errno2(__FILE__,__LINE__,extracted_filename); } static int mk_timestamp() *************** *** 418,423 **** --- 428,434 ---- if ((s = strrchr(extracted_filename, '.')) != NULL) *s = '\0'; sprintf(cmd, "%s < \"%s\" > \"%s\"", CMD_UNCOMPRESS, object->url->filename, extracted_filename); + Debug(65,1,("extract_Compressed: %s\n", cmd)); return(do_system(cmd)); } *************** *** 428,433 **** --- 439,445 ---- sprintf(cmd, "%s < \"%s\" | %s -xf - ", CMD_UNCOMPRESS, object->url->filename, CMD_TAR); + Debug(65,1,("extract_CompressedTar: %s\n", cmd)); return(do_system(cmd)); } *************** *** 441,446 **** --- 453,459 ---- *s = '\0'; sprintf(cmd, "%s -dc \"%s\" > \"%s\"", CMD_GZIP, object->url->filename, extracted_filename); + Debug(65,1,("extract_GNUCompressed: %s\n", cmd)); return(do_system(cmd)); } *************** *** 451,456 **** --- 464,470 ---- sprintf(cmd, "%s -dc \"%s\" | %s -xf - ", CMD_GZIP, object->url->filename, CMD_TAR); + Debug(65,1,("extract_GNUCompressedTar: %s\n", cmd)); return(do_system(cmd)); } *************** *** 460,465 **** --- 474,480 ---- char cmd[BUFSIZ]; sprintf(cmd, "%s -xf - < \"%s\" ", CMD_TAR, object->url->filename); + Debug(65,1,("extract_Tar: %s\n", cmd)); return(do_system(cmd)); } *************** *** 469,474 **** --- 484,490 ---- char cmd[BUFSIZ]; sprintf(cmd, "uudecode \"%s\" ", object->url->filename); + Debug(65,1,("extract_Uuencoded: %s\n", cmd)); return(do_system(cmd)); } *************** *** 478,483 **** --- 494,500 ---- char cmd[BUFSIZ]; sprintf(cmd, "unshar < \"%s\" ", object->url->filename); + Debug(65,1,("extract_ShellArchive: %s\n", cmd)); return(do_system(cmd)); } *************** *** 487,492 **** --- 504,510 ---- char cmd[BUFSIZ]; sprintf(cmd, "hexbin < \"%s\" ", object->url->filename); + Debug(65,1,("extract_MacBinHex: %s\n", cmd)); return(do_system(cmd)); } *************** *** 501,506 **** --- 519,525 ---- char cmd[BUFSIZ]; sprintf(cmd, "%s -qq -n -o -a \"%s\"", CMD_UNZIP,object->url->filename); + Debug(65,1,("extract_PCZipped: %s\n", cmd)); return(do_system(cmd)); }