diff options
Diffstat (limited to 'debian/htdig/htdig-3.2.0b6/htnet')
30 files changed, 7576 insertions, 0 deletions
diff --git a/debian/htdig/htdig-3.2.0b6/htnet/.cvsignore b/debian/htdig/htdig-3.2.0b6/htnet/.cvsignore new file mode 100644 index 00000000..09dc8ef2 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/.cvsignore @@ -0,0 +1,7 @@ +Makefile +*.lo +*.la +.purify +.pure +.deps +.libs diff --git a/debian/htdig/htdig-3.2.0b6/htnet/Connection.cc b/debian/htdig/htdig-3.2.0b6/htnet/Connection.cc new file mode 100644 index 00000000..e6781db9 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/Connection.cc @@ -0,0 +1,861 @@ +// +// Connection.cc +// +// Connection: This class forms a easy to use interface to the berkeley +// tcp socket library. All the calls are basically the same, +// but the parameters do not have any stray _addr or _in +// mixed in... +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1999-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: Connection.cc,v 1.10 2004/05/28 13:15:22 lha Exp $ +// +#ifdef HAVE_CONFIG_H +#include "htconfig.h" +#endif /* HAVE_CONFIG_H */ + +#include "Connection.h" +#include "Object.h" +#include "List.h" + +#include <errno.h> +#include <stdio.h> +#include <sys/types.h> + +#ifdef _MSC_VER /* _WIN32 */ +#include <windows.h> +#include <winsock.h> +#define EALREADY WSAEALREADY +#define EISCONN WSAEISCONN +#else +#include <sys/socket.h> +#include <arpa/inet.h> // For inet_ntoa +#include <netinet/in.h> +#include <netdb.h> +#include <sys/ioctl.h> +#include <sys/uio.h> +#endif + +#ifndef _MSC_VER /* _WIN32 */ +#include <sys/file.h> +#include <sys/time.h> +#else +#include <io.h> +#endif + +#include <signal.h> +#include <fcntl.h> +#include <stdlib.h> + +#ifndef _MSC_VER /* _WIN32 */ +#include <unistd.h> +#endif + +#ifdef HAVE_STRINGS_H +#include <strings.h> +#endif +#ifdef HAVE_SYS_SELECT_H +#include <sys/select.h> +#endif + +typedef void (*SIGNAL_HANDLER) (...); + +#ifndef _MSC_VER /* _WIN32 */ +extern "C" { + int rresvport(int *); +} +#endif + +#undef MIN +#define MIN(a,b) ((a)<(b)?(a):(b)) + +List all_connections; + +//************************************************************************* +// Connection::Connection(int socket) +// - Default constructor +// PURPOSE: +// Create a connection from just a socket. +// PARAMETERS: +// int socket: obvious!!!! +// +//************************************************************************* +Connection::Connection(int socket) +: pos(0), pos_max(0), + sock(socket), connected(0), peer(""), server_name(""), server_ip_address(""), + need_io_stop(0), timeout_value(0), retry_value(1), + wait_time(5) // wait 5 seconds after a failed connection attempt +{ + Win32Socket_Init(); + + if (socket > 0) + { + GETPEERNAME_LENGTH_T length = sizeof(server); + if (getpeername(socket, (struct sockaddr *)&server, &length) < 0) + perror("getpeername"); + } + + all_connections.Add(this); +} + +// Copy constructor +Connection::Connection(const Connection& rhs) +: pos(rhs.pos), pos_max(rhs.pos_max), + sock(rhs.sock), connected(rhs.connected), + peer(rhs.peer), server_name(rhs.server_name), server_ip_address(rhs.server_ip_address), + need_io_stop(rhs.need_io_stop), timeout_value(rhs.timeout_value), + retry_value(rhs.retry_value), + wait_time(rhs.wait_time) // wait 5 seconds after a failed connection attempt +{ + all_connections.Add(this); +} + + +//***************************************************************************** +// Connection::~Connection() +// +Connection::~Connection() +{ + all_connections.Remove(this); + this->Close(); +} + + +//***************************************************************************** +// int Connection::Win32Socket_init(void) +// +// This function is only used when Code is compiled as a Native Windows +// application +// +// The native Windows socket system needs to be initialized. +// +int Connection::Win32Socket_Init(void) +{ +#ifdef _MSC_VER /* _WIN32 */ + WORD wVersionRequested; + WSADATA wsaData; + + wVersionRequested = MAKEWORD(2, 2); + + if (WSAStartup(wVersionRequested, &wsaData)) + return(-1); + + if (LOBYTE(wsaData.wVersion) != 2 || HIBYTE(wsaData.wVersion) != 2 ) { + WSACleanup(); + return(-1); + } +#endif + + return(0); +} +//***************************************************************************** +// int Connection::Open(int priv) +// +int Connection::Open(int priv) +{ + if (priv) + { + int aport = IPPORT_RESERVED - 1; + +// Native Windows (MSVC) has no rresvport +#ifndef _MSC_VER /* _WIN32 */ + sock = rresvport(&aport); +#else + return NOTOK; +#endif + } + else + { + sock = socket(AF_INET, SOCK_STREAM, 0); + //cout << "socket() sock=" << sock << endl; + } + + if (sock == NOTOK) + return NOTOK; + + int on = 1; + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *) &on, sizeof(on)); + server.sin_family = AF_INET; + + return OK; +} + + +//***************************************************************************** +// int Connection::Ndelay() +// +int Connection::Ndelay() +{ +#ifndef _MSC_VER /* _WIN32 */ + return fcntl(sock, F_SETFL, FNDELAY); +#else + // Note: This function is never called + // TODO: Look into ioctsocket(..) of Win32 Socket API + return(0); +#endif +} + + +//***************************************************************************** +// int Connection::Nondelay() +// +int Connection::Nondelay() +{ +#ifndef _MSC_VER /* _WIN32 */ + return fcntl(sock, F_SETFL, 0); +#else + // Note: This function is never called + // TODO: Look into ioctsocket(..) of Win32 Socket API + return(0); +#endif +} + +//***************************************************************************** +// int Connection::Timeout(int value) +// +int Connection::Timeout(int value) +{ + int oval = timeout_value; + timeout_value = value; + return oval; +} + +//***************************************************************************** +// int Connection::retries(int value) +// +int Connection::Retries(int value) +{ + int oval = retry_value; + retry_value = value; + return oval; +} + +//***************************************************************************** +// int Connection::Close() +// +int Connection::Close() +{ + connected = 0; + if (sock >= 0) + { + int ret = close(sock); + sock = -1; + return ret; + } + return NOTOK; +} + + +//***************************************************************************** +// int Connection::Assign_Port(int port) +// +int Connection::Assign_Port(int port) +{ + server.sin_port = htons(port); + return OK; +} + + +//***************************************************************************** +// int Connection::Assign_Port(char *service) +// +int Connection::Assign_Port(const String &service) +{ + struct servent *sp; + + sp = getservbyname(service, "tcp"); + if (sp == 0) + { + return NOTOK; + } + server.sin_port = sp->s_port; + return OK; +} + +//***************************************************************************** +// int Connection::Assign_Server(unsigned int addr) +// +int Connection::Assign_Server(unsigned int addr) +{ + server.sin_addr.s_addr = addr; + return OK; +} + +#ifndef _MSC_VER /* _WIN32 */ +//extern "C" unsigned int inet_addr(char *); +#endif + +//***************************************************************************** +// +int Connection::Assign_Server(const String& name) +{ + struct hostent *hp; + char **alias_list; + unsigned int addr; + + // + // inet_addr arg IS const char even though prototype says otherwise + // + addr = inet_addr((char*)name.get()); + if (addr == (unsigned int)~0) + { + // Gets the host given a string + hp = gethostbyname(name); + + if (hp == 0) + return NOTOK; + + alias_list = hp->h_aliases; + memcpy((char *)&server.sin_addr, (char *)hp->h_addr, hp->h_length); + } + else + { + memcpy((char *)&server.sin_addr, (char *)&addr, sizeof(addr)); + } + + server_name = name.get(); + server_ip_address = inet_ntoa(server.sin_addr); + + return OK; +} + +// +// Do nothing, we are only interested in the EINTR return of the +// running system call. +// +static void handler_timeout(int) { +} + +//***************************************************************************** +// int Connection::Connect() +// +int Connection::Connect() +{ + int status; + int retries = retry_value; + + while (retries--) + { +#ifndef _MSC_VER /* _WIN32 */ + // + // Set an alarm to make sure the connect() call times out + // appropriately This ensures the call won't hang on a + // dead server or bad DNS call. + // Save the previous alarm signal handling policy, if any. + // + struct sigaction action; + struct sigaction old_action; + memset((char*)&action, '\0', sizeof(struct sigaction)); + memset((char*)&old_action, '\0', sizeof(struct sigaction)); + action.sa_handler = handler_timeout; + sigaction(SIGALRM, &action, &old_action); + alarm(timeout_value); +#endif + + status = connect(sock, (struct sockaddr *)&server, sizeof(server)); + + // + // Disable alarm and restore previous policy if any + // +#ifndef _MSC_VER /* _WIN32 */ + alarm(0); + sigaction(SIGALRM, &old_action, 0); +#endif + + if (status == 0 || errno == EALREADY || errno == EISCONN) + { + connected = 1; + return OK; + } + + // + // Only loop if timed out. Other errors are fatal. + // + if (status < 0 && errno != EINTR) + break; + + // cout << " <" << ::strerror(errno) << "> "; + close(sock); + Open(); + + sleep(wait_time); + + } + +#if 0 + if (status == ECONNREFUSED) + { + // + // For the case where the connection attempt is refused, we need + // to close the socket and create a new one in order to do any + // more with it. + // + Close(sock); + Open(); + } +#else + close(sock); + Open(0); +#endif + + connected = 0; + return NOTOK; +} + + +//***************************************************************************** +// int Connection::Bind() +// +int Connection::Bind() +{ + if (bind(sock, (struct sockaddr *)&server, sizeof(server)) == NOTOK) + { + return NOTOK; + } + return OK; +} + + +//***************************************************************************** +// int Connection::Get_Port() +// +int Connection::Get_Port() +{ + GETPEERNAME_LENGTH_T length = sizeof(server); + + if (getsockname(sock, (struct sockaddr *)&server, &length) == NOTOK) + { + return NOTOK; + } + return ntohs(server.sin_port); +} + + +//***************************************************************************** +// int Connection::Listen(int n) +// +int Connection::Listen(int n) +{ + return listen(sock, n); +} + + +//***************************************************************************** +// Connection *Connection::Accept(int priv) +// +Connection *Connection::Accept(int priv) +{ + int newsock; + + while (1) + { + newsock = accept(sock, (struct sockaddr *)0, (GETPEERNAME_LENGTH_T *)0); + if (newsock == NOTOK && errno == EINTR) + continue; + break; + } + if (newsock == NOTOK) + return (Connection *)0; + + Connection *newconnect = new Connection; + newconnect->sock = newsock; + + GETPEERNAME_LENGTH_T length = sizeof(newconnect->server); + getpeername(newsock, (struct sockaddr *)&newconnect->server, &length); + + if (priv && newconnect->server.sin_port >= IPPORT_RESERVED) + { + delete newconnect; + return (Connection *)0; + } + + return newconnect; +} + + +//************************************************************************* +// Connection *Connection::Accept_Privileged() +// PURPOSE: +// Accept in incoming connection but only if it is from a +// privileged port +// +Connection * Connection::Accept_Privileged() +{ + return Accept(1); +} + +//***************************************************************************** +// int Connection::read_char() +// +int Connection::Read_Char() +{ + if (pos >= pos_max) + { + pos_max = Read_Partial(buffer, sizeof(buffer)); + pos = 0; + if (pos_max <= 0) + { + return -1; + } + } + return buffer[pos++] & 0xff; +} + + +//***************************************************************************** +// String *Connection::Read_Line(String &s, char *terminator) +// +String *Connection::Read_Line(String &s, char *terminator) +{ + int termseq = 0; + s = 0; + + for (;;) + { + int ch = Read_Char(); + if (ch < 0) + { + // + // End of file reached. If we still have stuff in the input buffer + // we need to return it first. When we get called again we will + // return 0 to let the caller know about the EOF condition. + // + if (s.length()) + break; + else + return (String *) 0; + } + else if (terminator[termseq] && ch == terminator[termseq]) + { + // + // Got one of the terminator characters. We will not put + // it in the string but keep track of the fact that we + // have seen it. + // + termseq++; + if (!terminator[termseq]) + break; + } + else + { + s << (char) ch; + } + } + + return &s; +} + + +//***************************************************************************** +// String *Connection::read_line(char *terminator) +// +String *Connection::Read_Line(char *terminator) +{ + String *s; + + s = new String; + return Read_Line(*s, terminator); +} + + +//***************************************************************************** +// char *Connection::read_line(char *buffer, int maxlength, char *terminator) +// +char *Connection::Read_Line(char *buffer, int maxlength, char *terminator) +{ + char *start = buffer; + int termseq = 0; + + while (maxlength > 0) + { + int ch = Read_Char(); + if (ch < 0) + { + // + // End of file reached. If we still have stuff in the input buffer + // we need to return it first. When we get called again, we will + // return 0 to let the caller know about the EOF condition. + // + if (buffer > start) + break; + else + return (char *) 0; + } + else if (terminator[termseq] && ch == terminator[termseq]) + { + // + // Got one of the terminator characters. We will not put + // it in the string but keep track of the fact that we + // have seen it. + // + termseq++; + if (!terminator[termseq]) + break; + } + else + { + *buffer++ = ch; + maxlength--; + } + } + *buffer = '\0'; + + return start; +} + + +//***************************************************************************** +// int Connection::write_line(char *str, char *eol) +// +int Connection::Write_Line(char *str, char *eol) +{ + int n, nn; + + if ((n = Write(str)) < 0) + return -1; + + if ((nn = Write(eol)) < 0) + return -1; + + return n + nn; +} + + +//***************************************************************************** +// int Connection::Write(char *buffer, int length) +// +int Connection::Write(char *buffer, int length) +{ + int nleft, nwritten; + + if (length == -1) + length = strlen(buffer); + + nleft = length; + while (nleft > 0) + { + nwritten = Write_Partial(buffer, nleft); + if (nwritten < 0 && errno == EINTR) + continue; + if (nwritten <= 0) + return nwritten; + nleft -= nwritten; + buffer += nwritten; + } + return length - nleft; +} + + +//***************************************************************************** +// int Connection::Read(char *buffer, int length) +// +int Connection::Read(char *buffer, int length) +{ + int nleft, nread; + + nleft = length; + + // + // If there is data in our internal input buffer, use that first. + // + if (pos < pos_max) + { + int n = MIN(length, pos_max - pos); + + memcpy(buffer, &this->buffer[pos], n); + pos += n; + buffer += n; + nleft -= n; + } + + while (nleft > 0) + { + nread = Read_Partial(buffer, nleft); + if (nread < 0 && errno == EINTR) + continue; + if (nread < 0) + return -1; + else if (nread == 0) + break; + + nleft -= nread; + buffer += nread; + } + return length - nleft; +} + + +void Connection::Flush() +{ + pos = pos_max = 0; +} + +//************************************************************************* +// int Connection::Read_Partial(char *buffer, int maxlength) +// PURPOSE: +// Read at most <maxlength> from the current TCP connection. +// This is equivalent to the workings of the standard read() +// system call +// PARAMETERS: +// char *buffer: Buffer to read the data into +// int maxlength: Maximum number of bytes to read into the buffer +// RETURN VALUE: +// The actual number of bytes read in. +// ASSUMPTIONS: +// The connection has been previously established. +// FUNCTIONS USED: +// read() +// +int Connection::Read_Partial(char *buffer, int maxlength) +{ + int count; + + need_io_stop = 0; + do + { + errno = 0; + + if (timeout_value > 0) { + FD_SET_T fds; + FD_ZERO(&fds); + FD_SET(sock, &fds); + + timeval tv; + tv.tv_sec = timeout_value; + tv.tv_usec = 0; + + int selected = select(sock+1, &fds, 0, 0, &tv); + if (selected <= 0) + need_io_stop++; + } + + if (!need_io_stop) + count = recv(sock, buffer, maxlength, 0); + else + count = -1; // Input timed out + } + while (count <= 0 && errno == EINTR && !need_io_stop); + need_io_stop = 0; + + return count; +} + + +//************************************************************************* +// int Connection::Write_Partial(char *buffer, int maxlength) +// +int Connection::Write_Partial(char *buffer, int maxlength) +{ + int count; + + do + { + + count = send(sock, buffer, maxlength, 0); + + } + while (count < 0 && errno == EINTR && !need_io_stop); + need_io_stop = 0; + + return count; +} + + +//************************************************************************* +// char * Connection::Socket_as_String() +// PURPOSE: +// Return the numeric ASCII equivalent of the socket number. +// This is needed to pass the socket to another program +// +char * Connection::Socket_as_String() +{ + char *buffer = new char[20]; + + sprintf(buffer, "%d", sock); + return buffer; +} + +#ifndef _MSC_VER /* _WIN32 */ +extern "C" char *inet_ntoa(struct in_addr); +#endif + +//************************************************************************* +// char *Connection::Get_Peername() +// +const char* Connection::Get_Peername() +{ + if (peer.empty()) + { + struct sockaddr_in p; + GETPEERNAME_LENGTH_T length = sizeof(p); + struct hostent *hp; + + if (getpeername(sock, (struct sockaddr *) &p, &length) < 0) + { + return 0; + } + + length = sizeof(p.sin_addr); + hp = gethostbyaddr((const char *) &p.sin_addr, length, AF_INET); + if (hp) + peer = (char *) hp->h_name; + else + peer = (char *) inet_ntoa(p.sin_addr); + } + return (const char*) peer.get(); +} + + +//************************************************************************* +// char *Connection::Get_PeerIP() +// +const char* Connection::Get_PeerIP() const +{ + struct sockaddr_in p; + GETPEERNAME_LENGTH_T length = sizeof(p); + + if (getpeername(sock, (struct sockaddr *) &p, &length) < 0) + { + return 0; + } + return (const char*) inet_ntoa(p.sin_addr); +} + +#ifdef NEED_PROTO_GETHOSTNAME +extern "C" int gethostname(char *name, int namelen); +#endif + +//************************************************************************* +// unsigned int GetHostIP(char *ip, int length) +// +unsigned int GetHostIP(char *ip, int length) +{ + char hostname[100]; + if (gethostname(hostname, sizeof(hostname)) == NOTOK) + return 0; + + struct hostent *ent = gethostbyname(hostname); + if (!ent) + return 0; + + struct in_addr addr; + memcpy((char *) &addr.s_addr, ent->h_addr, sizeof(addr)); + if (ip) + strncpy(ip, inet_ntoa(addr), length); + return addr.s_addr; +} + + + +//************************************************************************* +// int Connection::WaitTime(unsigned int _wt) +// +int Connection::WaitTime(unsigned int _wt) +{ + wait_time = _wt; + return OK; +} diff --git a/debian/htdig/htdig-3.2.0b6/htnet/Connection.h b/debian/htdig/htdig-3.2.0b6/htnet/Connection.h new file mode 100644 index 00000000..6a352230 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/Connection.h @@ -0,0 +1,146 @@ +// +// Connection.h +// +// Connection: This class forms a easy to use interface to the berkeley +// tcp socket library. All the calls are basically the same, +// but the parameters do not have any stray _addr or _in +// mixed in... +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1995-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: Connection.h,v 1.10 2004/05/28 13:15:22 lha Exp $ +// + +#ifndef _Connection_h_ +#define _Connection_h_ + +#include "Object.h" +#include "htString.h" + +#include <stdlib.h> +#include <sys/types.h> + +#ifdef _MSC_VER /* _WIN32 */ +#include <winsock.h> +#else +#include <sys/socket.h> +#include <netinet/in.h> +#include <netdb.h> +#endif + +class Connection : public Object +{ +public: + // Constructors & Destructors + Connection(int socket = -1); // Default constructor + Connection(const Connection& rhs); // Copy constructor + ~Connection(); + + // (De)initialization + int Win32Socket_Init(void); + int Open(int priv = 0); + virtual int Close(); + int Ndelay(); + int Nondelay(); + int Timeout(int value); + int Retries(int value); + int WaitTime(unsigned int _wt); + + // Port stuff + int Assign_Port(int port = 0); + int Assign_Port(const String& service); + int Get_Port(); + inline int Is_Privileged(); + + // Host stuff + int Assign_Server(const String& name); + int Assign_Server(unsigned int addr = INADDR_ANY); + const String &Get_Server() const { return server_name; } + const String &Get_Server_IPAddress() const { return server_ip_address; } + + // Connection establishment + virtual int Connect(); + Connection *Accept(int priv = 0); + Connection *Accept_Privileged(); + + // Registration things + int Bind(); + int Listen(int n = 5); + + // IO + String* Read_Line(String &, char *terminator = (char *)"\n"); + char* Read_Line(char *buffer, int maxlength, char *terminator=(char *)"\n"); + String* Read_Line(char *terminator = (char *)"\n"); + virtual int Read_Char(); + int Write_Line(char *buffer, char *eol = (char *)"\n"); + + int Write(char *buffer, int maxlength = -1); + int Read(char *buffer, int maxlength); + + virtual int Read_Partial(char *buffer, int maxlength); + virtual int Write_Partial(char *buffer, int maxlength); + void Stop_IO() {need_io_stop = 1;} + + // Access to socket number + char *Socket_as_String(); + int Get_Socket() { return sock; } + int IsOpen() { return sock >= 0; } + int IsConnected() { return connected; } + + // Access to info about remote socket + const char* Get_PeerIP() const; + const char* Get_Peername(); + + // A method to re-initialize the buffer + virtual void Flush(); + +private: + // + // For buffered IO we will need a buffer + // + enum {BUFFER_SIZE = 8192}; + char buffer[BUFFER_SIZE]; + int pos, pos_max; + // Assignment operator declared private for preventing any use + Connection& operator+ (const Connection& rhs) { return *this; } + +protected: + int sock; + struct sockaddr_in server; + int connected; + String peer; + String server_name; + String server_ip_address; + int need_io_stop; + int timeout_value; + int retry_value; + unsigned int wait_time; // time to wait after an + // unsuccessful connection +}; + + +//************************************************************************* +// inline int Connection::Is_Privileged() +// PURPOSE: +// Return whether the port is priveleged or not. +// +inline int Connection::Is_Privileged() +{ + return server.sin_port < 1023; +} + + +// +// Get arround the lack of gethostip() library call... There is a gethostname() +// call but we want the IP address, not the name! +// The call will put the ASCII string representing the IP address in the supplied +// buffer and it will also return the 4 byte unsigned long equivalent of it. +// The ip buffer can be null... +// +unsigned int gethostip(char *ip = 0, int length = 0); + +#endif diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtCookie.cc b/debian/htdig/htdig-3.2.0b6/htnet/HtCookie.cc new file mode 100644 index 00000000..8a3f8348 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtCookie.cc @@ -0,0 +1,513 @@ +// +// HtCookie.cc +// +// HtCookie: This class represents a HTTP cookie. +// +// HtCookie.cc +// +// by Robert La Ferla. Started 12/5/2000. +// Reviewed by G.Bartolini - since 24 Feb 2001 +// Cookies input file by G.Bartolini - since 27 Jan 2003 +// +//////////////////////////////////////////////////////////// +// +// The HtCookie class represents a single HTTP cookie. +// +// See "PERSISTENT CLIENT STATE HTTP COOKIES" Specification +// at http://www.netscape.com/newsref/std/cookie_spec.html +// Modified according to RFC2109 (max age and version attributes) +// +// This class also manages the creation of a cookie from a line +// of a cookie file format, which is a text file as proposed by Netscape; +// each line contains a name-value pair for a cookie. +// Fields within a single line are separated by the 'tab' character; +// +/////// +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Part of the ht://Check package <http://htcheck.sourceforge.net/> +// Copyright (c) 2001-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: HtCookie.cc,v 1.14 2004/05/28 13:15:22 lha Exp $ +// + +#ifdef HAVE_CONFIG_H +#include "htconfig.h" +#endif /* HAVE_CONFIG_H */ + +#include "HtCookie.h" +#include <stdlib.h> +#include <ctype.h> + +#ifdef HAVE_STD +#include <iostream> +#ifdef HAVE_NAMESPACES +using namespace std; +#endif +#else +#include <iostream.h> +#endif /* HAVE_STD */ + +/////// + // Static variables initialization +/////// + + // Debug level + int HtCookie::debug = 0; + +// Precompiled constants regarding the cookies file format (field order) +#define COOKIES_FILE_DOMAIN 0 +#define COOKIES_FILE_FLAG 1 +#define COOKIES_FILE_PATH 2 +#define COOKIES_FILE_SECURE 3 +#define COOKIES_FILE_EXPIRES 4 +#define COOKIES_FILE_NAME 5 +#define COOKIES_FILE_VALUE 6 + + +// Default constructor +HtCookie::HtCookie() +: name(0), + value(0), + path(0), + domain(0), + expires(0), + isSecure(false), + isDomainValid(true), + srcURL(0), + issue_time(), + max_age(-1), + rfc_version(0) +{ +} + + +// Constructor that accepts a name and a value +// and the calling URL +HtCookie::HtCookie(const String &aName, const String &aValue, + const String& aURL) +: name(aName), + value(aValue), + path(0), + domain(0), + expires(0), + isSecure(false), + isDomainValid(true), + srcURL(aURL), + issue_time(), + max_age(-1), + rfc_version(0) +{ +} + + +// Constructor from a server response header +HtCookie::HtCookie(const String &setCookieLine, const String& aURL) +: name(0), + value(0), + path(0), + domain(0), + expires(0), + isSecure(false), + isDomainValid(true), + srcURL(aURL), + issue_time(), + max_age(-1), + rfc_version(0) +{ + + String cookieLineStr(setCookieLine); + char * token; + const char * str; + + if (debug > 5) + cout << "Creating cookie from response header: " << cookieLineStr << endl; + + // Parse the cookie line + token = strtok(cookieLineStr, "="); + if (token != NULL) + { + SetName(token); + token = strtok(NULL, ";"); + SetValue(token); + } + + // Get all the fields returned by the server + while ((str = strtok(NULL, "="))) + { + const char * ctoken; + + token = stripAllWhitespace(str); + + if (mystrcasecmp(token, "path") == 0) + { + // Let's grab the path + ctoken = strtok(NULL, ";"); + SetPath(ctoken); + } + else if (mystrcasecmp(token, "expires") == 0) + { + // Let's grab the expiration date + HtDateTime dt; + + ctoken = strtok(NULL, ";"); + + if (ctoken && SetDate(ctoken, dt)) + SetExpires(&dt); + else + SetExpires(0); + } else if (mystrcasecmp(token, "secure") == 0) + SetIsSecure(true); + else if (mystrcasecmp(token, "domain") == 0) + { + ctoken = strtok(NULL, ";"); + SetDomain(ctoken); + } + else if (mystrcasecmp(token, "max-age") == 0) + { + ctoken = strtok(NULL, ";"); + SetMaxAge(atoi(ctoken)); + } + else if (mystrcasecmp(token, "version") == 0) + { + ctoken = strtok(NULL, ";"); + SetVersion(atoi(ctoken)); + } + + if (token) + delete[](token); + + } + + if (debug>3) + printDebug(); + +} + + +// Constructor from a line of a cookie file (according to Netscape format) +HtCookie::HtCookie(const String &CookieFileLine) +: name(0), + value(0), + path(0), + domain(0), + expires(0), + isSecure(false), + isDomainValid(true), + srcURL(0), + issue_time(), + max_age(-1), + rfc_version(0) +{ + + String cookieLineStr(CookieFileLine); + char * token; + const char * str; + + if (debug > 5) + cout << "Creating cookie from a cookie file line: " << cookieLineStr << endl; + + // Parse the cookie line + if ((str = strtok(cookieLineStr, "\t"))) + { + int num_field = 0; + int expires_value; // Holds the expires value that will be read + + // According to the field number, set the appropriate object member's value + do + { + + token = stripAllWhitespace(str); + + switch(num_field) + { + case COOKIES_FILE_DOMAIN: + SetDomain(token); + break; + case COOKIES_FILE_FLAG: + // Ignored + break; + case COOKIES_FILE_PATH: + SetPath(token); + break; + case COOKIES_FILE_SECURE: + if (mystrcasecmp(token, "false")) + SetIsSecure(true); + else + SetIsSecure(false); + break; + case COOKIES_FILE_EXPIRES: + if ((expires_value = atoi(token) > 0)) // Sets the expires value only if > 0 + { + time_t tmp = atoi(token); // avoid ambiguous arg list + expires = new HtDateTime(tmp); + } + break; + case COOKIES_FILE_NAME: + SetName(token); + break; + case COOKIES_FILE_VALUE: + SetValue(token); + break; + } + + ++num_field; + } while((str = strtok(NULL, "\t"))); + } + + if (debug>3) + printDebug(); + +} + + +// Copy constructor +HtCookie::HtCookie(const HtCookie& rhs) +: name(rhs.name), + value(rhs.value), + path(rhs.path), + domain(rhs.domain), + expires(0), + isSecure(rhs.isSecure), + isDomainValid(rhs.isDomainValid), + srcURL(rhs.srcURL), + issue_time(rhs.issue_time), + max_age(rhs.max_age), + rfc_version(rhs.rfc_version) +{ + if (rhs.expires) + expires = new HtDateTime(*rhs.expires); +} + +// Destructor +HtCookie::~HtCookie() +{ + // Delete the DateTime info + if (expires) + delete expires; +} + + +// Set the expires datetime +void HtCookie::SetExpires(const HtDateTime *aDateTime) +{ + // + // If expires has not yet been set, + // we just copy the reference + // otherwise, we just change the contents + // of our internal attribute + // + + // We don't have a valid datetime, it's null + if (!aDateTime) + { + if (expires) + delete expires; + + expires=0; + + } + else + { + // We do have a valid datetime + + // Let's check whether expires has already been created + if (!expires) + expires = new HtDateTime(*aDateTime); // No ... let's create it and copy it + + } + +} + +// Strip all the whitespaces +char * HtCookie::stripAllWhitespace(const char * str) +{ + int len; + int i; + int j; + char * newstr; + + len = strlen(str); + newstr = new char[len + 1]; + j = 0; + for (i = 0; i < len; i++) { + char c; + + c = str[i]; + if (isspace(c) == 0) + newstr[j++] = c; + } + newstr[j++] = (char)0; + return newstr; +} + + +// Copy operator overload +const HtCookie &HtCookie::operator = (const HtCookie &rhs) +{ + + // Prevent from copying itself + if (this == &rhs) + return *this; + + // Copy all the values + + name = rhs.name; + value = rhs.value; + path = rhs.path; + domain = rhs.domain; + srcURL = rhs.srcURL; + + // Set the expiration time + SetExpires(rhs.expires); + + isSecure = rhs.isSecure; + isDomainValid = rhs.isDomainValid; + + issue_time = rhs.issue_time; + max_age = rhs.max_age; + + return *this; +} + + +// Print a debug message +ostream& HtCookie::printDebug(ostream &out) +{ + + out << " - "; + + out << "NAME=" << name << " VALUE=" << value << " PATH=" << path; + + if (expires) + out << " EXPIRES=" << expires->GetRFC850(); + + if (domain.length()) + out << " DOMAIN=" << domain << " (" + << (isDomainValid?"VALID":"INVALID") + << ")"; + + if (max_age >= 0) + out << " MAX-AGE=" << max_age; + + if (isSecure) + out << " SECURE"; + + if (srcURL.length() > 0) + out << " - Issued by: " << srcURL; + + out << endl; + + return out; + +} + + +// +// Set the date time value of a cookie's expires +// Given an HtDateTime object and a datestring +// It returns true if everything goes ok +// and false otherwise. +// +int HtCookie::SetDate(const char *datestring, HtDateTime &dt) +{ + if (!datestring) // for any reason we don't have a string for the date + return 0; // and we exit + + DateFormat df; + + while (*datestring && isspace(*datestring)) + datestring++; // skip initial spaces + + df = RecognizeDateFormat(datestring); + if (df == DateFormat_NotRecognized) + { + // Not recognized + if (debug > 0) + cout << "Cookie '" << name + << "' date format not recognized: " << datestring << endl; + + return false; + } + + dt.ToGMTime(); // Set to GM time + + switch(df) + { + // Asc Time format + case DateFormat_AscTime: + dt.SetAscTime((char *)datestring); + break; + + // RFC 1123 + case DateFormat_RFC1123: + dt.SetRFC1123((char *)datestring); + break; + + // RFC 850 + case DateFormat_RFC850: + dt.SetRFC850((char *)datestring); + break; + + default: + if (debug > 0) + cout << "Cookie '" << name + << "' date format not handled: " << (int)df << endl; + break; + } + + return !(df==DateFormat_NotRecognized); + +} + + +// Recognize the date sent by the server +// +// The expires attribute specifies a date string that defines the valid life time +// of that cookie. Once the expiration date has been reached, the cookie will no +// longer be stored or given out. +// +// The date string is formatted as: +// Wdy, DD-Mon-YYYY HH:MM:SS GMT +// This is based on RFC 822, RFC 850, RFC 1036, and RFC 1123, with the variations +// that the only legal time zone is GMT and the separators between the elements +// of the date must be dashes. +// + +HtCookie::DateFormat HtCookie::RecognizeDateFormat(const char *datestring) +{ + + register const char *s; + + if (datestring) + { + + if ((s=strchr(datestring, ','))) + { + // A comma is present. + // Two chances: RFC1123 or RFC850 + + if(strchr(s, '-')) + return DateFormat_RFC850; // RFC 850 recognized + else return DateFormat_RFC1123; // RFC 1123 recognized + } + else + { + // No comma present + + // Let's try C Asctime: Sun Nov 6 08:49:37 1994 + if (strlen(datestring) == 24) + { + return DateFormat_AscTime; + } + } + } + + return DateFormat_NotRecognized; + +} + + diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtCookie.h b/debian/htdig/htdig-3.2.0b6/htnet/HtCookie.h new file mode 100644 index 00000000..934600d8 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtCookie.h @@ -0,0 +1,145 @@ +// +// HtCookie.h +// +// HtCookie: Class for cookies +// +// by Robert La Ferla. Started 12/5/2000. +// Reviewed by G.Bartolini - since 24 Feb 2001 +// Cookies input file by G.Bartolini - since 27 Jan 2003 +// +//////////////////////////////////////////////////////////// +// +// The HtCookie class represents a single HTTP cookie. +// +// See "PERSISTENT CLIENT STATE HTTP COOKIES" Specification +// at http://www.netscape.com/newsref/std/cookie_spec.html +// Modified according to RFC2109 (max age and version attributes) +// +// This class also manages the creation of a cookie from a line +// of a cookie file format, which is a text file as proposed by Netscape; +// each line contains a name-value pair for a cookie. +// Fields within a single line are separated by the 'tab' character; +// +/////// +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Part of the ht://Check package <http://htcheck.sourceforge.net/> +// Copyright (c) 2001-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: HtCookie.h,v 1.10 2004/05/28 13:15:23 lha Exp $ +// + +#ifndef _HTCOOKIE_H +#define _HTCOOKIE_H + +#ifdef HAVE_CONFIG_H +#include "htconfig.h" +#endif + +#include "Object.h" +#include "htString.h" +#include "HtDateTime.h" + +class HtCookie : public Object +{ + public: + + /////// + // Construction/Destruction + /////// + + HtCookie(); // default constructor + HtCookie(const String &setCookieLine, const String& aURL); + HtCookie(const String &aName, const String &aValue, const String& aURL); + HtCookie(const String &line); // From a line of cookie file + HtCookie(const HtCookie& rhs); // default constructor + + ~HtCookie(); // Destructor + + /////// + // Public Interface + /////// + + void SetName(const String &aName) { name = aName; } + void SetValue(const String &aValue) { value = aValue; } + void SetPath(const String &aPath) { path = aPath; } + void SetDomain(const String &aDomain) { domain = aDomain; } + void SetExpires(const HtDateTime *aDateTime); + void SetIsSecure(const bool flag) { isSecure = flag; } + void SetIsDomainValid(const bool flag) { isDomainValid = flag; } + void SetSrcURL(const String &aURL) { srcURL = aURL; } + void SetMaxAge(const int ma) { max_age = ma; } + void SetVersion(const int vs) { rfc_version = vs; } + + const String &GetName() const { return name; } + const String &GetValue()const { return value; } + const String &GetPath()const { return path; } + const String &GetDomain()const { return domain; } + const HtDateTime *GetExpires() const { return expires; } + const bool getIsSecure() const { return isSecure; } + const bool getIsDomainValid() const { return isDomainValid; } + const String &GetSrcURL()const { return srcURL; } + const int GetMaxAge()const { return max_age; } + const HtDateTime &GetIssueTime() const { return issue_time; } + const int GetVersion() const { return rfc_version; } + + // Print debug info +#ifndef _MSC_VER /* _WIN32 */ + virtual ostream &printDebug(ostream &out = std::cout); +#else + virtual ostream &printDebug(ostream &out = cout); +#endif + + // Set the debug level + static void SetDebugLevel (int d) { debug=d;} + + // Copy operator overload + const HtCookie &operator = (const HtCookie &rhs); + + protected: + + /////// + // Date formats enumeration + /////// + + enum DateFormat + { + DateFormat_RFC1123, + DateFormat_RFC850, + DateFormat_AscTime, + DateFormat_NotRecognized + }; + + /////// + // Protected methods + /////// + + char * stripAllWhitespace(const char * str); + int SetDate(const char * datestring, HtDateTime &dt); + DateFormat RecognizeDateFormat(const char * datestring); + + String name; + String value; + String path; + String domain; + HtDateTime * expires; + bool isSecure; + bool isDomainValid; + String srcURL; + HtDateTime issue_time; // When the cookie has been created + int max_age; // rfc2109: lifetime of the cookie, in seconds + int rfc_version; + + /////// + // Debug level + /////// + + static int debug; + +}; + +#endif + diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtCookieInFileJar.cc b/debian/htdig/htdig-3.2.0b6/htnet/HtCookieInFileJar.cc new file mode 100644 index 00000000..e18fca0c --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtCookieInFileJar.cc @@ -0,0 +1,155 @@ +/////////////////////////////////////////////////////////////// +// +// File: HtCookieInFileJar.cc - Definition of class 'HtCookieInFileJar' +// +// Author: Gabriele Bartolini <angusgb@users.sf.net> +// Started: Mon Jan 27 14:38:42 CET 2003 +// +// Class which allows a cookie file to be imported in memory +// for ht://Check and ht://Dig applications. +// +// The cookie file format is a text file, as proposed by Netscape, +// and each line contains a name-value pair for a cookie. +// Fields within a single line are separated by the 'tab' character; +// Here is the format for a line, as taken from http://www.cookiecentral.com/faq/#3.5: +// +// domain - The domain that created AND that can read the variable. +// flag - A TRUE/FALSE value indicating if all machines within a given domain +// can access the variable. This value is set automatically by the browser, +// depending on the value you set for domain. +// path - The path within the domain that the variable is valid for. +// secure - A TRUE/FALSE value indicating if a secure connection with the +// domain is needed to access the variable. +// expiration - The UNIX time that the variable will expire on. UNIX time is +// defined as the number of seconds since Jan 1, 1970 00:00:00 GMT. +// name - The name of the variable. +// value - The value of the variable. +// +// +/////////////////////////////////////////////////////////////// +// +// Part of the ht://Check <http://htcheck.sourceforge.net/> +// Copyright (c) 1999-2004 Comune di Prato, Italia +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1999-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +/////////////////////////////////////////////////////////////// +// $Id: HtCookieInFileJar.cc,v 1.5 2004/05/28 13:15:23 lha Exp $ +/////////////////////////////////////////////////////////////// + +#ifndef __HtCookieInFileJar_H +#include "HtCookieInFileJar.h" +#endif + +#include <stdio.h> + +#define MAX_COOKIE_LINE 16384 + +// Costruttore (default constructor) +HtCookieInFileJar::HtCookieInFileJar(const String& fn, int& result) +: _filename(fn) +{ + result = Load(); +} + +// Costruttore di copia (copy constructor) +HtCookieInFileJar::HtCookieInFileJar(const HtCookieInFileJar& rhs) +{ +} + +// Distruttore +HtCookieInFileJar::~HtCookieInFileJar() +{ +} + +// Operatore di assegnamento (assignment operator) +HtCookieInFileJar& HtCookieInFileJar::operator=(const HtCookieInFileJar& rhs) +{ + if (this == &rhs) + return *this; + + // Code for attributes copy + + return *this; // ritorna se stesso +} + + +// Loads the contents of a cookies file into memory +int HtCookieInFileJar::Load() +{ + FILE *f = fopen((const char *)_filename, "r"); + + if (f == NULL) + return -1; + + char buf[MAX_COOKIE_LINE]; + while(fgets(buf, MAX_COOKIE_LINE, f)) + { + if (*buf && *buf != '#' && (strlen(buf) > 10)) // 10 is an indicative value + { + HtCookie *Cookie = new HtCookie(buf); + + // Interface to the insert method + // If the cookie is not valid or has not been added, we'd better delete it + if (!Cookie->GetName().length() + || !AddCookieForHost (Cookie, Cookie->GetSrcURL())) + { + if (debug > 2) + cout << "Discarded cookie line: " << buf; + delete Cookie; + } + + } + } + + return 0; + +} + + +// Outputs a summary of the cookies that have been imported +ostream &HtCookieInFileJar::ShowSummary(ostream &out) +{ + + char * key; + int num_cookies = 0; // Global number of cookies + + cookieDict->Start_Get(); + + out << endl << "Cookies that have been correctly imported from: " << _filename << endl; + + while ((key = cookieDict->Get_Next())) + { + List * list; + HtCookie * cookie; + + list = (List *)cookieDict->Find(key); + list->Start_Get(); + + while ((cookie = (HtCookie *)list->Get_Next())) + { + ++num_cookies; + out << " " << num_cookies << ". " << cookie->GetName() + << ": " << cookie->GetValue() << " (Domain: " << cookie->GetDomain(); + if (debug > 1) + { + out << " - Path: " << cookie->GetPath(); + if (cookie->GetExpires()) + out << " - Expires: " << cookie->GetExpires()->GetRFC850(); + } + out << ")" << endl; + } + + + // Global number of cookies + } + + return out; + +} + +/////////////////////////////////////////////////////////////// diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtCookieInFileJar.h b/debian/htdig/htdig-3.2.0b6/htnet/HtCookieInFileJar.h new file mode 100644 index 00000000..53b0dd06 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtCookieInFileJar.h @@ -0,0 +1,83 @@ +/////////////////////////////////////////////////////////////// +// +// File: HtCookieInFileJar.h - Declaration of class 'HtCookieInFileJar' +// +// Author: Gabriele Bartolini <angusgb@users.sf.net> +// Started: Mon Jan 27 14:38:42 CET 2003 +// +// Class which allows a cookie file to be imported in memory +// for ht://Check and ht://Dig applications. +// +// The cookie file format is a text file, as proposed by Netscape, +// and each line contains a name-value pair for a cookie. +// Fields within a single line are separated by the 'tab' character; +// Here is the format for a line, as taken from http://www.cookiecentral.com/faq/#3.5: +// +// domain - The domain that created AND that can read the variable. +// flag - A TRUE/FALSE value indicating if all machines within a given domain +// can access the variable. This value is set automatically by the browser, +// depending on the value you set for domain. +// path - The path within the domain that the variable is valid for. +// secure - A TRUE/FALSE value indicating if a secure connection with the +// domain is needed to access the variable. +// expiration - The UNIX time that the variable will expire on. UNIX time is +// defined as the number of seconds since Jan 1, 1970 00:00:00 GMT. +// name - The name of the variable. +// value - The value of the variable. +// +/////////////////////////////////////////////////////////////// +// +// Part of the ht://Check <http://htcheck.sourceforge.net/> +// Copyright (c) 1999-2004 Comune di Prato, Italia +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1999-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +/////////////////////////////////////////////////////////////// +// $Id: HtCookieInFileJar.h,v 1.5 2004/05/28 13:15:23 lha Exp $ +/////////////////////////////////////////////////////////////// + +#ifndef __HtCookieInFileJar_H +#define __HtCookieInFileJar_H + +#include "HtCookieMemJar.h" +#include "htString.h" + +class HtCookieInFileJar: public HtCookieMemJar +{ + +// Public Interface +public: + + // Default constructor + HtCookieInFileJar(const String& fn, int& result); + + // Copy constructor + HtCookieInFileJar(const HtCookieInFileJar& rhs); + + // Destructor + ~HtCookieInFileJar(); + + // Assignment operator + HtCookieInFileJar& operator=(const HtCookieInFileJar& rhs); + + // Show stats +#ifdef _MSC_VER /* _WIN32 */ + virtual ostream &ShowSummary (ostream &out = cout); +#else + virtual ostream &ShowSummary (ostream &out = std::cout); +#endif + +// Protected attributes +protected: + String _filename; // Filename + + int Load(); // Load the contents of a cookies file into memory +}; + +#endif + +/////////////////////////////////////////////////////////////// diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtCookieJar.cc b/debian/htdig/htdig-3.2.0b6/htnet/HtCookieJar.cc new file mode 100644 index 00000000..eea9731e --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtCookieJar.cc @@ -0,0 +1,150 @@ +// +// HtCookieJar.cc +// +// HtCookieJar: This class stores/retrieves cookies. +// +// by Robert La Ferla. Started 12/9/2000. +// Reviewed by G.Bartolini - since 24 Feb 2001 +// +//////////////////////////////////////////////////////////// +// +// The HtCookieJar class stores/retrieves cookies. +// It's an abstract class though, which has to be the interface +// for HtHTTP class. +// +// +// See "PERSISTENT CLIENT STATE HTTP COOKIES" Specification +// at http://www.netscape.com/newsref/std/cookie_spec.html +// Modified according to RFC2109 (max age and version attributes) +// +/////// +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Part of the ht://Check package <http://htcheck.sourceforge.net/> +// Copyright (c) 2001-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: HtCookieJar.cc,v 1.6 2004/05/28 13:15:23 lha Exp $ +// + +#include "HtCookieJar.h" + +/////// + // Static variables initialization +/////// + + // Debug level + int HtCookieJar::debug = 0; + +/////// + // Writes the HTTP request line given a cookie + // in a flexible way (chooses between the RFC2109 + // and the specification given by Netscape) +/////// + // + // RFC2109: The syntax for the header is: + // cookie = "Cookie:" cookie-version + // 1*((";" | ",") cookie-value) + // cookie-value = NAME "=" VALUE [";" path] [";" domain] + // cookie-version = "$Version" "=" value + // NAME = attr + // VALUE = value + // path = "$Path" "=" value + // domain = "$Domain" "=" value + // + + +int HtCookieJar::WriteCookieHTTPRequest(const HtCookie &Cookie, + String &RequestString, const int &NumCookies) +{ + + switch (Cookie.GetVersion()) + { + // RFC2109 Version + case 1: + // Writes the string to be sent to the web server + if (NumCookies == 1) + RequestString << "Cookie: $Version=\"1\"; "; + else + RequestString << "; " ; + + // Print complete debug info + if (debug > 6) + { + cout << "Cookie (RFC2109) info: NAME=" << Cookie.GetName() + << " VALUE="<< Cookie.GetValue() + << " PATH=" << Cookie.GetPath(); + + if (Cookie.GetExpires()) + cout << " EXPIRES=" << Cookie.GetExpires()->GetRFC850(); + + cout << endl; + } + + // Prepare cookie line for HTTP protocol + RequestString << Cookie.GetName() << "=" << Cookie.GetValue(); + + if (Cookie.GetPath().length() > 0) + RequestString << " ;$Path=" << Cookie.GetPath(); + + if (Cookie.GetDomain().length() > 0) + RequestString << " ;$Domain=" << Cookie.GetDomain(); + break; + + // Netscape specification + case 0: + // Writes the string to be sent to the web server + if (NumCookies == 1) + RequestString << "Cookie: "; + else + RequestString << "; " ; + + // Print complete debug info + if (debug > 6) + { + cout << "Cookie (Netscape spec) info: NAME=" << Cookie.GetName() + << " VALUE=" << Cookie.GetValue() + << " PATH=" << Cookie.GetPath(); + + if (Cookie.GetExpires()) + cout << " EXPIRES=" << Cookie.GetExpires()->GetRFC850(); + + cout << endl; + } + + // Prepare cookie line for HTTP protocol + RequestString << Cookie.GetName() << "=" << Cookie.GetValue(); + + break; + } + + return true; + +} + + +int HtCookieJar::GetDomainMinNumberOfPeriods(const String& domain) const +{ + // Well ... if a domain has been specified, we need some check-ups + // as the standard says. + static char* TopLevelDomains[] = { "com", "edu", "net", "org", + "gov", "mil", "int", 0}; + + const char* s = strrchr(domain.get(), '.'); + + if (!s) // no 'dot' has been found. Not valid + return 0; + + if (! *(++s)) // nothing after the dot. Not Valid + return 0; + + for (char** p = TopLevelDomains; *p; ++p) + { + if (!strncmp(*p, s, strlen(*p))) + return 2; + } + + return 3; // By default the minimum value +} diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtCookieJar.h b/debian/htdig/htdig-3.2.0b6/htnet/HtCookieJar.h new file mode 100644 index 00000000..cb7829dd --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtCookieJar.h @@ -0,0 +1,128 @@ +// +// HtCookieJar.h +// +// HtCookieJar: Abstract Class for storing/retrieving cookies +// +// by Robert La Ferla. Started 12/9/2000. +// Reviewed by G.Bartolini - since 24 Feb 2001 +// +//////////////////////////////////////////////////////////// +// +// The HtCookieJar class stores/retrieves cookies. +// It's an abstract class though, which has to be the interface +// for HtHTTP class. +// +// The class has only 2 access point from the outside: +// - a method for cookies insertion (AddCookie()); +// - a method for getting the HTTP request for cookies +// (SetHTTPRequest_CookiesString). +// +// See "PERSISTENT CLIENT STATE HTTP COOKIES" Specification +// at http://www.netscape.com/newsref/std/cookie_spec.html +// Modified according to RFC2109 (max age and version attributes) +// +/////// +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Part of the ht://Check package <http://htcheck.sourceforge.net/> +// Copyright (c) 2001-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: HtCookieJar.h,v 1.6 2004/05/28 13:15:23 lha Exp $ +// + +#ifndef _HTCOOKIE_JAR_H +#define _HTCOOKIE_JAR_H + +#ifdef HAVE_CONFIG_H +#include "htconfig.h" +#endif + +#include "Object.h" +#include "htString.h" +#include "HtCookie.h" +#include "URL.h" + +// for ShowSummary() +#ifdef HAVE_STD +#include <iostream> +#ifdef HAVE_NAMESPACES +using namespace std; +#endif +#else +#include <iostream.h> +#endif /* HAVE_STD */ + + +class HtCookieJar : public Object +{ + + public: + + /////// + // Construction/Destruction + /////// + + HtCookieJar() {}; // empty + virtual ~HtCookieJar() {}; // empty + + /////// + // Interface methods + /////// + + // This method allow the insertion of a cookie + // into the jar. + virtual int AddCookie(const String &CookieString, + const URL &url) = 0; + + // Set the request string to be sent to an HTTP server + // for cookies. It manages all the process regarding + // domains and subdomains. + virtual int SetHTTPRequest_CookiesString(const URL &_url, + String &RequestString) = 0; + + // Get the next cookie + virtual const HtCookie* NextCookie() = 0; + + // Reset the iterator + virtual void ResetIterator() = 0; + + // Get the minimum number of periods from a specified domain + // returns 0 if not valid + virtual int GetDomainMinNumberOfPeriods(const String& domain) const; + + // Set its debug level and HtCookie class' + static void SetDebugLevel (int d) + { + debug=d; // internal one + HtCookie::SetDebugLevel(d); // HtCookie's debug level + } + + // Show summary (abstract) + virtual ostream &ShowSummary (ostream &out) = 0; + + protected: + + /////// + // Protected attributes + /////// + + // Writes the HTTP request line given a cookie + virtual int WriteCookieHTTPRequest(const HtCookie &Cookie, + String &RequestString, const int &NumCookies); + + // Print debug info + virtual void printDebug() = 0; + + /////// + // Debug level + /////// + + static int debug; + +}; + +#endif + diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtCookieMemJar.cc b/debian/htdig/htdig-3.2.0b6/htnet/HtCookieMemJar.cc new file mode 100644 index 00000000..25922b27 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtCookieMemJar.cc @@ -0,0 +1,576 @@ + +// HtCookieMemJar.cc +// +// HtCookieMemJar: This class stores/retrieves cookies. +// +// by Robert La Ferla. Started 12/9/2000. +// Reviewed by G.Bartolini - since 24 Feb 2001 +// +//////////////////////////////////////////////////////////// +// +// The HtCookieMemJar class stores/retrieves cookies +// directly into memory. It is derived from HtCookieJar class. +// +// See "PERSISTENT CLIENT STATE HTTP COOKIES" Specification +// at http://www.netscape.com/newsref/std/cookie_spec.html +// Modified according to RFC2109 (max age and version attributes) +// +/////// +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Part of the ht://Check package <http://htcheck.sourceforge.net/> +// Copyright (c) 2001-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: HtCookieMemJar.cc,v 1.10 2004/05/28 13:15:23 lha Exp $ +// + +#ifdef HAVE_CONFIG_H +#include "htconfig.h" +#endif + +#include "HtCookieMemJar.h" +#include "HtCookie.h" +#include "List.h" +#include "Dictionary.h" +#include <stdlib.h> +#include <ctype.h> + +#ifdef HAVE_STD +#include <iostream> +#ifdef HAVE_NAMESPACES +using namespace std; +#endif +#else +#include <iostream.h> +#endif /* HAVE_STD */ + +// Constructor +HtCookieMemJar::HtCookieMemJar() +: _key(0), _list(0), _idx(0) +{ + cookieDict = new Dictionary(); + cookieDict->Start_Get(); // reset the iterator +} + +// Copy constructor +HtCookieMemJar::HtCookieMemJar(const HtCookieMemJar& rhs) +: _key(0), _list(0), _idx(0) +{ + + if (rhs.cookieDict) + { + // Let's perform a deep copy of the 'jar' + cookieDict = new Dictionary(); + rhs.cookieDict->Start_Get(); + + // Let's walk the domains + while (char* d = rhs.cookieDict->Get_Next()) + { + List* l = new List(); + cookieDict->Add(d, l); // add that domain + + // Let's walk the cookies for that domain + if (List* rhsl = (List*) rhs.cookieDict->Find(d)) + { + + rhsl->Start_Get(); + + while (HtCookie* cookie = ((HtCookie *)rhsl->Get_Next())) + { + HtCookie* new_cookie = new HtCookie(*cookie); + l->Add((Object *)new_cookie); // add this cookie + } + } + } + } + else + cookieDict = new Dictionary(); + + cookieDict->Start_Get(); // reset the iterator +} + +// Destructor +HtCookieMemJar::~HtCookieMemJar() +{ + if (debug>4) + printDebug(); + + if (cookieDict) + delete cookieDict; +} + +// Add a cookie to the Jar +int HtCookieMemJar::AddCookie(const String &CookieString, const URL &url) +{ + + // Builds a new Cookie object + HtCookie *Cookie = new HtCookie(CookieString, url.get()); + + // Interface to the insert method + // If the cookie has not been added, we'd better delete it + if (!AddCookieForHost (Cookie, url.host())) + delete Cookie; + + return true; + +} + + +// Add a cookie to a host +int HtCookieMemJar::AddCookieForHost(HtCookie *cookie, String HostName) +{ + + List *list; // pointer to the Cookie list of an exact host + HtCookie *theCookie; + bool inList = false; + +///////////////////////////////////////////////////////////// +// That's an abstract from the Netscape Cookies specification +///////////////////////////////////////////////////////////// +// +// When searching the cookie list for valid cookies, +// a comparison of the domain attributes of the cookie +// is made with the Internet domain name of the host from which the URL +// will be fetched. If there is a tail match, then the cookie +// will go through path matching to see if it should be sent. +// +// "Tail matching" means that domain attribute is matched against +// the tail of the fully qualified domain name of the host. +// A domain attribute of "acme.com" would match host names "anvil.acme.com" +// as well as "shipping.crate.acme.com". +// +// Only hosts within the specified domain can set a cookie +// for a domain and domains must have at least two (2) +// or three (3) periods in them to prevent domains of +// the form: ".com", ".edu", and "va.us". +// +// Any domain that fails within one of the seven special top level domains +// listed below only require two periods. +// Any other domain requires at least three. +// +// The seven special top level domains are: +// "COM", "EDU", "NET", "ORG", "GOV", "MIL", and "INT". +// +// The default value of domain is the host name of the +// server which generated the cookie response. +// +///////////////////////////////////////////////////////////// + + + // Let's get the domain of the cookie + String Domain(cookie->GetDomain()); + + // Lowercase the HostName + HostName.lowercase(); + + if (!Domain.length()) + Domain = HostName; + else + { + Domain.lowercase(); // lowercase the domain + + // The cookie's domain must have a minimum number of periods + // inside, as stated by the abstract cited above + int minimum_periods = GetDomainMinNumberOfPeriods(Domain); + + if (!minimum_periods) + { + if (debug > 2) + cout << "Cookie - Invalid domain " + << "(minimum number of periods): " << Domain << endl; + + cookie->SetIsDomainValid(false); + } + else + { + // Let's see if the domain is now valid + const char* s = Domain.get(); + const char* r = s + strlen(s) - 1; // go to the last char + int num_periods = 1; // at minimum is one + + while (r > s && *r) + { + if (*r == '.' && *(r+1) && *(r+1) != '.') + ++num_periods; // when a 'dot' is found increment + // the number of periods + --r; + } + + if (num_periods >= minimum_periods) // here is a so-far valid domain + { + while (*r && *r == '.') + ++r; // goes beyond the first dot + + if (r>s) + Domain.set((char*) r); // Set the new 'shorter' domain + + + if (HostName.indexOf(Domain.get()) != -1) + { + if (debug > 2) + cout << "Cookie - valid domain: " + << Domain << endl; + } + else if (HostName.length() == 0) + { + if (debug > 2) + cout << "Imported cookie - valid domain: " + << Domain << endl; + } + else + { + cookie->SetIsDomainValid(false); + if (debug > 2) + cout << "Cookie - Invalid domain " + << "(host not within the specified domain): " << Domain << endl; + } + } + else + { + cookie->SetIsDomainValid(false); + if (debug > 2) + cout << "Cookie - Invalid domain " + << "(minimum number of periods): " << Domain << endl; + } + } + } + + if (! cookie->getIsDomainValid()) // Not a valid domain + Domain = HostName; // Set the default + + // Is the host in the dictionary? + if (cookieDict->Exists(Domain) == 0) + { + // No, add a list instance + list = new List(); + cookieDict->Add(Domain, list); + } + else list = (List *)cookieDict->Find(Domain); + + // Is cookie already in list? + list->Start_Get(); + + // Let's start looking for it + // The match is made on the name and the path + + if (debug > 5) + cout << "- Let's go searching for the cookie '" + << cookie->GetName() << "' in the list" << endl; + + while (!inList && (theCookie = (HtCookie *)list->Get_Next())) + { + if ( (theCookie->GetName().compare(cookie->GetName()) == 0 ) + && ( theCookie->GetPath().compare(cookie->GetPath()) == 0 )) + { + // The cookie has been found + inList = true; + + // Let's update the expiration datetime + if (debug > 5) + cout << " - Found: Update cookie expire time." << endl; + + theCookie->SetExpires(cookie->GetExpires()); + + } + } + + // Well ... the cookie wasn't in the list. Until now! ;-) + // Let's go add it! + if (inList == false) + { + if (debug > 5) + cout << " - Not Found: let's go add it." << endl; + + list->Add((Object *)cookie); + } + + return !inList; +} + + +// Retrieve all cookies that are valid for a domain +List * HtCookieMemJar::cookiesForDomain(const String &DomainName) +{ + List * list; + + list = (List *)cookieDict->Find(DomainName); + return list; +} + + + +int HtCookieMemJar::SetHTTPRequest_CookiesString(const URL &_url, + String &RequestString) +{ + + // Let's split the URL domain and get all of the subdomains. + // For instance: + // - bar.com + // - foo.bar.com + // - www.foo.bar.com + + String Domain(_url.host()); + Domain.lowercase(); + + int minimum_periods = GetDomainMinNumberOfPeriods(Domain); + + if (debug > 3) + cout << "Looking for cookies - Domain: " + << Domain + << " (Minimum periods: " << minimum_periods << ")" << endl; + + // Let's get the subdomains, starting from the end + const char* s = Domain.get(); + const char* r = s + strlen(s) - 1; // go to the last char + int num_periods = 1; // at minimum is one + + while (r > s && *r) + { + if (*r == '.' && *(r+1) && *(r+1) != '.') + { + ++num_periods; // when a 'dot' is found increment + // the number of periods + + if (num_periods > minimum_periods) // here is a so-far valid domain + { + const String SubDomain(r+1); + if (debug > 3) + cout << "Trying to find cookies for subdomain: " + << SubDomain << endl; + + if (cookieDict->Exists(SubDomain)) + WriteDomainCookiesString(_url, SubDomain, RequestString); + } + } + + --r; + } + + if (num_periods >= minimum_periods + && cookieDict->Exists(Domain)) + // Let's send cookies for this domain to the Web server ... + WriteDomainCookiesString(_url, Domain, RequestString); + + return true; +} + + + +///////////////////////////////////////////////////////////// +// That's an abstract from the Netscape Cookies specification +///////////////////////////////////////////////////////////// +// +// +// When requesting a URL from an HTTP server, the browser will match +// the URL against all cookies and if any of them match, +// a line containing the name/value pairs of all matching cookies +// will be included in the HTTP request. +// +// Here is the format of that line: +// Cookie: NAME1=OPAQUE_STRING1; NAME2=OPAQUE_STRING2 ... +// +// This method writes on a string (RequestString) the headers +// for cookies settings as defined by Netscape standard +// +///////////////////////////////////////////////////////////// + +int HtCookieMemJar::WriteDomainCookiesString(const URL &_url, + const String &Domain, String &RequestString) +{ + + // Cookie support. We need a list of cookies and a cookie object + List *cookieList; + HtCookie *cookie; + const HtDateTime now; // Instant time, used for checking + // cookies expiration time + + // Let's find all the valid cookies depending on the specified domain + cookieList = cookiesForDomain(Domain); + + if (cookieList) + { + // Let's store the number of cookies eventually sent + int NumCookies = 0; + + if (debug > 5) + cout << "Found a cookie list for: '" << Domain << "'" << endl; + + // Let's crawl the list for getting the 'path' matching ones + cookieList->Start_Get(); + + while ((cookie = (HtCookie *)cookieList->Get_Next())) + { + const String cookiePath = cookie->GetPath(); + const String urlPath = _url.path(); + + // + // Let's see if the cookie has expired + // by checking the Expires value of it + // If it's not empty and the datetime + // is before now. + // + // Another way of determining whether a + // cookie is expired is checking the + // max_age property that is to say: + // (now - issuetime <= maxage). + // + const bool expired = + (cookie->GetExpires() && (*(cookie->GetExpires()) < now)) // Expires + || (HtDateTime::GetDiff(now, cookie->GetIssueTime()) + <= cookie->GetMaxAge()); // Max-age + + if (debug > 5) + cout << "Trying to match paths and expiration time: " + << urlPath << " in " << cookiePath; + + // Is the path matching + if (!expired && !strncmp(cookiePath, urlPath, cookiePath.length())) + { + + if (debug > 5) + cout << " (passed)" << endl; + + ++NumCookies; + + // Write the string by passing the cookie to the superclass' method + WriteCookieHTTPRequest(*cookie, RequestString, NumCookies); + + } + else if (debug > 5) cout << " (discarded)" << endl; + + } + + // Have we sent one cookie at least? + if (NumCookies > 0) + RequestString <<"\r\n"; + + } + + // That's the end of function + return true; +} + + +// Debug info +void HtCookieMemJar::printDebug() +{ + char * key; + + cookieDict->Start_Get(); + + cout << "Summary of the cookies stored so far" << endl; + + while ((key = cookieDict->Get_Next())) + { + List * list; + HtCookie * cookie; + + cout << " - View cookies for: '" << key << "'" << endl; + list = (List *)cookieDict->Find(key); + list->Start_Get(); + + while ((cookie = (HtCookie *)list->Get_Next())) + cookie->printDebug(); + } +} + + +/////// + // Show the summary of the stored cookies +/////// + +ostream &HtCookieMemJar::ShowSummary(ostream &out) +{ + + char * key; + int num_cookies = 0; // Global number of cookies + int num_server = 0; // Number of servers with cookies + + cookieDict->Start_Get(); + + out << endl << "Summary of the cookies" << endl; + out << "======================" << endl; + + while ((key = cookieDict->Get_Next())) + { + List * list; + HtCookie * cookie; + int num_cookies_server = 0; + + ++num_server; // Number of servers with cookies + + out << " Host: '" << key << "'" << endl; + list = (List *)cookieDict->Find(key); + list->Start_Get(); + + while ((cookie = (HtCookie *)list->Get_Next())) + { + ++num_cookies_server; + cookie->printDebug(); + } + + out << " Number of cookies: " << num_cookies_server << endl << endl; + + // Global number of cookies + num_cookies += num_cookies_server; + } + + out << "Total number of cookies: " << num_cookies << endl; + out << "Servers with cookies: " << num_server << endl << endl; + + return out; + +} + + +// Get the next cookie. It is a bit tricky, but for now it is good +const HtCookie* HtCookieMemJar::NextCookie() +{ + if (!cookieDict) + return 0; + + if (!_idx && (_key = cookieDict->Get_Next()) + && (_list = (List *)cookieDict->Find(_key))) + _list->Start_Get(); // the first time we position at the beginning + + ++_idx; + + if (!_key) + return 0; // ends + + if (!_list) + return 0; // ends + +#ifdef _MSC_VER /* _WIN32 */ + const HtCookie *cookie = ((const HtCookie*)_list->Get_Next()); // Cookie object +#else + const HtCookie* cookie( (const HtCookie*)(_list->Get_Next()) ); // Cookie object +#endif + + if (cookie) + return cookie; + else + { + // Non ci sono cookie per l'host. Si passa a quello seguente + if ((_key = cookieDict->Get_Next()) && + (_list = (List *)cookieDict->Find(_key))) + { + _list->Start_Get(); + if ((cookie = (const HtCookie*)_list->Get_Next())) + return cookie; + } + } + + return 0; +} + +// Reset the iterator +void HtCookieMemJar::ResetIterator() +{ + cookieDict->Start_Get(); + _idx = 0; +} + diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtCookieMemJar.h b/debian/htdig/htdig-3.2.0b6/htnet/HtCookieMemJar.h new file mode 100644 index 00000000..811397a5 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtCookieMemJar.h @@ -0,0 +1,127 @@ +// +// HtCookieMemJar.h +// +// HtCookieMemJar: Class for storing/retrieving cookies +// +// by Robert La Ferla. Started 12/9/2000. +// Reviewed by G.Bartolini - since 24 Feb 2001 +// +//////////////////////////////////////////////////////////// +// +// The HtCookieMemJar class stores/retrieves cookies +// directly into memory. +// +// See "PERSISTENT CLIENT STATE HTTP COOKIES" Specification +// at http://www.netscape.com/newsref/std/cookie_spec.html +// Modified according to RFC2109 (max age and version attributes) +// +/////// +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Part of the ht://Check package <http://htcheck.sourceforge.net/> +// Copyright (c) 2001-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: HtCookieMemJar.h,v 1.9 2004/05/28 13:15:23 lha Exp $ +// + +#ifndef _HTCOOKIE_MEM_JAR_H +#define _HTCOOKIE_MEM_JAR_H + +#ifdef HAVE_CONFIG_H +#include "htconfig.h" +#endif + +#include "Object.h" +#include "htString.h" +#include "Dictionary.h" +#include "List.h" +#include "HtCookieJar.h" + +// for ShowSummary() +#ifdef HAVE_STD +#include <iostream> +#ifdef HAVE_NAMESPACES +using namespace std; +#endif +#else +#include <iostream.h> +#endif /* HAVE_STD */ + +class HtCookieMemJar : public HtCookieJar +{ + + public: + + /////// + // Construction/Destruction + /////// + + HtCookieMemJar(); + HtCookieMemJar(const HtCookieMemJar& rhs); + virtual ~HtCookieMemJar(); + + /////// + // Interface methods + /////// + + // Set the request string to be sent to an HTTP server + // for cookies. It manages all the process regarding + // domains and subdomains. + virtual int SetHTTPRequest_CookiesString(const URL &_url, + String &RequestString); + + virtual int AddCookie(const String &CookieString, + const URL &url); + + // Get the next cookie + virtual const HtCookie* NextCookie(); + + // Reset the iterator + virtual void ResetIterator(); + + // Show stats +#ifdef _MSC_VER /* _WIN32 */ + virtual ostream &ShowSummary (ostream &out = cout); +#else + virtual ostream &ShowSummary (ostream &out = std::cout); +#endif + + void printDebug(); + + protected: + + /////// + // Protected methods + /////// + + // Passed a domain, this method writes all the cookies + // directly in the request string for HTTP. + int WriteDomainCookiesString(const URL &_url, + const String &Domain, String &RequestString); + + // Get a list of the cookies for a domain + List *cookiesForDomain(const String &DomainName); + + // Add a cookie in memory + int AddCookieForHost(HtCookie *cookie, String HostName); + + /////// + // Protected attributes + /////// + + /////// + // Internal dictionary of cookies + /////// + + Dictionary * cookieDict; + char* _key; // For iteration purposes + List* _list; // ditto + int _idx; // ditto + +}; + +#endif + diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtFTP.cc b/debian/htdig/htdig-3.2.0b6/htnet/HtFTP.cc new file mode 100644 index 00000000..e89c1319 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtFTP.cc @@ -0,0 +1,248 @@ +// +// HtFTP.cc +// +// HtFTP: Interface classes for retriving documents from FTP servers +// +// Including: +// - Generic class +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1995-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: HtFTP.cc,v 1.7 2004/05/28 13:15:23 lha Exp $ +// + +#ifdef HAVE_CONFIG_H +#include "htconfig.h" +#endif /* HAVE_CONFIG_H */ + +#include "lib.h" +#include "Transport.h" +#include "HtFTP.h" +#include "Dictionary.h" +#include "StringList.h" +#include "defaults.h" // for config + +#include <signal.h> +#include <sys/types.h> +#include <ctype.h> +#include <stdio.h> // for sscanf +#include <sys/stat.h> + +#ifdef HAVE_STD +#include <iostream> +#include <fstream> +#ifdef HAVE_NAMESPACES +using namespace std; +#endif +#else +#include <iostream.h> +#include <fstream.h> +#endif /* HAVE_STD */ + +#ifndef _MSC_VER /* _WIN32 */ +#include <unistd.h> +#endif + +#ifdef _MSC_VER /* _WIN32 */ +#include "dirent_local.h" +#else +#include <dirent.h> // for scandir +#endif + + +/////// + // HtFTP_Response class +/////// + + +// Construction + +HtFTP_Response::HtFTP_Response() +{ +} + + +// Destruction + +HtFTP_Response::~HtFTP_Response() +{ +} + +/////// + // HtFTP generic class + // + // +/////// + + +// Construction + +HtFTP::HtFTP() +{ +} + +// Destruction + +HtFTP::~HtFTP() +{ + // It's empty +} + + +/////// + // Manages the requesting process +/////// + +//NB: HtFTP::DocStatus = return-value (defineret i Transport.h +// Husk HtFTP er nedarvet fra Transport klassen + +HtFTP::DocStatus HtFTP::Request() +{ + HtConfiguration* config= HtConfiguration::config(); + static Dictionary *mime_map = 0; + + // har vi brug for denne mime_map i denne klasse? + // hvis ja, burde den ikke laves til function i HtConfiguration + + + if (!mime_map) // generate mime_map from the current configuration + { + mime_map = new Dictionary(); + ifstream in(config->Find("mime_types").get()); + if (in) + { + String line; + while (in >> line) + { + line.chop("\n\r \t"); + int cmt; + if ((cmt = line.indexOf('#')) >= 0) + line = line.sub(0, cmt); + StringList split_line(line, "\t "); + // Let's cache mime type to lesser the number of + // operator [] callings + String mime_type = split_line[0]; + // Fill map with values. + for (int i = 1; i < split_line.Count(); i++) + mime_map->Add(split_line[i], new String(mime_type)); + } + } + } + + // Reset the response + _response.Reset(); + + struct stat stat_buf; + // Check that it exists, and is a regular file or directory + // Should we allow FIFO's? + if ( stat(_url.path(), &stat_buf) != 0 || + !(S_ISREG(stat_buf.st_mode) || S_ISDIR(stat_buf.st_mode)) ) + return Transport::Document_not_found; + + // Now handle directories with a pseudo-HTML document (and appropriate noindex) + if ( S_ISDIR(stat_buf.st_mode) ) + { + _response._content_type = "text/html"; + _response._contents = "<html><head><meta name=\"robots\" content=\"noindex\">\n"; + + struct dirent *namelist; + DIR *dirList; + String filename; + + if (( dirList = opendir(_url.path()) )) + { + while (( namelist = readdir(dirList) )) + { + filename = _url.path(); + filename << namelist->d_name; + + if ( namelist->d_name[0] != '.' + && stat(filename.get(), &stat_buf) == 0 ) + { + if (S_ISDIR(stat_buf.st_mode)) + _response._contents << "<link href=\"file://" << _url.path() + << "/" << namelist->d_name << "/\">\n"; + else + _response._contents << "<link href=\"file://" << _url.path() + << "/" << namelist->d_name << "\">\n"; + } + } + closedir(dirList); + } + + _response._contents << "</head><body></body></html>\n"; + + if (debug > 4) + cout << " Directory listing: " << endl << _response._contents << endl; + + _response._content_length = stat_buf.st_size; + _response._document_length = _response._contents.length(); + _response._modification_time = new HtDateTime(stat_buf.st_mtime); + _response._status_code = 0; + return Transport::Document_ok; + } + + if (_modification_time && *_modification_time >= HtDateTime(stat_buf.st_mtime)) + return Transport::Document_not_changed; + + const char *ext = strrchr(_url.path(), '.'); + if (ext == NULL) + return Transport::Document_not_local; + + if (mime_map && mime_map->Count()) + { + String *mime_type = (String *)mime_map->Find(ext + 1); + if (mime_type) + _response._content_type = *mime_type; + else + return Transport::Document_not_local; + } + else + { + if ((mystrcasecmp(ext, ".html") == 0) || (mystrcasecmp(ext, ".htm") == 0)) + _response._content_type = "text/html"; + else if (mystrcasecmp(ext, ".txt") == 0) + _response._content_type = "text/plain"; + else + return Transport::Document_not_local; + } + + _response._modification_time = new HtDateTime(stat_buf.st_mtime); + + FILE *f = fopen((const char *)_url.path(), "r"); + if (f == NULL) + return Document_not_found; + + char docBuffer[8192]; + int bytesRead; + while ((bytesRead = fread(docBuffer, 1, sizeof(docBuffer), f)) > 0) + { + if (_response._contents.length() + bytesRead > _max_document_size) + bytesRead = _max_document_size - _response._contents.length(); + _response._contents.append(docBuffer, bytesRead); + if (_response._contents.length() >= _max_document_size) + break; + } + fclose(f); + + _response._content_length = stat_buf.st_size; + _response._document_length = _response._contents.length(); + _response._status_code = 0; + + if (debug > 2) + cout << "Read a total of " << _response._document_length << " bytes\n"; + return Transport::Document_ok; +} + +HtFTP::DocStatus HtFTP::GetDocumentStatus() +{ + // Let's give a look at the return status code + if (_response._status_code == -1) + return Transport::Document_not_found; + return Transport::Document_ok; +} + diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtFTP.h b/debian/htdig/htdig-3.2.0b6/htnet/HtFTP.h new file mode 100644 index 00000000..a0456bf5 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtFTP.h @@ -0,0 +1,120 @@ +// +// HtFTP.h +// +// HtFTP: Class for FTP protocol access (derived from Transport) +// +// Søren Vejrup Carlsen, based on from HtFTP.h by Alexis Mikhailov +// started: 26.08.2002 +// +// //////////////////////////////////////////////////////////// +// +// The HtFTP class should provide an interface for retrieving documents +// from FTP-servers. It derives from Transport class. +// +/////// +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1995-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: HtFTP.h,v 1.3 2004/05/28 13:15:23 lha Exp $ +// + +#ifndef _HTFTP_H +#define _HTFTP_H + +#include "Transport.h" +#include "URL.h" +#include "htString.h" + + +// In advance declarations + +class HtFTP; + +class HtFTP_Response : public Transport_Response +{ + + friend class HtFTP; // declaring friendship + + public: +/////// + // Construction / Destruction +/////// + + HtFTP_Response(); + ~HtFTP_Response(); +}; + +class HtFTP : public Transport +{ +public: + +/////// + // Construction/Destruction +/////// + + HtFTP(); + ~HtFTP(); + + // Information about the method to be used in the request + + // manages a Transport request (method inherited from Transport class) + virtual DocStatus Request (); + + /////// + // Interface for resource retrieving + /////// + + // Set and get the document to be retrieved + void SetRequestURL(URL &u) { _url = u;} + URL GetRequestURL () { return _url;} + + + // Set and get the referring URL + void SetRefererURL (URL u) { _referer = u;} + URL GetRefererURL () { return _referer;} + + + /////// + // Interface for the HTTP Response + /////// + + // We have a valid response only if the status code is not equal to + // initialization value + + Transport_Response *GetResponse() + { + if (_response._status_code != -1) + return &_response; + else return NULL;} + + + // Get the document status + virtual DocStatus GetDocumentStatus(); + +protected: + +/////// + // Member attributes +/////// + + /////// + // Http single Request information (Member attributes) + /////// + + URL _url; // URL to retrieve + URL _referer; // Referring URL + + /////// + // Http Response information + /////// + + HtFTP_Response _response; // Object where response + // information will be stored into +}; + +#endif + diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtFile.cc b/debian/htdig/htdig-3.2.0b6/htnet/HtFile.cc new file mode 100644 index 00000000..45f1632e --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtFile.cc @@ -0,0 +1,341 @@ +// +// HtFile.cc +// +// HtFile: Interface classes for retriving local documents +// +// Including: +// - Generic class +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1995-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: HtFile.cc,v 1.13 2004/05/28 13:15:23 lha Exp $ +// + +#ifdef HAVE_CONFIG_H +#include "htconfig.h" +#endif /* HAVE_CONFIG_H */ + +#include "lib.h" +#include "Transport.h" +#include "HtFile.h" +#include "Dictionary.h" +#include "StringList.h" +#include "defaults.h" // for config + +#include <signal.h> +#include <sys/types.h> +#include <ctype.h> + +#ifdef HAVE_STD +#include <iostream> +#include <fstream> +#ifdef HAVE_NAMESPACES +using namespace std; +#endif +#else +#include <iostream.h> +#include <fstream.h> +#endif /* HAVE_STD */ + +#include <stdio.h> // for sscanf +#include <sys/stat.h> + +#ifndef _MSC_VER /* _WIN32 */ +#include <unistd.h> +#endif + +#ifdef _MSC_VER /* _WIN32 */ +#include "dirent_local.h" +#else +#include <dirent.h> // for scandir +#endif + +#ifdef _MSC_VER /* _WIN32 */ +#define popen _popen +#define pclose _pclose +#define lstat stat +#define readlink(x,y,z) {-1} +#endif + + +/////// + // HtFile_Response class +/////// + + +// Construction + +HtFile_Response::HtFile_Response() +{ +} + + +// Destruction + +HtFile_Response::~HtFile_Response() +{ +} + +/////// + // HtFile generic class + // + // +/////// + + +// Construction + +HtFile::HtFile() +{ +} + +// Destruction + +HtFile::~HtFile() +{ + // It's empty +} + + +// Return mime type indicated by extension ext (which is assumed not +// to contain the '.'), or NULL if ext is not a know mime type. +const String *HtFile::Ext2Mime (const char *ext) +{ + static Dictionary *mime_map = 0; + + if (!mime_map) + { + HtConfiguration* config= HtConfiguration::config(); + mime_map = new Dictionary(); + if (!mime_map) + return NULL; + + if (debug > 2) + cout << "MIME types: " << config->Find("mime_types").get() << endl; + ifstream in(config->Find("mime_types").get()); + if (in) + { + String line; + while (in >> line) + { + line.chop("\n\r \t"); + int cmt; + if ((cmt = line.indexOf('#')) >= 0) + line = line.sub(0, cmt); + StringList split_line(line, "\t "); + // Let's cache mime type to lesser the number of + // operator [] callings + String mime_type = split_line[0]; + // Fill map with values. + for (int i = 1; i < split_line.Count(); i++) + { + if (debug > 3) + cout << "MIME: " << split_line[i] + << "\t-> " << mime_type << endl; + mime_map->Add(split_line[i], new String(mime_type)); + } + } + } + else + { + if (debug > 2) + cout << "MIME types file not found. Using default types.\n"; + mime_map->Add(String("html"), new String("text/html")); + mime_map->Add(String("htm"), new String("text/html")); + mime_map->Add(String("txt"), new String("text/plain")); + mime_map->Add(String("asc"), new String("text/plain")); + mime_map->Add(String("pdf"), new String("application/pdf")); + mime_map->Add(String("ps"), new String("application/postscript")); + mime_map->Add(String("eps"), new String("application/postscript")); + } + } + + // return MIME type, or NULL if not found + return (String *)mime_map->Find(ext); +} + +// Return mime type of the file named 'fname'. +// If the type can't be determined, "application/x-unknown" is returned. +String HtFile::File2Mime (const char *fname) +{ + HtConfiguration* config= HtConfiguration::config(); + + // default to "can't identify" + char content_type [100] = "application/x-unknown\n"; + + String cmd = config->Find ("content_classifier"); + if (cmd.get() && *cmd) + { + cmd << " \"" << fname << '\"'; // allow file names to have spaces + FILE *fileptr; + if ( (fileptr = popen (cmd.get(), "r")) != NULL ) + { + fgets (content_type, sizeof (content_type), fileptr); + pclose (fileptr); + } + } + + // Remove trailing newline, charset or language information + int delim = strcspn (content_type, ",; \n\t"); + content_type [delim] = '\0'; + + if (debug > 1) + cout << "Mime type: " << fname << ' ' << content_type << endl; + return (String (content_type)); +} + +/////// + // Manages the requesting process +/////// + +HtFile::DocStatus HtFile::Request() +{ + // Reset the response + _response.Reset(); + + struct stat stat_buf; + + String path (_url.path()); + decodeURL (path); // Convert '%20' to ' ' etc + + // Check that it exists, and is a regular file or directory + // Don't allow symbolic links to directories; they mess up '../'. + // Should we allow FIFO's? + if ( stat(path.get(), &stat_buf) != 0 || + !(S_ISREG(stat_buf.st_mode) || S_ISDIR(stat_buf.st_mode)) ) + { + return Transport::Document_not_found; + } + + // Now handle directories with a pseudo-HTML document (and appropriate noindex) + if ( S_ISDIR(stat_buf.st_mode) ) + { + _response._content_type = "text/html"; + _response._contents = "<html><head><meta name=\"robots\" content=\"noindex\">\n"; + + struct dirent *namelist; + DIR *dirList; + String filename; + String encodedName; + + if (( dirList = opendir(path.get()) )) + { + while (( namelist = readdir(dirList) )) + { + filename = path; + filename << namelist->d_name; + + if ( namelist->d_name[0] != '.' + && lstat(filename.get(), &stat_buf) == 0 ) + { + // Recursively resolve symbolic links. + // Could leave "absolute" links, or even all not + // containing '../'. That would allow "aliasing" of + // directories without causing loops. + + int i; // avoid infinite loops + for (i=0; (stat_buf.st_mode & S_IFMT) == S_IFLNK && i<10; i++) + { + char link [100]; + int count = readlink(filename.get(), link, sizeof(link)-1); + + if (count < 0) + break; + link [count] = '\0'; + encodedName = link; + encodeURL (encodedName); + URL newURL (encodedName, _url); // resolve relative paths + filename = newURL.path(); + decodeURL (filename); + if (debug > 2) + cout << "Link to " << link << " gives " + << filename.get() << endl; + lstat(filename.get(), &stat_buf); + } + // filename now only sym-link if nested too deeply or I/O err. + + encodeURL (filename, UNRESERVED "/"); // convert ' ' to '%20' etc., but leave "/" intact + if (S_ISDIR(stat_buf.st_mode)) + _response._contents << "<link href=\"file://" + << filename.get() << "/\">\n"; + else if (S_ISREG(stat_buf.st_mode)) + _response._contents << "<link href=\"file://" + << filename.get() << "\">\n"; + } + } + closedir(dirList); + } + + _response._contents << "</head><body></body></html>\n"; + + if (debug > 4) + cout << " Directory listing: " << endl << _response._contents << endl; + + _response._content_length = stat_buf.st_size; + _response._document_length = _response._contents.length(); + _response._modification_time = new HtDateTime(stat_buf.st_mtime); + _response._status_code = 0; + return Transport::Document_ok; + } + + if (_modification_time && *_modification_time >= HtDateTime(stat_buf.st_mtime)) + return Transport::Document_not_changed; + + bool unknown_ext = false; + char *ext = strrchr(path.get(), '.'); + if (ext == NULL) + unknown_ext = true; + else + { + const String *mime_type = Ext2Mime(ext + 1); + if (mime_type) + _response._content_type = *mime_type; + else + unknown_ext = true; + } + if (unknown_ext) + { + _response._content_type = File2Mime (path.get()); + if (!strncmp (_response._content_type.get(), "application/x-", 14)) + return Transport::Document_not_local; + } + + _response._modification_time = new HtDateTime(stat_buf.st_mtime); + + FILE *f = fopen((const char *)path.get(), "r"); + if (f == NULL) + return Document_not_found; + + char docBuffer[8192]; + int bytesRead; + while ((bytesRead = fread(docBuffer, 1, sizeof(docBuffer), f)) > 0) + { + if (_response._contents.length() + bytesRead > _max_document_size) + bytesRead = _max_document_size - _response._contents.length(); + _response._contents.append(docBuffer, bytesRead); + if (_response._contents.length() >= _max_document_size) + break; + } + fclose(f); + + _response._content_length = stat_buf.st_size; + _response._document_length = _response._contents.length(); + _response._status_code = 0; + + if (debug > 2) + cout << "Read a total of " << _response._document_length << " bytes\n"; + return Transport::Document_ok; +} + +HtFile::DocStatus HtFile::GetDocumentStatus() +{ + // Let's give a look at the return status code + if (_response._status_code == -1) + return Transport::Document_not_found; + return Transport::Document_ok; +} + diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtFile.h b/debian/htdig/htdig-3.2.0b6/htnet/HtFile.h new file mode 100644 index 00000000..c595e5cc --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtFile.h @@ -0,0 +1,130 @@ +// +// HtFile.h +// +// HtFile: Class for local files (derived from Transport) +// +// Alexis Mikhailov, from HtHTTP.h by Gabriele Bartolini - Prato - Italia +// started: 03.05.1999 +// +// //////////////////////////////////////////////////////////// +// +// The HtFile class should provide an interface for retrieving local +// documents. It derives from Transport class. +// +/////// +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1995-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: HtFile.h,v 1.6 2004/05/28 13:15:23 lha Exp $ +// + +#ifndef _HTFILE_H +#define _HTFILE_H + +#include "Transport.h" +#include "URL.h" +#include "htString.h" + + +// In advance declarations + +class HtFile; + +class HtFile_Response : public Transport_Response +{ + + friend class HtFile; // declaring friendship + + public: +/////// + // Construction / Destruction +/////// + + HtFile_Response(); + ~HtFile_Response(); +}; + +class HtFile : public Transport +{ +public: + +/////// + // Construction/Destruction +/////// + + HtFile(); + ~HtFile(); + + // Information about the method to be used in the request + + // manages a Transport request (method inherited from Transport class) + virtual DocStatus Request (); + + // Determine Mime type of file from its extension + static const String *Ext2Mime (const char *); + + // Determine Mime type of file from its contents + static String File2Mime (const char *); + + /////// + // Interface for resource retrieving + /////// + + /////// + // Interface for resource retrieving + /////// + + // Set and get the document to be retrieved + void SetRequestURL(URL &u) { _url = u;} + URL GetRequestURL () { return _url;} + + + // Set and get the referring URL + void SetRefererURL (URL u) { _referer = u;} + URL GetRefererURL () { return _referer;} + + + /////// + // Interface for the HTTP Response + /////// + + // We have a valid response only if the status code is not equal to + // initialization value + + Transport_Response *GetResponse() + { + if (_response._status_code != -1) + return &_response; + else return NULL;} + + + // Get the document status + virtual DocStatus GetDocumentStatus(); + +protected: + +/////// + // Member attributes +/////// + + /////// + // Http single Request information (Member attributes) + /////// + + URL _url; // URL to retrieve + URL _referer; // Referring URL + + /////// + // Http Response information + /////// + + HtFile_Response _response; // Object where response + // information will be stored into +}; + +#endif + diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtHTTP.cc b/debian/htdig/htdig-3.2.0b6/htnet/HtHTTP.cc new file mode 100644 index 00000000..2a52ef54 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtHTTP.cc @@ -0,0 +1,1084 @@ +// +// HtHTTP.cc +// +// HtHTTP: Interface classes for HTTP messaging +// +// Including: +// - Generic class +// - Response message class +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1995-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: HtHTTP.cc,v 1.27 2004/05/28 13:15:23 lha Exp $ +// + +#ifdef HAVE_CONFIG_H +#include "htconfig.h" +#endif /* HAVE_CONFIG_H */ + +#include "lib.h" +#include "Transport.h" +#include "HtHTTP.h" + +#include <signal.h> +#include <sys/types.h> +#include <ctype.h> +#include <stdio.h> // for sscanf + +// for setw() +#ifdef HAVE_STD +#include <iomanip> +#ifdef HAVE_NAMESPACES +using namespace std; +#endif +#else +#include <iomanip.h> +#endif /* HAVE_STD */ + +#if 1 +typedef void (*SIGNAL_HANDLER) (...); +#else +typedef SIG_PF SIGNAL_HANDLER; +#endif + + // User Agent + String HtHTTP::_user_agent = 0; + + // Stats information + int HtHTTP::_tot_seconds = 0; + int HtHTTP::_tot_requests = 0; + int HtHTTP::_tot_bytes = 0; + + // flag that manage the option of 'HEAD' before 'GET' + bool HtHTTP::_head_before_get = true; + + // Handler of the CanParse function + + int (* HtHTTP::CanBeParsed) (char *) = 0; + + // Cookies jar + HtCookieJar *HtHTTP::_cookie_jar = 0; // Set to 0 by default + +/////// + // HtHTTP_Response class + // + // Response message sent by the remote HTTP server +/////// + + +// Construction + +HtHTTP_Response::HtHTTP_Response() +: _version(0), + _transfer_encoding(0), + _server(0), + _hdrconnection(0), + _content_language(0) +{ +} + + +// Destruction + +HtHTTP_Response::~HtHTTP_Response() +{ +} + + +void HtHTTP_Response::Reset() +{ + + // Call the base class method in order to reset + // the base class attributes + + Transport_Response::Reset(); + + // Initialize the version, transfer-encoding, location and server strings + _version.trunc(); + _transfer_encoding.trunc(); + _hdrconnection.trunc(); + _server.trunc(); + _content_language.trunc(); + +} + + + + +/////// + // HtHTTP generic class + // + // +/////// + + +// Construction + +HtHTTP::HtHTTP(Connection& connection) +: Transport(&connection), + _Method(Method_GET), // Default Method Request + _bytes_read(0), + _accept_language(0), + _persistent_connection_allowed(true), + _persistent_connection_possible(false), + _send_cookies(true) +{ +} + +// Destruction + +HtHTTP::~HtHTTP() +{ +} + + +/////// + // Manages the requesting process +/////// + +Transport::DocStatus HtHTTP::Request() +{ + + DocStatus result = Document_ok; + +/////// + // We make a double request (HEAD and, maybe, GET) + // Depending on the +/////// + + if (HeadBeforeGet() && // Option value to true + _Method == Method_GET) // Initial request method is GET + { + + if (debug>3) + cout << " Making a HEAD call before the GET" << endl; + + _Method = Method_HEAD; + + result = HTTPRequest(); + + _Method = Method_GET; + } + + if (result == Document_ok) + result = HTTPRequest(); + + if(result == Document_no_header + && isPersistentConnectionAllowed()) + { + + // Sometimes, the parsing phase of the header of the response + // that the server gives us back, fails and a <no header> + // error is raised. This happens with HTTP/1.1 persistent + // connections, usually because the previous response stream + // has not yet been flushed, so the buffer still contains + // data regarding the last document retrieved. That sucks alot! + // The only thing to do is to lose persistent connections benefits + // for this document, so close the connection and 'GET' it again. + + CloseConnection(); // Close a previous connection + + if (debug>0) + cout << "! Impossible to get the HTTP header line." << endl + << " Connection closed. Try to get it again." << endl; + + result = HTTPRequest(); // Get the document again + + } + + return result; +} + + +/////// + // Sends an HTTP 1/1 request +/////// + +Transport::DocStatus HtHTTP::HTTPRequest() +{ + + static Transport::DocStatus DocumentStatus; + bool ShouldTheBodyBeRead = true; + + SetBodyReadingController(&HtHTTP::ReadBody); + + // Reset the response + _response.Reset(); + + // Flush the connection + FlushConnection(); + + _bytes_read=0; + + if( debug > 4) + cout << "Try to get through to host " + << _url.host() << " (port " << _url.port() << ")" << endl; + + ConnectionStatus result; + + // Assign the timeout + AssignConnectionTimeOut(); + + // Assign number of retries + AssignConnectionRetries(); + + // Assign connection wait time + AssignConnectionWaitTime(); + + // Start the timer + _start_time.SettoNow(); + + result = EstablishConnection(); + + if(result != Connection_ok && result != Connection_already_up) + { + + switch (result) + { + // Open failed + + case Connection_open_failed: + if (debug>1) + cout << "Unable to open the connection with host: " + << _url.host() << " (port " << _url.port() << ")" << endl; + CloseConnection(); + return FinishRequest(Document_no_connection); + break; + + // Server not reached + case Connection_no_server: + if (debug>1) + cout << "Unable to find the host: " + << _url.host() << " (port " << _url.port() << ")" << endl; + CloseConnection(); + return FinishRequest(Document_no_host); + break; + + // Port not reached + case Connection_no_port: + if (debug>1) + cout << "Unable to connect with the port " << _url.port() + << " of the host: " << _url.host() << endl; + CloseConnection(); + return FinishRequest(Document_no_port); + break; + + // Connection failed + case Connection_failed: + if (debug>1) + cout << "Unable to establish the connection with host: " + << _url.host() << " (port " << _url.port() << ")" << endl; + CloseConnection(); + return FinishRequest(Document_no_connection); + break; + + // Other reason + default: + if (debug>1) + cout << "connection failed with unexpected result: result = " + << (int)result << ", " + << _url.host() << " (port " << _url.port() << ")" << endl; + CloseConnection(); + return FinishRequest(Document_other_error); + break; + } + + return FinishRequest(Document_other_error); + + } + + // Visual comments about the result of the connection + if (debug > 5) + switch(result) + { + case Connection_already_up: + cout << "Taking advantage of persistent connections" << endl; + break; + case Connection_ok: + cout << "New connection open successfully" << endl; + break; + default: + cout << "Unexptected value: " << (int)result << endl; + break; + } + + String command; + + switch(_Method) + { + case Method_GET: + command = "GET "; + break; + case Method_HEAD: + command = "HEAD "; + ShouldTheBodyBeRead = false; + break; + } + + // Set the request command + + SetRequestCommand(command); + + if (debug > 6) + cout << "Request\n" << command; + + // Writes the command + ConnectionWrite(command); + + // Parse the header + if (ParseHeader() == -1) // Connection down + { + // The connection probably fell down !?! + if ( debug > 4 ) + cout << setw(5) << Transport::GetTotOpen() << " - " + << "Connection fell down ... let's close it" << endl; + + CloseConnection(); // Let's close the connection which is down now + + // Return that the connection has fallen down during the request + return FinishRequest(Document_connection_down); + } + + + if (_response._status_code == -1) + { + // Unable to retrieve the status line + + if ( debug > 4 ) + cout << "Unable to retrieve or parse the status line" << endl; + + return FinishRequest(Document_no_header); + } + + + if (debug > 3) + { + + cout << "Retrieving document " << _url.path() << " on host: " + << _url.host() << ":" << _url.port() << endl; + + cout << "Http version : " << _response._version << endl; + cout << "Server : " << _response._version << endl; + cout << "Status Code : " << _response._status_code << endl; + cout << "Reason : " << _response._reason_phrase << endl; + + if (_response.GetAccessTime()) + cout << "Access Time : " << _response.GetAccessTime()->GetRFC1123() << endl; + + if (_response.GetModificationTime()) + cout << "Modification Time : " << _response.GetModificationTime()->GetRFC1123() << endl; + + cout << "Content-type : " << _response.GetContentType() << endl; + + if (_response._transfer_encoding.length()) + cout << "Transfer-encoding : " << _response._transfer_encoding << endl; + + if (_response._content_language.length()) + cout << "Content-Language : " << _response._content_language << endl; + + if (_response._hdrconnection.length()) + cout << "Connection : " << _response._hdrconnection << endl; + + } + + // Check if persistent connection are possible + CheckPersistentConnection(_response); + + if (debug > 4) + cout << "Persistent connection: " + << (_persistent_connection_possible ? "would be accepted" : "not accepted") + << endl; + + DocumentStatus = GetDocumentStatus(_response); + + // We read the body only if the document has been found + if (DocumentStatus != Document_ok) + { + ShouldTheBodyBeRead=false; + } + + // For now a chunked response MUST BE retrieved + if (mystrncasecmp ((char*)_response._transfer_encoding, "chunked", 7) == 0) + { + // Change the controller of the body reading + SetBodyReadingController(&HtHTTP::ReadChunkedBody); + } + + // If "ShouldTheBodyBeRead" is set to true and + // If the document is parsable, we can read the body + // otherwise it is not worthwhile + + if (ShouldTheBodyBeRead) + { + if ( debug > 4 ) + cout << "Reading the body of the response" << endl; + + // We use a int (HtHTTP::*)() function pointer + if ( (this->*_readbody)() == -1 ) + { + // The connection probably fell down !?! + if ( debug > 4 ) + cout << setw(5) << Transport::GetTotOpen() << " - " + << "Connection fell down ... let's close it" << endl; + + CloseConnection(); // Let's close the connection which is down now + + // Return that the connection has fallen down during the request + return FinishRequest(Document_connection_down); + } + + if ( debug > 6 ) + cout << "Contents:" << endl << _response.GetContents(); + + // Check if the stream returned by the server has not been completely read + + if (_response._document_length != _response._content_length && + _response._document_length == _max_document_size) + { + // Max document size reached + + if (debug > 4) + cout << "Max document size (" << GetRequestMaxDocumentSize() + << ") reached "; + + if (isPersistentConnectionUp()) + { + // Only have to close persistent connection when we didn't read + // all the input. For now, we always read all chunked input... + if (mystrncasecmp ((char*)_response._transfer_encoding, "chunked", 7) != 0) + { + if (debug > 4) + cout << "- connection closed. "; + + CloseConnection(); + } + } + + if (debug > 4) + cout << endl; + } + + // Make sure our content-length makes sense, if none given... + if (_response._content_length < _response._document_length) + _response._content_length = _response._document_length; + + } + else if ( debug > 4 ) + cout << "Body not retrieved" << endl; + + + // Close the connection (if there's no persistent connection) + + if( ! isPersistentConnectionUp() ) + { + if ( debug > 4 ) + cout << setw(5) << Transport::GetTotOpen() << " - " + << "Connection closed (No persistent connection)" << endl; + + CloseConnection(); + } + else + { + // Persistent connection is active + + // If the document is not parsable and we asked for it with a 'GET' + // method, the stream's not been completely read. + + if (DocumentStatus == Document_not_parsable && _Method == Method_GET) + { + // We have to close the connection. + if ( debug > 4 ) + cout << "Connection must be closed (stream not completely read)" + << endl; + + CloseConnection(); + + } + else + if ( debug > 4 ) + cout << "Connection stays up ... (Persistent connection)" << endl; + } + + + // Check the doc_status and return a value + + return FinishRequest(DocumentStatus); + +} + + + +HtHTTP::ConnectionStatus HtHTTP::EstablishConnection() +{ + + int result; + + // Open the connection + result=OpenConnection(); + + if (!result) + return Connection_open_failed; // Connection failed + else if(debug > 4) + { + cout << setw(5) << Transport::GetTotOpen() << " - "; + + if (result == -1) + cout << "Connection already open. No need to re-open." << endl; + else + cout << "Open of the connection ok" << endl; + } + + + if(result==1) // New connection open + { + + // Assign the remote host to the connection + if ( !AssignConnectionServer() ) + return Connection_no_server; + else if (debug > 4) + cout << "\tAssigned the remote host " << _url.host() << endl; + + // Assign the port of the remote host + if ( !AssignConnectionPort() ) + return Connection_no_port; + else if (debug > 4) + cout << "\tAssigned the port " << _url.port() << endl; + } + + // Connect + if (! (result = Connect())) + return Connection_failed; + else if (result == -1) return Connection_already_up; // Persistent + else return Connection_ok; // New connection + +} + + + +// Set the string of the HTTP message request + +void HtHTTP::SetRequestCommand(String &cmd) +{ + + // Initialize it + + if (_useproxy) { + cmd << _url.get() << " HTTP/1.1\r\n"; + } else + cmd << _url.path() << " HTTP/1.1\r\n"; + + // Insert the "virtual" host to which ask the document + + cmd << "Host: " << _url.host(); + if (_url.port() != 0 && _url.port() != _url.DefaultPort()) + cmd << ":" << _url.port(); + cmd << "\r\n"; + + + // Insert the User Agent + + if (_user_agent.length()) + cmd << "User-Agent: " << _user_agent << "\r\n"; + + + // Referer + if (_referer.get().length()) + cmd << "Referer: " << _referer.get() << "\r\n"; + + // Accept-Language + if (_accept_language.length()) + cmd << "Accept-language: " << _accept_language << "\r\n"; + + // Authentication + if (_credentials.length()) + cmd << "Authorization: Basic " << _credentials << "\r\n"; + + // Proxy Authentication + if (_useproxy && _proxy_credentials.length()) + cmd << "Proxy-Authorization: Basic " << _proxy_credentials << "\r\n"; + + // Accept-Encoding: waiting to handle the gzip and compress formats, we + // just send an empty header which, according to the HTTP 1/1 standard, + // should let the server know that we only accept the 'identity' case + // (no encoding of the document) + cmd << "Accept-Encoding: \r\n"; + + // A date has been passed to check if the server one is newer than + // the one we already own. + + if(_modification_time && *_modification_time > 0) + { + _modification_time->ToGMTime(); + cmd << "If-Modified-Since: " << _modification_time->GetRFC1123() << "\r\n"; + } + +/////// + // Cookies! Let's go eat them! ;-) +/////// + + // The method returns all the valid cookies and writes them + // directly into the request string, as a list of headers + if (_send_cookies && _cookie_jar) + _cookie_jar->SetHTTPRequest_CookiesString(_url, cmd); + + + // Let's close the command + cmd << "\r\n"; + +} + + + + +//***************************************************************************** +// int HtHTTP::ParseHeader() +// Parse the header of the document +// +int HtHTTP::ParseHeader() +{ + String line = 0; + int inHeader = 1; + + if (_response._modification_time) + { + delete _response._modification_time; + _response._modification_time=0; + } + while (inHeader) + { + + line.trunc(); + + if(! _connection->Read_Line(line, "\n")) + return -1; // Connection down + + _bytes_read+=line.length(); + line.chop('\r'); + + if (line.length() == 0) + inHeader = 0; + else + { + // Found a not-empty line + + if (debug > 2) + cout << "Header line: " << line << endl; + + // Status - Line check + char *token = line.get(); + + while (*token && !isspace(*token) && *token != ':') + ++token; + + while (*token && (isspace(*token) || *token == ':')) + ++token; + + if(!strncmp((char*)line, "HTTP/", 5)) + { + // Here is the status-line + + // store the HTTP version returned by the server + _response._version = strtok(line, " "); + + // Store the status code + _response._status_code = atoi(strtok(0, " ")); + + // Store the reason phrase + _response._reason_phrase = strtok(0, "\n"); + + } + else if( ! mystrncasecmp((char*)line, "server:", 7)) + { + // Server info + + // Set the server info + token = strtok(token, "\n\t"); + + if (token && *token) + _response._server = token; + + } + else if( ! mystrncasecmp((char*)line, "last-modified:", 14)) + { + // Modification date sent by the server + + // Set the response modification time + token = strtok(token, "\n\t"); + + if (token && *token) + _response._modification_time = NewDate(token); + + } + else if( ! mystrncasecmp((char*)line, "date:", 5)) + { + // Access date time sent by the server + + // Set the response access time + token = strtok(token, "\n\t"); + + if (token && *token) + _response._access_time = NewDate(token); + + } + else if( ! mystrncasecmp((char*)line, "content-type:", 13)) + { + // Content - type + + token = strtok(token, "\n\t"); + + if (token && *token) + _response._content_type = token; + + } + else if( ! mystrncasecmp((char*)line, "content-length:", 15)) + { + // Content - length + + token = strtok(token, "\n\t"); + + if (token && *token) + _response._content_length = atoi(token); + + } + else if( ! mystrncasecmp((char*)line, "transfer-encoding:", 18)) + { + // Transfer-encoding + + token = strtok(token, "\n\t"); + + if (token && *token) + _response._transfer_encoding = token; + + } + else if( ! mystrncasecmp((char*)line, "location:", 9)) + { + // Found a location directive - redirect in act + + token = strtok(token, "\n\t"); + + if (token && *token) + _response._location = token; + + } + else if( ! mystrncasecmp((char*)line, "connection:", 11)) + { + // Ooops ... found a Connection clause + + token = strtok(token, "\n\t"); + + if (token && *token) + _response._hdrconnection = token; + + } + else if( ! mystrncasecmp((char*)line, "content-language:", 17)) + { + // Found a content-language directive + + token = strtok(token, "\n\t"); + + if (token && *token) + _response._content_language = token; + + } + else if( ! mystrncasecmp((char*)line, "set-cookie:", 11)) + { + // Found a cookie + + // Are cookies enabled? + if (_send_cookies && _cookie_jar) + { + token = strtok(token, "\n\t"); + + if (token && *token) + { + // Insert the cookie into the jar + _cookie_jar->AddCookie(token, _url); + } + } + + } + else + { + // Discarded + + if (debug > 3) + cout << "Discarded header line: " << line << endl; + } + } + } + + if (_response._modification_time == 0) + { + if (debug > 3) + cout << "No modification time returned: assuming now" << endl; + + //Set the modification time + _response._modification_time = new HtDateTime; + _response._modification_time->ToGMTime(); // Set to GM time + + } + + return 1; + +} + + +// Check for a document to be parsable +// It all depends on the content-type directive returned by the server + +bool HtHTTP::isParsable(const char *content_type) +{ + + // Here I can decide what kind of document I can parse + // depending on the value of Transport:_default_parser_content_type + // and the rest are determined by the external_parser settings + + if( ! mystrncasecmp (_default_parser_content_type.get(), content_type, + _default_parser_content_type.length()) ) + return true; + + // External function that checks if a document is parsable or not. + // CanBeParsed should point to a function that returns an int value, + // given a char * containing the content-type. + + if (CanBeParsed && (*CanBeParsed)( (char *) content_type) ) + return true; + + return false; + +} + + +// Check for a possibile persistent connection +// on the return message's HTTP version basis + +void HtHTTP::CheckPersistentConnection(HtHTTP_Response &response) +{ + + const char *version = response.GetVersion(); + + if( ! mystrncasecmp ("HTTP/1.1", version, 8)) + { + const char *connection = response.GetConnectionInfo(); + + if( ! mystrncasecmp ("close", connection, 5)) + _persistent_connection_possible=false; // Server wants to close + else _persistent_connection_possible=true; + + } + else + _persistent_connection_possible=false; + +} + + +HtHTTP::DocStatus HtHTTP::FinishRequest (HtHTTP::DocStatus ds) +{ + + int seconds; + + // Set the finish time + _end_time.SettoNow(); + + // Let's add the number of seconds needed by the request + seconds=HtDateTime::GetDiff(_end_time, _start_time); + + _tot_seconds += seconds; + _tot_requests ++; + _tot_bytes += _bytes_read; + + if (debug > 2) + cout << "Request time: " << seconds << " secs" << endl; + + return ds; + +} + + +HtHTTP::DocStatus HtHTTP::GetDocumentStatus(HtHTTP_Response &r) +{ + + // Let's give a look at the return status code + + HtHTTP::DocStatus returnStatus=Document_not_found; + int statuscode; + + statuscode=r.GetStatusCode(); + + if(statuscode==200) + { + returnStatus = Document_ok; // OK + + // Is it parsable? + + if (! isParsable ((const char*)r.GetContentType()) ) + returnStatus=Document_not_parsable; + } + else if(statuscode > 200 && statuscode < 300) + returnStatus = Document_ok; // Successful 2xx + else if(statuscode==304) + returnStatus = Document_not_changed; // Not modified + else if(statuscode > 300 && statuscode < 400) + returnStatus = Document_redirect; // Redirection 3xx + else if(statuscode==401) + returnStatus = Document_not_authorized; // Unauthorized + + // Exit the function + return returnStatus; + +} + +void HtHTTP::SetCredentials (const String& s) +{ + Transport::SetHTTPBasicAccessAuthorizationString(_credentials, s); +} + + +void HtHTTP::SetProxyCredentials (const String& s) +{ + Transport::SetHTTPBasicAccessAuthorizationString(_proxy_credentials, s); +} + +int HtHTTP::ReadBody() +{ + + _response._contents = 0; // Initialize the string + + char docBuffer[8192]; + int bytesRead = 0; + int bytesToGo = _response._content_length; + + if (bytesToGo < 0 || bytesToGo > _max_document_size) + bytesToGo = _max_document_size; + + while (bytesToGo > 0) + { + int len = bytesToGo< (int)sizeof(docBuffer) ? bytesToGo : (int)sizeof(docBuffer); + bytesRead = _connection->Read(docBuffer, len); + if (bytesRead <= 0) + break; + + _response._contents.append(docBuffer, bytesRead); + + bytesToGo -= bytesRead; + + _bytes_read+=bytesRead; + + } + + // Set document length + _response._document_length = _response._contents.length(); + + return bytesRead; + +} + + +int HtHTTP::ReadChunkedBody() +{ + // Chunked Transfer decoding + // as shown in the RFC2616 (HTTP/1.1) - 19.4.6 + +#define BSIZE 8192 + + int length = 0; // initialize the length + unsigned int chunk_size; + String ChunkHeader = 0; + char buffer[BSIZE+1]; + int chunk, rsize; + + _response._contents.trunc(); // Initialize the string + + // Read chunk-size and CRLF + if (!_connection->Read_Line(ChunkHeader, "\r\n")) + return -1; + + sscanf ((char *)ChunkHeader, "%x", &chunk_size); + + if (debug>4) + cout << "Initial chunk-size: " << chunk_size << endl; + + while (chunk_size > 0) + { + chunk = chunk_size; + + do { + if (chunk > BSIZE) { + rsize = BSIZE; + if (debug>4) + cout << "Read chunk partial: left=" << chunk << endl; + } else { + rsize = chunk; + } + chunk -= rsize; + + // Read Chunk data + if (_connection->Read(buffer, rsize) == -1) + return -1; + + length+=rsize; + + // Append the chunk-data to the contents of the response + // ... but not more than _max_document_size... + if (rsize > _max_document_size-_response._contents.length()) + rsize = _max_document_size-_response._contents.length(); + buffer[rsize] = 0; + _response._contents.append(buffer, rsize); + + } while (chunk); + + // if (_connection->Read(buffer, chunk_size) == -1) + // return -1; + + // Read CRLF - to be ignored + if (!_connection->Read_Line(ChunkHeader, "\r\n")) + return -1; + + // Read chunk-size and CRLF + if (!_connection->Read_Line(ChunkHeader, "\r\n")) + return -1; + + sscanf ((char *)ChunkHeader, "%x", &chunk_size); + + if (debug>4) + cout << "Chunk-size: " << chunk_size << endl; + } + + ChunkHeader = 0; + + // Ignoring next part of the body - the TRAILER + // (it contains further headers - not implemented) + + // Set content length + _response._content_length = length; + + // Set document length + _response._document_length = _response._contents.length(); + + return length; + +} + + +/////// + // Show the statistics +/////// + +ostream &HtHTTP::ShowStatistics (ostream &out) +{ + Transport::ShowStatistics(out); // call the base class method + + out << " HTTP Requests : " << GetTotRequests() << endl; + out << " HTTP KBytes requested : " << (double)GetTotBytes()/1024 << endl; + out << " HTTP Average request time : " << GetAverageRequestTime() + << " secs" << endl; + + out << " HTTP Average speed : " << GetAverageSpeed()/1024 + << " KBytes/secs" << endl; + + return out; +} diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtHTTP.h b/debian/htdig/htdig-3.2.0b6/htnet/HtHTTP.h new file mode 100644 index 00000000..b4b052fb --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtHTTP.h @@ -0,0 +1,452 @@ +// +// HtHTTP.h +// +// HtHTTP: Class for HTTP messaging (derived from Transport) +// +// Gabriele Bartolini - Prato - Italia +// started: 03.05.1999 +// +// //////////////////////////////////////////////////////////// +// +// The HtHTTP class should provide (as I hope) an interface for +// retrieving document on the Web. It derives from Transport class. +// +// It should be HTTP/1.1 compatible. +// +// It also let us take advantage of persitent connections use, +// and optimize request times (specially if directed to the same +// server). +// +// HtHTTP use another class to store the response returned by the +// remote server. +// +// Now cookies management is enabled. +// +/////// +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1995-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: HtHTTP.h,v 1.15 2004/05/28 13:15:23 lha Exp $ +// + +#ifndef _HTHTTP_H +#define _HTHTTP_H + +#ifdef HAVE_CONFIG_H +#include "htconfig.h" +#endif /* HAVE_CONFIG_H */ + +#include "Transport.h" + +// Cookie support +#include "HtCookie.h" +#include "HtCookieJar.h" + +#include "URL.h" +#include "htString.h" + +// for HtHTTP::ShowStatistics#ifdef HAVE_STD +#ifdef HAVE_STD +#include <iostream> +#ifdef HAVE_NAMESPACES +using namespace std; +#endif +#else +#include <iostream.h> +#endif /* HAVE_STD */ + +// In advance declarations + +class HtHTTP; + + +class HtHTTP_Response : public Transport_Response +{ + + friend class HtHTTP; // declaring friendship + + public: +/////// + // Construction / Destruction +/////// + + HtHTTP_Response(); + ~HtHTTP_Response(); + + +/////// + // Interface +/////// + + // Reset + void Reset(); + + // Get the HTTP version + const String &GetVersion() const { return _version; } + + // Get the Transfer-encoding + const String &GetTransferEncoding() const + { return _transfer_encoding; } + + // Get server info + const String &GetServer() const { return _server; } + + // Get Connection info + const String &GetConnectionInfo() const { return _hdrconnection; } + + // Get Content language + const String &GetContentLanguage() const { return _content_language; } + + + protected: + + // Status line information + + String _version; // HTTP Version + + // Other header information + + String _transfer_encoding; // Transfer-encoding + String _server; // Server string returned + String _hdrconnection; // Connection header + String _content_language; // Content-language + +}; + + + +class HtHTTP : public Transport +{ +private: + HtHTTP() {} // Declared private - avoids default constructor to be created + // in some cases by the compiler. +public: + +/////// + // Construction/Destruction +/////// + + HtHTTP(Connection&); + virtual ~HtHTTP() = 0; + + // Information about the method to be used in the request + + enum Request_Method + { + Method_GET, + Method_HEAD + }; + + + +/////// + // Sends an HTTP request message +/////// + + // manages a Transport request (method inherited from Transport class) + virtual DocStatus Request (); + + // Sends a request message for HTTP + virtual DocStatus HTTPRequest (); + + +/////// + // Control of member the variables +/////// + + /////// + // Set the Request Method + /////// + + void SetRequestMethod (Request_Method rm) { _Method = rm; } + Request_Method GetRequestMethod() { return _Method; } + + + /////// + // Interface for resource retrieving + /////// + + // Set and get the document to be retrieved + void SetRequestURL(const URL &u) { _url = u;} + URL GetRequestURL () { return _url;} + + + // Set and get the referring URL + void SetRefererURL (const URL& u) { _referer = u;} + URL GetRefererURL () { return _referer;} + + // Set and get the accept-language string + void SetAcceptLanguage (const String& al) { _accept_language = al; } + URL GetAcceptLanguage () { return _accept_language; } + + // Info for multiple requests (static) + // Get the User agent string + static void SetRequestUserAgent (const String &s) { _user_agent=s; } + static const String &GetRequestUserAgent() { return _user_agent; } + + // Set (Basic) Authentication Credentials + virtual void SetCredentials (const String& s); + + // Set (Basic) Authentication Credentials for the HTTP Proxy + virtual void SetProxyCredentials (const String& s); + + /////// + // Interface for the HTTP Response + /////// + + // We have a valid response only if the status code is not equal to + // initialization value + + Transport_Response *GetResponse() + { + if (_response._status_code != -1) + return &_response; + else return 0;} + + + // Get the document status + virtual DocStatus GetDocumentStatus() + { return GetDocumentStatus (_response); } + + // It's a static method + static DocStatus GetDocumentStatus(HtHTTP_Response &); + + + +/////// + // Persistent connection choices interface +/////// + + // Is allowed + bool isPersistentConnectionAllowed() {return _persistent_connection_allowed;} + + // Is possible + bool isPersistentConnectionPossible() {return _persistent_connection_possible;} + + // Check if a persistent connection is possible depending on the HTTP response + void CheckPersistentConnection(HtHTTP_Response &); + + // Is Up (is both allowed and permitted by the server too) + bool isPersistentConnectionUp() + { return isConnected() && isPersistentConnectionAllowed() && + isPersistentConnectionPossible(); } + + // Allow Persistent Connection + void AllowPersistentConnection() { _persistent_connection_allowed=true; } + + // Disable Persistent Connection + void DisablePersistentConnection() { _persistent_connection_allowed=false; } + + // Allow Cookies + void AllowCookies() { _send_cookies=true; } + + // Disable Persistent Connection + void DisableCookies() { _send_cookies=false; } + + +/////// + // Set the cookie manager class (that is to say the class) +/////// + + // It's set only if not done before + static void SetCookieJar(HtCookieJar *cj) { _cookie_jar = cj; } + + +/////// + // Manage statistics +/////// + + static int GetTotSeconds () { return _tot_seconds; } + + static int GetTotRequests () { return _tot_requests; } + + static int GetTotBytes () { return _tot_bytes; } + + static double GetAverageRequestTime () + { return _tot_seconds?( ((double) _tot_seconds) / _tot_requests) : 0; } + + static float GetAverageSpeed () + { return _tot_bytes?( ((double) _tot_bytes) / _tot_seconds) : 0; } + + static void ResetStatistics () + { _tot_seconds=0; _tot_requests=0; _tot_bytes=0;} + + // Show stats + static ostream &ShowStatistics (ostream &out); + + + +/////// + // Set the _head_before_get option + // make a request to be made up of a HEAD call and then, + // if necessary, a GET call +/////// + + static void EnableHeadBeforeGet() { _head_before_get = true; } + static void DisableHeadBeforeGet() { _head_before_get = false; } + + static bool HeadBeforeGet() { return _head_before_get; } + + +/////// + // Set the controller for the parsing check. That is to say + // that External function that checks if a document is parsable or not. + // CanBeParsed static attribute should point to a function + // that returns an int value, given a char * containing the content-type. +/////// + + static void SetParsingController (int (*f)(char*)) { CanBeParsed = f; } + +protected: + +/////// + // Member attributes +/////// + + Request_Method _Method; + + /////// + // Http single Request information (Member attributes) + /////// + + int _bytes_read; // Bytes read + URL _url; // URL to retrieve + URL _referer; // Referring URL + + String _accept_language; // accept-language directive + + /////// + // Http multiple Request information + /////// + + static String _user_agent; // User agent + + + + /////// + // Http Response information + /////// + + HtHTTP_Response _response; // Object where response + // information will be stored into + + + /////// + // Allow or not a persistent connection (user choice) + /////// + + bool _persistent_connection_allowed; + + + /////// + // Is a persistent connection possible (with this http server)? + /////// + + bool _persistent_connection_possible; + + /////// + // Are cookies enabled? + /////// + + bool _send_cookies; + + /////// + // Option that, if set to true, make a request to be made up + // of a HEAD call and then, if necessary, a GET call + /////// + + static bool _head_before_get; + +/////// + // Manager of the body reading +/////// + + int (HtHTTP::*_readbody) (); + + +/////// + // Enum +/////// + + // Information about the status of a connection + + enum ConnectionStatus + { + Connection_ok, + Connection_already_up, + Connection_open_failed, + Connection_no_server, + Connection_no_port, + Connection_failed + }; + + +/////// + // Protected Services or method (Hidden by outside) +/////// + + + /////// + // Establish the connection + /////// + + ConnectionStatus EstablishConnection (); + + + /////// + // Set the string of the command containing the request + /////// + + void SetRequestCommand(String &); + + + /////// + // Parse the header returned by the server + /////// + + int ParseHeader(); + + /////// + // Check if a document is parsable looking the content-type info + /////// + + static bool isParsable(const char *); + + /////// + // Read the body returned by the server + /////// + + void SetBodyReadingController (int (HtHTTP::*f)()) { _readbody = f; } + int ReadBody(); + int ReadChunkedBody(); // Read the body of a chunked encoded-response + + + // Finish the request and return a DocStatus value; + + DocStatus FinishRequest (DocStatus); + + +/////// + // Static attributes and methods +/////// + + // Unique cookie Jar + static HtCookieJar *_cookie_jar; // Jar containing all of the cookies + + static int _tot_seconds; // Requests last (in seconds) + static int _tot_requests; // Number of requests + static int _tot_bytes; // Number of bytes read + + // This is a pointer to function that check if a ContentType + // is parsable or less. + + static int (*CanBeParsed) (char *); + +}; + +#endif + diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtHTTPBasic.cc b/debian/htdig/htdig-3.2.0b6/htnet/HtHTTPBasic.cc new file mode 100644 index 00000000..f1ab2f2e --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtHTTPBasic.cc @@ -0,0 +1,43 @@ +// +// HtHTTPBasic.cc +// +// HtHTTPBasic: Class for HTTP messaging (derived from Transport) +// Does not handle HTTPS connections -- use HtHTTPSecure +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1995-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: HtHTTPBasic.cc,v 1.5 2004/05/28 13:15:23 lha Exp $ +// + +#ifdef HAVE_CONFIG_H +#include "htconfig.h" +#endif /* HAVE_CONFIG_H */ + +#ifdef HAVE_STD +#include <iostream> +#ifdef HAVE_NAMESPACES +using namespace std; +#endif +#else +#include <iostream.h> +#endif /* HAVE_STD */ + +#include "HtHTTPBasic.h" + +// HtHTTPBasic constructor +// +HtHTTPBasic::HtHTTPBasic() +: HtHTTP(*(new Connection())) // Creates a new connection +{ +} + +// HtHTTPBasic destructor +// +HtHTTPBasic::~HtHTTPBasic() +{ +} + diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtHTTPBasic.h b/debian/htdig/htdig-3.2.0b6/htnet/HtHTTPBasic.h new file mode 100644 index 00000000..6e993a3a --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtHTTPBasic.h @@ -0,0 +1,34 @@ +// +// HtHTTPBasic.h +// +// HtHTTPBasic: Class for HTTP messaging (derived from Transport) +// Does not handle HTTPS connections -- use HtHTTPSecure +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1995-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: HtHTTPBasic.h,v 1.4 2004/05/28 13:15:23 lha Exp $ +// + +#ifndef _HTHTTPBASIC_H +#define _HTHTTPBASIC_H + +#include "HtHTTP.h" // We inherrit from this +#include "Transport.h" +#include "Connection.h" +#include "URL.h" +#include "htString.h" + +class HtHTTPBasic : public HtHTTP +{ + public: + + HtHTTPBasic(); + ~HtHTTPBasic(); + +}; + +#endif diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtHTTPSecure.cc b/debian/htdig/htdig-3.2.0b6/htnet/HtHTTPSecure.cc new file mode 100644 index 00000000..9449d1cd --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtHTTPSecure.cc @@ -0,0 +1,46 @@ +// +// HtHTTPSecure.cc +// +// HtHTTPSecure: Class for HTTP/HTTPS messaging (derived from Transport) +// Uses an SSLConnection for secure connections. +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1995-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: HtHTTPSecure.cc,v 1.5 2004/05/28 13:15:23 lha Exp $ +// + +#ifdef HAVE_CONFIG_H +#include "htconfig.h" +#endif /* HAVE_CONFIG_H */ + +#ifdef HAVE_SSL_H + +#include "HtHTTPSecure.h" + +#ifdef HAVE_STD +#include <iostream> +#ifdef HAVE_NAMESPACES +using namespace std; +#endif +#else +#include <iostream.h> +#endif /* HAVE_STD */ + +// HtHTTPSecure constructor +// +HtHTTPSecure::HtHTTPSecure() +: HtHTTP(*(new SSLConnection())) // Create a new "secure" connection +{ +} + +// HtHTTPSecure destructor +// +HtHTTPSecure::~HtHTTPSecure() +{ +} + +#endif diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtHTTPSecure.h b/debian/htdig/htdig-3.2.0b6/htnet/HtHTTPSecure.h new file mode 100644 index 00000000..0f6001e7 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtHTTPSecure.h @@ -0,0 +1,37 @@ +// +// HtHTTPSecure.h +// +// HtHTTPSecure: Class for HTTP/HTTPS messaging (derived from Transport) +// Uses an SSLConnection for secure connections. +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1995-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: HtHTTPSecure.h,v 1.4 2004/05/28 13:15:23 lha Exp $ +// + +#ifndef _HTHTTPSECURE_H +#define _HTHTTPSECURE_H + +#include "HtHTTP.h" +#include "Transport.h" +#include "SSLConnection.h" +#include "URL.h" +#include "htString.h" + +#ifdef HAVE_SSL_H + +class HtHTTPSecure : public HtHTTP +{ + public: + HtHTTPSecure(); + ~HtHTTPSecure(); +}; + +#endif + +#endif + diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtNNTP.cc b/debian/htdig/htdig-3.2.0b6/htnet/HtNNTP.cc new file mode 100644 index 00000000..2a79bf41 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtNNTP.cc @@ -0,0 +1,286 @@ +// +// HtNNTP.cc +// +// HtNNTP: Interface classes for NNTP messaging +// +// Gabriele Bartolini - Prato - Italia +// started: 01.08.2000 +// +// Including: +// - Generic class +// - Response message class +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 2000-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: HtNNTP.cc,v 1.5 2004/05/28 13:15:23 lha Exp $ +// + +#ifdef HAVE_CONFIG_H +#include "htconfig.h" +#endif /* HAVE_CONFIG_H */ + +#include "lib.h" +#include "Transport.h" +#include "HtNNTP.h" + +#include <signal.h> +#include <sys/types.h> +#include <ctype.h> +#include <stdio.h> // for sscanf + +// for setw() +#ifdef HAVE_STD +#include <iomanip> +#ifdef HAVE_NAMESPACES +using namespace std; +#endif +#else +#include <iomanip.h> +#endif /* HAVE_STD */ + +#if 1 +typedef void (*SIGNAL_HANDLER) (...); +#else +typedef SIG_PF SIGNAL_HANDLER; +#endif + + // Stats information + int HtNNTP::_tot_seconds = 0; + int HtNNTP::_tot_requests = 0; + int HtNNTP::_tot_bytes = 0; + +/////// + // HtNNTP_Response class + // + // Response message sent by the remote NNTP server +/////// + + +// Construction + +HtNNTP_Response::HtNNTP_Response() +{ + +} + + +// Destruction + +HtNNTP_Response::~HtNNTP_Response() +{ +} + + +void HtNNTP_Response::Reset() +{ + + // Call the base class method in order to reset + // the base class attributes + + Transport_Response::Reset(); + +} + + + + +/////// + // HtNNTP generic class + // + // +/////// + + +// Construction + +HtNNTP::HtNNTP() +: Transport(new Connection()), + _bytes_read(0), + _useproxy(0) +{ +} + +// Destruction + +HtNNTP::~HtNNTP() +{ + // Free the connection + // + CloseConnection(); + if (_connection) + delete _connection; + _connection = 0; +} + + +/////// + // Manages the requesting process +/////// + +Transport::DocStatus HtNNTP::Request() +{ + + DocStatus result = Document_ok; + _response.Reset(); // Reset the response + + return result; + +} + + +void HtNNTP::SetRequestCommand(String &cmd) +{ + + cmd << "\r\n"; + +} + + + + +//***************************************************************************** +// int HtNNTP::ParseHeader() +// Parse the header of the document +// +int HtNNTP::ParseHeader() +{ + String line = 0; + int inHeader = 1; + + if (_response._modification_time) + { + delete _response._modification_time; + _response._modification_time=NULL; + } + while (inHeader) + { + + line.trunc(); + + if(! _connection->Read_Line(line, "\n")) + return -1; // Connection down + + _bytes_read+=line.length(); + line.chop('\r'); + + if (line.length() == 0) + inHeader = 0; + else + { + // Found a not-empty line + + if (debug > 3) + cout << "Header line: " << line << endl; + + // Status - Line check + char *token = line.get(); + + while (*token && !isspace(*token)) + token++; + + while (*token && isspace(*token)) + token++; + } + } + + if (_response._modification_time == NULL) + { + if (debug > 3) + cout << "No modification time returned: assuming now" << endl; + + //Set the modification time + _response._modification_time = new HtDateTime; + _response._modification_time->ToGMTime(); // Set to GM time + + } + + return 1; + +} + + +HtNNTP::DocStatus HtNNTP::GetDocumentStatus(HtNNTP_Response &r) +{ + + // Let's give a look at the return status code + + HtNNTP::DocStatus returnStatus=Document_not_found; + int statuscode; + + statuscode=r.GetStatusCode(); + + if(statuscode==200) + { + returnStatus = Document_ok; // OK + } + + // Exit the function + return returnStatus; + +} + + +int HtNNTP::ReadBody() +{ + + _response._contents = 0; // Initialize the string + + char docBuffer[8192]; + int bytesRead = 0; + int bytesToGo = _response._content_length; + + if (bytesToGo < 0 || bytesToGo > _max_document_size) + bytesToGo = _max_document_size; + + if( _connection == NULL ) + { + cout << "HtNNTP::ReadBody: _connection is NULL\n"; + exit(0); + } + + + while (bytesToGo > 0) + { + int len = bytesToGo< (int)sizeof(docBuffer) ? bytesToGo : (int)sizeof(docBuffer); + bytesRead = _connection->Read(docBuffer, len); + if (bytesRead <= 0) + break; + + _response._contents.append(docBuffer, bytesRead); + + bytesToGo -= bytesRead; + + _bytes_read+=bytesRead; + + } + + // Set document length + _response._document_length = _response._contents.length(); + + return bytesRead; + +} + + +/////// + // Show the statistics +/////// + +ostream &HtNNTP::ShowStatistics (ostream &out) +{ + Transport::ShowStatistics(out); // call the base class method + + out << " NNTP Requests : " << GetTotRequests() << endl; + out << " NNTP KBytes requested : " << (double)GetTotBytes()/1024 << endl; + out << " NNTP Average request time : " << GetAverageRequestTime() + << " secs" << endl; + + out << " NNTP Average speed : " << GetAverageSpeed()/1024 + << " KBytes/secs" << endl; + + return out; +} diff --git a/debian/htdig/htdig-3.2.0b6/htnet/HtNNTP.h b/debian/htdig/htdig-3.2.0b6/htnet/HtNNTP.h new file mode 100644 index 00000000..07b0c87b --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/HtNNTP.h @@ -0,0 +1,195 @@ +// +// HtNNTP.h +// +// HtNNTP: Class for NNTP messaging (derived from Transport) +// +// Gabriele Bartolini - Prato - Italia +// started: 01.08.2000 +// +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1995-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the General GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: HtNNTP.h,v 1.5 2004/05/28 13:15:23 lha Exp $ +// + +#ifndef _HTNNTP_H +#define _HTNNTP_H + +#ifdef HAVE_CONFIG_H +#include "htconfig.h" +#endif + +#include "Transport.h" +#include "URL.h" +#include "htString.h" + +// for HtNNTP::ShowStatistics +#ifdef HAVE_STD +#include <iostream> +#ifdef HAVE_NAMESPACES +using namespace std; +#endif +#else +#include <iostream.h> +#endif /* HAVE_STD */ + + +// In advance declarations + +class HtNNTP; + + +class HtNNTP_Response : public Transport_Response +{ + + friend class HtNNTP; // declaring friendship + + public: +/////// + // Construction / Destruction +/////// + + HtNNTP_Response(); + ~HtNNTP_Response(); + + +/////// + // Interface +/////// + + // Reset + void Reset(); + + protected: + + // Other header information + +}; + + + +class HtNNTP : public Transport +{ +public: + +/////// + // Construction/Destruction +/////// + + HtNNTP(); + ~HtNNTP(); + +/////// + // Sends an NNTP request message +/////// + + // manages a Transport request (method inherited from Transport class) + virtual DocStatus Request (); + +/////// + // Control of member the variables +/////// + + /////// + // Interface for resource retrieving + /////// + + // Set and get the document to be retrieved + void SetRequestURL(URL &u) { _url = u;} + URL GetRequestURL () { return _url;} + + + Transport_Response *GetResponse() + { + if (_response._status_code != -1) + return &_response; + else return NULL; + } + + // Get the document status + virtual DocStatus GetDocumentStatus() + { return GetDocumentStatus (_response); } + + // It's a static method + static DocStatus GetDocumentStatus(HtNNTP_Response &); + + +// Manage statistics + + static int GetTotSeconds () { return _tot_seconds; } + + static int GetTotRequests () { return _tot_requests; } + + static int GetTotBytes () { return _tot_bytes; } + + static double GetAverageRequestTime () + { return _tot_seconds?( ((double) _tot_seconds) / _tot_requests) : 0; } + + static float GetAverageSpeed () + { return _tot_bytes?( ((double) _tot_bytes) / _tot_seconds) : 0; } + + static void ResetStatistics () + { _tot_seconds=0; _tot_requests=0; _tot_bytes=0;} + + // Show stats + static ostream &ShowStatistics (ostream &out); + + // Proxy settings + void SetProxy(int aUse) { _useproxy=aUse; } + +protected: + +/////// + // Member attributes +/////// + + /////// + // NNTP single Request information (Member attributes) + /////// + + int _bytes_read; // Bytes read + URL _url; // URL to retrieve + int _useproxy; // Shall we use a proxy? + + + /////// + // NNTP Response information + /////// + + HtNNTP_Response _response; // Object where response + // information will be stored into + + /////// + // Set the string of the command containing the request + /////// + + void SetRequestCommand(String &); + + /////// + // Parse the header returned by the server + /////// + + int ParseHeader(); + + /////// + // Read the body returned by the server + /////// + + int ReadBody(); + +/////// + // Static attributes and methods +/////// + + static int _tot_seconds; // Requests last (in seconds) + static int _tot_requests; // Number of requests + static int _tot_bytes; // Number of bytes read + +}; + +#endif + diff --git a/debian/htdig/htdig-3.2.0b6/htnet/Makefile.am b/debian/htdig/htdig-3.2.0b6/htnet/Makefile.am new file mode 100644 index 00000000..40269e18 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/Makefile.am @@ -0,0 +1,25 @@ + +include $(top_srcdir)/Makefile.config + +pkglib_LTLIBRARIES = libhtnet.la + +libhtnet_la_SOURCES = Connection.cc Transport.cc HtHTTP.cc HtFile.cc \ + HtNNTP.cc HtFTP.cc HtCookie.cc HtCookieJar.cc HtCookieMemJar.cc \ + HtCookieInFileJar.cc HtHTTPBasic.cc HtHTTPSecure.cc SSLConnection.cc + +libhtnet_la_LDFLAGS = -release $(HTDIG_MAJOR_VERSION).$(HTDIG_MINOR_VERSION).$(HTDIG_MICRO_VERSION) ${extra_ldflags} + +pkginclude_HEADERS = \ + Connection.h \ + Transport.h \ + HtHTTP.h \ + HtFile.h \ + HtFTP.h \ + HtNNTP.h \ + HtCookie.h \ + HtCookieJar.h \ + HtCookieMemJar.h \ + HtCookieInFileJar.h \ + HtHTTPBasic.h \ + HtHTTPSecure.h \ + SSLConnection.h diff --git a/debian/htdig/htdig-3.2.0b6/htnet/Makefile.in b/debian/htdig/htdig-3.2.0b6/htnet/Makefile.in new file mode 100644 index 00000000..4d931c44 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/Makefile.in @@ -0,0 +1,515 @@ +# Makefile.in generated by automake 1.7.9 from Makefile.am. +# @configure_input@ + +# Copyright 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003 +# Free Software Foundation, Inc. +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + +# +# To compile with profiling do the following: +# +# make CFLAGS=-g CXXFLAGS=-g PROFILING=-p all +# + +srcdir = @srcdir@ +top_srcdir = @top_srcdir@ +VPATH = @srcdir@ +pkgdatadir = $(datadir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +top_builddir = .. + +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +INSTALL = @INSTALL@ +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +host_triplet = @host@ +ACLOCAL = @ACLOCAL@ +ALLOCA = @ALLOCA@ +AMDEP_FALSE = @AMDEP_FALSE@ +AMDEP_TRUE = @AMDEP_TRUE@ +AMTAR = @AMTAR@ +APACHE = @APACHE@ +APACHE_MODULES = @APACHE_MODULES@ +AR = @AR@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CGIBIN_DIR = @CGIBIN_DIR@ +COMMON_DIR = @COMMON_DIR@ +CONFIG_DIR = @CONFIG_DIR@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DATABASE_DIR = @DATABASE_DIR@ +DEFAULT_CONFIG_FILE = @DEFAULT_CONFIG_FILE@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +ECHO = @ECHO@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +F77 = @F77@ +FFLAGS = @FFLAGS@ +FIND = @FIND@ +GUNZIP = @GUNZIP@ +HAVE_SSL = @HAVE_SSL@ +HTDIG_MAJOR_VERSION = @HTDIG_MAJOR_VERSION@ +HTDIG_MICRO_VERSION = @HTDIG_MICRO_VERSION@ +HTDIG_MINOR_VERSION = @HTDIG_MINOR_VERSION@ +IMAGE_DIR = @IMAGE_DIR@ +IMAGE_URL_PREFIX = @IMAGE_URL_PREFIX@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +LDFLAGS = @LDFLAGS@ +LEX = @LEX@ +LEXLIB = @LEXLIB@ +LEX_OUTPUT_ROOT = @LEX_OUTPUT_ROOT@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBTOOL = @LIBTOOL@ +LN_S = @LN_S@ +LTLIBOBJS = @LTLIBOBJS@ +MAINT = @MAINT@ +MAINTAINER_MODE_FALSE = @MAINTAINER_MODE_FALSE@ +MAINTAINER_MODE_TRUE = @MAINTAINER_MODE_TRUE@ +MAKEINFO = @MAKEINFO@ +MV = @MV@ +OBJEXT = @OBJEXT@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PERL = @PERL@ +RANLIB = @RANLIB@ +RRDTOOL = @RRDTOOL@ +SEARCH_DIR = @SEARCH_DIR@ +SEARCH_FORM = @SEARCH_FORM@ +SED = @SED@ +SENDMAIL = @SENDMAIL@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +STRIP = @STRIP@ +TAR = @TAR@ +TESTS_FALSE = @TESTS_FALSE@ +TESTS_TRUE = @TESTS_TRUE@ +TIME = @TIME@ +TIMEV = @TIMEV@ +USER = @USER@ +VERSION = @VERSION@ +YACC = @YACC@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_F77 = @ac_ct_F77@ +ac_ct_RANLIB = @ac_ct_RANLIB@ +ac_ct_STRIP = @ac_ct_STRIP@ +am__fastdepCC_FALSE = @am__fastdepCC_FALSE@ +am__fastdepCC_TRUE = @am__fastdepCC_TRUE@ +am__fastdepCXX_FALSE = @am__fastdepCXX_FALSE@ +am__fastdepCXX_TRUE = @am__fastdepCXX_TRUE@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +datadir = @datadir@ +exec_prefix = @exec_prefix@ +extra_ldflags = @extra_ldflags@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +oldincludedir = @oldincludedir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +subdirs = @subdirs@ +sysconfdir = @sysconfdir@ +target_alias = @target_alias@ + +AUTOMAKE_OPTIONS = foreign no-dependencies + +INCLUDES = -DDEFAULT_CONFIG_FILE=\"$(DEFAULT_CONFIG_FILE)\" \ + -I$(top_srcdir)/include -I$(top_srcdir)/htlib \ + -I$(top_srcdir)/htnet -I$(top_srcdir)/htcommon \ + -I$(top_srcdir)/htword \ + -I$(top_srcdir)/db -I$(top_builddir)/db \ + $(LOCAL_DEFINES) $(PROFILING) + + +HTLIBS = $(top_builddir)/htnet/libhtnet.la \ + $(top_builddir)/htcommon/libcommon.la \ + $(top_builddir)/htword/libhtword.la \ + $(top_builddir)/htlib/libht.la \ + $(top_builddir)/htcommon/libcommon.la \ + $(top_builddir)/htword/libhtword.la \ + $(top_builddir)/db/libhtdb.la \ + $(top_builddir)/htlib/libht.la + + +pkglib_LTLIBRARIES = libhtnet.la + +libhtnet_la_SOURCES = Connection.cc Transport.cc HtHTTP.cc HtFile.cc \ + HtNNTP.cc HtFTP.cc HtCookie.cc HtCookieJar.cc HtCookieMemJar.cc \ + HtCookieInFileJar.cc HtHTTPBasic.cc HtHTTPSecure.cc SSLConnection.cc + + +libhtnet_la_LDFLAGS = -release $(HTDIG_MAJOR_VERSION).$(HTDIG_MINOR_VERSION).$(HTDIG_MICRO_VERSION) ${extra_ldflags} + +pkginclude_HEADERS = \ + Connection.h \ + Transport.h \ + HtHTTP.h \ + HtFile.h \ + HtFTP.h \ + HtNNTP.h \ + HtCookie.h \ + HtCookieJar.h \ + HtCookieMemJar.h \ + HtCookieInFileJar.h \ + HtHTTPBasic.h \ + HtHTTPSecure.h \ + SSLConnection.h + +subdir = htnet +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +mkinstalldirs = $(SHELL) $(top_srcdir)/mkinstalldirs +CONFIG_HEADER = $(top_builddir)/include/config.h +CONFIG_CLEAN_FILES = +LTLIBRARIES = $(pkglib_LTLIBRARIES) + +libhtnet_la_LIBADD = +am_libhtnet_la_OBJECTS = Connection.lo Transport.lo HtHTTP.lo HtFile.lo \ + HtNNTP.lo HtFTP.lo HtCookie.lo HtCookieJar.lo HtCookieMemJar.lo \ + HtCookieInFileJar.lo HtHTTPBasic.lo HtHTTPSecure.lo \ + SSLConnection.lo +libhtnet_la_OBJECTS = $(am_libhtnet_la_OBJECTS) + +DEFAULT_INCLUDES = -I. -I$(srcdir) -I$(top_builddir)/include +depcomp = +am__depfiles_maybe = +CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) +LTCXXCOMPILE = $(LIBTOOL) --mode=compile $(CXX) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CXXFLAGS) $(CXXFLAGS) +CXXLD = $(CXX) +CXXLINK = $(LIBTOOL) --mode=link $(CXXLD) $(AM_CXXFLAGS) $(CXXFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +DIST_SOURCES = $(libhtnet_la_SOURCES) +HEADERS = $(pkginclude_HEADERS) + +DIST_COMMON = $(pkginclude_HEADERS) $(srcdir)/Makefile.in \ + $(top_srcdir)/Makefile.config Makefile.am +SOURCES = $(libhtnet_la_SOURCES) + +all: all-am + +.SUFFIXES: +.SUFFIXES: .cc .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ Makefile.am $(top_srcdir)/Makefile.config $(top_srcdir)/configure.in $(ACLOCAL_M4) + cd $(top_srcdir) && \ + $(AUTOMAKE) --foreign htnet/Makefile +Makefile: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.in $(top_builddir)/config.status + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe) +pkglibLTLIBRARIES_INSTALL = $(INSTALL) +install-pkglibLTLIBRARIES: $(pkglib_LTLIBRARIES) + @$(NORMAL_INSTALL) + $(mkinstalldirs) $(DESTDIR)$(pkglibdir) + @list='$(pkglib_LTLIBRARIES)'; for p in $$list; do \ + if test -f $$p; then \ + f="`echo $$p | sed -e 's|^.*/||'`"; \ + echo " $(LIBTOOL) --mode=install $(pkglibLTLIBRARIES_INSTALL) $(INSTALL_STRIP_FLAG) $$p $(DESTDIR)$(pkglibdir)/$$f"; \ + $(LIBTOOL) --mode=install $(pkglibLTLIBRARIES_INSTALL) $(INSTALL_STRIP_FLAG) $$p $(DESTDIR)$(pkglibdir)/$$f; \ + else :; fi; \ + done + +uninstall-pkglibLTLIBRARIES: + @$(NORMAL_UNINSTALL) + @list='$(pkglib_LTLIBRARIES)'; for p in $$list; do \ + p="`echo $$p | sed -e 's|^.*/||'`"; \ + echo " $(LIBTOOL) --mode=uninstall rm -f $(DESTDIR)$(pkglibdir)/$$p"; \ + $(LIBTOOL) --mode=uninstall rm -f $(DESTDIR)$(pkglibdir)/$$p; \ + done + +clean-pkglibLTLIBRARIES: + -test -z "$(pkglib_LTLIBRARIES)" || rm -f $(pkglib_LTLIBRARIES) + @list='$(pkglib_LTLIBRARIES)'; for p in $$list; do \ + dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \ + test "$$dir" = "$$p" && dir=.; \ + echo "rm -f \"$${dir}/so_locations\""; \ + rm -f "$${dir}/so_locations"; \ + done +libhtnet.la: $(libhtnet_la_OBJECTS) $(libhtnet_la_DEPENDENCIES) + $(CXXLINK) -rpath $(pkglibdir) $(libhtnet_la_LDFLAGS) $(libhtnet_la_OBJECTS) $(libhtnet_la_LIBADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) core *.core + +distclean-compile: + -rm -f *.tab.c + +.cc.o: + $(CXXCOMPILE) -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$< + +.cc.obj: + $(CXXCOMPILE) -c -o $@ `if test -f '$<'; then $(CYGPATH_W) '$<'; else $(CYGPATH_W) '$(srcdir)/$<'; fi` + +.cc.lo: + $(LTCXXCOMPILE) -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs + +distclean-libtool: + -rm -f libtool +uninstall-info-am: +pkgincludeHEADERS_INSTALL = $(INSTALL_HEADER) +install-pkgincludeHEADERS: $(pkginclude_HEADERS) + @$(NORMAL_INSTALL) + $(mkinstalldirs) $(DESTDIR)$(pkgincludedir) + @list='$(pkginclude_HEADERS)'; for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + f="`echo $$p | sed -e 's|^.*/||'`"; \ + echo " $(pkgincludeHEADERS_INSTALL) $$d$$p $(DESTDIR)$(pkgincludedir)/$$f"; \ + $(pkgincludeHEADERS_INSTALL) $$d$$p $(DESTDIR)$(pkgincludedir)/$$f; \ + done + +uninstall-pkgincludeHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginclude_HEADERS)'; for p in $$list; do \ + f="`echo $$p | sed -e 's|^.*/||'`"; \ + echo " rm -f $(DESTDIR)$(pkgincludedir)/$$f"; \ + rm -f $(DESTDIR)$(pkgincludedir)/$$f; \ + done + +ETAGS = etags +ETAGSFLAGS = + +CTAGS = ctags +CTAGSFLAGS = + +tags: TAGS + +ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) ' { files[$$0] = 1; } \ + END { for (i in files) print i; }'`; \ + mkid -fID $$unique + +TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + tags=; \ + here=`pwd`; \ + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) ' { files[$$0] = 1; } \ + END { for (i in files) print i; }'`; \ + test -z "$(ETAGS_ARGS)$$tags$$unique" \ + || $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$tags $$unique + +ctags: CTAGS +CTAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + tags=; \ + here=`pwd`; \ + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) ' { files[$$0] = 1; } \ + END { for (i in files) print i; }'`; \ + test -z "$(CTAGS_ARGS)$$tags$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$tags $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && cd $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) $$here + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) + +top_distdir = .. +distdir = $(top_distdir)/$(PACKAGE)-$(VERSION) + +distdir: $(DISTFILES) + $(mkinstalldirs) $(distdir)/.. + @srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's|.|.|g'`; \ + list='$(DISTFILES)'; for file in $$list; do \ + case $$file in \ + $(srcdir)/*) file=`echo "$$file" | sed "s|^$$srcdirstrip/||"`;; \ + $(top_srcdir)/*) file=`echo "$$file" | sed "s|^$$topsrcdirstrip/|$(top_builddir)/|"`;; \ + esac; \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + dir=`echo "$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test "$$dir" != "$$file" && test "$$dir" != "."; then \ + dir="/$$dir"; \ + $(mkinstalldirs) "$(distdir)$$dir"; \ + else \ + dir=''; \ + fi; \ + if test -d $$d/$$file; then \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -pR $(srcdir)/$$file $(distdir)$$dir || exit 1; \ + fi; \ + cp -pR $$d/$$file $(distdir)$$dir || exit 1; \ + else \ + test -f $(distdir)/$$file \ + || cp -p $$d/$$file $(distdir)/$$file \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(LTLIBRARIES) $(HEADERS) + +installdirs: + $(mkinstalldirs) $(DESTDIR)$(pkglibdir) $(DESTDIR)$(pkgincludedir) +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + `test -z '$(STRIP)' || \ + echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -rm -f $(CONFIG_CLEAN_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-pkglibLTLIBRARIES \ + mostlyclean-am + +distclean: distclean-am + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-libtool distclean-tags + +dvi: dvi-am + +dvi-am: + +info: info-am + +info-am: + +install-data-am: install-pkgincludeHEADERS + +install-exec-am: install-pkglibLTLIBRARIES + +install-info: install-info-am + +install-man: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-info-am uninstall-pkgincludeHEADERS \ + uninstall-pkglibLTLIBRARIES + +.PHONY: CTAGS GTAGS all all-am check check-am clean clean-generic \ + clean-libtool clean-pkglibLTLIBRARIES ctags distclean \ + distclean-compile distclean-generic distclean-libtool \ + distclean-tags distdir dvi dvi-am info info-am install \ + install-am install-data install-data-am install-exec \ + install-exec-am install-info install-info-am install-man \ + install-pkgincludeHEADERS install-pkglibLTLIBRARIES \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-compile mostlyclean-generic mostlyclean-libtool pdf \ + pdf-am ps ps-am tags uninstall uninstall-am uninstall-info-am \ + uninstall-pkgincludeHEADERS uninstall-pkglibLTLIBRARIES + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/debian/htdig/htdig-3.2.0b6/htnet/Makefile.win32 b/debian/htdig/htdig-3.2.0b6/htnet/Makefile.win32 new file mode 100644 index 00000000..ce0e5f50 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/Makefile.win32 @@ -0,0 +1,21 @@ + +TARGET = $(LIBDIR)/libhtnet$(LIBSFX) + +# ---------------------------------------------------------------------------- +# add new library members to this list + +# ---------------------------------------------------------------------------- + +include ../Makedefs.win32 + +CXXSRC = Connection.cc Transport.cc HtHTTP.cc HtFile.cc HtNNTP.cc \ + HtCookie.cc HtCookieJar.cc HtCookieMemJar.cc HtHTTPBasic.cc \ + HtHTTPSecure.cc SSLConnection.cc HtFTP.cc HtCookieInFileJar.cc + +CPPFLAGS += -DHAVE_CONFIG_H -I../db -I../htlib -I../htword -I../htcommon + +$(TARGET): $(OBJDIRDEP) $(LIBDIRDEP) $(OBJS) + $(AR) $(ARFLAGS) $(OBJS) + +include ../Makerules.win32 + diff --git a/debian/htdig/htdig-3.2.0b6/htnet/SSLConnection.cc b/debian/htdig/htdig-3.2.0b6/htnet/SSLConnection.cc new file mode 100644 index 00000000..766d69c1 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/SSLConnection.cc @@ -0,0 +1,177 @@ +// +// SSLConnection.cc +// +// SSLConnection: This class forms a easy to use interface to the +// socket interface using SSL connection encryption. +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1999-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: SSLConnection.cc,v 1.6 2004/05/28 13:15:23 lha Exp $ +// + +#ifdef HAVE_CONFIG_H +#include "htconfig.h" +#endif /* HAVE_CONFIG_H */ + +#ifdef HAVE_SSL_H +#include "SSLConnection.h" + +#ifdef HAVE_STD +#include <iostream> +#ifdef HAVE_NAMESPACES +using namespace std; +#endif +#else +#include <iostream.h> +#endif /* HAVE_STD */ + +// Global needed only once in HtDig +// +SSL_CTX *SSLConnection::ctx = NULL; + +SSLConnection::SSLConnection() +{ + InitSSL(); +} + +SSLConnection::SSLConnection(int socket) +{ + InitSSL(); +} + + +void SSLConnection::InitSSL() +{ + if( ctx == NULL ) + { + // Add in the OpenSSL algorithms ?? + // + OpenSSL_add_ssl_algorithms(); + + // Load the error strings ... openssl says so + // + SSL_load_error_strings(); + + // New CTX, either v3 but can default to v2 + // + ctx = SSL_CTX_new (SSLv23_client_method()); + + if( ctx == NULL ) + { + cout << "ctx NULL" << endl; + exit(1); + } + } +} + +SSLConnection::~SSLConnection() +{ + if( ctx != NULL ) + SSL_CTX_free (ctx); + ctx = NULL; +} + +int SSLConnection::Connect() +{ + // Run the standard connect + // + int ret = Connection::Connect(); + + if( ret != OK ) + return ret; + + // Now start SSL negotiation + // + ssl = SSL_new (ctx); + + // Attach ssl to socket + // + SSL_set_fd (ssl, sock); + int err = SSL_connect (ssl); + if( err == -1) + { + // cout << "SSL_connect failed!" << endl; + // Close the socket + // + Connection::Close(); + + return NOTOK; + } + return OK; +} + +int SSLConnection::Close() +{ + // First kill the SSL stuff + // + SSL_shutdown (ssl); /* send SSL/TLS close_notify */ + + /* Clean up. */ + + // Now call the Connection Close + // + int ret = Connection::Close(); + if( ret == OK ) + { + SSL_free (ssl); + } + return ret; +} + +int SSLConnection::Read_Partial(char *buffer, int maxlength) +{ + int count; + + need_io_stop = 0; + do + { + errno = 0; + + if (timeout_value > 0) { + FD_SET_T fds; + FD_ZERO(&fds); + FD_SET(sock, &fds); + + timeval tv; + tv.tv_sec = timeout_value; + tv.tv_usec = 0; + + int selected = select(sock+1, &fds, 0, 0, &tv); + if (selected <= 0) + need_io_stop++; + } + + if (!need_io_stop) + count = SSL_read(ssl, buffer, maxlength); + else + count = -1; // Input timed out + } + while (count <= 0 && errno == EINTR && !need_io_stop); + need_io_stop = 0; + + return count; +} + + +//************************************************************************* +// int Connection::Write_Partial(char *buffer, int maxlength) +// +int SSLConnection::Write_Partial(char *buffer, int maxlength) +{ + int count; + + do + { + count = SSL_write(ssl, buffer, maxlength); + } + while (count < 0 && errno == EINTR && !need_io_stop); + need_io_stop = 0; + + return count; +} + +#endif diff --git a/debian/htdig/htdig-3.2.0b6/htnet/SSLConnection.h b/debian/htdig/htdig-3.2.0b6/htnet/SSLConnection.h new file mode 100644 index 00000000..725220d7 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/SSLConnection.h @@ -0,0 +1,48 @@ +// +// SSLConnection.cc +// +// SSLConnection: This class forms a easy to use interface to the +// socket interface using SSL connection encryption. +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1999-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: SSLConnection.h,v 1.5 2004/05/28 13:15:23 lha Exp $ +// + +#ifndef _SSLConnection_h_ +#define _SSLConnection_h_ + +#include "Connection.h" + +#ifdef HAVE_SSL_H +#include <openssl/crypto.h> +#include <openssl/x509.h> +#include <openssl/pem.h> +#include <openssl/ssl.h> +#include <openssl/err.h> + +class SSLConnection : public Connection +{ + public: + SSLConnection(); + SSLConnection(int socket); + ~SSLConnection(); + + protected: + static SSL_CTX *ctx; + SSL *ssl; + + void InitSSL(); + int Connect(); + int Close(); + + int Read_Partial( char *buffer, int maxlength ); + int Write_Partial( char *buffer, int maxlength ); +}; +#endif + +#endif diff --git a/debian/htdig/htdig-3.2.0b6/htnet/Transport.cc b/debian/htdig/htdig-3.2.0b6/htnet/Transport.cc new file mode 100644 index 00000000..f768c731 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/Transport.cc @@ -0,0 +1,512 @@ +// +// Transport.cc +// +// Transport: A virtual transport interface class for accessing +// remote documents. Used to grab URLs based on the +// scheme (e.g. http://, ftp://...) +// +// Keep constructor and destructor in a file of its own. +// Also takes care of the lower-level connection code. +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1995-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: Transport.cc,v 1.12 2004/05/28 13:15:23 lha Exp $ +// +// + +#ifdef HAVE_CONFIG_H +#include "htconfig.h" +#endif /* HAVE_CONFIG_H */ + +#include "Transport.h" + +#ifdef HAVE_STD +#include <iomanip> +#ifdef HAVE_NAMESPACES +using namespace std; +#endif +#else +#include <iomanip.h> +#endif /* HAVE_STD */ + +#include <ctype.h> + +#define DEFAULT_CONNECTION_TIMEOUT 15 + +/////// + // Static variables initialization +/////// + + // Debug level + int Transport::debug = 0; + + // Default parser content-type string + String Transport::_default_parser_content_type = 0; + + // Statistics + int Transport::_tot_open = 0; + int Transport::_tot_close = 0; + int Transport::_tot_changes = 0; + + +/////// + // Transport_Response class definition +/////// + + /////// + // Class Constructor + /////// + +Transport_Response::Transport_Response() +{ + + // Initialize the pointers to the HtDateTime objs + _modification_time = 0; + _access_time = 0; + + // Set the content length and the return status code to negative values + _content_length = -1; + _status_code = -1; + + // Also set the document length, but to zero instead of -1 + _document_length = 0; + + // Zeroes the contents and the content-type + _contents = 0; + _content_type = 0; + + // Initialize the reason_phrase + _reason_phrase = 0; + + // Initialize the location + _location = 0; + +} + + + /////// + // Empty destructor + /////// + +Transport_Response::~Transport_Response() +{ + + // Free memory correctly + + if(_modification_time) + { + delete _modification_time; + _modification_time=0; + } + + if(_access_time) + { + delete _access_time; + _access_time=0; + } + +} + + + +void Transport_Response::Reset() +{ + // Reset all the field of the object + + // Check if an HtDateTime object exists, and delete it + if(_modification_time) + { + delete _modification_time; + _modification_time=0; + } + + if(_access_time) + { + delete _access_time; + _access_time=0; + } + + // Set the content length to a negative value + _content_length=-1; + + // Also set the document length, but to zero instead of -1 + _document_length=0; + + // Zeroes the contents and content type strings + _contents.trunc(); + _content_type.trunc(); + + // Set the return status code to a negative value + _status_code=-1; + + // Zeroes the reason phrase of the s.c. + _reason_phrase.trunc(); + + // Reset the location + _location.trunc(); + +} + + + +/////// + // Transport class definition +/////// + + /////// + // Constructor + /////// + +Transport::Transport(Connection* connection) +: _connection(connection), + _host(0), _ip_address(0), _port(-1), _timeout(DEFAULT_CONNECTION_TIMEOUT), + _retries(1), _wait_time(5), + _modification_time(0), _max_document_size(0), + _credentials(0), _useproxy(0), _proxy_credentials(0) +{ +} + + + /////// + // Destructor + /////// + +Transport::~Transport() +{ + + // Close the connection that was still up + if (CloseConnection()) + if ( debug > 4 ) + cout << setw(5) << GetTotOpen() << " - " + << "Closing previous connection with the remote host" << endl; + + if (_connection) + delete (_connection); + +} + +/////// + // Show the statistics +/////// + +ostream &Transport::ShowStatistics (ostream &out) +{ + out << " Connections opened : " << GetTotOpen() << endl; + out << " Connections closed : " << GetTotClose() << endl; + out << " Changes of server : " << GetTotServerChanges() << endl; + + return out; + +} + + + /////// + // Connection Management + /////// + + // Open the connection + // Returns + // - 0 if failed + // - -1 if already open + // - 1 if ok + +int Transport::OpenConnection() +{ + if (!_connection) return 0; + + if(_connection->IsOpen() && _connection->IsConnected()) + return -1; // Already open and connection is up + + // No open connection + // Let's open a new one + + if(_connection->Open() == NOTOK) return 0; // failed + + _tot_open ++; + return 1; +} + + + // Assign the server to the connection + +int Transport::AssignConnectionServer() +{ + if (debug > 5) + cout << "\tAssigning the server (" << _host << ") to the TCP connection" << endl; + + if( _connection == 0 ) + { + cout << "Transport::AssignConnectionServer: _connection is NULL\n"; + exit(0); + } + + if (_connection->Assign_Server(_host) == NOTOK) return 0; + + _ip_address = _connection->Get_Server_IPAddress(); + + return 1; +} + + + // Assign the remote server port to the connection + +int Transport::AssignConnectionPort() +{ + if (debug > 5) + cout << "\tAssigning the port (" << _port << ") to the TCP connection" << endl; + + if( _connection == 0 ) + { + cout << "Transport::AssignConnectionPort: _connection is NULL\n"; + exit(0); + } + + if (_connection->Assign_Port(_port) == NOTOK) return 0; + + return 1; +} + + + + + // Connect + // Returns + // - 0 if failed + // - -1 if already connected + // - 1 if ok + +int Transport::Connect() +{ + if (debug > 5) + cout << "\tConnecting via TCP to (" << _host << ":" << _port << ")" << endl; + + if (isConnected()) return -1; // Already connected + + if( _connection == 0 ) + { + cout << "Transport::Connection: _connection is NULL\n"; + exit(0); + } + + if ( _connection->Connect() == NOTOK) return 0; // Connection failed + + return 1; // Connected +} + + + // Flush the connection + +void Transport::FlushConnection() +{ + + if(_connection) + { + _connection->Flush(); + } + +} + + + // Close the connection + // Returns + // - 0 if not open + // - 1 if closed ok + + +int Transport::CloseConnection() +{ + if( _connection == 0 ) + { + // We can't treat this as a fatal error, because CloseConnection() + // may be called from our destructor after _connection already deleted. + // cout << "Transport::CloseConnection: _connection is NULL\n"; + // exit(0); + return 0; + } + + if(_connection->IsOpen()) + _connection->Close(); // Close the connection + else return 0; + + _tot_close ++; + + return 1; +} + + +void Transport::SetConnection (const String &host, int port) +{ + + if (_port != -1) + { + // Already initialized + // Let's check if the server or the port are changed + + bool ischanged = false; + + // Checking the connection server + if(_host != host) // server is gonna change + ischanged=true; + + // Checking the connection port + if( _port != port ) // the port is gonna change + ischanged=true; + + if (ischanged) + { + // Let's close any pendant connection with the old + // server / port pair + + _tot_changes ++; + + if ( debug > 4 ) + cout << setw(5) << GetTotOpen() << " - " + << "Change of server. Previous connection closed." << endl; + + CloseConnection(); + } + } + + // Copy the host and port information to the object + _host = host; + _port = port; + +} + + +// Create a new date time object containing the date specified in a string +HtDateTime *Transport::NewDate(const char *datestring) +{ + + while(isspace(*datestring)) datestring++; // skip initial spaces + + DateFormat df = RecognizeDateFormat (datestring); + + if(df == DateFormat_NotRecognized) + { + // Not recognized + if(debug > 0) + cout << "Date Format not recognized: " << datestring << endl; + + return 0; + } + + HtDateTime *dt = new HtDateTime; + + dt->ToGMTime(); // Set to GM time + + switch(df) + { + // Asc Time format + case DateFormat_AscTime: + dt->SetAscTime((char *)datestring); + break; + // RFC 1123 + case DateFormat_RFC1123: + dt->SetRFC1123((char *)datestring); + break; + // RFC 850 + case DateFormat_RFC850: + dt->SetRFC850((char *)datestring); + break; + default: + cout << "Date Format not handled: " << (int)df << endl; + break; + } + + return dt; + +} + + +// Recognize the possible date format sent by the server +Transport::DateFormat Transport::RecognizeDateFormat (const char *datestring) +{ + register const char *s; + + if((s=strchr(datestring, ','))) + { + // A comma is present. + // Two chances: RFC1123 or RFC850 + + if(strchr(s, '-')) + return DateFormat_RFC850; // RFC 850 recognized + else + return DateFormat_RFC1123; // RFC 1123 recognized + } + else + { + // No comma present + + // Let's try C Asctime: Sun Nov 6 08:49:37 1994 + if(strlen(datestring) == 24) return DateFormat_AscTime; + } + + return DateFormat_NotRecognized; + +} + + +// This method is used to write into 'dest' the credentials contained in 's' +// according to the HTTP Basic access authorization [RFC2617] +// It is written in this abstract class because it is used also +// when dealing with HTTP proxies, no matter what protocol we are +// using (HTTP now, but FTP in the future). + +void Transport::SetHTTPBasicAccessAuthorizationString(String &dest, const String& s) +{ + static char tbl[64] = + { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', + 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/' + }; + dest.trunc(); + const char *p; + int n = s.length(); + int ch; + + for (p = s.get(); n > 2; n -= 3, p += 3) + { + ch = *p >> 2; + dest << tbl[ch & 077]; + ch = ((*p << 4) & 060) | ((p[1] >> 4) & 017); + dest << tbl[ch & 077]; + ch = ((p[1] << 2) & 074) | ((p[2] >> 6) & 03); + dest << tbl[ch & 077]; + ch = p[2] & 077; + dest << tbl[ch & 077]; + } + + if (n != 0) + { + char c1 = *p; + char c2 = n == 1 ? 0 : p[1]; + + ch = c1 >> 2; + dest << tbl[ch & 077]; + + ch = ((c1 << 4) & 060) | ((c2 >> 4) & 017); + dest << tbl[ch & 077]; + + if (n == 1) + dest << '='; + else + { + ch = (c2 << 2) & 074; + dest << tbl[ch & 077]; + } + dest << '='; + } +} + +// End of Transport.cc (it's a virtual class anyway!) diff --git a/debian/htdig/htdig-3.2.0b6/htnet/Transport.h b/debian/htdig/htdig-3.2.0b6/htnet/Transport.h new file mode 100644 index 00000000..cf6b28f7 --- /dev/null +++ b/debian/htdig/htdig-3.2.0b6/htnet/Transport.h @@ -0,0 +1,371 @@ +// +// Transport.h +// +// Transport: A virtual transport interface class for accessing +// remote documents. Used to grab URLs based on the +// scheme (e.g. http://, ftp://...) +// +// Part of the ht://Dig package <http://www.htdig.org/> +// Copyright (c) 1995-2004 The ht://Dig Group +// For copyright details, see the file COPYING in your distribution +// or the GNU Library General Public License (LGPL) version 2 or later +// <http://www.gnu.org/copyleft/lgpl.html> +// +// $Id: Transport.h,v 1.12 2004/05/28 13:15:23 lha Exp $ +// +// + +#ifndef _Transport_H +#define _Transport_H + +#ifdef HAVE_CONFIG_H +#include "htconfig.h" +#endif + +#include "Object.h" +#include "HtDateTime.h" +#include "htString.h" +#include "URL.h" +#include "Connection.h" + +#ifdef HAVE_STD +#include <iostream> +#ifdef HAVE_NAMESPACES +using namespace std; +#endif +#else +#include <iostream.h> +#endif /* HAVE_STD */ + +// Declare in advance +class Transport; + +// But first, something completely different. Here's the response class +class Transport_Response : public Object +{ + friend class Transport; // declaring friendship + + public: + +/////// + // Construction / Destruction +/////// + + Transport_Response(); + virtual ~Transport_Response(); + + // Reset the information stored + virtual void Reset(); // This function must be defined + + // Get the contents + virtual const String &GetContents() const { return _contents; } + + // Get the modification time object pointer + virtual HtDateTime *GetModificationTime() const { return _modification_time; } + + // Get the access time object pointer + virtual HtDateTime *GetAccessTime() const { return _access_time; } + + // Get the Content type + virtual const String &GetContentType() const { return _content_type; } + + // Get the Content length + virtual int GetContentLength() const { return _content_length; } + + // Get the Document length (really stored) + virtual int GetDocumentLength() const { return _document_length; } + + // Get the Status Code + virtual int GetStatusCode() const { return _status_code; } + + // Get the Status Code reason phrase + virtual const String &GetReasonPhrase() { return _reason_phrase; } + + // Get the location (redirect) + virtual const String &GetLocation() { return _location; } + + + protected: + + // Body of the response message + + String _contents; // Contents of the document + + HtDateTime *_modification_time; // Modification time returned by the server + HtDateTime *_access_time; // Access time returned by the server + + String _content_type; // Content-type returned by the server + int _content_length; // Content-length returned by the server + int _document_length; // Length really stored + + int _status_code; // return Status code + String _reason_phrase; // status code reason phrase + + String _location; // Location (in case of redirect) + +}; + + +/////// + // Transport class declaration +/////// + +class Transport : public Object +{ + + public: + +/////// + // Construction / Destruction +/////// + + Transport(Connection* _connection = 0); + virtual ~Transport(); + +/////// + // Enumeration of possible return status of a resource retrieving +/////// + + enum DocStatus + { + Document_ok, + Document_not_changed, + Document_not_found, + Document_not_parsable, + Document_redirect, + Document_not_authorized, + Document_no_connection, + Document_connection_down, + Document_no_header, + Document_no_host, + Document_no_port, + Document_not_local, + Document_not_recognized_service, // Transport service not recognized + Document_other_error // General error (memory) + }; + + +/////// + // Connects to an host and a port + // Overloaded methods provided in order to take + // the info from a URL obj or ptr +/////// + + // Set Connection parameters + virtual void SetConnection (const String &host, int port); + + // from a URL pointer + virtual void SetConnection (URL *u) + { SetConnection (u->host(), u->port()); } + + // from a URL object + virtual void SetConnection (URL &u) + { SetConnection (&u); } + + + + // Make the request + virtual DocStatus Request() = 0; // different in derived classes + + + + // Get the date time information about the request + const HtDateTime *GetStartTime() const { return &_start_time; } + const HtDateTime *GetEndTime() const { return &_end_time; } + + // Set and get the connection time out value + void SetTimeOut ( int t ) { _timeout=t; } + int GetTimeOut () { return _timeout; } + + // Set and get the connection retry number + void SetRetry ( int r ) { _retries=r; } + int GetRetry () { return _retries; } + + // Set and get the wait time after a failed connection + void SetWaitTime ( unsigned int t ) { _wait_time = t; } + unsigned int GetWaitTime () { return _wait_time; } + + // Get the Connection Host + const String &GetHost() { return _host; } + + // Get the Connection IP Address + const String &GetHostIPAddress() { return _ip_address; } + + // Get the Connection Port + int GetPort() { return _port; } + + // Set and get the credentials + // Likely to vary based on transport protocol + virtual void SetCredentials (const String& s) { _credentials = s;} + virtual String GetCredentials () { return _credentials;} + + // Proxy settings + virtual void SetProxy(int aUse) { _useproxy=aUse; } + + // Proxy credentials + virtual void SetProxyCredentials (const String& s) { _proxy_credentials = s;} + virtual String GetProxyCredentials () { return _proxy_credentials;} + + // Set the modification date and time for If-Modified-Since + void SetRequestModificationTime (HtDateTime *p) { _modification_time=p; } + void SetRequestModificationTime (HtDateTime &p) + { SetRequestModificationTime (&p) ;} + + // Get the modification date time + HtDateTime *GetRequestModificationTime () { return _modification_time; } + + // Get and set the max document size to be retrieved + void SetRequestMaxDocumentSize (int s) { _max_document_size=s; } + int GetRequestMaxDocumentSize() const { return _max_document_size; } + + virtual Transport_Response *GetResponse() = 0; + + virtual DocStatus GetDocumentStatus() = 0; + +/////// + // Querying the status of the connection +/////// + + // Are we still connected? + // This is the only part regarding + // a connection that's got a public access + + virtual bool isConnected(){ return _connection?_connection->IsConnected():0; } + +// Set the default parser string for the content-type + static void SetDefaultParserContentType (const String &ct) + { _default_parser_content_type = ct; } + +// Set the debug level + static void SetDebugLevel (int d) { debug=d;} + +// Get statistics info + static int GetTotOpen () { return _tot_open; } + static int GetTotClose () { return _tot_close; } + static int GetTotServerChanges () { return _tot_changes; } + + +protected: + + /////// + // Services about a Transport layer connection + // They're invisible from outside + /////// + + // Open the connection + + virtual int OpenConnection(); + + // Assign the host and the port for the connection + + int AssignConnectionServer(); + int AssignConnectionPort(); + + // Connect to the specified host and port + int Connect(); + + // Write a message + inline int ConnectionWrite(char *cmd) + { return _connection?_connection->Write(cmd):0; } + + + // Assign the timeout to the connection (returns the old value) + + inline int AssignConnectionTimeOut() + { return _connection?_connection->Timeout(_timeout):0; } + + // Assign the retry number to the connection (returns the old value) + + inline int AssignConnectionRetries() + { return _connection?_connection->Retries(_retries):0; } + + // Assign the wait time (after a failure) to the connection + + inline int AssignConnectionWaitTime() + { return _connection?_connection->WaitTime(_wait_time):0; } + + // Flush the connection + void FlushConnection(); + + // Close the connection + + int CloseConnection(); + + // Reset Stats + static void ResetStatistics () + { _tot_open=0; _tot_close=0; _tot_changes=0;} + + // Show stats + static ostream &ShowStatistics (ostream &out); + + // Methods for manipulating date strings -- useful for subclasses + + enum DateFormat + { + DateFormat_RFC1123, + DateFormat_RFC850, + DateFormat_AscTime, + DateFormat_NotRecognized + }; + + // Create a new HtDateTime object + HtDateTime *NewDate(const char *); + + // Recognize Date Format + DateFormat RecognizeDateFormat (const char *); + + protected: + + Connection *_connection; // Connection object + + String _host; // TCP Connection host + String _ip_address; // TCP Connection host (IP Address) + int _port; // TCP Connection port + + int _timeout; // Connection timeout + int _retries; // Connection retry limit + unsigned int _wait_time; // Connection wait time (if failed) + + HtDateTime *_modification_time; // Stored modification time if avail. + int _max_document_size; // Max document size to retrieve + + String _credentials; // Credentials for this connection + + int _useproxy; // if true, GET should include full url, + // not path only + String _proxy_credentials; // Credentials for this proxy connection + + HtDateTime _start_time; // Start time of the request + HtDateTime _end_time; // end time of the request + + + /////// + // Default parser content-type + // This string is matched in order to determine + // what content type can be considered parsed + // directly by the internal indexer (not by using + // any external parser) + /////// + + static String _default_parser_content_type; + + + /////// + // Debug level + /////// + + static int debug; + + // Statistics about requests + static int _tot_open; // Number of connections opened + static int _tot_close; // Number of connections closed + static int _tot_changes; // Number of server changes + + // Use the HTTP Basic Digest Access Authentication method to write a String + // to be used for credentials (both HTTP and HTTP PROXY authentication) + static void SetHTTPBasicAccessAuthorizationString(String &dest, const String& s); + +}; + +#endif + |