/************************************************************ ** $Id: database.c,v 0.8 2003/12/29 16:45:38 bcx Exp bcx $ ** ** Copyright Bryan Costales and Perforce Software, Inc. 2003 ** ** This code is "open source" as defined by version 1.9 of ** the Open Source Definition from: ** ** http://www.opensource.org/docs/definition.php. ** ** Entry points defined in this file: ** ** open_database(): ** Open a database, creating it if asked to. ** close_database(): ** Close a database. ** put_ip_database(): ** Add a date to the primary database of IP numbers. ** Create a new entry of the IP number is not found. ** get_ip_database(): ** Look up an ip number in the primary database ** del_ip_database(): ** Remove and IP record and all its events ** or just one date's record. ** put_event_database(): ** Add an ip/date item to the secondary database. ** get_event_database(): ** Read an ip/date item from the secondary database. ** dump_database(): ** Print the database contents to a file pointer. ** event_to_str(): ** Format the event for printing or logging. *************************************************************/ # define EXTERN extern # include "slow.h" DB * open_database(char *fname, int force) { static DB *dp; u_int32_t flags = DB_THREAD; int ret; ret = db_create(&dp, (DB_ENV *)NULL, 0); if (ret != 0) { errno = ret; return NULL; } if (force == TRUE) flags |= DB_CREATE; ret = dp->open(dp, NULL, fname, NULL, DB_HASH, flags, 0644); if (ret != 0) { errno = ret; return NULL; } return dp; } pthread_mutex_t dbio_mutex; void init_database_mutex(void) { pthread_mutex_init(&dbio_mutex, NULL); return; } void close_database(DB *dp) { (void)dp->close(dp, 0); } /* ** Append a new date to the list of dates for a given IP address. ** The dates will be in ascending order, with the oldest at the ** start. This scheme will only break if the date is manually ** reset on the machine (and even worse if the date is set backwards). ** ** If the IP key doesn't exist yet, it is begun with a single date ** record. The IP/date combination must be added to the secondary ** database for this to work. An unreferenced IP/date key in that ** database will produce a warning. ** ** Adds and delets must be in this one routine to avoid thread ** contention. */ int put_ip_database(DB *dp, PRIMARY_KEY ip, PRIMARY_DATUM date, bool drop, int log) { DBT key, data; int ret; PRIMARY_DATUM *new, *d, *n; int i, got; extern int errno; if (dp == NULL) return EFAULT; (void) memset(&data, 0, sizeof data); (void) memset(&key, 0, sizeof key); key.data = &ip; key.size = sizeof(PRIMARY_KEY); data.data = NULL; data.size = 0; data.ulen = 0; data.flags = DB_DBT_MALLOC; pthread_mutex_lock(&dbio_mutex); ret = dp->get(dp, NULL, &key, &data, 0); if (ret != 0) { if (ret != DB_NOTFOUND) { return ret; pthread_mutex_unlock(&dbio_mutex); } if (drop == TRUE) { pthread_mutex_unlock(&dbio_mutex); return DB_NOTFOUND; } data.size = 0; data.data = try_alloc(__FILE__, __LINE__, sizeof(PRIMARY_DATUM), log); if (data.data == NULL) { pthread_mutex_unlock(&dbio_mutex); return DB_NOTFOUND; } } else { /* * Never delete/update the version. */ if (ip == 0) { pthread_mutex_unlock(&dbio_mutex); return 0; } /* * You can only whitelist new entries. */ if (date == TYPE_RECORD_WHITELIST && drop == FALSE) { pthread_mutex_unlock(&dbio_mutex); if (log == FALSE) return DB_NOTFOUND; else return 0; } if (drop == FALSE) { /* * Is the date already in the database for this * IP number. If so, just update the record. */ got = FALSE; d = (PRIMARY_DATUM *)data.data; if (*d == TYPE_RECORD_WHITELIST) { /* * Handle whitelist */ pthread_mutex_unlock(&dbio_mutex); if (log == FALSE) return DB_NOTFOUND; else return 0; } for (i = 0; i < data.size; i += sizeof(PRIMARY_DATUM)) { if (*d == date) { got = TRUE; break; } ++d; } if (got == TRUE) { pthread_mutex_unlock(&dbio_mutex); return 0; } data.data = try_realloc(__FILE__, __LINE__, data.data, data.size + sizeof(PRIMARY_DATUM), log); if (data.data == NULL) { pthread_mutex_unlock(&dbio_mutex); return DB_NOTFOUND; } } else { new = (PRIMARY_DATUM *)try_alloc(__FILE__, __LINE__, data.size, log); n = new; if (n == NULL) { pthread_mutex_unlock(&dbio_mutex); return DB_NOTFOUND; } d = (PRIMARY_DATUM *)data.data; got = FALSE; for (i = 0; i < data.size; i += sizeof(PRIMARY_DATUM)) { if (*d == date) { ++d; got = TRUE; continue; } *n = *d; ++n; ++d; } if (got == FALSE) { (void) free(data.data); (void) free(new); pthread_mutex_unlock(&dbio_mutex); return DB_NOTFOUND; } (void) free(data.data); data.data = new; data.size = data.size - sizeof(PRIMARY_DATUM); } } if (drop == FALSE) { d = (PRIMARY_DATUM *)data.data; /* * This memcpy works for whitelisting too. */ (void)memcpy(((char *)(data.data)) + data.size, &date, sizeof(PRIMARY_DATUM)); data.size += sizeof(PRIMARY_DATUM); } if (data.size > 0) ret = dp->put(dp, NULL, &key, &data, 0); else ret = dp->del(dp, NULL, &key, 0); if (ret != 0) { (void)free(data.data); pthread_mutex_unlock(&dbio_mutex); return ret; } (void)dp->sync(dp, 0); pthread_mutex_unlock(&dbio_mutex); (void)free(data.data); return 0; } int get_db_version(DB *dp, int log) { PRIMARY_DATUM *rp; int ndates; rp = get_ip_database(dp, 0, &ndates, log); if (rp == NULL) return 0; return *(int *)rp; } /* ** Look up an IP number in the primary database and return ** its list of dates. The list is in allocated memory, so ** it can be freed. ** ** If the IP number is not found, return NULL and set ndates to 0. ** On error return NULL, and sets ndates to non-zero. ** No data returns a valid address but set ndates to 0. ** Otherwise return address of list of dates and set ndates to how many dates. */ PRIMARY_DATUM * get_ip_database(DB *dp, PRIMARY_KEY ip, int *ndates, int log) { DBT key, data; int ret; extern int errno; if (dp == NULL) { errno = *ndates = EFAULT; return NULL; } (void) memset(&key, 0, sizeof key); (void) memset(&data, 0, sizeof data); key.data = &ip; key.size = sizeof(PRIMARY_KEY); data.data = NULL; data.size = 0; data.ulen = 0; data.flags = DB_DBT_MALLOC; ret = dp->get(dp, NULL, &key, &data, 0); if (ret != 0) { if (ret == DB_NOTFOUND) { *ndates = 0; return NULL; } errno = *ndates = ret; data.data = try_alloc(__FILE__, __LINE__, sizeof(PRIMARY_DATUM), log); return data.data; } *ndates = data.size / sizeof(PRIMARY_DATUM); return data.data; } /* ** Delete a record for an IP number or the entire ** IP number and all its records. ** ** If the IP number is not found, return FALSE. ** On error return FALSE. ** If the date is zero (no date specified) remove everything. */ int del_ip_database(DB *dp1, DB *dp2, PRIMARY_KEY ip, time_t date, int log) { SECONDARY_KEY k; PRIMARY_DATUM *dates, *d; int ndates, i, ret; extern int errno; if (dp1 == NULL || dp2 == NULL) return (errno = EINVAL); errno = 0; ndates = 0; if ((dates = get_ip_database(dp1, ip, &ndates, log)) == NULL) { if (ndates == 0) { errno = EINVAL; return DB_NOTFOUND; } return ndates; } if (date != 0) { for(d = dates, i = 0; i < ndates; ++i, ++d) { if (date == *d) break; } if (i == ndates) return DB_NOTFOUND; errno = 0; ret = put_ip_database(dp1, ip, date, TRUE, log); if (ret != 0) return ret; k.ip = ip; k.date = date; errno = 0; return del_event_database(dp2, &k); } if (ndates == 1 && *dates == TYPE_RECORD_WHITELIST) { return (put_ip_database(dp1, ip, *dates, TRUE, log)); } for(d = dates, i = 0; i < ndates; ++i, ++d) { errno = 0; ret = put_ip_database(dp1, ip, *d, TRUE, log); if (ret != 0) return ret; k.ip = ip; k.date = *d; errno = 0; ret = del_event_database(dp2, &k); if (ret != 0) return ret; } return 0; } int put_event_database(DB *dp, SECONDARY_KEY *k, SECONDARY_DATUM *event) { DBT key, data; int ret; extern int errno; if (dp == NULL) return EFAULT; (void) memset(&data, 0, sizeof data); (void) memset(&key, 0, sizeof key); key.data = k; key.size = sizeof(SECONDARY_KEY); data.data = event; data.size = sizeof(SECONDARY_DATUM); /* ** If IP,date already exists as a key, this ** will silently overwrite the old entry. */ ret = dp->put(dp, NULL, &key, &data, 0); if (ret != 0) { return ret; } (void)dp->sync(dp, 0); return 0; } int del_event_database(DB *dp, SECONDARY_KEY *k) { DBT key, data; int ret; extern int errno; if (dp == NULL) return EFAULT; (void) memset(&data, 0, sizeof data); (void) memset(&key, 0, sizeof key); key.data = k; key.size = sizeof(SECONDARY_KEY); /* ** If IP,date already exists as a key, this ** will silently overwrite the old entry. */ ret = dp->del(dp, NULL, &key, 0); if (ret != 0) { return ret; } (void)dp->sync(dp, 0); return 0; } SECONDARY_DATUM * get_event_database(DB *dp, SECONDARY_KEY *k, bool verbose, int *ret) { DBT key, data; extern int errno; if (dp == NULL) return NULL; (void) memset(&key, 0, sizeof key); (void) memset(&data, 0, sizeof data); key.data = k; key.size = sizeof(SECONDARY_KEY); data.data = NULL; data.size = 0; data.ulen = 0; data.flags = DB_DBT_MALLOC; *ret = 0; *ret = dp->get(dp, NULL, &key, &data, 0); if (*ret != 0) { if (verbose) { dp->err(dp, *ret, "%s", SecondaryDatabaseLoc); } return NULL; } return (SECONDARY_DATUM *)(data.data); } /* ** Walk the databases, producing a report of its contents. ** The fp pointer is where to print the report, and is usually ** the standard output. ** ** This routine is intended for use from the command-line, and is ** not thread safe. */ int dump_database(DB *ipdp, DB *recdp, FILE *fp, int clean) { DBT key, data; int i, ret, records; DBC *curs; SECONDARY_DATUM *event; SECONDARY_KEY k; struct in_addr in; time_t *tp; char ipbuf[MAXHOSTNAMELEN]; extern int errno; if (ipdp == NULL) return 0; ret = ipdp->cursor(ipdp, NULL, &curs, 0); if (ret != 0) { return ret; } (void) memset(&data, 0, sizeof data); (void) memset(&key, 0, sizeof key); key.flags = DB_DBT_REALLOC; data.flags = DB_DBT_REALLOC; for (;;) { ret = curs->c_get(curs, &key, &data, DB_NEXT); if (ret == DB_NOTFOUND) break; if (ret != 0) { (void) curs->c_close(curs); return ret; } if (key.data == NULL) continue; /* * Always silently skip the version. */ if (*(PRIMARY_KEY *)(key.data) == 0) continue; records = data.size / sizeof(PRIMARY_DATUM); (void) memcpy(&in.s_addr, ((char *)(key.data)), sizeof(in.s_addr)); in.s_addr = ntohl(in.s_addr); (void) sprintf(ipbuf, "%s", inet_ntoa(in)); tp = (time_t *)data.data; if (*tp == TYPE_RECORD_WHITELIST) { fprintf(fp, "%s\n", event_to_str(ipbuf, *tp, NULL, FALSE)); continue; } for (i = 0; i < records; i++, tp++) { int ret; k.ip = *((PRIMARY_KEY *)(key.data)); k.date = *tp; event = get_event_database(recdp, &k, FALSE, &ret); if (event == NULL && clean == TRUE) continue; fprintf(fp, "%s\n", event_to_str(ipbuf, k.date, event, FALSE)); (void) free((char *)event); } } (void) curs->c_close(curs); return 0; } /* ALIAS */ char * alias_to_str(char *ipbuf, char *alias, int log) { char buf[BUFSIZ]; if (ipbuf == NULL || alias == NULL) return NULL; (void) sprintf(buf, "%s aliased to %s", ipbuf, alias); return try_strdup(__FILE__, __LINE__, buf, log); } char * event_to_str(char *ipbuf, time_t date, SECONDARY_DATUM *event, int log) { char buf[BUFSIZ]; struct tm *t; if (ipbuf == NULL) return NULL; if (date == TYPE_RECORD_WHITELIST) { (void) sprintf(buf, "%s: Whitelisted", ipbuf); return try_strdup(__FILE__, __LINE__, buf, log); } t = localtime(&date); (void) sprintf(buf, "%s: %02d/%02d/%d.%02d:%02d:%02d ", ipbuf, t->tm_mon + 1, t->tm_mday, t->tm_year + 1900, t->tm_hour, t->tm_min, t->tm_sec); if (event == NULL) { if (errno != 0) { (void) sprintf(buf+strlen(buf), "%s", errno == DB_NOTFOUND ? "Item not in database (need to garbage collect)" : "Lookup error while processing item"); } else { (void) sprintf(buf+strlen(buf), "Garbage Collected"); } return try_strdup(__FILE__, __LINE__, buf, log); } (void) sprintf(buf+strlen(buf), "er=%d,", event->envrcpts); (void) sprintf(buf+strlen(buf), "hr=%d,", event->headrcpts); (void) sprintf(buf+strlen(buf), "ho=%d,", event->honeyrcpts); (void) sprintf(buf+strlen(buf), "br=%d,", event->badrcpts); if (event->eventmap != 0) { if (isbitset(event->eventmap, BIT_TOO_MANY_HDR_RCPTS)) (void) strcat(buf, "hed,"); if (isbitset(event->eventmap, BIT_TOO_MANY_ENV_RCPTS)) (void) strcat(buf, "env,"); if (isbitset(event->eventmap, BIT_GOT_HONEYPOT)) (void) strcat(buf, "hon,"); if (isbitset(event->eventmap, BIT_BAD_MSG_ID)) (void) strcat(buf, "mid,"); if (isbitset(event->eventmap, BIT_BAD_FROM_HEAD)) (void) strcat(buf, "frm,"); if (isbitset(event->eventmap, BIT_BAD_HOST)) (void) strcat(buf, "bho,"); if (isbitset(event->eventmap, BIT_RBL_BAD)) (void) strcat(buf, "rbl,"); if (isbitset(event->eventmap, BIT_MILTER_ABORTED)) (void) strcat(buf, "abo,"); if (isbitset(event->eventmap, BIT_ADVANCEWRITE)) (void) strcat(buf, "pip,"); if (isbitset(event->eventmap, BIT_FORGED)) (void) strcat(buf, "for,"); if (isbitset(event->eventmap, BIT_FROMMX)) (void) strcat(buf, "mmx,"); if (isbitset(event->eventmap, BIT_NOTINET)) (void) strcat(buf, "net,"); if (isbitset(event->eventmap, BIT_ADDEDIP)) (void) strcat(buf, "aip,"); } (void) sprintf(buf+strlen(buf), "msgid=%s", event->msgid); return try_strdup(__FILE__, __LINE__, buf, log); } /* ** Walk the databases, removig garbage as we go. ** The fp pointer is where to print what was done. ** ** This routine is intended for use from the command-line, and is ** not thread safe. */ int garbage_database(DB *ipdp, DB *recdp, FILE *fp) { DBT key, data; int i, ret, records; DBC *curs; SECONDARY_DATUM *event; SECONDARY_KEY k, *kp; struct in_addr in; time_t *tp, t; char ipbuf[MAXHOSTNAMELEN]; extern int errno; /* * First walk the IP database, looking for events * that do not exist in the events database, and * removing them. */ fprintf(fp, "===== Phase 1, IP database\n"); if (ipdp == NULL) return 0; ret = ipdp->cursor(ipdp, NULL, &curs, 0); if (ret != 0) { return ret; } (void) memset(&data, 0, sizeof data); (void) memset(&key, 0, sizeof key); key.flags = DB_DBT_REALLOC; data.flags = DB_DBT_REALLOC; for (;;) { ret = curs->c_get(curs, &key, &data, DB_NEXT); if (ret == DB_NOTFOUND) break; if (ret != 0) { (void) curs->c_close(curs); return ret; } if (key.data == NULL) continue; /* * Always silently skip the version. */ if (*(PRIMARY_KEY *)(key.data) == 0) continue; records = data.size / sizeof(PRIMARY_DATUM); (void) memcpy(&in.s_addr, ((char *)(key.data)), sizeof(in.s_addr)); in.s_addr = ntohl(in.s_addr); (void) sprintf(ipbuf, "%s", inet_ntoa(in)); tp = (time_t *)data.data; if (*tp == TYPE_RECORD_WHITELIST) { continue; } for (i = 0; i < records; i++, tp++) { k.ip = *((PRIMARY_KEY *)(key.data)); k.date = *tp; event = get_event_database(recdp, &k, TRUE, &ret); if (event != NULL) continue; if (ret != DB_NOTFOUND) continue; ret = put_ip_database(ipdp, k.ip, k.date, TRUE, TRUE); errno = ret; fprintf(fp, "%s\n", event_to_str(ipbuf, *tp, NULL, FALSE)); (void) free((char *)event); } } (void) curs->c_close(curs); /* * Second, walk the events database, looking for * IP numbers that do not exist in the IP database, and * removing them. */ fprintf(fp, "===== Phase 2, Events database\n"); if (recdp == NULL) return 0; ret = recdp->cursor(recdp, NULL, &curs, 0); if (ret != 0) { return ret; } (void) memset(&data, 0, sizeof data); (void) memset(&key, 0, sizeof key); key.flags = DB_DBT_REALLOC; data.flags = DB_DBT_REALLOC; for (;;) { ret = curs->c_get(curs, &key, &data, DB_NEXT); if (ret == DB_NOTFOUND) break; if (ret != 0) { (void) curs->c_close(curs); return ret; } if (key.data == NULL) continue; /* * Always silently skip the version. */ kp = (SECONDARY_KEY *)(key.data); records = data.size / sizeof(PRIMARY_DATUM); (void) memcpy(&in.s_addr, (char *)&(kp->ip), sizeof(in.s_addr)); in.s_addr = ntohl(in.s_addr); (void) sprintf(ipbuf, "%s", inet_ntoa(in)); t = kp->date; if (get_ip_database(ipdp, kp->ip, &ret, FALSE) == NULL) { if (ret != 0) continue; ret = del_event_database(recdp, kp); errno = ret; fprintf(fp, "%s\n", event_to_str(ipbuf, t, NULL, FALSE)); } } (void) curs->c_close(curs); return 0; }
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#7 | 4222 | bryan_costales |
Massive rewrite to speed up the database writes. Using a single database now with duplicate keys where the keys are the IP numbers. Added a purge command and removed the garbage command. Fixed some leaking memory bugs and properly closed the database in a few places were it was not properly closed. Updated the docs to reflect this and bumped both the database version and release number. Running on a FreeBSD 3.x machine and a Solaris 9 machine. |
||
#6 | 4052 | bryan_costales |
Implimented: whitelisting AddMXHost for MX servers that lie Converted to thread safe DNS routines garbage collection RunAsUser and RunAsGroup for root startups rebuild the database summarize by IP number Finished all documentation. Moved release from alpha to beta |
||
#5 | 4030 | bryan_costales |
Finished documenting the configuration file Fixed a race condition and a core dump bug. Added hooks for whitelisting and IP aliasing Added support for Berkeley DB 4.2 Converted to htonl() and ntohl() Known Bugs: ip.db cannot be shared over NFS IP tracking from MX hosts can fail A RunAsUser config option is needed. |
||
#4 | 3998 | bryan_costales |
Brought the whole distribution up to V0.9 Added a huge abount of documentation. Added slowedit find Created startup scripts to launch for testing Fixed numerous bugs. Fixed a few portablity issues. Installed hooks for whitelisting and IP aliases. |
||
#3 | 3957 | bryan_costales |
Added rbl lookup support and testing for same. Folded in support for smfi_stop(). Added lots of slowedit commands Fixed a serious bug in MX lookups. Added to documentation. |
||
#2 | 3890 | bryan_costales |
This is the 0.6 release. The following have been added with the uses indicated: Source files: edit.c -- the slowedit functions compat.c -- missing system files Autoconf: configure.ac, makefile.am config.h aclocal.m4 acinclude.m4 build/ Documents: doc/ -- html and man(1) documents Testing: tests/ -- regressive testing TODO: -- revised to show actual progress |
||
#1 | 3838 | bryan_costales |
This is pre-release 0.5 (rcs numbering) which includes: The milter source files and Makefile A regressive testing subdirectory with Makefile and /bin/sh scripts. A patches subdirectory with a patch file for V8.13 sendmail All have been compiled and tested on a 64bit Sun Solaris 9 machine. |