0.1.11

scorpion · Dec 27, 2011 · b06fae4 · b06fae4
1 parent da99ff3
commit b06fae4
Show file tree

Hide file tree

Showing 14 changed files with 199 additions and 42 deletions.
diff --git a/Makefile.in b/Makefile.in
@@ -67,11 +67,12 @@ $(RECURSIVE_TARGETS):
 installdirs: mkinstalldirs
 	$(srcdir)/mkinstalldirs \
 		$(DESTDIR)$(bindir) $(DESTDIR)$(sbindir) $(DESTDIR)$(libdir) $(DESTDIR)$(libexecdir)/piler $(DESTDIR)$(sysconfdir) \
-		$(DESTDIR)$(datarootdir)/piler $(DESTDIR)$(includedir)/piler $(DESTDIR)$(localstatedir)/lib/piler/data \
-		$(DESTDIR)$(localstatedir)/spool
+		$(DESTDIR)$(datarootdir)/piler $(DESTDIR)$(includedir)/piler $(DESTDIR)$(localstatedir)/piler/store \
+		$(DESTDIR)$(localstatedir)/piler/stat $(DESTDIR)$(localstatedir)/spool
 
 	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/run/piler
-	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/lib/piler/stat
+	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/store
+	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/stat
 
 
 install-am:
@@ -83,7 +84,7 @@ uninstall:
 		$(DESTDIR)$(sysconfdir)/piler.conf
 
 	rm -rf $(DESTDIR)$(libexecdir)/piler $(DESTDIR)$(includedir)/piler $(DESTDIR)$(datarootdir)/piler
-	@echo "I left the $(DESTDIR)$(localstatedir)/lib/piler directory, remove it for yourself"
+	@echo "I left the $(DESTDIR)$(localstatedir)/piler directory, remove it for yourself"
 
 
 all-am: Makefile

diff --git a/etc/cron.jobs.in b/etc/cron.jobs.in
@@ -1,8 +1,29 @@
+#########################################################
+###     all the cron jobs you may need for piler      ###
+### be sure to review it and adjust it for your needs ###
+#########################################################
 
+root's crontab:
 
-sphinx cronjob:
+### optional: query postfix queue statistics
+*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape > LOCALSTATEDIR/piler/stat/active+incoming
+*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape -s > LOCALSTATEDIR/piler/stat/active+incoming-sender
+*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape deferred > LOCALSTATEDIR/piler/stat/deferred
+*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape -s deferred > LOCALSTATEDIR/piler/stat/deferred-sender
 
-*/2 * * * * /usr/local/bin/indexer --quiet delta1 --rotate && sleep 2 && /usr/local/bin/indexer --quiet --merge main1 delta1 --merge-dst-range deleted 0 0 --rotate
 
+piler's crontab:
+
+### optional: show cpu usage
+*/5 * * * * LC_ALL=C mpstat | tail -1 | awk '{print $11}' > /var/piler/stat/cpu.stat
+
+### optional: regular AD sync
+0 8 * * * /usr/bin/php LIBEXECDIR/piler/ldap_sync.php /srv/www/webui.yourdomain.com > LOCALSTATEDIR/piler/stat/adsync.stat
+
+
+
+sphinx's cronjob:
+
+*/5 * * * * /usr/local/bin/indexer --quiet delta1 --rotate && sleep 2 && /usr/local/bin/indexer --quiet --merge main1 delta1 --merge-dst-range deleted 0 0 --rotate
 */5 * * * * /usr/local/bin/indexer --quiet tag1 --rotate
 
diff --git a/src/Makefile.in b/src/Makefile.in
@@ -76,6 +76,7 @@ install-piler:
 
 	$(INSTALL) -m 0755 piler $(DESTDIR)$(sbindir)
 	$(INSTALL) -m 0755 pilerconf $(DESTDIR)$(sbindir)
+	$(INSTALL) -m 0755 pilerget $(DESTDIR)$(bindir)
 
 clean:
 	rm -f *.o *.a libpiler.so* piler pilerconf pilerget pilertest

diff --git a/src/cfg.c b/src/cfg.c
@@ -63,7 +63,6 @@ struct _parse_rule config_parse_rules[] =
    { "clamd_addr", "string", (void*) string_parser, offsetof(struct __config, clamd_addr), "", MAXVAL-1},
    { "clamd_port", "integer", (void*) int_parser, offsetof(struct __config, clamd_port), "0", sizeof(int)},
    { "clamd_socket", "string", (void*) string_parser, offsetof(struct __config, clamd_socket), CLAMD_SOCKET, MAXVAL-1},
-   { "deferdir", "string", (void*) string_parser, offsetof(struct __config, deferdir), DEFER_DIR, MAXVAL-1},
    { "hostid", "string", (void*) string_parser, offsetof(struct __config, hostid), HOSTID, MAXVAL-1},
    { "iv", "string", (void*) string_parser, offsetof(struct __config, iv), "", MAXVAL-1},
    { "listen_addr", "string", (void*) string_parser, offsetof(struct __config, listen_addr), "127.0.0.1", MAXVAL-1},

diff --git a/src/cfg.h b/src/cfg.h
@@ -33,7 +33,6 @@ struct __config {
 
    char workdir[MAXVAL];
    char queuedir[MAXVAL];
-   char deferdir[MAXVAL];
 
    int verbosity;
    char locale[MAXVAL];

diff --git a/src/config.h b/src/config.h
@@ -11,16 +11,15 @@
 
 #define PROGNAME "piler"
 
-#define VERSION "0.1.10"
+#define VERSION "0.1.11"
 
-#define PROGINFO VERSION ", Janos SUTO <sj@acts.hu>\n\n" CONFIGURE_PARAMS "\n\nSend bugs/issues to https://jira.acts.hu:8443/\n"
+#define PROGINFO VERSION ", Janos SUTO <sj@acts.hu>\n\n" CONFIGURE_PARAMS "\n"
 
 #define HOSTID "mailarchiver"
 
 #define CONFIG_FILE CONFDIR "/piler.conf"
 #define WORK_DIR DATADIR "/spool/piler/tmp"
 #define QUEUE_DIR DATADIR "/piler/store"
-#define DEFER_DIR DATADIR "/spool/piler/deferred"
 
 #define CLAMD_SOCKET "/tmp/clamd"
 
@@ -53,6 +52,7 @@
 #define MEMCACHED_MSGS_RCVD MEMCACHED_CLAPF_PREFIX ":rcvd"
 #define MEMCACHED_MSGS_VIRUS MEMCACHED_CLAPF_PREFIX ":virus"
 #define MEMCACHED_MSGS_DUPLICATE MEMCACHED_CLAPF_PREFIX ":duplicate"
+#define MEMCACHED_MSGS_IGNORE MEMCACHED_CLAPF_PREFIX ":ignore"
 
 
 #define LOG_PRIORITY LOG_INFO

diff --git a/src/counters.c b/src/counters.c
@@ -16,7 +16,7 @@ struct __counters loadCounters(struct session_data *sdata, struct __config *cfg)
 
    bzero(&counters, sizeof(counters));
 
-   snprintf(buf, SMALLBUFSIZE-1, "SELECT rcvd, virus, duplicate FROM %s", SQL_COUNTER_TABLE);
+   snprintf(buf, SMALLBUFSIZE-1, "SELECT rcvd, virus, duplicate, ignore FROM %s", SQL_COUNTER_TABLE);
 
 #ifdef NEED_MYSQL
    MYSQL_RES *res;
@@ -30,6 +30,7 @@ struct __counters loadCounters(struct session_data *sdata, struct __config *cfg)
             counters.c_rcvd = strtoull(row[0], NULL, 10);
             counters.c_virus = strtoull(row[1], NULL, 10);
             counters.c_duplicate = strtoull(row[2], NULL, 10);
+            counters.c_ignore = strtoull(row[3], NULL, 10);
          }
          mysql_free_result(res);
       }
@@ -59,25 +60,27 @@ void updateCounters(struct session_data *sdata, struct __data *data, struct __co
          if(counters->c_ham > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_HAM, strlen(MEMCACHED_MSGS_HAM), counters->c_ham, &mc);
          if(counters->c_virus > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_VIRUS, strlen(MEMCACHED_MSGS_VIRUS), counters->c_virus, &mc);
          if(counters->c_duplicate > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_DUPLICATE, strlen(MEMCACHED_MSGS_DUPLICATE), counters->c_duplicate, &mc);
+         if(counters->c_duplicate > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_IGNORE, strlen(MEMCACHED_MSGS_IGNORE), counters->c_ignore, &mc);
 
 
          bzero(&c, sizeof(c)); 
 
-         snprintf(buf, MAXBUFSIZE-1, "%s %s %s %s", MEMCACHED_MSGS_RCVD, MEMCACHED_MSGS_VIRUS, MEMCACHED_MSGS_DUPLICATE, MEMCACHED_COUNTERS_LAST_UPDATE);
+         snprintf(buf, MAXBUFSIZE-1, "%s %s %s %s %s", MEMCACHED_MSGS_RCVD, MEMCACHED_MSGS_VIRUS, MEMCACHED_MSGS_DUPLICATE, MEMCACHED_MSGS_IGNORE, MEMCACHED_COUNTERS_LAST_UPDATE);
 
          if(memcached_mget(&(data->memc), buf) == MEMCACHED_SUCCESS){
             while((memcached_fetch_result(&(data->memc), &key[0], &buf[0], &flags))){
                if(!strcmp(key, MEMCACHED_MSGS_RCVD)) c.c_rcvd = strtoull(buf, NULL, 10);
                else if(!strcmp(key, MEMCACHED_MSGS_VIRUS)) c.c_virus = strtoull(buf, NULL, 10);
                else if(!strcmp(key, MEMCACHED_MSGS_DUPLICATE)) c.c_duplicate = strtoull(buf, NULL, 10);
+               else if(!strcmp(key, MEMCACHED_MSGS_IGNORE)) c.c_ignore = strtoull(buf, NULL, 10);
                else if(!strcmp(key, MEMCACHED_COUNTERS_LAST_UPDATE)) mc = strtoull(buf, NULL, 10);
             }
 
 
             if(sdata->now - mc > cfg->memcached_to_db_interval && c.c_rcvd > 0 && c.c_rcvd >= rcvd){
                snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_set(&(data->memc), MEMCACHED_COUNTERS_LAST_UPDATE, strlen(MEMCACHED_COUNTERS_LAST_UPDATE), buf, strlen(buf), 0, 0);
 
-               snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET rcvd=%llu, virus=%llu, duplicate=%llu", c.c_rcvd, c.c_virus, c.c_duplicate);
+               snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET rcvd=%llu, virus=%llu, duplicate=%llu, ignore=%llu", c.c_rcvd, c.c_virus, c.c_duplicate, c.c_ignore);
 
                //if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: update counters: %s", sdata->ttmpfile, buf);
 
@@ -95,12 +98,13 @@ void updateCounters(struct session_data *sdata, struct __data *data, struct __co
          snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_virus + counters->c_virus); memcached_add(&(data->memc), MEMCACHED_MSGS_VIRUS, strlen(MEMCACHED_MSGS_VIRUS), buf, strlen(buf), 0, 0);
          snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_rcvd + counters->c_rcvd); memcached_add(&(data->memc), MEMCACHED_MSGS_RCVD, strlen(MEMCACHED_MSGS_RCVD), buf, strlen(buf), 0, 0);
          snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_duplicate + counters->c_duplicate); memcached_add(&(data->memc), MEMCACHED_MSGS_DUPLICATE, strlen(MEMCACHED_MSGS_DUPLICATE), buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_ignore + counters->c_ignore); memcached_add(&(data->memc), MEMCACHED_MSGS_IGNORE, strlen(MEMCACHED_MSGS_IGNORE), buf, strlen(buf), 0, 0);
       }
 
    }
    else {
 #endif
-      snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET rcvd=rcvd+%llu, virus=virus+%llu, duplicate=duplicate+%llu", SQL_COUNTER_TABLE, counters->c_rcvd, counters->c_virus, counters->c_duplicate);
+      snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET rcvd=rcvd+%llu, virus=virus+%llu, duplicate=duplicate+%llu, ignore=ignore+%llu", SQL_COUNTER_TABLE, counters->c_rcvd, counters->c_virus, counters->c_duplicate, counters->c_ignore);
 
       //if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: update counters: %s", sdata->ttmpfile, buf);
 

diff --git a/src/defs.h b/src/defs.h
@@ -203,6 +203,7 @@ struct __counters {
    unsigned long long c_rcvd;
    unsigned long long c_virus;
    unsigned long long c_duplicate;
+   unsigned long long c_ignore;
 };
 
 #endif /* _DEFS_H */

diff --git a/src/dirs.c b/src/dirs.c
@@ -35,14 +35,6 @@ void check_and_create_directories(struct __config *cfg, uid_t uid, gid_t gid){
    }
    createdir(cfg->queuedir, uid, gid, 0700);
 
-   p = strrchr(cfg->deferdir, '/');
-   if(p){
-      *p = '\0';
-      createdir(cfg->deferdir, uid, gid, 0755);
-      *p = '/';
-   }
-   createdir(cfg->deferdir, uid, gid, 0700);
-
    p = strrchr(cfg->sqlite3, '/');
    if(p){
       *p = '\0';

diff --git a/src/parser.c b/src/parser.c
@@ -379,10 +379,10 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, stru
 
       len = strlen(puf);
 
-      if(state->message_state == MSG_FROM && strchr(puf, '@') && state->is_1st_header == 1 && state->b_from[0] == '\0' && strlen(state->b_from) < SMALLBUFSIZE-len-1)
+      if(state->message_state == MSG_FROM && strchr(puf, '@') && strlen(puf) > 5 && state->is_1st_header == 1 && state->b_from[0] == '\0' && strlen(state->b_from) < SMALLBUFSIZE-len-1)
          memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
 
-      else if((state->message_state == MSG_TO || state->message_state == MSG_CC) && state->is_1st_header == 1 && strchr(puf, '@') && strlen(state->b_to) < SMALLBUFSIZE-len-1){
+      else if((state->message_state == MSG_TO || state->message_state == MSG_CC) && state->is_1st_header == 1 && strchr(puf, '@') && strlen(puf) > 5 && strlen(state->b_to) < SMALLBUFSIZE-len-1){
 
          if(is_string_on_list(state->rcpt, puf) == 0){
             append_list(&(state->rcpt), puf);

diff --git a/src/pilerget.c b/src/pilerget.c
@@ -113,30 +113,18 @@ int inf(unsigned char *in, int len, FILE *dest){
 }
 
 
-int main(int argc, char **argv){
+int retrieve_file_from_archive(char *filename, struct __config *cfg){
    int rc, n, olen, tlen, len;
    unsigned char inbuf[BIGBUFSIZE];
-   struct __config cfg;
    struct stat st;
 
 
-   cfg = read_config(configfile);
-
-   if(read_key(&cfg)){
-      printf("%s\n", ERR_READING_KEY);
-      return 1;
-   }
-
-
-   if(argc != 2){
-      printf("usage: $0 <encrypted file>\n");
-      return 1;
-   }
+   if(filename == NULL) return 1;
 
 
-   fd = open(argv[1], O_RDONLY);
+   fd = open(filename, O_RDONLY);
    if(fd == -1){
-      printf("error reading file: %s\n", argv[1]);
+      printf("error reading file: %s\n", filename);
       return 1;
    }
 
@@ -149,7 +137,7 @@ int main(int argc, char **argv){
 
 
    EVP_CIPHER_CTX_init(&ctx);
-   EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg.key, cfg.iv);
+   EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
 
    len = st.st_size+EVP_MAX_BLOCK_LENGTH;
 
@@ -192,3 +180,27 @@ int main(int argc, char **argv){
 
    return 0;
 }
+
+
+int main(int argc, char **argv){
+   struct __config cfg;
+
+
+   cfg = read_config(configfile);
+
+   if(read_key(&cfg)){
+      printf("%s\n", ERR_READING_KEY);
+      return 1;
+   }
+
+
+   if(argc != 2){
+      printf("usage: %s <encrypted file>\n", argv[0]);
+      return 1;
+   }
+
+
+   retrieve_file_from_archive(argv[1], &cfg);
+
+   return 0;
+}
diff --git a/src/session.c b/src/session.c
@@ -201,6 +201,7 @@ void handle_smtp_session(int new_sd, struct __data *data, struct __config *cfg){
                         if(arule){
                            syslog(LOG_PRIORITY, "%s: discarding message by archiving policy: *%s*", sdata.ttmpfile, arule);
                            inj = OK;
+                           counters.c_ignore++;
                         }
                         else {
                            inj = processMessage(&sdata, &sstate, cfg);

diff --git a/util/db-mysql.sql b/util/db-mysql.sql
@@ -122,6 +122,7 @@ create table if not exists `counter` (
    `rcvd` bigint unsigned default 0,
    `virus` bigint unsigned default 0,
    `duplicate` bigint unsigned default 0
+   `ignore` bigint unsigned default 0
 ) Engine=InnoDB;
 
 insert into `counter` values(0, 0, 0);
@@ -147,3 +148,51 @@ create table if not exists `user_settings` (
 
 create index `user_settings_idx` on `user_settings`(`username`);
 
+
+
+create table if not exists `user` (
+   `uid` int unsigned not null primary key,
+   `gid` int unsigned not null,
+   `username` char(64) not null unique,
+   `realname` char(64) default null,
+   `password` char(48) default null,
+   `domain` char(64) default null,
+   `dn` char(255) default '*',
+   `policy_group` int(4) default 0,
+   `isadmin` tinyint default 0
+) Engine=InnoDB;
+
+insert into `user` (`uid`, `gid`, `username`, `realname`, `password`, `policy_group`, `isadmin`, `domain`) values (0, 0, 'admin', 'built-in piler admin', '$1$PItc7d$zsUgON3JRrbdGS11t9JQW1', 0, 1, 'local');
+
+create table if not exists `email` (
+   `uid` int unsigned not null,
+   `email` char(128) not null primary key
+) ENGINE=InnoDB;
+
+insert into `email` (`uid`, `email`) values(0, 'admin@local');
+
+
+create table if not exists `email_groups` (
+   `uid` int unsigned not null,
+   `gid` int unsigned not null,
+   unique key `uid` (`uid`,`gid`),
+   key `email_groups_idx` (`uid`,`gid`)
+) ENGINE=InnoDB;
+
+
+create table if not exists `remote` (
+   `remotedomain` char(64) not null primary key,
+   `remotehost` char(64) not null,
+   `basedn` char(64) not null,
+   `binddn` char(64) not null,
+   `sitedescription` char(64) default null
+) ENGINE=InnoDB;
+
+
+create table if not exists `domain` (
+   `domain` char(64) not null primary key,
+   `mapped` char(64) not null
+) ENGINE=InnoDB;
+
+insert into `domain` (`domain`, `mapped`) values('local', 'local');
+