first demo of the new architecture

Change-Id: Id76636f30173465fbd0e8516d99017b4336de4db Signed-off-by: SJ <sj@acts.hu>
jsuto · Oct 23, 2016 · 5f5585d · 5f5585d
1 parent 6783f9e
commit 5f5585d
Show file tree

Hide file tree

Showing 18 changed files with 948 additions and 236 deletions.
diff --git a/configure b/configure
@@ -4645,7 +4645,7 @@ fi
 
 if test "$os" = "FreeBSD"; then
    defs="$defs -DFREEBSD"
-   antispam_libs="-lz -lm -lcrypto -lssl"
+   antispam_libs="-lz -lm -lcrypto -lssl -liconv"
    MAKE="gmake"
 fi
 
@@ -4866,7 +4866,7 @@ echo; echo
 
 CFLAGS="$static -O2 -Wall -g"
 LIBS="$antispam_libs $sunos_libs "
-OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o mydomains.o $objs"
+OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o mydomains.o $objs"
 
 ac_config_files="$ac_config_files Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile unit_tests/Makefile contrib/imap/Makefile"
 

diff --git a/configure.in b/configure.in
@@ -544,7 +544,7 @@ echo; echo
 
 CFLAGS="$static -O2 -Wall -g"
 LIBS="$antispam_libs $sunos_libs "
-OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o mydomains.o $objs"
+OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o mydomains.o $objs"
 
 AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile unit_tests/Makefile contrib/imap/Makefile])
 AC_OUTPUT

diff --git a/src/Makefile.in b/src/Makefile.in
@@ -33,7 +33,7 @@ MAKE = `which make`
 
 INSTALL = @INSTALL@
 
-all: libpiler.a piler pilerconf pilerget pileraget pilerimport pilerexport pilerpurge reindex test
+all: libpiler.a piler pilerconf pilerget pileraget pilerimport pilerexport pilerpurge reindex test piler-smtp
 install: install-piler
 
 
@@ -47,6 +47,8 @@ libpiler.a: $(OBJS) $(SQL_OBJS)
 	ln -sf libpiler.so.$(LIBPILER_VERSION) libpiler.so
 	ln -sf libpiler.so.$(LIBPILER_VERSION) libpiler.so.$(PILER_VERSION)
 
+piler-smtp: piler-smtp.c libpiler.a
+	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< cfg.o misc.o tai.o smtp.o dirs.o sig.o bdat.o $(LIBS) $(LIBDIR)
 
 pilerget: pilerget.c libpiler.a
 	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR)
@@ -83,6 +85,7 @@ install-piler:
 	(cd $(DESTDIR)$(libdir) && ln -sf libpiler.so.$(LIBPILER_VERSION) libpiler.so.$(PILER_VERSION))
 
 	$(INSTALL) -m 0755 piler $(DESTDIR)$(sbindir)
+	$(INSTALL) -m 0755 piler-smtp $(DESTDIR)$(sbindir)
 	$(INSTALL) -m 0755 pilerconf $(DESTDIR)$(sbindir)
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerget $(DESTDIR)$(bindir)
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pileraget $(DESTDIR)$(bindir)
@@ -93,7 +96,7 @@ install-piler:
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilertest $(DESTDIR)$(bindir)
 
 clean:
-	rm -f *.o *.a libpiler.so* piler pilerconf pilerget pileraget pilerimport pilerexport pilerpurge pilertest reindex
+	rm -f *.o *.a libpiler.so* piler pilerconf pilerget pileraget pilerimport pilerexport pilerpurge pilertest reindex piler-smtp
 
 distclean: clean
 	rm -f Makefile

diff --git a/src/bdat.c b/src/bdat.c
@@ -0,0 +1,136 @@
+/*
+ * bdat.c, SJ
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <signal.h>
+#include <syslog.h>
+#include <time.h>
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+#include <piler.h>
+#include <smtp.h>
+
+
+void reset_bdat_counters(struct smtp_session *session){
+   session->bdat_rounds = 0;
+   session->bdat_last_round = 0;
+   session->bdat_bytes_to_read = 0;
+   session->bad = 0;
+}
+
+
+void get_bdat_size_to_read(struct smtp_session *session, char *buf){
+   char *p;
+
+   session->bdat_rounds++;
+   session->bdat_bytes_to_read = 0;
+
+   session->protocol_state = SMTP_STATE_BDAT;
+
+   // determine if this is the last BDAT command
+
+   p = strcasestr(buf, " LAST");
+   if(p){
+      session->bdat_last_round = 1;
+      syslog(LOG_INFO, "%s: BDAT LAST", session->ttmpfile);
+      *p = '\0';
+   }
+
+   // determine the size to be read
+
+   p = strchr(buf, ' ');
+   if(p){
+      session->bdat_bytes_to_read = atoi(p);
+      if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: BDAT len=%d", session->ttmpfile, session->bdat_bytes_to_read);
+   }
+
+   if(!p || session->bdat_bytes_to_read <= 0){
+      session->bdat_bytes_to_read = 0;
+      syslog(LOG_INFO, "%s: malformed BDAT command", session->ttmpfile);
+   }
+}
+
+
+void process_bdat(struct smtp_session *session, char *readbuf, int readlen){
+   int i;
+   char buf[SMALLBUFSIZE];
+
+   if(readlen <= 0) return;
+
+   //printf("readbuf in process_bdat (%d): *%s*\n", readlen, readbuf);
+
+   if(session->bdat_rounds == 1){
+      session->fd = open(session->ttmpfile, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR|S_IRGRP);
+      if(session->fd == -1){
+         syslog(LOG_PRIORITY, "%s: %s", ERR_OPEN_TMP_FILE, session->ttmpfile);
+      }
+   }
+
+   session->bdat_bytes_to_read -= readlen;
+
+   if(session->fd != -1){
+      write(session->fd, readbuf, readlen);
+      session->tot_len += readlen;
+
+      if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: wrote %d bytes, %d bytes to go", session->ttmpfile, readlen, session->bdat_bytes_to_read);
+   }
+
+
+   if(session->bdat_bytes_to_read < 0){
+      // malformed data from client: we got more data then had been told in BDAT argument
+      syslog(LOG_PRIORITY, "ERROR: invalid BDAT data. Expected %d, got %d bytes", session->bdat_bytes_to_read + readlen, readlen);
+      session->bad = 1;
+
+      close(session->fd);
+      unlink(session->ttmpfile);
+   }
+
+   if(session->bdat_bytes_to_read == 0){
+
+      if(session->bdat_last_round == 1){
+         if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: read all bdat data in %d rounds", session->ttmpfile, session->bdat_rounds);
+
+         // send back the smtp answers
+         for(i=0; i<session->bdat_rounds; i++){
+            if(session->fd == -1){
+               send_smtp_response(session, SMTP_RESP_421_ERR_WRITE_FAILED);
+            }
+            else {
+               if(i == 0){
+                  fsync(session->fd);
+                  close(session->fd);
+
+                  move_email(session);
+
+                  snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
+                  send_smtp_response(session, buf);
+                  syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d", session->ttmpfile, session->mailfrom, session->tot_len);
+               }
+               else send_smtp_response(session, SMTP_RESP_250_BDAT);
+            }
+         }
+
+         // technically we are not in the PERIOD state, but it's good enough
+         // to quit the BDAT processing state
+         session->protocol_state = SMTP_STATE_PERIOD;
+
+         session->fd = -1;
+      }
+
+      else {
+         // this is not the last BDAT round, let's go back
+         // after the rcpt state, and wait for the next
+         // BDAT command
+         session->protocol_state = SMTP_STATE_RCPT_TO;
+      }
+   }
+}
diff --git a/src/cfg.c b/src/cfg.c
@@ -101,6 +101,7 @@ struct _parse_rule config_parse_rules[] =
    { "process_rcpt_to_addresses", "integer", (void*) int_parser, offsetof(struct __config, process_rcpt_to_addresses), "0", sizeof(int)},
    { "queuedir", "string", (void*) string_parser, offsetof(struct __config, queuedir), QUEUE_DIR, MAXVAL-1},
    { "server_id", "integer", (void*) int_parser, offsetof(struct __config, server_id), "0", sizeof(int)},
+   { "smtp_timeout", "integer", (void*) int_parser, offsetof(struct __config, smtp_timeout), "60", sizeof(int)},
    { "spam_header_line", "string", (void*) string_parser, offsetof(struct __config, spam_header_line), "", MAXVAL-1},
    { "syslog_recipients", "integer", (void*) int_parser, offsetof(struct __config, syslog_recipients), "0", sizeof(int)},
    { "tls_enable", "integer", (void*) int_parser, offsetof(struct __config, tls_enable), "0", sizeof(int)},

diff --git a/src/cfg.h b/src/cfg.h
@@ -48,6 +48,7 @@ struct __config {
    int verbosity;
    char locale[MAXVAL];
 
+   int smtp_timeout;
    int helper_timeout;
    int extract_attachments;
 

diff --git a/src/config.h b/src/config.h
@@ -11,9 +11,9 @@
 
 #define PROGNAME "piler"
 
-#define VERSION "1.2.0"
+#define VERSION "1.3.0-master"
 
-#define BUILD 952
+#define BUILD 975
 
 #define HOSTID "mailarchiver"
 

diff --git a/src/defs.h b/src/defs.h
@@ -22,6 +22,7 @@
 
 #include <openssl/sha.h>
 #include <openssl/ssl.h>
+#include <netinet/in.h>
 #include "tai.h"
 #include "config.h"
 
@@ -64,9 +65,6 @@
 #define RULE_NO_MATCH -100
 
 
-typedef void signal_func (int);
-
-
 struct child {
    pid_t pid;
    int serial;
@@ -386,6 +384,26 @@ struct session_ctx {
    struct counters *counters;
 };
 
+struct smtp_session {
+   char ttmpfile[SMALLBUFSIZE];
+   char mailfrom[SMALLBUFSIZE];
+   char buf[SMALLBUFSIZE];
+   char remote_host[INET6_ADDRSTRLEN];
+   time_t lasttime;
+   int protocol_state;
+   int fd;
+   int bad;
+   int buflen;
+   int tot_len;
+   int bdat_rounds;
+   int bdat_last_round;
+   int bdat_bytes_to_read;
+   int socket;
+   struct __config *cfg;
+   SSL_CTX *ctx;
+   SSL *ssl;
+   int use_ssl;
+   int starttls;
+};
 
 #endif /* _DEFS_H */
-
diff --git a/src/dirs.c b/src/dirs.c
@@ -18,6 +18,7 @@ void createdir(char *path, uid_t uid, gid_t gid, mode_t mode);
 
 void check_and_create_directories(struct __config *cfg, uid_t uid, gid_t gid){
    char *p, s[SMALLBUFSIZE];
+   int i;
 
    p = strrchr(cfg->workdir, '/');
    if(p){
@@ -45,6 +46,11 @@ void check_and_create_directories(struct __config *cfg, uid_t uid, gid_t gid){
       *p = '/';
    }
 
+   for(i=0; i<cfg->number_of_worker_processes; i++){
+      snprintf(s, sizeof(s)-1, "%s/%d", cfg->workdir, i);
+      createdir(s, uid, gid, 0700);
+   }
+
 }
 
 

diff --git a/src/misc.c b/src/misc.c
@@ -107,6 +107,21 @@ int searchStringInBuffer(char *s, int len1, char *what, int len2){
 }
 
 
+int search_char_backward(char *buf, int buflen, char c){
+   int n, m;
+
+   m = buflen - 1 - 5;
+   if(m < 0) m = 0;
+
+   for(n=m; n<buflen; n++){
+      if(*(buf + n) == c){
+         return n;
+      }
+   }
+   return -1;
+}
+
+
 /*
  * count a character in buffer
  */
@@ -627,6 +642,17 @@ int can_i_write_current_directory(){
 }
 
 
+void move_email(struct smtp_session *session){
+   char buf[SMALLBUFSIZE];
+
+   snprintf(buf, sizeof(buf)-1, "%d/%s", session->ttmpfile[RND_STR_LEN-1] % session->cfg->number_of_worker_processes, session->ttmpfile);
+
+   if(rename(session->ttmpfile, buf)){
+      syslog(LOG_PRIORITY, "ERROR: couldn't rename %s to %s", session->ttmpfile, buf);
+   }
+}
+
+
 #ifndef _GNU_SOURCE
 char *strcasestr(const char *s, const char *find){
    char c, sc;
@@ -647,4 +673,3 @@ char *strcasestr(const char *s, const char *find){
    return((char*)s);
 }
 #endif
-
diff --git a/src/misc.h b/src/misc.h
@@ -20,6 +20,7 @@ void get_extractor_list();
 void __fatal(char *s);
 long tvdiff(struct timeval a, struct timeval b);
 int searchStringInBuffer(char *s, int len1, char *what, int len2);
+int search_char_backward(char *buf, int buflen, char c);
 int countCharacterInBuffer(char *p, char c);
 void replaceCharacterInBuffer(char *p, char from, char to);
 char *split(char *str, int ch, char *buf, int buflen, int *result);
@@ -46,6 +47,8 @@ void *get_in_addr(struct sockaddr *sa);
 
 int can_i_write_current_directory();
 
+void move_email(struct smtp_session *session);
+
 #ifndef _GNU_SOURCE
    char *strcasestr(const char *s, const char *find);
 #endif