added reimport option for pilerimport

mhoffmann75 · Apr 5, 2016 · 65fc0b3 · 65fc0b3
1 parent 6789108
commit 65fc0b3
Show file tree

Hide file tree

Showing 47 changed files with 314 additions and 208 deletions.
diff --git a/src/Makefile.in b/src/Makefile.in
@@ -33,7 +33,7 @@ MAKE = `which make`
 
 INSTALL = @INSTALL@
 
-all: libpiler.a piler pilerconf pilerget pileraget pilerimport pilerexport pilerpurge reindex test fixmetadata
+all: libpiler.a piler pilerconf pilerget pileraget pilerimport pilerexport pilerpurge reindex test
 install: install-piler
 
 

diff --git a/src/archive.c b/src/archive.c
@@ -222,7 +222,7 @@ int retrieve_email_from_archive(struct session_data *sdata, struct __data *data,
       return 1;
    }
 
-   attachments = query_attachments(sdata, data, &ptr_arr[0], cfg);
+   attachments = query_attachments(sdata, data, &ptr_arr[0]);
 
    if(attachments == -1){
       printf("problem querying the attachment of %s\n", sdata->ttmpfile);

diff --git a/src/attachment.c b/src/attachment.c
@@ -21,8 +21,8 @@ int store_attachments(struct session_data *sdata, struct parser_state *state, st
    int i, rc=1, found, affected_rows;
 
 
-   if(prepare_sql_statement(sdata, &(data->stmt_insert_into_attachment_table), SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE, cfg) == ERR) return rc;
-   if(prepare_sql_statement(sdata, &(data->stmt_get_attachment_id_by_signature), SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE, cfg) == ERR) return rc;
+   if(prepare_sql_statement(sdata, &(data->stmt_insert_into_attachment_table), SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE) == ERR) return rc;
+   if(prepare_sql_statement(sdata, &(data->stmt_get_attachment_id_by_signature), SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE) == ERR) return rc;
 
 
    for(i=1; i<=state->n_attachments; i++){
@@ -42,13 +42,13 @@ int store_attachments(struct session_data *sdata, struct parser_state *state, st
 
             data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
 
-            p_store_results(sdata, data->stmt_get_attachment_id_by_signature, data);
+            p_store_results(data->stmt_get_attachment_id_by_signature, data);
             if(p_fetch_results(data->stmt_get_attachment_id_by_signature) == OK) found = 1;
             p_free_results(data->stmt_get_attachment_id_by_signature);
          }
 
          if(found == 0){
-            if(store_file(sdata, state->attachments[i].internalname, 0, 0, cfg) == 0){
+            if(store_file(sdata, state->attachments[i].internalname, 0, cfg) == 0){
                syslog(LOG_PRIORITY, "%s: error storing attachment: %s", sdata->ttmpfile, state->attachments[i].internalname);
                goto CLOSE;
             }
@@ -91,10 +91,10 @@ int store_attachments(struct session_data *sdata, struct parser_state *state, st
 }
 
 
-int query_attachment_pointers(struct session_data *sdata, struct __data *data, uint64 ptr, char *piler_id, int *id, struct __config *cfg){
+int query_attachment_pointers(struct session_data *sdata, struct __data *data, uint64 ptr, char *piler_id, int *id){
    int rc=0;
 
-   if(prepare_sql_statement(sdata, &(data->stmt_get_attachment_pointer), SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER, cfg) == ERR) return rc;
+   if(prepare_sql_statement(sdata, &(data->stmt_get_attachment_pointer), SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER) == ERR) return rc;
 
 
    p_bind_init(data);
@@ -108,7 +108,7 @@ int query_attachment_pointers(struct session_data *sdata, struct __data *data, u
       data->sql[data->pos] = piler_id; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = RND_STR_LEN; data->pos++;
       data->sql[data->pos] = (char *)id; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
 
-      p_store_results(sdata, data->stmt_get_attachment_pointer, data);
+      p_store_results(data->stmt_get_attachment_pointer, data);
 
       if(p_fetch_results(data->stmt_get_attachment_pointer) == OK) rc = 1;
       p_free_results(data->stmt_get_attachment_pointer);
@@ -120,13 +120,13 @@ int query_attachment_pointers(struct session_data *sdata, struct __data *data, u
 }
 
 
-int query_attachments(struct session_data *sdata, struct __data *data, struct ptr_array *ptr_arr, struct __config *cfg){
+int query_attachments(struct session_data *sdata, struct __data *data, struct ptr_array *ptr_arr){
    int i, rc, id, attachments=0;
    uint64 ptr;
 
    for(i=0; i<MAX_ATTACHMENTS; i++) memset((char*)&ptr_arr[i], 0, sizeof(struct ptr_array));
 
-   if(prepare_sql_statement(sdata, &(data->stmt_query_attachment), SQL_PREPARED_STMT_QUERY_ATTACHMENT, cfg) == ERR) return attachments;
+   if(prepare_sql_statement(sdata, &(data->stmt_query_attachment), SQL_PREPARED_STMT_QUERY_ATTACHMENT) == ERR) return attachments;
 
    p_bind_init(data);
 
@@ -139,14 +139,14 @@ int query_attachments(struct session_data *sdata, struct __data *data, struct pt
    data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
    data->sql[data->pos] = (char *)&ptr; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
 
-   p_store_results(sdata, data->stmt_query_attachment, data);
+   p_store_results(data->stmt_query_attachment, data);
 
    while(p_fetch_results(data->stmt_query_attachment) == OK){
 
       if(id > 0 && id < MAX_ATTACHMENTS){
          if(ptr > 0){
             ptr_arr[id].ptr = ptr;
-            rc = query_attachment_pointers(sdata, data, ptr, &(ptr_arr[id].piler_id[0]), &(ptr_arr[id].attachment_id), cfg);
+            rc = query_attachment_pointers(sdata, data, ptr, &(ptr_arr[id].piler_id[0]), &(ptr_arr[id].attachment_id));
             if(!rc){
                attachments = -1;
                goto CLOSE;

diff --git a/src/cfg.c b/src/cfg.c
@@ -17,7 +17,7 @@ int string_parser(char *src, char *target, int limit){
    return 0;
 };
 
-int multi_line_string_parser(char *src, char *target, int limit){
+int multi_line_string_parser(char *src, char *target, unsigned int limit){
    if(strlen(src) > 0 && strlen(target) + strlen(src) + 3 < limit){
       strncat(target, src, limit-strlen(target));
       strncat(target, "\r\n", limit-strlen(target));
@@ -28,14 +28,14 @@ int multi_line_string_parser(char *src, char *target, int limit){
    return 1;
 };
 
-int int_parser(char *src, int *target, int limit){
+int int_parser(char *src, int *target){
    *target = strtol(src, (char **) NULL, 10);
 
    return 0;
 };
 
 
-int float_parser(char *src, float *target, int limit){
+int float_parser(char *src, float *target){
    *target = strtof(src, (char **) NULL);
 
    return 0;
@@ -110,7 +110,7 @@ struct _parse_rule config_parse_rules[] =
    { "verbosity", "integer", (void*) int_parser, offsetof(struct __config, verbosity), "1", sizeof(int)},
    { "workdir", "string", (void*) string_parser, offsetof(struct __config, workdir), WORK_DIR, MAXVAL-1},
 
-   {NULL, NULL, NULL, 0, 0}
+   {NULL, NULL, NULL, 0, 0, 0}
 };
 
 

diff --git a/src/config.h b/src/config.h
@@ -13,7 +13,7 @@
 
 #define VERSION "1.2.0-master"
 
-#define BUILD 941
+#define BUILD 942
 
 #define HOSTID "mailarchiver"
 
@@ -94,18 +94,18 @@
 #define SQL_ATTACHMENTS_VIEW "v_attachment"
 
 #define SQL_PREPARED_STMT_GET_DOMAINS                "SELECT `domain` FROM `" SQL_DOMAIN_TABLE "`"
-#define SQL_PREPARED_STMT_GET_META_ID_BY_MESSAGE_ID  "SELECT id FROM " SQL_METADATA_TABLE " WHERE message_id=?"
+#define SQL_PREPARED_STMT_GET_META_ID_BY_MESSAGE_ID  "SELECT id, piler_id FROM " SQL_METADATA_TABLE " WHERE message_id=?"
 #define SQL_PREPARED_STMT_INSERT_INTO_RCPT_TABLE     "INSERT INTO " SQL_RECIPIENT_TABLE " (`id`,`to`,`todomain`) VALUES(?,?,?)"
 #define SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE   "INSERT INTO " SQL_SPHINX_TABLE " (`id`, `from`, `to`, `fromdomain`, `todomain`, `subject`, `body`, `arrived`, `sent`, `size`, `direction`, `folder`, `attachments`, `attachment_types`) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
 #define SQL_PREPARED_STMT_INSERT_INTO_META_TABLE     "INSERT INTO " SQL_METADATA_TABLE " (`from`,`fromdomain`,`subject`,`spam`,`arrived`,`sent`,`retained`,`size`,`hlen`,`direction`,`attachments`,`piler_id`,`message_id`,`reference`,`digest`,`bodydigest`,`vcode`) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
+#define SQL_PREPARED_STMT_UPDATE_META_TABLE          "UPDATE " SQL_METADATA_TABLE " SET `from`=?,`fromdomain`=?,`subject`=?,`spam`=?,`arrived`=?,`sent`=?,`retained`=?,`size`=?,`hlen`=?,`direction`=?,`attachments`=?,`reference`=?,`digest`=?,`bodydigest`=?,`vcode`=? WHERE id=?"
 #define SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE     "INSERT INTO " SQL_ATTACHMENT_TABLE " (`piler_id`,`attachment_id`,`sig`,`name`,`type`,`size`,`ptr`) VALUES(?,?,?,?,?,?,?)"
 #define SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE   "SELECT `id` FROM `" SQL_ATTACHMENT_TABLE "` WHERE `sig`=? AND `ptr`=0 AND `size`=?"
 #define SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER     "SELECT `piler_id`, `attachment_id` FROM " SQL_ATTACHMENT_TABLE " WHERE id=?"
 #define SQL_PREPARED_STMT_QUERY_ATTACHMENT           "SELECT `attachment_id`, `ptr` FROM " SQL_ATTACHMENT_TABLE " WHERE piler_id=? ORDER BY attachment_id ASC"
 #define SQL_PREPARED_STMT_GET_FOLDER_ID              "SELECT `id` FROM " SQL_FOLDER_TABLE " WHERE `name`=? AND `parent_id`=?"
 #define SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE   "INSERT INTO `" SQL_FOLDER_TABLE "` (`name`, `parent_id`) VALUES(?,?)"
 #define SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE  "UPDATE " SQL_METADATA_TABLE " SET reference=? WHERE message_id=? AND reference=''"
-#define SQL_PREPARED_STMT_UPDATE_META_TABLE          "UPDATE " SQL_METADATA_TABLE " SET subject=?, attachments=? WHERE id=?"
 #define SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS        "SELECT id, type, username, password, server FROM " SQL_IMPORT_TABLE " WHERE started=0 ORDER BY id LIMIT 0,1"
 #define SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE      "INSERT INTO " SQL_FOLDER_MESSAGE_TABLE " (`folder_id`, `id`) VALUES(?,?)"
 

diff --git a/src/counters.c b/src/counters.c
@@ -11,7 +11,7 @@
 #include <piler.h>
 
 
-struct __counters load_counters(struct session_data *sdata, struct __data *data, struct __config *cfg){
+struct __counters load_counters(struct session_data *sdata, struct __data *data){
    char buf[SMALLBUFSIZE];
    struct __counters counters;
 
@@ -20,7 +20,7 @@ struct __counters load_counters(struct session_data *sdata, struct __data *data,
    snprintf(buf, SMALLBUFSIZE-1, "SELECT `rcvd`, `virus`, `duplicate`, `ignore`, `size`, `stored_size` FROM `%s`", SQL_COUNTER_TABLE);
 
 
-   if(prepare_sql_statement(sdata, &(data->stmt_generic), buf, cfg) == ERR) return counters;
+   if(prepare_sql_statement(sdata, &(data->stmt_generic), buf) == ERR) return counters;
 
 
    p_bind_init(data);
@@ -36,7 +36,7 @@ struct __counters load_counters(struct session_data *sdata, struct __data *data,
       data->sql[data->pos] = (char *)&counters.c_size; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
       data->sql[data->pos] = (char *)&counters.c_stored_size; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
 
-      p_store_results(sdata, data->stmt_generic, data);
+      p_store_results(data->stmt_generic, data);
       p_fetch_results(data->stmt_generic);
       p_free_results(data->stmt_generic);
    }
@@ -101,7 +101,7 @@ void update_counters(struct session_data *sdata, struct __data *data, struct __c
       }
       else {
 
-         c = load_counters(sdata, data, cfg);
+         c = load_counters(sdata, data);
 
          snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_add(&(data->memc), MEMCACHED_COUNTERS_LAST_UPDATE, strlen(MEMCACHED_COUNTERS_LAST_UPDATE), buf, strlen(buf), 0, 0);
 

diff --git a/src/decoder.c b/src/decoder.c
@@ -158,7 +158,8 @@ int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){
 
 
 void decodeQP(char *p){
-   int i, k=0, a, b;
+   unsigned int i;
+   int k=0, a, b;
    char c;
 
    if(p == NULL) return;
@@ -247,7 +248,8 @@ void decodeHTML(char *p, int utf8){
 
 
 void decodeURL(char *p){
-   int i, c, k=0, a, b;
+   unsigned int i;
+   int c, k=0, a, b;
 
    if(p == NULL) return;
 
@@ -303,30 +305,21 @@ inline void utf8_encode_char(unsigned char c, unsigned char *buf, int buflen, in
          count++;
       }
 
-      else if(c <= 0x7FF){
+      else {
          *(buf+count) = ( 0xC0 | (c >> 6) );
          count++;
          *(buf+count) = ( 0x80 | (c & 0x3F) );
          count++;
       }
 
-
-      else if (c <= 0xFFFF){
-         *(buf+count) = ( 0xE0 | (c >> 12) );
-         count++;
-         *(buf+count) = ( 0x80 | ((c >> 6) & 0x3F) );
-         count++;
-         *(buf+count) = ( 0x80 | (c & 0x3F) );
-         count++;
-      }
-
    *len = count;
 }
 
 
 int utf8_encode(char *inbuf, int inbuflen, char *outbuf, int outbuflen, char *encoding){
    iconv_t cd;
-   size_t size, inbytesleft, outbytesleft;
+   size_t inbytesleft, outbytesleft;
+   int ret = ERR;
 
    memset(outbuf, 0, outbuflen);
 
@@ -336,13 +329,14 @@ int utf8_encode(char *inbuf, int inbuflen, char *outbuf, int outbuflen, char *en
       inbytesleft = inbuflen;
       outbytesleft = outbuflen-1;
 
-      size = iconv(cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
+      if(iconv(cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft) == (size_t) -1)
+         ret = ERR;
+      else
+         ret = OK;
 
       iconv_close(cd);
-
-      if(size >= 0) return OK;
    }
 
-   return ERR;
+   return ret;
 }
 
diff --git a/src/defs.h b/src/defs.h
@@ -294,6 +294,7 @@ struct import {
    int download_only;
    int keep_eml;
    int timeout;
+   int reimport;
    int cap_uidplus;
    long total_size;
    time_t started, updated, finished;

diff --git a/src/fixmetadata.c b/src/fixmetadata.c
@@ -30,7 +30,7 @@ int update_meta_data(struct session_data *sdata, struct parser_state *state, str
    subj = state->b_subject;
    if(*subj == ' ') subj++;
 
-   if(prepare_sql_statement(sdata, &(data->stmt_update_metadata), SQL_PREPARED_STMT_UPDATE_META_TABLE, cfg) == ERR) return ret;
+   if(prepare_sql_statement(sdata, &(data->stmt_update_metadata), SQL_PREPARED_STMT_UPDATE_META_TABLE) == ERR) return ret;
 
    p_bind_init(data);
 
@@ -81,17 +81,8 @@ int main(int argc, char **argv){
 
    inithash(data.mydomains);
 
-   //initrules(data.archiving_rules);
-   //initrules(data.retention_rules);
-   //initrules(data.folder_rules);
-
    load_mydomains(&sdata, &data, &cfg);
 
-   //load_rules(&sdata, &data, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE, &cfg);
-   //load_rules(&sdata, &data, data.retention_rules, SQL_RETENTION_RULE_TABLE, &cfg);
-   //load_rules(&sdata, &data, data.folder_rules, SQL_FOLDER_RULE_TABLE, &cfg);
-
-
    init_session_data(&sdata, &cfg);
 
    sdata.delivered = 0;

diff --git a/src/hash.c b/src/hash.c
@@ -40,7 +40,7 @@ void clearhash(struct node *xhash[]){
 }
 
 
-struct node *makenewnode(struct node *xhash[], char *s){
+struct node *makenewnode(char *s){
    struct node *h;
    int len;
 
@@ -82,7 +82,7 @@ int addnode(struct node *xhash[], char *s){
    key = DJBHash(s, len);
 
    if(xhash[hash(key)] == NULL){
-      xhash[hash(key)] = makenewnode(xhash, s);
+      xhash[hash(key)] = makenewnode(s);
    }
    else {
       q = xhash[hash(key)];
@@ -95,7 +95,7 @@ int addnode(struct node *xhash[], char *s){
             q = q->r;
          }
       }
-      if(p) p->r = makenewnode(xhash, s);
+      if(p) p->r = makenewnode(s);
    }
 
    return 1;

diff --git a/src/hash.h b/src/hash.h
@@ -11,7 +11,7 @@
 
 void inithash(struct node *xhash[]);
 void clearhash(struct node *xhash[]);
-struct node *makenewnode(struct node *xhash[], char *s);
+struct node *makenewnode(char *s);
 int addnode(struct node *xhash[], char *s);
 struct node *findnode(struct node *xhash[], char *s);
 int is_substr_in_hash(struct node *xhash[], char *s);

diff --git a/src/imap.c b/src/imap.c
@@ -124,8 +124,8 @@ int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sda
 
 
    if(data->recursive_folder_names == 1){
-      data->folder = get_folder_id(sdata, data, folder, 0, cfg);
-      if(data->folder == ERR_FOLDER) data->folder = add_new_folder(sdata, data, folder, 0, cfg);
+      data->folder = get_folder_id(sdata, data, folder, 0);
+      if(data->folder == ERR_FOLDER) data->folder = add_new_folder(sdata, data, folder, 0);
    }
 
 
@@ -307,7 +307,7 @@ int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sda
 }
 
 
-int connect_to_imap_server(int sd, int *seq, char *username, char *password, int port, struct __data *data, int use_ssl){
+int connect_to_imap_server(int sd, int *seq, char *username, char *password, struct __data *data, int use_ssl){
    int n;
    char buf[MAXBUFSIZE];
    X509* server_cert;