pilerimport uses libcurl to download emails from imap server (#211)

* pilerimport uses libcurl to download emails from imap server Signed-off-by: Janos SUTO <sj@acts.hu> * Curl SSL setup moved to a macro Signed-off-by: Janos SUTO <sj@acts.hu> * Introduced verifyssl option to piler.conf Signed-off-by: Janos SUTO <sj@acts.hu> * Dont verify imap ssl cert for github actions tests Signed-off-by: Janos SUTO <sj@acts.hu> * pilerimport shall remove processed emails after download Signed-off-by: Janos SUTO <sj@acts.hu> --------- Signed-off-by: Janos SUTO <sj@acts.hu>
jsuto · Dec 20, 2024 · 26b754f · 26b754f
1 parent 0a468c9
commit 26b754f
Show file tree

Hide file tree

Showing 11 changed files with 484 additions and 53 deletions.
diff --git a/configure b/configure
@@ -3570,6 +3570,7 @@ have_psql="no"
 have_tre="no"
 have_zip="no"
 have_zlib="no"
+have_curl="no"
 have_tweak_sent_time="no"
 
 database=""
@@ -4242,6 +4243,102 @@ if test "$have_zlib" = "no"; then
 fi
 
 
+       for ac_header in curl/curl.h
+do :
+  ac_fn_c_check_header_compile "$LINENO" "curl/curl.h" "ac_cv_header_curl_curl_h" "$ac_includes_default"
+if test "x$ac_cv_header_curl_curl_h" = xyes
+then :
+  printf "%s\n" "#define HAVE_CURL_CURL_H 1" >>confdefs.h
+ have_curl=yes
+else $as_nop
+  echo "curl.h is not found"
+fi
+
+done
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for main in -lcurl" >&5
+printf %s "checking for main in -lcurl... " >&6; }
+if test ${ac_cv_lib_curl_main+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lcurl  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+
+int
+main (void)
+{
+return main ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  ac_cv_lib_curl_main=yes
+else $as_nop
+  ac_cv_lib_curl_main=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curl_main" >&5
+printf "%s\n" "$ac_cv_lib_curl_main" >&6; }
+if test "x$ac_cv_lib_curl_main" = xyes
+then :
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curl_easy_setopt in -lcurl" >&5
+printf %s "checking for curl_easy_setopt in -lcurl... " >&6; }
+if test ${ac_cv_lib_curl_curl_easy_setopt+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lcurl  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+char curl_easy_setopt ();
+int
+main (void)
+{
+return curl_easy_setopt ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  ac_cv_lib_curl_curl_easy_setopt=yes
+else $as_nop
+  ac_cv_lib_curl_curl_easy_setopt=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curl_curl_easy_setopt" >&5
+printf "%s\n" "$ac_cv_lib_curl_curl_easy_setopt" >&6; }
+if test "x$ac_cv_lib_curl_curl_easy_setopt" = xyes
+then :
+  have_curl=yes
+else $as_nop
+  echo "curl.so is not found"; have_curl=no
+fi
+
+fi
+ac_cv_lib_curl=ac_cv_lib_curl_main
+
+if test "$have_curl" = "no"; then
+   echo "please install the libcurl development packages"
+   exit 1;
+fi
+
 # Extract the first word of "mariadb_config", so it can be a program name with args.
 set dummy mariadb_config; ac_word=$2
 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
@@ -4537,6 +4634,13 @@ else
    echo "zip library: no"
 fi
 
+if test "$have_curl" = "yes"; then
+   echo "curl library: yes"
+   defs="$defs -DHAVE_CURL"
+   antispam_libs="$antispam_libs -lcurl"
+else
+   echo "curl library: no"
+fi
 
 echo
 

diff --git a/configure.in b/configure.in
@@ -37,6 +37,7 @@ have_psql="no"
 have_tre="no"
 have_zip="no"
 have_zlib="no"
+have_curl="no"
 have_tweak_sent_time="no"
 
 database=""
@@ -151,6 +152,15 @@ if test "$have_zlib" = "no"; then
    exit 1;
 fi
 
+dnl libcurl
+
+AC_CHECK_HEADERS(curl/curl.h, have_curl=yes, echo "curl.h is not found")
+AC_CHECK_LIB([curl],[main],[AC_CHECK_LIB(curl, curl_easy_setopt, have_curl=yes, echo "curl.so is not found"; have_curl=no)],[],[])ac_cv_lib_curl=ac_cv_lib_curl_main
+
+if test "$have_curl" = "no"; then
+   echo "please install the libcurl development packages"
+   exit 1;
+fi
 
 AC_CHECK_PROG(MYSQL_CONFIG, mariadb_config, yes)
 
@@ -350,6 +360,13 @@ else
    echo "zip library: no"
 fi
 
+if test "$have_curl" = "yes"; then
+   echo "curl library: yes"
+   defs="$defs -DHAVE_CURL"
+   antispam_libs="$antispam_libs -lcurl"
+else
+   echo "curl library: no"
+fi
 
 echo
 

diff --git a/etc/example.conf b/etc/example.conf
@@ -274,3 +274,7 @@ max_message_size=50000000
 ; when this limit is exceeded, no new emails will be accepted
 ; until the used memory decreases below this level
 max_smtp_memory=500000000
+
+; When connecting to imap server whether to verify (1) the
+; SSL/TLS certificate or not (1)
+verifyssl=1
diff --git a/src/cfg.c b/src/cfg.c
@@ -104,6 +104,7 @@ struct _parse_rule config_parse_rules[] =
    { "update_counters_to_memcached", "integer", (void*) int_parser, offsetof(struct config, update_counters_to_memcached), "0", sizeof(int)},
    { "username", "string", (void*) string_parser, offsetof(struct config, username), "piler", MAXVAL-1},
    { "verbosity", "integer", (void*) int_parser, offsetof(struct config, verbosity), "1", sizeof(int)},
+   { "verifyssl", "integer", (void*) int_parser, offsetof(struct config, verifyssl), "1", sizeof(int)},
    { "workdir", "string", (void*) string_parser, offsetof(struct config, workdir), WORK_DIR, MAXVAL-1},
 
    {NULL, NULL, NULL, 0, 0, 0}

diff --git a/src/cfg.h b/src/cfg.h
@@ -107,6 +107,8 @@ struct config {
 
    int max_message_size;
    uint64 max_smtp_memory;
+
+   int verifyssl;
 };
 
 

diff --git a/src/decoder.c b/src/decoder.c
@@ -341,3 +341,31 @@ int utf8_encode(char *inbuf, int inbuflen, char *outbuf, int outbuflen, char *en
 
    return ret;
 }
+
+
+char *url_encode(const char *str){
+   size_t len = strlen(str);
+
+   char *result = malloc(3*len + 1);  // Worst case scenario: each char becomes %XX
+   if(!result){
+      fprintf(stderr, "malloc() error in %s:%d", __func__, __LINE__);
+      return NULL;
+   }
+
+   char *out = result;
+
+   while(*str){
+      unsigned char c = *str;
+      if(isalnum(c) || c == '_' || c == '-' || c == '.' || c == '~') {
+         *out++ = c;
+      } else {
+         sprintf(out, "%%%02X", c);
+         out += 3;
+      }
+      str++;
+   }
+
+   *out = '\0';
+
+   return result;
+}
diff --git a/src/decoder.h b/src/decoder.h
@@ -11,5 +11,6 @@ void decodeQP(char *p);
 void decodeHTML(char *p, int utf8);
 void decodeURL(char *p);
 int utf8_encode(char *inbuf, int inbuflen, char *outbuf, int outbuflen, char *encoding);
+char *url_encode(const char *str);
 
 #endif /* _DECODER_H */
diff --git a/src/defs.h b/src/defs.h
@@ -322,6 +322,7 @@ struct import {
    int table_id;
    int delay;
    int la_limit;
+   int verifyssl;
    char *server;
    char *username;
    char *password;
@@ -429,4 +430,15 @@ struct tls_protocol {
    int version;
 };
 
+struct MemoryStruct {
+   char *memory;
+   size_t size;
+   size_t written_size;
+};
+
+struct FolderList {
+   char **folders;
+   size_t count;
+};
+
 #endif /* _DEFS_H */
-Original file line number
+Diff line change
@@ Expand Up / @@ -107,6 +107,8 @@ struct config { @@
        int max_message_size;
        uint64 max_smtp_memory;
+       int verifyssl;
     };
@@ Expand Down @@