Fixed piler rt index in pilerpurge (#194)

* Fixed piler rt index in pilerpurge Signed-off-by: Janos SUTO <sj@acts.hu> * Added purge test to workflow Signed-off-by: Janos SUTO <sj@acts.hu> * Added queuedir to example.conf Signed-off-by: Janos SUTO <sj@acts.hu> * Fixed pilerpurge Signed-off-by: Janos SUTO <sj@acts.hu> --------- Signed-off-by: Janos SUTO <sj@acts.hu>
jsuto · Oct 16, 2024 · 0f662c7 · 0f662c7
1 parent d5edf67
commit 0f662c7
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 1 deletion.
diff --git a/etc/example.conf b/etc/example.conf
@@ -83,6 +83,7 @@ extract_attachments=1
 ; Please also note that the meaning of this variable depends on your Unix implementation
 backlog=20
 
+queuedir=/var/piler/store
 workdir=/var/piler/tmp
 
 ; whether to enable writing folder_message table (1) or not (0)

diff --git a/tests/run.sh b/tests/run.sh
@@ -185,6 +185,44 @@ test_retrieved_messages_are_the_same() {
    echo "tested ${i} messages"
 }
 
+purge_300_emails() {
+   local container="$1"
+   local manticore_host="$2"
+   local piler_id
+   local found
+
+   echo "${FUNCNAME[0]}"
+
+   piler_id="$(docker exec -i "$container" mysql --defaults-file=/etc/piler/.my.cnf -N piler <<< "select piler_id from metadata where id=1" 2>/dev/null)"
+
+   echo "piler id: ${piler_id}"
+
+   found=$(docker exec -i "$container" find "/var/piler/store/00/" -name "${piler_id}.m" 2>/dev/null | wc -l)
+   [[ $found -eq 1 ]] || set_verdict $RESULT_CRITICAL
+
+   echo "Purging emails"
+
+   docker exec -i "$container" mysql --defaults-file=/etc/piler/.my.cnf piler <<< "update metadata set retained=1000000 where id<=300" 2>/dev/null
+   docker exec "$container" su piler -c "/usr/libexec/piler/purge.sh" 2>/dev/null
+
+   echo "Test if emails are purge from manticore"
+
+   if [[ $(docker exec -i "$container" mysql -N -h "$manticore_host" -P9306 <<< "select count(*) as count from piler1 where id<=300") -gt 0 ]]; then
+      echo "Purge has not removed all messages from 1-300"
+      docker exec -i "$container" mysql -N -h "$manticore_host" -P9306 <<< "select count(*) as count from piler1 where id<=300"
+      set_verdict $RESULT_CRITICAL
+   fi
+
+   echo "Test if ${piler_id} still exists"
+
+   found=$(docker exec -i "$container" find "/var/piler/store/00" -name "${piler_id}.m" 2>/dev/null | wc -l)
+   if [[ $found -gt 0 ]]; then
+      echo "Still found ${found} messages that supposed to be deleted"
+      set_verdict $RESULT_CRITICAL
+   fi
+}
+
+
 for i in Inbox Inbox2 Levelszemet Levelszemet2 spam0 spam1 spam2 journal deduptest special; do
    "$SMTP_SOURCE_PROG" -s "$SMTP_HOST" -r "archive@${ARCHIVE_HOST}" -p 25 -t 20 --dir "$EML_DIR/$i" --no-counter
 done
@@ -208,6 +246,8 @@ docker exec "$CONTAINER" su piler -c 'php /usr/libexec/piler/sign.php --webui /v
 
 run_import_job
 
+purge_300_emails "$CONTAINER" "manticore"
+
 docker exec "$CONTAINER" tail -30 /var/log/nginx/error.log
 
 docker exec -i "$CONTAINER" bash -c 'cat >>/root/.bashrc' <<< "alias n='tail -f /var/log/nginx/error.log'"

diff --git a/util/pilerpurge.py b/util/pilerpurge.py
@@ -34,6 +34,7 @@ def read_options(filename="", opts={}):
     opts['database'] = config.get('piler', 'mysqldb')
     opts['storedir'] = config.get('piler', 'queuedir')
     opts['rtindex'] = config.getint('piler', 'rtindex', fallback=0)
+    opts['sphxdb'] = config.get('piler', 'sphxdb', fallback='piler1')
     opts['sphxhost'] = config.get('piler', 'sphxhost', fallback='127.0.0.1')
     opts['sphxport'] = config.getint('piler', 'sphxport', fallback=9306)
     opts['server_id'] = "%02x" % config.getint('piler', 'server_id')
@@ -194,7 +195,7 @@ def purge_index_data(ids=[], opts={}):
     if opts['rtindex'] == 1 and opts['dry_run'] is False:
         cursor = opts['sphx'].cursor()
         a = "," . join([str(x) for x in ids])
-        cursor.execute("DELETE FROM piler WHERE id IN (%s)" % (a))
+        cursor.execute("DELETE FROM %s WHERE id IN (%s)" % (opts['sphxdb'], a))
 
 
 def main():