improved indexing

scorpion · Jul 9, 2013 · 7e65be8 · 7e65be8
1 parent bc16df8
commit 7e65be8
Show file tree

Hide file tree

Showing 7 changed files with 77 additions and 6 deletions.
diff --git a/Makefile.in b/Makefile.in
@@ -110,7 +110,7 @@ key:
 	dd if=/dev/urandom bs=56 count=1 of=piler.key
 
 postinstall:
-	@sh util/postinstall.sh $(RUNNING_USER) $(RUNNING_GROUP) $(sysconfdir) $(localstatedir)
+	@sh util/postinstall.sh $(RUNNING_USER) $(RUNNING_GROUP) $(sysconfdir) $(localstatedir) $(libexecdir)
 
 
 
diff --git a/etc/cron.jobs.in b/etc/cron.jobs.in
@@ -17,8 +17,8 @@ piler's crontab:
 ### mandatory
 
 */5  * * * * LC_ALL=C mpstat | tail -1 | awk '{print $11}' > /var/piler/stat/cpu.stat
-5,35 2-23 * * * /usr/local/bin/indexer --quiet delta1 --rotate && sleep 2 && /usr/local/bin/indexer --quiet --merge dailydelta1 delta1 --merge-dst-range deleted 0 0 --rotate
-15   0 * * * /usr/local/bin/indexer --quiet --merge main1 dailydelta1 --merge-dst-range deleted 0 0 --rotate && sleep 2 && /usr/local/bin/indexer --quiet dailydelta1 --rotate
+5,35 * * * * LIBEXECDIR/piler/indexer.delta.sh
+15   2 * * * LIBEXECDIR/piler/indexer.main.sh
 */15 * * * * /usr/local/bin/indexer --quiet tag1 --rotate
 */15 * * * * /usr/local/bin/indexer --quiet note1 --rotate
 

diff --git a/util/Makefile.in b/util/Makefile.in
@@ -31,6 +31,8 @@ install:
 	$(INSTALL) -m 0755 $(srcdir)/daily-report.php $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/gmail-imap-import.php $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/generate_stats.php $(DESTDIR)$(libexecdir)/piler
+	$(INSTALL) -m 0755 $(srcdir)/indexer.delta.sh $(DESTDIR)$(libexecdir)/piler
+	$(INSTALL) -m 0755 $(srcdir)/indexer.main.sh $(DESTDIR)$(libexecdir)/piler
 
 clean:
 

diff --git a/util/indexer.delta.sh b/util/indexer.delta.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+MAINTMPFILE=/var/run/piler/main.indexer.tmp
+DELTATMPFILE=/var/run/piler/delta.indexer.tmp
+INDEXER=`which indexer`
+PRIORITY=mail.error
+
+if [ -f $MAINTMPFILE ]; then echo "INDEXER ERROR: indexer merging to main index is already running. It started at "`cat $MAINTMPFILE` | logger -p $PRIORITY ; exit 1; fi
+
+if [ -f $DELTATMPFILE ]; then echo "INDEXER ERROR: delta indexing and merging is already running. It started at "`cat $DELTATMPFILE` | logger -p $PRIORITY ; exit 1; fi
+
+date > $DELTATMPFILE
+
+function finish {
+   rm -f $DELTATMPFILE
+}
+
+trap finish EXIT
+
+echo "INDEXER INFO: indexing delta1 started" | logger -p $PRIORITY
+
+5,35 2-23 * * * /usr/local/bin/indexer --quiet delta1 --rotate && sleep 2 && /usr/local/bin/indexer --quiet --merge dailydelta1 delta1 --merge-dst-range deleted 0 0 --rotate
+
+$INDEXER --quiet delta1 --rotate
+
+echo "INDEXER INFO: indexing delta1 finished" | logger -p $PRIORITY
+
+sleep 5
+
+echo "INDEXER INFO: merging delta to dailydelta started" | logger -p $PRIORITY
+
+$INDEXER --quiet --merge dailydelta1 delta1 --merge-dst-range deleted 0 0 --rotate
+
+echo "INDEXER INFO: merging delta to dailydelta finished" | logger -p $PRIORITY
+
diff --git a/util/indexer.main.sh b/util/indexer.main.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+MAINTMPFILE=/var/run/piler/main.indexer.tmp
+#MAINTMPFILE=/tmp/main.indexer.tmp
+INDEXER=`which indexer`
+PRIORITY=mail.error
+
+if [ -f $MAINTMPFILE ]; then echo "ERROR: indexer merging to main index is already running. It started at "`cat $MAINTMPFILE` | logger -p $PRIORITY ; exit 1; fi
+
+date > $MAINTMPFILE
+
+function finish {
+   rm -f $MAINTMPFILE
+}
+
+trap finish EXIT
+
+echo "INFO: merging to main started" | logger -p $PRIORITY
+
+$INDEXER --quiet --merge main1 dailydelta1 --merge-dst-range deleted 0 0 --rotate
+
+echo "INFO: merging to main finished" | logger -p $PRIORITY
+
+sleep 5
+
+echo "INFO: resetting daily delta started" | logger -p $PRIORITY
+
+$INDEXER --quiet dailydelta1 --rotate
+
+echo "INFO: resetting daily delta finished" | logger -p $PRIORITY
+
diff --git a/util/postinstall.sh b/util/postinstall.sh
@@ -75,7 +75,7 @@ gather_smtp_relay_data() {
    ask "Please enter smtp relay" "$SMARTHOST"
    SMARTHOST=$response
 
-   askNonBlankNoEcho "Please enter smtp relay port" "$SMARTHOST_PORT"
+   ask "Please enter smtp relay port" "$SMARTHOST_PORT"
    SMARTHOST_PORT=$response
 }
 
@@ -100,7 +100,9 @@ make_cron_entries() {
    echo ""
    echo "### PILERSTART" >> $CRON_TMP
    echo "*/5 * * * * LC_ALL=C mpstat | tail -1 | awk '{print \$11}' > $LOCALSTATEDIR/piler/stat/cpu.stat" >> $CRON_TMP
-   echo "5,35 * * * * $INDEXER --quiet delta1 --rotate && sleep 2 && $INDEXER --quiet --merge main1 delta1 --merge-dst-range deleted 0 0 --rotate" >> $CRON_TMP
+
+   echo "5,35 * * * * $LIBEXECDIR/piler/indexer.delta.sh" >> $CRON_TMP
+   echo "15   2 * * * $LIBEXECDIR/piler/indexer.main.sh" >> $CRON_TMP
    echo "*/15 * * * * $INDEXER --quiet tag1 --rotate" >> $CRON_TMP
    echo "*/15 * * * * $INDEXER --quiet note1 --rotate" >> $CRON_TMP
    echo "### PILEREND" >> $CRON_TMP
@@ -255,7 +257,7 @@ PILERUSER=$1
 PILERGROUP=$2
 SYSCONFDIR=$3
 LOCALSTATEDIR=$4
-
+LIBEXECDIR=$5
 
 #LOGFILE="/tmp/piler-install.log.$$"
 #touch $LOGFILE

diff --git a/util/utilfunc.sh b/util/utilfunc.sh
@@ -11,6 +11,7 @@ load_default_values() {
    PILERUSER="piler"
    SYSCONFDIR="/usr/local/etc"
    LOCALSTATEDIR="/var"
+   LIBEXECDIR="/usr/local/libexec"
 
    KEYTMPFILE="piler.key"
    KEYFILE="$SYSCONFDIR/piler.key"