Initial kafkatee

Change-Id: I86230e961e64090da091bf5e69bdf0d46d3fdcd5
wikimedia · Jan 30, 2014 · ab859cb · ab859cb
1 parent 3ddb03a
commit ab859cb
Show file tree

Hide file tree

Showing 33 changed files with 5,077 additions and 0 deletions.
diff --git a/.dir-locals.el b/.dir-locals.el
@@ -0,0 +1 @@
+( (c-mode . ((c-file-style . "linux"))) )
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,8 @@
+kafkatee
+*~
+.emacs*
+*.d
+*.o
+core
+*.pid
+*.offset
diff --git a/Makefile b/Makefile
@@ -0,0 +1,46 @@
+
+NAME     =kafkatee
+VER     := `git describe --abbrev=6 --tags HEAD --always`
+CFLAGS  +=-DKAFKATEE_VERSION="\"$(VER)\""
+DESTDIR ?=/usr/local
+
+SRCS=	kafkatee.c config.c queue.c input.c output.c exec.c format.c ezd.c
+
+OBJS=	$(SRCS:.c=.o)
+DEPS=	${OBJS:%.o=%.d}
+
+LIBS=	-lyajl -lrdkafka -lrt -lpthread -lz
+
+CFLAGS+=-O2 -Wall -Werror -Wfloat-equal -Wpointer-arith -g
+
+
+# Profiling
+#CFLAGS+=-O0
+#CFLAGS += -pg
+#LDFLAGS += -pg
+
+.PHONY:
+
+all: $(NAME)
+
+%.o: %.c
+	$(CC) -MD -MP $(CFLAGS) -c $<
+
+$(NAME): $(OBJS)
+	$(CC) $(LDFLAGS) $(OBJS) -o $(NAME) $(LIBS)
+
+install:
+	if [ "$(DESTDIR)" != "/usr/local" ]; then \
+		DESTDIR="$(DESTDIR)/usr"; \
+	else \
+		DESTDIR="$(DESTDIR)" ; \
+	fi ; \
+	install -t $$DESTDIR/bin $(NAME)
+
+test: .PHONY
+	make -C test
+
+clean:
+	rm -f $(OBJS) $(DEPS) $(NAME)
+
+-include $(DEPS)
diff --git a/README.md b/README.md
@@ -0,0 +1,60 @@
+kafkatee - Apache Kafka consumer with multiple inputs and outputs
+=================================================================
+
+Copyright (c) 2014 [Wikimedia Foundation](http://www.wikimedia.org)
+
+Copyright (c) 2014 [Magnus Edenhill](http://www.edenhill.se/)
+
+
+# Description
+
+kafkatee consumes messages from one or more Kafka topics and
+writes the messages to one or more outputs - either command pipes or files.
+
+It provides simple transformation from JSON to arbitrary string output
+controlled through configuration.
+
+Each output has a configurable sample rate.
+
+
+Features:
+ * Supported input types:  Kafka consumer or piped command.
+ * Supported output types: Piped command or file.
+ * Configuration file syntax is backwards compatible with Wikimedia's udp2log
+ * Configurable output queue size
+ * Memory frugal: message payload is shared by all output queues
+ * Configured with configuration file
+ * Operates as a daemon (daemonization, pidfile)
+ * Closes/stops and reopens/restarts all inputs and outputs on SIGHUP.
+
+
+# Documentation
+
+See `kafkatee.conf.example` for an annotated configuration file explaining
+available configuration properties.
+
+
+# Build
+
+## Dependencies
+
+ * librdkafka-dev >= 0.8.3
+ * libyajl-dev (yajl1 or yajl2)
+
+## Compile
+
+    make
+
+## Install
+
+    make install
+    # or:
+    DESTDIR=/alternate/buildroot make install
+
+
+## Run tests
+
+    make test
+
+
+