Merging hogarth into master. This will 'finalize' my work on Redexer.…

… The merge is simpler than expected and I'm confident Redexer will continue to work properly on the master branch
plum-umd · Apr 27, 2021 · f4876c5 · f4876c5
2 parents 25a2922 + 4359be6
commit f4876c5
Show file tree

Hide file tree

Showing 7,014 changed files with 2,811,521 additions and 11,831 deletions.
diff --git a/Makefile b/Makefile
@@ -8,10 +8,11 @@ redexer: main.native
 
 # debugging code targets; could also (more likely) build *.d.byte separately
 debug: main.d.byte
+	mv main.d.byte redexer
 
 # auxiliary targets
 doc:
-	ocamlbuild -use-ocamlfind redexer.docdir/index.html
+	ocamlbuild -no-hygiene -use-ocamlfind redexer.docdir/index.html
 	mkdir -p docs
 	rm -rf docs/api
 	mv redexer.docdir docs/api
@@ -27,12 +28,12 @@ allclean: clean
 
 # x.native; strip the .native extension after building
 %.native:
-	ocamlbuild -use-ocamlfind $@
+	ocamlbuild -no-hygiene -use-ocamlfind -tag thread $@
 	mv $@ $*
 
 # x.d.byte or x.byte
 %.byte:
-	ocamlbuild $@
+	ocamlbuild -use-ocamlfind -tag thread $@
 
 .PHONY: doc debug all model test clean modelclean allclean default
 
diff --git a/README.md b/README.md
@@ -1,5 +1,10 @@
 [![Build Status](https://travis-ci.org/plum-umd/redexer.svg?branch=master)](https://travis-ci.org/plum-umd/redexer)
 
+# This branch
+
+This branch is to support logging with Hogarth. A system for sparse
+logging of Android apps and subsequent analysis of those logs. 
+
 # redexer
 
 Redexer is a reengineering tool that manipulates Android app binaries.

diff --git a/data/logging-protos.txt b/data/logging-protos.txt
@@ -0,0 +1 @@
+Lorg/umd/logging/Logger;.log
diff --git a/data/logging.dex b/data/logging.dex
diff --git a/data/logging.json b/data/logging.json
diff --git a/data/loggingFull.dex b/data/loggingFull.dex
diff --git a/data/skip.txt b/data/skip.txt
@@ -1,26 +1,7 @@
-org.acra
-org.afree
-org.apache
-org.jboss
-org.json
-org.slf4j
-oauth.signpost
-android.annotation
 android.support
-com.android.vending
-com.google
-com.ebay
-twitter4j
-winterwell.jtwitter
-com.flurry.android
-com.millennialmedia.android
-com.MASTAdView
-com.atti.mobile.hyperlocalad
-com.mopub.mobileads
-com.noqoush.adfalcon
-com.appsflyer
-com.tapjoy
-com.dvmobile.spyderlynk
-net.hockeyapp.android
-ru.begun.adlib
-ru.wapstart.plus1.sdk
+a.a.a
+com.google.protobuf
+ProtoDefs
+com.facebook.GraphRequestBatch
+com.google.android.gms.measurement.internal
+com.google.android.gms.internal
diff --git a/facebook_survey/PermissionCounts.csv b/facebook_survey/PermissionCounts.csv
@@ -0,0 +1,19 @@
+"","combo","total"
+"1","read:'activity'",1
+"2","read:'email'",11
+"3","read:'input_method'",1
+"4","read:'observer'",1
+"5","read:'public_profile'",15
+"6","read:'publish_actions'",1
+"7","read:'user_birthday'",2
+"8","read:'user_friends'",3
+"9","read:'user_photos'",2
+"10","read:'user_status'",1
+"11","read:'user_videos'",1
+"12","read:'wishlists'",1
+"13","write:'activity'",1
+"14","write:'logging_in'",1
+"15","write:'public_profile'",4
+"16","write:'publish_actions'",11
+"17","write:'publish'",3
+"18","write:'user_friends'",1
diff --git a/facebook_survey/PermissionRequests.csv b/facebook_survey/PermissionRequests.csv
@@ -0,0 +1,35 @@
+"","app","permlist","total"
+"1","bn.ereader.apk","read:'', write:''",2
+"2","com.amazon.mShop.android.shopping.apk","read:''",1
+"3","com.aws.android.apk","read:'public_profile', read:'email', read:'user_birthday'",3
+"4","com.bitstrips.imoji","write:'logging_in'",1
+"5","com.booking.apk","read:'wishlists', read:'email'",2
+"6","com.buzzfeed.android.apk","read:'email', write:'publish_actions'",2
+"7","com.clearchannel.iheartradio.controller.apk","read:'activity', read:'observer', write:'activity'",3
+"8","com.cmcm.live","read:'public_profile', read:'user_birthday', read:'email', read:'user_friends', write:'publish_actions'",5
+"9","com.cnn.mobile.android.phone.apk","write:'publish'",1
+"10","com.contacts1800.ecomapp.apk","write:'publish'",1
+"11","com.digidust.elokence.akinator.freemium.apk","write:'user_friends', write:'publish_actions', read:'user_friends'",3
+"12","com.duolingo","read:''",1
+"13","com.emn8.mobilem8.nativeapp.bk","write:'', read:''",2
+"14","com.energysh.drawshow","write:'publish_actions'",1
+"15","com.expedia.bookings.apk","read:'email'",1
+"16","com.fishbrain.app","write:'publish_actions', read:'publish_actions'",2
+"17","com.hp.pregnancy.lite.apk","write:''",1
+"18","com.jb.gosms.apk","write:'publish_actions'",1
+"19","com.move.realtor.apk","read:'user_status'",1
+"20","com.nascar.nascarmobile","write:'publish'",1
+"21","com.oovoo.apk","read:'', write:''",2
+"22","com.ovuline.pregnancy","read:'input_method', read:'public_profile', read:'email'",3
+"23","com.picsart.studio.apk","write:'publish_actions'",1
+"24","com.pinterest.apk","read:''",1
+"25","com.quvideo.xiaoying.apk","read:'public_profile', read:'user_photos', read:'user_videos'",3
+"26","com.relayrides.android.relayrides.apk","read:'email'",1
+"27","com.roidapp.photogrid.apk","read:'user_photos'",1
+"28","com.sgiggle.production.apk","read:'', write:'publish_actions'",2
+"29","com.sirma.mobile.bible.android","read:'email', read:'public_profile'",2
+"30","com.smule.magicpiano","write:'publish_actions', read:''",2
+"31","com.smule.singandroid.apk","write:'publish_actions', read:''",2
+"32","com.target.socsav","read:'public_profile', read:'user_friends', read:'email', write:'publish_actions'",4
+"33","com.wishabi.flipp","read:'email'",1
+"34","org.khanacademy.android","read:'email'",1
diff --git a/facebook_survey/facebook_analyze.r b/facebook_survey/facebook_analyze.r
@@ -0,0 +1,34 @@
+# For `melt`
+library(reshape2)
+library(DT)
+library(dplyr)
+library(ggplot2)
+library(ggthemes)
+library(lattice)
+library(latticeExtra)
+library(extrafont)
+library(grid)
+library(gtable)
+
+data <- read.csv("/Users/dvotipka/Documents/Projects/UMD/AndroidStudies/android-logging/facebook_survey/final.csv")
+
+chart_sub <- subset(data, select=c("app"))
+chart_sub <- unique(chart_sub)
+
+print(paste("There are ",nrow(chart_sub)," apps using Facebook permissions"))
+
+chart_sub <- unique(data)
+chart_sub$combo <- paste(chart_sub$type,chart_sub$permission,sep=":")
+chart_sub$count <- 1
+chart_sub_long <- summarise(group_by(chart_sub, app), permlist = toString(combo), total = sum(count))
+chart_sub_long$app <- reorder(chart_sub_long$app,chart_sub_long$total)
+write.csv(chart_sub_long, file="/Users/dvotipka/Documents/Projects/UMD/AndroidStudies/android-logging/facebook_survey/PermissionRequests.csv")
+
+# postscript("/Users/dvotipka/Documents/Projects/UMD/AndroidStudies/android-logging/facebook_survey/PermissionRequests.eps",width=12,heigh=17,horizontal=FALSE,onefile=FALSE,paper="special",colormodel="cmyk",family="Arial")
+# #cols <- c("chocolate2","darkgoldenrod2", "darkgreen", "dodgerblue3", "antiquewhite1","hotpink2")
+# p10 <- ggplot(chart_sub_long, aes(y=total, x=app)) + geom_bar(stat="identity") + coord_flip()
+# p10 <- p10 + geom_text(data=chart_sub_long, aes(x = app, y=total+.3, label = total), 
+#           size=6) + theme(text=element_text(family="Arial",size=22),legend.title = element_blank(),axis.title.y=element_blank(),legend.position="bottom")
+# p10 <- p10 + labs(y="Permission Count") + ggtitle("Permissions Requested per App")
+# print(p10)
+# dev.off
diff --git a/facebook_survey/facebook_check.py b/facebook_survey/facebook_check.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python2
+
+import androguard
+from androguard.session import Session
+import multiprocessing
+import re
+from os import listdir
+from os.path import isfile, join
+
+def getPermissions(instrs):
+	perms = []
+	for instr in instrs:
+		m = re.search('\'(.*)\'',instr.show_buff(1))
+		if m:
+			perms.append(m.group(0))
+	return perms
+
+def findFacebook(filename, d):
+	foundPulls = ""
+	classes = d.get_classes_names()
+	for class_name in classes:
+		if 'com/facebook/' not in class_name:
+			methods = d.get_class(class_name).get_methods()
+			for meth in methods:
+				instrs = [i for i in meth.get_instructions()]
+				counter = 0
+				last_counter = 0
+				for instr in instrs:
+					if "Lcom/facebook/login/LoginManager;->logInWithReadPermissions" in instr.show_buff(1):
+						for perm in getPermissions([i for i in meth.get_instructions()][last_counter:counter]):
+							foundPulls = foundPulls + filename + ", read, " + perm  + "\n"	
+						last_counter = counter			
+					if "Lcom/facebook/login/LoginManager;->logInWithPublishPermissions" in instr.show_buff(1):
+						for perm in getPermissions([i for i in meth.get_instructions()][last_counter:counter]):
+							foundPulls = foundPulls + filename + ", write, " + perm + "\n"      
+                                       		last_counter = counter
+					counter = counter + 1				
+	return foundPulls
+
+def processFile(filename):
+	s = Session()
+	mypath = '/Users/dvotipka/Documents/Projects/UMD/AndroidStudies/android-logging/apks/'
+	resultspath = '/Users/dvotipka/Documents/Projects/UMD/AndroidStudies/android-logging/facebook_survey/results/'
+	with open(mypath + filename, "r") as fd:
+		s.add(mypath + filename, fd.read())
+	a,d,dx = s.get_objects_apk(mypath + filename)
+	foundPulls = ""
+	if "androguard.core.bytecodes.dvm.DalvikVMFormat" in str(type(d)):
+		foundPulls = foundPulls + findFacebook(filename[:-4], d)
+	else:
+		print type(d)
+		for class_set in d:
+			foundPulls = foundPulls + findFacebook(filename, class_set)
+	#if foundPulls != "":
+	resultsFile = open(resultspath+filename[:-4],'wb')
+	resultsFile.write(foundPulls)
+	resultsFile.close()	
+
+def runFiles(files):
+	for file in files:
+		processFile(file)
+
+if __name__ == '__main__':
+	jobs = []
+	bound = 0
+	processors = 1
+#	mypath = '/Users/dvotipka/Documents/Projects/UMD/AndroidStudies/apks' 
+	mypath = '/Users/dvotipka/Documents/Projects/UMD/AndroidStudies/android-logging/apks'
+	filenames = [f for f in listdir(mypath) if isfile(join(mypath, f))]
+	distance = int((len(filenames)-bound)/processors)+1
+	print "1"
+	print filenames
+	sum = 0
+	for i in range(processors):
+		end = bound + distance
+		if end > len(filenames):
+			end = len(filenames)
+		print "2"
+		print filenames[bound:end]
+		p = multiprocessing.Process(target=runFiles, args=([filenames[bound:end]]))
+		jobs.append(p)
+		p.start()
+		bound = bound + distance
+
diff --git a/facebook_survey/facebook_obfs_check.py b/facebook_survey/facebook_obfs_check.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python2
+
+import multiprocessing
+import re
+from os import listdir
+from os.path import isfile, join
+import commands
+
+def processFile(filename):
+	mypath = '/Users/dvotipka/Documents/Projects/UMD/AndroidStudies/apks/'
+	resultspath = '/Users/dvotipka/Documents/Projects/UMD/AndroidStudies/android-logging/facebook_survey/results/'
+	foundPulls = ""
+    	foundFacebook = False
+    	foundFacebookLoginManager = False
+    	output = commands.getstatusoutput('dexdump ' + mypath + filename + ' | grep "com/facebook"')[1]
+    	if output != "":
+        	foundFacebook = True
+    	output = commands.getstatusoutput('dexdump ' + mypath + filename + ' | grep "com/facebook/login/LoginManager"')[1]
+    	if output != "":
+        	foundFacebookLoginManager = True
+    	if foundFacebook and foundFacebookLoginManager:
+        	foundPulls = filename + ",Facbook found not obfuscated\n"
+    	elif foundFacebook and not foundFacebookLoginManager:
+        	foundPulls = filename + ",Facebook found obfuscated\n"
+    	else:
+        	foundPulls = filename + ",Facebook not found\n"
+
+	resultsFile = open(resultspath+filename[:-4],'wb')
+	resultsFile.write(foundPulls)
+	resultsFile.close()	
+
+def runFiles(files):
+	for file in files:
+		processFile(file)
+
+if __name__ == '__main__':
+	jobs = []
+	bound = 0
+	processors = 1
+	mypath = '/Users/dvotipka/Documents/Projects/UMD/AndroidStudies/apks' 
+#	mypath = '/Users/dvotipka/Documents/Projects/UMD/AndroidStudies/android-logging/apks'
+	filenames = [f for f in listdir(mypath) if isfile(join(mypath, f))]
+	distance = int((len(filenames)-bound)/processors)+1
+	print "1"
+	print filenames
+	sum = 0
+	for i in range(processors):
+		end = bound + distance
+		if end > len(filenames):
+			end = len(filenames)
+		print "2"
+		print filenames[bound:end]
+		p = multiprocessing.Process(target=runFiles, args=([filenames[bound:end]]))
+		jobs.append(p)
+		p.start()
+		bound = bound + distance
diff --git a/facebook_survey/final.csv b/facebook_survey/final.csv
@@ -0,0 +1,77 @@
+app,type,permission
+bn.ereader.apk,read,'public_profile'
+bn.ereader.apk,write,'public_profile'
+com.amazon.mShop.android.shopping.apk,read,'public_profile'
+com.aws.android.apk,read,'public_profile'
+com.aws.android.apk,read,'public_profile'
+com.aws.android.apk,read,'email'
+com.aws.android.apk,read,'user_birthday'
+com.aws.android.apk,read,'public_profile'
+com.aws.android.apk,read,'email'
+com.aws.android.apk,read,'user_birthday'
+com.bitstrips.imoji,write,'logging_in'
+com.booking.apk,read,'wishlists'
+com.booking.apk,read,'email'
+com.buzzfeed.android.apk,read,'email'
+com.buzzfeed.android.apk,write,'publish_actions'
+com.clearchannel.iheartradio.controller.apk,read,'activity'
+com.clearchannel.iheartradio.controller.apk,read,'observer'
+com.clearchannel.iheartradio.controller.apk,write,'activity'
+com.cmcm.live,read,'public_profile'
+com.cmcm.live,read,'user_birthday'
+com.cmcm.live,read,'email'
+com.cmcm.live,read,'user_friends'
+com.cmcm.live,write,'publish_actions'
+com.cnn.mobile.android.phone.apk,write,'publish'
+com.contacts1800.ecomapp.apk,write,'publish'
+com.digidust.elokence.akinator.freemium.apk,write,'user_friends'
+com.digidust.elokence.akinator.freemium.apk,write,'publish_actions'
+com.digidust.elokence.akinator.freemium.apk,write,'publish_actions'
+com.digidust.elokence.akinator.freemium.apk,read,'user_friends'
+com.duolingo,read,'public_profile'
+com.emn8.mobilem8.nativeapp.bk,write,'public_profile'
+com.emn8.mobilem8.nativeapp.bk,read,'public_profile'
+com.energysh.drawshow,write,'publish_actions'
+com.expedia.bookings.apk,read,'email'
+com.fishbrain.app,write,'publish_actions'
+com.fishbrain.app,read,'publish_actions'
+com.hp.pregnancy.lite.apk,write,'public_profile'
+com.jb.gosms.apk,write,'publish_actions'
+com.move.realtor.apk,read,'user_status'
+com.nascar.nascarmobile,write,'publish'
+com.oovoo.apk,read,'public_profile'
+com.oovoo.apk,write,'public_profile'
+com.ovuline.pregnancy,read,'input_method'
+com.ovuline.pregnancy,read,'public_profile'
+com.ovuline.pregnancy,read,'email'
+com.picsart.studio.apk,write,'publish_actions'
+com.pinterest.apk,read,'public_profile'
+com.quvideo.xiaoying.apk,read,'public_profile'
+com.quvideo.xiaoying.apk,read,'user_photos'
+com.quvideo.xiaoying.apk,read,'user_videos'
+com.relayrides.android.relayrides.apk,read,'email'
+com.roidapp.photogrid.apk,read,'user_photos'
+com.sgiggle.production.apk,read,'public_profile'
+com.sgiggle.production.apk,write,'publish_actions'
+com.sirma.mobile.bible.android,read,'email'
+com.sirma.mobile.bible.android,read,'public_profile'
+com.sirma.mobile.bible.android,read,'email'
+com.sirma.mobile.bible.android,read,'public_profile'
+com.smule.magicpiano,write,'publish_actions'
+com.smule.magicpiano,read,'public_profile'
+com.smule.singandroid.apk,write,'publish_actions'
+com.smule.singandroid.apk,read,'public_profile'
+com.target.socsav,read,'public_profile'
+com.target.socsav,read,'user_friends'
+com.target.socsav,read,'email'
+com.target.socsav,read,'user_friends'
+com.target.socsav,read,'email'
+com.target.socsav,read,'user_friends'
+com.target.socsav,write,'publish_actions'
+com.target.socsav,read,'email'
+com.target.socsav,read,'user_friends'
+com.target.socsav,read,'user_friends'
+com.target.socsav,read,'user_friends'
+com.wishabi.flipp,read,'email'
+com.wishabi.flipp,read,'email'
+org.khanacademy.android,read,'email'