Use Google finance API instead of Yahoo

sryza · Jul 16, 2017 · a09d441 · a09d441
1 parent 41135c8
commit a09d441
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 7 deletions.
diff --git a/ch09-risk/data/download-symbol.sh b/ch09-risk/data/download-symbol.sh
@@ -4,4 +4,4 @@
 #
 # See LICENSE file for further information.
 
-curl -o $2/$1.csv https://ichart.yahoo.com/table.csv?s=$1&a=0&b=1&c=2000&d=0&e=31&f=2013&g=d&ignore=.csv
+wget -O ${2}/${1}.csv "http://www.google.com/finance/historical?q=${1}&startdate=Jan+01%2C+2000&enddate=Dec+31%2C+2013&output=csv"
diff --git a/ch09-risk/src/main/scala/com/cloudera/datascience/risk/RunRisk.scala b/ch09-risk/src/main/scala/com/cloudera/datascience/risk/RunRisk.scala
@@ -57,10 +57,10 @@ class RunRisk(private val spark: SparkSession) {
   import spark.implicits._
 
   /**
-   * Reads a history in the Yahoo format
+   * Reads a history in the Google format
    */
-  def readYahooHistory(file: File): Array[(LocalDate, Double)] = {
-    val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd")
+  def readGoogleHistory(file: File): Array[(LocalDate, Double)] = {
+    val formatter = DateTimeFormatter.ofPattern("d-MMM-yy")
     val lines = scala.io.Source.fromFile(file).getLines().toSeq
     lines.tail.map { line =>
       val cols = line.split(',')
@@ -125,17 +125,17 @@ class RunRisk(private val spark: SparkSession) {
     val files = stocksDir.listFiles()
     val allStocks = files.iterator.flatMap { file =>
       try {
-        Some(readYahooHistory(file))
+        Some(readGoogleHistory(file))
       } catch {
         case e: Exception => None
       }
     }
     val rawStocks = allStocks.filter(_.size >= 260 * 5 + 10)
 
     val factorsPrefix = "factors/"
-    val rawFactors = Array("^GSPC.csv", "^IXIC.csv", "^TYX.csv", "^FVX.csv").
+    val rawFactors = Array("NYSEARCA%3AGLD.csv", "NASDAQ%3ATLT.csv", "NYSEARCA%3ACRED.csv").
       map(x => new File(factorsPrefix + x)).
-      map(readYahooHistory)
+      map(readGoogleHistory)
 
     val stocks = rawStocks.map(trimToRegion(_, start, end)).map(fillInHistory(_, start, end))