Skip to content

Commit

Permalink
chore: to format
Browse files Browse the repository at this point in the history
  • Loading branch information
@5000164
5000164 committed Apr 28, 2019
1 parent 748c276 commit a3174eb
Show file tree
Hide file tree
Showing 14 changed files with 220 additions and 190 deletions.
34 changes: 17 additions & 17 deletions build.sbt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,23 +61,23 @@ lazy val migration = project

lazy val dependencies =
new {
val logback = "ch.qos.logback" % "logback-classic" % "1.2.3"
val scalaLogging = "com.typesafe.scala-logging" %% "scala-logging" % "3.9.0"
val scalaCompiler = "org.scala-lang" % "scala-compiler" % "2.12.6"
val scalaXml = "org.scala-lang.modules" %% "scala-xml" % "1.0.6"
val slick = "com.typesafe.slick" %% "slick" % "3.2.3"
val slickHikaricp = "com.typesafe.slick" %% "slick-hikaricp" % "3.2.3"
val slickCodegen = "com.typesafe.slick" %% "slick-codegen" % "3.2.3"
val h2 = "com.h2database" % "h2" % "1.4.197"
val sttpCore = "com.softwaremill.sttp" %% "core" % "1.2.1"
val scalatest = "org.scalatest" %% "scalatest" % "3.0.5"
val scalactic = "org.scalactic" %% "scalactic" % "3.0.5"
val flywayCore = "org.flywaydb" % "flyway-core" % "5.1.4"
val akkaActor = "com.typesafe.akka" %% "akka-actor" % "2.5.11"
val akkaStream = "com.typesafe.akka" %% "akka-stream" % "2.5.11"
val akkaSlf4j = "com.typesafe.akka" %% "akka-slf4j" % "2.5.11"
val slackScalaClient = "com.github.gilbertw1" %% "slack-scala-client" % "0.2.3"
val scalaScraper = "net.ruippeixotog" %% "scala-scraper" % "2.1.0"
val logback = "ch.qos.logback" % "logback-classic" % "1.2.3"
val scalaLogging = "com.typesafe.scala-logging" %% "scala-logging" % "3.9.0"
val scalaCompiler = "org.scala-lang" % "scala-compiler" % "2.12.6"
val scalaXml = "org.scala-lang.modules" %% "scala-xml" % "1.0.6"
val slick = "com.typesafe.slick" %% "slick" % "3.2.3"
val slickHikaricp = "com.typesafe.slick" %% "slick-hikaricp" % "3.2.3"
val slickCodegen = "com.typesafe.slick" %% "slick-codegen" % "3.2.3"
val h2 = "com.h2database" % "h2" % "1.4.197"
val sttpCore = "com.softwaremill.sttp" %% "core" % "1.2.1"
val scalatest = "org.scalatest" %% "scalatest" % "3.0.5"
val scalactic = "org.scalactic" %% "scalactic" % "3.0.5"
val flywayCore = "org.flywaydb" % "flyway-core" % "5.1.4"
val akkaActor = "com.typesafe.akka" %% "akka-actor" % "2.5.11"
val akkaStream = "com.typesafe.akka" %% "akka-stream" % "2.5.11"
val akkaSlf4j = "com.typesafe.akka" %% "akka-slf4j" % "2.5.11"
val slackScalaClient = "com.github.gilbertw1" %% "slack-scala-client" % "0.2.3"
val scalaScraper = "net.ruippeixotog" %% "scala-scraper" % "2.1.0"
}

lazy val commonDependencies = Seq(
Expand Down
9 changes: 4 additions & 5 deletions collect/src/main/scala/domain/Articles.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import scala.xml.{Elem, Node, XML}

/** 記事に関する操作を抽象化して扱う。 */
object Articles {

/** 配信された記事一覧を取得する。
*
* @param feedUrl 抽出元の RSS の URL
Expand All @@ -13,13 +14,13 @@ object Articles {
def fetchDeliveredArticles(feedUrl: String, fetchContent: String => String): Seq[DeliveredArticle] = {
val getItems = (element: Elem) => element \ "item"
val getArticle = (node: Node) => {
val url = (node \ "link").text
val url = (node \ "link").text
val title = (node \ "title").text
DeliveredArticle(url, title)
}

val xmlString = fetchContent(feedUrl)
val xml = XML.loadString(xmlString)
val xml = XML.loadString(xmlString)
getItems(xml).map(getArticle)
}
}
Expand All @@ -29,6 +30,4 @@ object Articles {
* @param url URL
* @param title タイトル
*/
case class DeliveredArticle(
url: String,
title: String)
case class DeliveredArticle(url: String, title: String)
24 changes: 15 additions & 9 deletions collect/src/main/scala/interfaces/Application.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,21 @@ object Application extends App with LazyLogging {

val repository = new Repository()
try {
Await.ready(Future.sequence(for {
(settingsId, watchSettings) <- settings.watches
deliveredArticle <- Articles.fetchDeliveredArticles(watchSettings.feedUrl, Client.fetchContent)
} yield Future {
if (!repository.existsUrl(deliveredArticle.url, settingsId)) repository.save(deliveredArticle.url, settingsId) match {
case Right(_) =>
case Left(e) => logger.error(s"保存処理に失敗 article:$deliveredArticle, settingsId:$settingsId", e)
}
}), Duration.Inf)
Await.ready(
Future.sequence(
for {
(settingsId, watchSettings) <- settings.watches
deliveredArticle <- Articles.fetchDeliveredArticles(watchSettings.feedUrl, Client.fetchContent)
} yield
Future {
if (!repository.existsUrl(deliveredArticle.url, settingsId)) repository.save(deliveredArticle.url, settingsId) match {
case Right(_) =>
case Left(e) => logger.error(s"保存処理に失敗 article:$deliveredArticle, settingsId:$settingsId", e)
}
}
),
Duration.Inf
)
} catch {
case e: Throwable =>
logger.error("エラー発生", e)
Expand Down
1 change: 1 addition & 0 deletions collect/src/main/scala/interfaces/Client.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import com.softwaremill.sttp._

/** 通信に関する処理を行う。 */
object Client {

/** 指定された URL の内容を返す。
*
* @param url 取得対象の URL
Expand Down
70 changes: 37 additions & 33 deletions collect/src/test/scala/domain/ArticlesSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,39 +8,43 @@ import scala.io.Source
class ArticlesSpec extends FeatureSpec {
feature("記事を抽出できる") {
scenario("XML から記事を抽出する") {
assert(fetchDeliveredArticles(
feedUrl = "feedUrl",
fetchContent = _ => Source.fromResource("sample.rss").getLines.mkString) === Seq(
DeliveredArticle("http://gendai.ismedia.jp/articles/-/56258", "凶悪犯罪続発！アメリカを蝕む「非モテの過激化」という大問題（八田 真行） | 現代ビジネス | 講談社（1/4）"),
DeliveredArticle("http://hosyusokuhou.jp/archives/48819730.html", "保守速報からのお知らせ ｜ 保守速報"),
DeliveredArticle("https://togetter.com/li/1241708", "電車で見たお姉さんを描いたイラストが性的すぎて炎上 - Togetter"),
DeliveredArticle("https://www3.nhk.or.jp/news/html/20180701/k10011503371000.html", "13歳が車運転し事故 １人死亡 ４人大けが 岡山 | NHKニュース"),
DeliveredArticle("https://togetter.com/li/1242324", "入社2年目の若手エンジニアがものすごく優秀だったので上司が『彼を課長待遇に』と提案するも即座に却下されてしまうという日本企業の“常識的”マネージメントについての話 - Togetter"),
DeliveredArticle("https://anond.hatelabo.jp/20180701081730", "こういうブコメの書き方やめろ"),
DeliveredArticle("https://www3.nhk.or.jp/news/html/20180701/k10011503121000.html", "かぜに効かない抗菌薬 ６割超の医師が処方 | NHKニュース"),
DeliveredArticle("https://anond.hatelabo.jp/20180624140926", "外に出ろって言うけど何をすればいいの？"),
DeliveredArticle("https://natalie.mu/owarai/news/289169", "明石家さんま、“勘違い”で34年ぶりテレ東出演！出川哲朗と山形へ電動バイク旅（コメントあり） - お笑いナタリー"),
DeliveredArticle("https://note.mu/psychs/n/nc519c2c85801", "Twitter での6年間 1｜Satoshi Nakagawa｜note"),
DeliveredArticle("http://twitter.com/lullymiura/status/1013022237962588161", "三浦瑠麗 Lully MIURAさんのツイート: \"杉田水脈議員が詩織さんの件に落ち度云々とコメントしていますが、仮に財布がズボンのポケットからはみ出て気をつけてないうちにスられたとしても、"),
DeliveredArticle("https://anond.hatelabo.jp/20180701133232", "今年外科には誰も入局者がいなかった"),
DeliveredArticle("http://novtan.hatenablog.com/entry/2018/07/01/103710", "われわれは反省すべきか、萎縮すべきか、Hagexの遺志を継ぐべきか - novtanの日常"),
DeliveredArticle("http://bunshun.jp/articles/-/7957", "「正社員はいらない」“煽る人”竹中平蔵とは何者なのか？ | 文春オンライン"),
DeliveredArticle("https://togetter.com/li/1242449", "「怪物と戦っていると思ったら、その正体は人間だった…」というお話の起源等を巡る話（※ネタバレは自己責任で） - Togetter"),
DeliveredArticle("http://kabumatome.doorblog.jp/archives/65921871.html", "トヨタ自動車の内部留保、日本共産党と朝日新聞がナントカ埋蔵金の如く非難 : 市況かぶ全力２階建"),
DeliveredArticle("https://anond.hatelabo.jp/20180629224012", "態度の悪い４０代無職を誰が救えるの？"),
DeliveredArticle("https://www3.nhk.or.jp/news/html/20180701/k10011503031000.html", "経済的に厳しい家庭も保護者の関与で子どもの学力向上 | NHKニュース"),
DeliveredArticle("https://anond.hatelabo.jp/20180701121356", "ぶっちゃけ2人っきりの飲み食い奢られて"),
DeliveredArticle("https://anond.hatelabo.jp/20180701010648", "女医さんがやってきた"),
DeliveredArticle("https://anond.hatelabo.jp/20180701155712", "今日もブクマカは増田叩き"),
DeliveredArticle("https://anond.hatelabo.jp/20180701063754", "あいつが低能先生だった"),
DeliveredArticle("https://togetter.com/li/1242309", "「私28歳。結婚してない恋もしてない。仕事もダメ。なにもない」朝ドラ『半分青い』のセリフが賛否両論　しかもこれの続きは男性を狙った二段構え - Togetter"),
DeliveredArticle("https://anond.hatelabo.jp/20180701012637", "酒が嫌いだというだけの愚痴"),
DeliveredArticle("https://mainichi.jp/articles/20180701/k00/00m/040/093000c", "ＩＴ講師刺殺１週間：ネット憎悪がリアル暴力に - 毎日新聞"),
DeliveredArticle("http://blog.inouetakuya.info/entry/2018/06/30/204134", "49インチ 4K ディスプレイの感想 - 彼女からは、おいちゃんと呼ばれています"),
DeliveredArticle("https://note.mu/psychs/n/nf073d58a2c4e", "Twitter での6年間 2｜Satoshi Nakagawa｜note"),
DeliveredArticle("https://okane.news/financial-movies/", "金融リテラシーが上がる！映画まとめ30選｜投資や経済を楽しく学べる | お得に節約お金ニュース"),
DeliveredArticle("https://samurai20.jp/2018/06/hosyusoku-2/", "保守速報など、まとめサイト群への救済処置（暫定版）【応援する人はシェア】 | 小坪しんやのHP～行橋市議会議員"),
DeliveredArticle("https://anond.hatelabo.jp/20180701000252", "たまごとうふには豆腐成分が一切入ってないと知った時の衝撃は忘れない")))
assert(
fetchDeliveredArticles(feedUrl = "feedUrl", fetchContent = _ => Source.fromResource("sample.rss").getLines.mkString) === Seq(
DeliveredArticle("http://gendai.ismedia.jp/articles/-/56258", "凶悪犯罪続発！アメリカを蝕む「非モテの過激化」という大問題（八田 真行） | 現代ビジネス | 講談社（1/4）"),
DeliveredArticle("http://hosyusokuhou.jp/archives/48819730.html", "保守速報からのお知らせ ｜ 保守速報"),
DeliveredArticle("https://togetter.com/li/1241708", "電車で見たお姉さんを描いたイラストが性的すぎて炎上 - Togetter"),
DeliveredArticle("https://www3.nhk.or.jp/news/html/20180701/k10011503371000.html", "13歳が車運転し事故 １人死亡 ４人大けが 岡山 | NHKニュース"),
DeliveredArticle("https://togetter.com/li/1242324", "入社2年目の若手エンジニアがものすごく優秀だったので上司が『彼を課長待遇に』と提案するも即座に却下されてしまうという日本企業の“常識的”マネージメントについての話 - Togetter"),
DeliveredArticle("https://anond.hatelabo.jp/20180701081730", "こういうブコメの書き方やめろ"),
DeliveredArticle("https://www3.nhk.or.jp/news/html/20180701/k10011503121000.html", "かぜに効かない抗菌薬 ６割超の医師が処方 | NHKニュース"),
DeliveredArticle("https://anond.hatelabo.jp/20180624140926", "外に出ろって言うけど何をすればいいの？"),
DeliveredArticle("https://natalie.mu/owarai/news/289169", "明石家さんま、“勘違い”で34年ぶりテレ東出演！出川哲朗と山形へ電動バイク旅（コメントあり） - お笑いナタリー"),
DeliveredArticle("https://note.mu/psychs/n/nc519c2c85801", "Twitter での6年間 1｜Satoshi Nakagawa｜note"),
DeliveredArticle(
"http://twitter.com/lullymiura/status/1013022237962588161",
"三浦瑠麗 Lully MIURAさんのツイート: \"杉田水脈議員が詩織さんの件に落ち度云々とコメントしていますが、仮に財布がズボンのポケットからはみ出て気をつけてないうちにスられたとしても、"
),
DeliveredArticle("https://anond.hatelabo.jp/20180701133232", "今年外科には誰も入局者がいなかった"),
DeliveredArticle("http://novtan.hatenablog.com/entry/2018/07/01/103710", "われわれは反省すべきか、萎縮すべきか、Hagexの遺志を継ぐべきか - novtanの日常"),
DeliveredArticle("http://bunshun.jp/articles/-/7957", "「正社員はいらない」“煽る人”竹中平蔵とは何者なのか？ | 文春オンライン"),
DeliveredArticle("https://togetter.com/li/1242449", "「怪物と戦っていると思ったら、その正体は人間だった…」というお話の起源等を巡る話（※ネタバレは自己責任で） - Togetter"),
DeliveredArticle("http://kabumatome.doorblog.jp/archives/65921871.html", "トヨタ自動車の内部留保、日本共産党と朝日新聞がナントカ埋蔵金の如く非難 : 市況かぶ全力２階建"),
DeliveredArticle("https://anond.hatelabo.jp/20180629224012", "態度の悪い４０代無職を誰が救えるの？"),
DeliveredArticle("https://www3.nhk.or.jp/news/html/20180701/k10011503031000.html", "経済的に厳しい家庭も保護者の関与で子どもの学力向上 | NHKニュース"),
DeliveredArticle("https://anond.hatelabo.jp/20180701121356", "ぶっちゃけ2人っきりの飲み食い奢られて"),
DeliveredArticle("https://anond.hatelabo.jp/20180701010648", "女医さんがやってきた"),
DeliveredArticle("https://anond.hatelabo.jp/20180701155712", "今日もブクマカは増田叩き"),
DeliveredArticle("https://anond.hatelabo.jp/20180701063754", "あいつが低能先生だった"),
DeliveredArticle("https://togetter.com/li/1242309", "「私28歳。結婚してない恋もしてない。仕事もダメ。なにもない」朝ドラ『半分青い』のセリフが賛否両論　しかもこれの続きは男性を狙った二段構え - Togetter"),
DeliveredArticle("https://anond.hatelabo.jp/20180701012637", "酒が嫌いだというだけの愚痴"),
DeliveredArticle("https://mainichi.jp/articles/20180701/k00/00m/040/093000c", "ＩＴ講師刺殺１週間：ネット憎悪がリアル暴力に - 毎日新聞"),
DeliveredArticle("http://blog.inouetakuya.info/entry/2018/06/30/204134", "49インチ 4K ディスプレイの感想 - 彼女からは、おいちゃんと呼ばれています"),
DeliveredArticle("https://note.mu/psychs/n/nf073d58a2c4e", "Twitter での6年間 2｜Satoshi Nakagawa｜note"),
DeliveredArticle("https://okane.news/financial-movies/", "金融リテラシーが上がる！映画まとめ30選｜投資や経済を楽しく学べる | お得に節約お金ニュース"),
DeliveredArticle("https://samurai20.jp/2018/06/hosyusoku-2/", "保守速報など、まとめサイト群への救済処置（暫定版）【応援する人はシェア】 | 小坪しんやのHP～行橋市議会議員"),
DeliveredArticle("https://anond.hatelabo.jp/20180701000252", "たまごとうふには豆腐成分が一切入ってないと知った時の衝撃は忘れない")
)
)
}
}
}
15 changes: 5 additions & 10 deletions common/SettingsSample.settings
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,15 @@ new SettingsType {
feedUrl = "http://b.hatena.ne.jp/hotentry/all.rss",
threshold = 500,
waitSeconds = 604800,
slack = SlackSettings(
postChannelId = "#channel_id",
userName = "USER NAME",
iconEmoji = ":+1:")
slack = SlackSettings(postChannelId = "#channel_id", userName = "USER NAME", iconEmoji = ":+1:")
),
2.toByte -> WatchSettings(
feedUrl = "http://b.hatena.ne.jp/hotentry/it.rss",
threshold = 100,
waitSeconds = 86400,
slack = SlackSettings(
postChannelId = "#channel_id",
userName = "USER NAME",
iconEmoji = ":+1:")
))
override val slackToken: String = "SLACK_TOKEN"
slack = SlackSettings(postChannelId = "#channel_id", userName = "USER NAME", iconEmoji = ":+1:")
)
)
override val slackToken: String = "SLACK_TOKEN"
override val parallelPostCount: Int = 10
}
33 changes: 18 additions & 15 deletions common/src/main/scala/infrastructure/Repository.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ class Repository extends LazyLogging {
*/
def fetchAllUnprocessed(): Seq[(Long, String, Byte, LocalDateTime)] = {
val articles = TableQuery[Articles]
val query = articles.filter(_.processed === false).map(t => (t.id, t.url, t.settingsId, t.createdAt))
val result = Await.result(db.run(query.result), Duration.Inf)
val query = articles.filter(_.processed === false).map(t => (t.id, t.url, t.settingsId, t.createdAt))
val result = Await.result(db.run(query.result), Duration.Inf)
result.map(r => (r._1, r._2, r._3, r._4.toLocalDateTime))
}

Expand All @@ -41,9 +41,9 @@ class Repository extends LazyLogging {
*/
def existsUrl(url: String, settingsId: Byte): Boolean = {
val articles = TableQuery[Articles]
val q = articles.filter(r => (r.url === url) && (r.settingsId === settingsId)).exists
val action = q.result
val result = db.run(action)
val q = articles.filter(r => (r.url === url) && (r.settingsId === settingsId)).exists
val action = q.result
val result = db.run(action)
Await.result(result, Duration.Inf)
}

Expand All @@ -55,14 +55,17 @@ class Repository extends LazyLogging {
*/
def save(url: String, settingsId: Byte): Either[Throwable, Unit] = {
val articles = TableQuery[Articles]
val date = new java.util.Date()
val insertActions = DBIO.seq(articles += ArticlesRow(
id = 0,
url = url,
settingsId = settingsId,
processed = false,
createdAt = new Timestamp(date.getTime),
updatedAt = new Timestamp(date.getTime)))
val date = new java.util.Date()
val insertActions = DBIO.seq(
articles += ArticlesRow(
id = 0,
url = url,
settingsId = settingsId,
processed = false,
createdAt = new Timestamp(date.getTime),
updatedAt = new Timestamp(date.getTime)
)
)
try {
Right(Await.result(db.run(insertActions.transactionally), Duration.Inf))
} catch {
Expand Down Expand Up @@ -106,8 +109,8 @@ class Repository extends LazyLogging {
* @return 実行結果
*/
def markProcessed(id: Long): Either[Throwable, Unit] = {
val articles = TableQuery[Articles]
val q = for {a <- articles if a.id === id} yield (a.processed, a.updatedAt)
val articles = TableQuery[Articles]
val q = for { a <- articles if a.id === id } yield (a.processed, a.updatedAt)
val updateAction = q.update((true, new Timestamp(System.currentTimeMillis())))

try {
Expand Down
Loading

0 comments on commit a3174eb

Please sign in to comment.