Initial sync support

atcol · Aug 6, 2020 · e12c664 · e12c664
1 parent dbbd438
commit e12c664
Show file tree

Hide file tree

Showing 7 changed files with 362 additions and 9 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,10 +1,6 @@
-# Generated by Cargo
-# will have compiled files and executables
-/target/
-
-# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
-# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
-Cargo.lock
-
-# These are backup files generated by rustfmt
 **/*.rs.bk
+Cargo.lock
+/target
+*.json
+*.7z
+*.fb
diff --git a/Cargo.toml b/Cargo.toml
@@ -0,0 +1,28 @@
+[package]
+name = "rustywow"
+version = "0.1.0"
+authors = ["Alex <squidnyan@pm.me>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+proptest = "0.10.0"
+actix-web = "2.0.0"
+tokio = { version = "0.2.21", features = ["full"] }
+serde_json = "1.0.56"
+reqwest = { version = "0.10.6", features = ["json"] }
+clap = "3.0.0-beta.1"
+oauth2 = { version = "3.0.0", features = ["futures-03", "reqwest-010"] }
+url = "2.1.1"
+serde = "1.0.114"
+flexbuffers = "0.1.1"
+chrono = "0.4.13"
+async-trait = "0.1.36"
+anyhow = "1.0.31"
+iced = "0.1.1"
+rust-lzma = "0.5.1"
+tokio-timer = "0.2.13"
+config = "0.10.1"
+env_logger = "0.7.1"
+log = "0.4.11"
diff --git a/Makefile b/Makefile
@@ -0,0 +1,36 @@
+DIR_BUILD     := target
+DIR_RELEASE   := ${DIR_BUILD}/release
+BIN_NAME      := "!set_this!"
+.DEFAULT_GOAL := all
+
+.PHONY: all
+all : build test
+
+.PHONY : ubuntu
+ubuntu : 
+	sudo apt-get install vflib3 vflib3-dev cmake liblzma-dev
+
+.PHONY: init
+init :
+	rustup toolchain install nightly
+	rustup override set nightly
+	rustup component add clippy
+	rustup component add rustfmt
+	cargo install cargo-watch
+	cargo install cargo-edit
+	cargo install cargo-tarpaulin
+	cargo install cargo-audit
+	cargo install cargo-outdated
+
+.PHONY: build
+build ${DIR_RELEASE}/${BIN_NAME} :
+	cargo build --release
+
+.PHONY: test
+test :
+	cargo test --verbose && \
+	  cargo rustdoc
+
+.PHONY: watch
+watch :
+	cargo-watch -x "test && cargo rustdoc"
diff --git a/Settings.toml b/Settings.toml
@@ -0,0 +1,5 @@
+realm_id = 1403
+client_id = "4b0ececf9c4248e5be5296669b39adda"
+client_secret = "3GBM4ZkzwsPsoC6mNxMfbuGlx4O3f2Dm"
+data_dir = "./data"
+delay_mins = 5
diff --git a/src/lib.rs b/src/lib.rs
@@ -0,0 +1,162 @@
+pub mod realm;
+use async_trait::async_trait;
+use chrono::{DateTime, Duration, Utc};
+use clap::Clap;
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Deserialize)]
+pub struct Settings {
+    /// Client access identifier
+    pub client_id: String,
+
+    /// Secret
+    pub client_secret: String,
+
+    /// The realm id, e.g. 1403 = Draenor
+    pub realm_id: u16,
+
+    /// The parent directory for all data
+    pub data_dir: String,
+
+    /// The time to delay between re-sync'ing data
+    pub delay_mins: u64,
+
+}
+
+impl Settings {
+
+    pub fn new() -> Result<Self, config::ConfigError> {
+        let mut settings = config::Config::default();
+        settings
+            // Add in `./Settings.toml`
+            .merge(config::File::with_name("Settings")).unwrap()
+            // Add in settings from the environment (with a prefix of APP)
+            // E.g. `APP_DEBUG=1 ./target/app` would set the `debug` key
+            .merge(config::Environment::with_prefix("APP")).unwrap();
+        settings.try_into()
+    }
+}
+
+#[derive(Clap, Clone)]
+#[clap(version = "1.0", author = "Alex Collins")]
+pub struct Opts {
+
+    /// The command
+    #[clap(subcommand)]
+    pub cmd: SubCmd,
+}
+
+#[derive(Clap, Clone)]
+pub enum SubCmd {
+    /// List auction house data
+    #[clap()]
+    Sync,
+    /// Load in to the database
+    Load(Database),
+}
+
+#[derive(Clap, Clone)]
+/// Load the dataset in to the database
+pub struct Database {
+    /// Load the data dumps from `--data-dir` into the database via `pg_string`
+    #[clap(long)]
+    pub pg_string: String,
+}
+
+/// An period of authenticated interaction with the battle.net APIs
+#[derive(Debug, Clone)]
+pub struct Session {
+    /// When the session opened, where `start_time + auth.expires_in / 60` = expired_date
+    start_time: DateTime<Utc>,
+
+    /// The client identifier
+    client_id: String,
+
+    /// Secret
+    client_secret: String,
+
+    /// The realm id, e.g. 1403 = Draenor
+    realm_id: u16,
+
+    auth: Auth,
+}
+
+impl Session {
+    pub fn has_expired(&self) -> bool {
+        (self.start_time + Duration::seconds(self.auth.expires_in.into())) < Utc::now()
+    }
+
+    fn auction_url(&self) -> String {
+        let url = format!("https://eu.api.blizzard.com/data/wow/connected-realm/{}/auctions?namespace=dynamic-eu&locale=en_US&access_token={}", self.realm_id, self.auth.access_token);
+        println!("url: {:?}", url);
+        url
+    }
+}
+
+/// See https://develop.battle.net/documentation/guides/using-oauth/client-credentials-flow
+/// curl -u {client_id}:{client_secret} -d grant_type=client_credentials https://us.battle.net/oauth/token
+pub async fn authenticate(
+    client_id: String,
+    client_secret: String,
+) -> Result<Auth, reqwest::Error> {
+    let client = reqwest::Client::new();
+    let auth = client
+        .post("https://eu.battle.net/oauth/token")
+        .basic_auth(client_id, Some(client_secret))
+        .query(&[("grant_type", "client_credentials")])
+        .send()
+        .await?
+        .text()
+        .await?;
+
+    println!("Response: {:?}", auth);
+    Ok(serde_json::from_str(&auth).expect("Failed parsing auth response"))
+}
+
+/// Authenticate and initiate a `Session`
+pub async fn get_session(opts: Settings) -> Result<Session, reqwest::Error> {
+    Ok(Session {
+        start_time: Utc::now(),
+        auth: authenticate(opts.client_id.clone(), opts.client_secret.clone()).await?,
+        client_id: opts.client_id,
+        client_secret: opts.client_secret,
+        realm_id: opts.realm_id,
+    })
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Auth {
+    access_token: String,
+
+    token_type: String,
+
+    expires_in: u32,
+    /// Optional scoping parameter e.g. wow.profile
+    scope: Option<String>,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub enum Error {
+    ApiFailure(String),
+    AuctionLookup(&'static str),
+    ConfigError(String),
+    IOError(String),
+}
+
+impl From<reqwest::Error> for Error {
+    fn from(e: reqwest::Error) -> Self {
+        Error::ApiFailure(format!("{:?}", e))
+    }
+}
+
+impl From<std::io::Error> for Error {
+    fn from(e: std::io::Error) -> Self {
+        Error::IOError(format!("{:?}", e))
+    }
+}
+
+impl From<config::ConfigError> for Error {
+    fn from(e: config::ConfigError) -> Self {
+        Error::ConfigError(format!("Configuration error - {:?}", e))
+    }
+}
diff --git a/src/main.rs b/src/main.rs
@@ -0,0 +1,61 @@
+use chrono::Utc;
+use clap::Clap;
+use lzma::compress;
+use rustywow::realm::{AuctionResponse, Realm};
+use rustywow::{get_session, Settings, Error, Opts, Session, SubCmd};
+use serde::ser::Serialize;
+use std::fs::File;
+use std::io::Write;
+use tokio::time::{delay_for, Duration};
+use tokio_timer::*;
+use log::{info,};
+
+#[tokio::main]
+async fn main() -> Result<(), Error> {
+    env_logger::init_from_env(
+        env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+    );
+
+    let settings = Settings::new()?;
+    let opts = Opts::parse();
+    match opts.cmd {
+        SubCmd::Sync => {
+            info!("Spawning auction thread");
+            tokio::spawn(async move {
+                loop {
+                    run(settings.clone()).await;
+                    delay_for(Duration::from_secs(60 * settings.delay_mins)).await;
+                }
+            })
+            .await;
+        },
+        SubCmd::Load(db) => {
+            info!("Loading to {} from {}", db.pg_string, settings.data_dir);      
+        },
+    }
+    Ok(())
+}
+
+async fn run(settings: Settings) -> Result<(), Error> {
+    let session = get_session(settings.clone())
+        .await
+        .expect("Failed to authenticate");
+    info!("Loading auctions");
+    let auc = session.auctions().await?;
+    save_auctions(settings.data_dir.clone().to_string(), &auc).await?;
+    Ok(())
+}
+
+async fn save_auctions(data_dir: String, auc: &AuctionResponse) -> Result<(), Error> {
+    info!("Saving auctions");
+    let mut s = flexbuffers::FlexbufferSerializer::new();
+    auc.serialize(&mut s).unwrap();
+    let timestamp = Utc::now().format("%+");
+    let mut raw = File::create(format!("{}/{}.fb", data_dir, timestamp.to_string()))?;
+    raw.write_all(s.view())?;
+
+    let mut compressed = File::create(format!("{}/{}.7z", data_dir, timestamp.to_string()))?;
+    compressed.write_all(&compress(s.view(), 9).expect("Failed to compress"))?;
+    info!("Auctions saved {}", timestamp.to_string());
+    Ok(())
+}
diff --git a/src/realm.rs b/src/realm.rs
@@ -0,0 +1,65 @@
+use crate::{Error, Session};
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+
+/// A WoW realm
+#[async_trait]
+pub trait Realm {
+    async fn auctions(&self) -> Result<AuctionResponse, Error>;
+}
+
+#[async_trait]
+impl Realm for Session {
+    async fn auctions(&self) -> Result<AuctionResponse, Error> {
+        let aurl = &self.auction_url();
+        let res = reqwest::get(aurl).await?;
+        match res.status() {
+            reqwest::StatusCode::OK => {
+                let ahd: AuctionResponse = res.json().await?;
+                println!("{:?}", ahd.auctions.len());
+                Ok(ahd)
+            }
+            sc => {
+                println!("Unexpected response status code: {:?}", sc);
+                Err(Error::AuctionLookup("Auction look-up failed"))
+            }
+        }
+    }
+}
+
+/// A link to the realm details
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct ConnectedRealmLink {
+    href: String,
+}
+
+/// The parent type for all auctions
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct AuctionResponse {
+    connected_realm: ConnectedRealmLink,
+    auctions: Vec<Auction>,
+}
+
+/// An individual auction
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct Auction {
+    id: u64,
+    item: Item,
+    buyout: Option<u64>,
+    quantity: u16,
+    time_left: AuctionTime,
+}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub enum AuctionTime {
+    SHORT,
+    MEDIUM,
+    LONG,
+    VERY_LONG,
+}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct Item {
+    id: u64,
+    context: Option<u16>,
+}