Merge pull request #237 from codeuniversity/clean-twitter

Move twitter models to twitter/models
codeuniversity · Dec 20, 2019 · ab6b64f · ab6b64f
2 parents 8ca575b + adbadfd
commit ab6b64f
Show file tree

Hide file tree

Showing 6 changed files with 9 additions and 9 deletions.
diff --git a/twitter/inserter/posts/inserter.go b/twitter/inserter/posts/inserter.go
@@ -12,7 +12,7 @@ import (
 	_ "github.com/jinzhu/gorm/dialects/postgres"
 
 	dbUtils "github.com/codeuniversity/smag-mvp/db"
-	"github.com/codeuniversity/smag-mvp/models"
+	"github.com/codeuniversity/smag-mvp/twitter/models"
 	"github.com/codeuniversity/smag-mvp/utils"
 	"github.com/codeuniversity/smag-mvp/worker"
 
@@ -40,7 +40,7 @@ func New(postgresHost, postgresPassword, dbName string, qReader *kafka.Reader) *
 
 	db, err := gorm.Open("postgres", connectionString)
 	utils.PanicIfNotNil(err)
-	i.db = db
+	i.db = db // use db.Debug() here to get detailed gorm logs
 
 	db.AutoMigrate(&models.TwitterPost{})
 

diff --git a/twitter/inserter/users/inserter.go b/twitter/inserter/users/inserter.go
@@ -12,7 +12,7 @@ import (
 	_ "github.com/jinzhu/gorm/dialects/postgres"
 
 	dbUtils "github.com/codeuniversity/smag-mvp/db"
-	"github.com/codeuniversity/smag-mvp/models"
+	"github.com/codeuniversity/smag-mvp/twitter/models"
 	"github.com/codeuniversity/smag-mvp/utils"
 	"github.com/codeuniversity/smag-mvp/worker"
 
@@ -40,7 +40,7 @@ func New(postgresHost, postgresPassword, dbName string, qReader *kafka.Reader) *
 
 	db, err := gorm.Open("postgres", connectionString)
 	utils.PanicIfNotNil(err)
-	i.db = db
+	i.db = db // use db.Debug() here to get detailed gorm logs
 
 	db.AutoMigrate(&models.TwitterUser{})
 

diff --git a/models/gorm.go → twitter/models/gorm.go b/models/gorm.go → twitter/models/gorm.go
diff --git a/models/twitter_post.go → twitter/models/twitter_post.go b/models/twitter_post.go → twitter/models/twitter_post.go
@@ -73,18 +73,18 @@ type TwitterPost struct {
 	Place    string
 
 	// content
-	Cashtags    pq.StringArray `gorm:"type:varchar(64)[]"`
-	Hashtags    pq.StringArray `gorm:"type:varchar(64)[]"`
+	Cashtags    pq.StringArray `gorm:"type:text[]"`
+	Hashtags    pq.StringArray `gorm:"type:text[]"`
 	Mentions    []*TwitterUser `gorm:"many2many:post_mentions;association_foreignkey:username;foreignkey:postIdentifier"`
-	Photos      pq.StringArray `gorm:"type:varchar(64)[]"`
+	Photos      pq.StringArray `gorm:"type:text[]"`
 	QuoteURL    string
 	ReplyTo     []*TwitterUser `gorm:"many2many:post_replies;association_foreignkey:username;foreignkey:postIdentifier"`
 	Retweet     bool
 	RetweetDate string
 	RetweetID   string
 	Source      string
 	Tweet       string
-	URLs        pq.StringArray `gorm:"type:varchar(64)[]"`
+	URLs        pq.StringArray `gorm:"type:text[]"`
 	Video       int
 
 	// reactions

diff --git a/models/twitter_user.go → twitter/models/twitter_user.go b/models/twitter_user.go → twitter/models/twitter_user.go
diff --git a/twitter/scraper/Dockerfile b/twitter/scraper/Dockerfile
@@ -8,4 +8,4 @@ RUN pip install -r requirements.txt
 COPY ./twitterscraper ./twitterscraper
 
 ENTRYPOINT [ "python" ]
-CMD [ "-c", "raise Exception('Please set the CMD to either tweets_scraper.py or user_scraper.py')" ]
+CMD [ "-c", "raise Exception('Please set the CMD to either `-m twitterscraper.posts_scraper.py`, `-m twitterscraper.users_scraper.py`, `-m twitterscraper.follwers_scraper.py` or `-m twitterscraper.follwing_scraper.py`')" ]