diff --git a/functions/query_mentions/main.go b/functions/query_mentions/main.go index f38bc69..41ffc36 100644 --- a/functions/query_mentions/main.go +++ b/functions/query_mentions/main.go @@ -50,7 +50,8 @@ func QueryMentions(twttr svc.Twitter, snsClient svc.SNSType) error { } billUrl, _ := tweetBill.SearchBill() tweetBill.URL = billUrl - cls, actions, _ := tweetBill.FetchBillData() + title, cls, actions, _ := tweetBill.FetchBillData() + tweetBill.Title = title tweetBill.Classification = cls actionJson, _ := json.Marshal(actions) tweetBill.Data = string(actionJson) diff --git a/functions/update_bill/main.go b/functions/update_bill/main.go index 06556d1..856a6a7 100644 --- a/functions/update_bill/main.go +++ b/functions/update_bill/main.go @@ -59,7 +59,7 @@ func handler(request events.SNSEvent) error { } // Get new data for bill, check if it's changed - _, actions, err := bill.FetchBillData() + _, _, actions, err := bill.FetchBillData() if err != nil { return err } diff --git a/pkg/models/bill.go b/pkg/models/bill.go index f851172..12d51dd 100644 --- a/pkg/models/bill.go +++ b/pkg/models/bill.go @@ -27,6 +27,7 @@ type Bill struct { TweetText string `gorm:"size:300" json:"tweet_text"` LastTweetID *int64 `json:"last_tweet_id,omitempty"` BillID string `gorm:"size:25" json:"id,omitempty"` + Title string `gorm:"size:250" json:"title"` Classification string `gorm:"size:250" json:"classification"` URL string `gorm:"size:250" json:"url"` Active bool `gorm:"default:true"` @@ -110,22 +111,23 @@ func (b *Bill) SearchBill() (string, error) { return fmt.Sprintf("https://chicago.legistar.com/%s", billUrl), nil } -func (b *Bill) FetchBillData() (string, []LegistarAction, error) { +func (b *Bill) FetchBillData() (string, string, []LegistarAction, error) { var actions []LegistarAction response, err := http.Get(b.URL) if err != nil { - return "", actions, err + return "", "", actions, err } defer response.Body.Close() document, err := goquery.NewDocumentFromReader(response.Body) if err != nil { - return "", actions, err + return "", "", actions, err } - status := strings.TrimSpace(document.Find("#ctl00_ContentPlaceHolder1_lblStatus2").First().Text()) + title := strings.TrimSpace(document.Find("#ctl00_ContentPlaceHolder1_lblTitle2").First().Text()) classification := strings.TrimSpace(document.Find("#ctl00_ContentPlaceHolder1_lblType2").First().Text()) + status := strings.TrimSpace(document.Find("#ctl00_ContentPlaceHolder1_lblStatus2").First().Text()) committee := strings.TrimSpace(document.Find("#ctl00_ContentPlaceHolder1_hypInControlOf2").First().Text()) document.Find(".rgMasterTable tbody tr").Each(func(index int, element *goquery.Selection) { @@ -147,50 +149,74 @@ func (b *Bill) FetchBillData() (string, []LegistarAction, error) { }) actions = append(actions, action) }) - return classification, actions, nil + return title, classification, actions, nil } func (b *Bill) CreateTweet() string { billId := b.GetCleanBillID() - billCls := b.Classification + billTitle := b.Title actions := b.GetActions() - if billCls != "" { - billCls = fmt.Sprintf("%s ", billCls) + if billTitle != "" { + billTitle = fmt.Sprintf("%s: %s", billId, billTitle) + } else { + billTitle = billId } - if len(actions) == 0 { - return fmt.Sprintf("%s%s. See more at %s #%s", billCls, billId, b.URL, b.BillID) + const TWEET_LEN = 280 // Max tweet length + const URL_LEN = 23 // Twitter cap for URL characters + baseChars := len(fmt.Sprintf(" See more at #%s", b.BillID)) + URL_LEN + + var actionStr string + var actionText string + var action LegistarAction + + // Pull the first action which is the most recent on Legistar, otherwise leave empty + if len(actions) > 0 { + action = actions[0] + actionStr = action.Action } - // Pull the first action which is the most recent on Legistar - action := actions[0] - actionText := fmt.Sprintf("%s%s", billCls, billId) - switch cls := action.Action; cls { + + switch cls := actionStr; cls { case "Introduced", "Direct Introduction": - actionText = fmt.Sprintf("%s%s was introduced in %s", billCls, billId, action.Actor) + actionText = fmt.Sprintf(" was introduced in %s", action.Actor) case "Placed on File": - actionText = fmt.Sprintf("%s%s was placed on file", billCls, billId) + actionText = " was placed on file" case "Referred", "Re-Referred": if action.Committee != "" { - actionText = fmt.Sprintf("%s%s was referred to the %s", billCls, billId, action.Committee) + actionText = fmt.Sprintf(" was referred to the %s", action.Committee) } else { - actionText = fmt.Sprintf("%s%s was referred to committee", billCls, billId) + actionText = " was referred to committee" } case "Recommended for Passage": - actionText = fmt.Sprintf("%s%s was recommended to pass by the %s", billCls, billId, action.Actor) + actionText = fmt.Sprintf(" was recommended to pass by the %s", action.Actor) case "Recommended Do Not Pass": - actionText = fmt.Sprintf("%s%s was recommended not to pass by the %s", billCls, billId, action.Actor) + actionText = fmt.Sprintf(" was recommended not to pass by the %s", action.Actor) case "Recommended for Re-referral": - actionText = fmt.Sprintf("%s%s was recommended for re-referral by the %s", billCls, billId, action.Actor) + actionText = fmt.Sprintf(" was recommended for re-referral by the %s", action.Actor) case "Passed", "Passed as Substitute": - actionText = fmt.Sprintf("%s%s passed", billCls, billId) + actionText = " passed" case "Failed to Pass": - actionText = fmt.Sprintf("%s%s failed to pass", billCls, billId) + actionText = " failed to pass" case "Introduced (Agreed Calendar)", "Adopted": - actionText = fmt.Sprintf("%s%s was adopted", billCls, billId) + actionText = " was adopted" case "Approved", "Repealed", "Vetoed", "Tabled", "Withdrawn": - actionText = fmt.Sprintf("%s%s was %s", billCls, billId, strings.ToLower(cls)) + actionText = fmt.Sprintf("was %s", strings.ToLower(cls)) + } + + tweetContent := fmt.Sprintf("%s%s.", billTitle, actionText) + if len(tweetContent)+baseChars > TWEET_LEN { + // Get the difference to remove (add characters for ellipsis) + tweetDiff := (baseChars + len(tweetContent) + 3) - TWEET_LEN + var ellipsis string + // Only include 2 periods for ellipsis if no action text, since it ends with a period + if actionText == "" { + ellipsis = ".." + } else { + ellipsis = "..." + } + tweetContent = fmt.Sprintf("%s%s%s.", strings.TrimSpace(billTitle[0:len(billTitle)-tweetDiff]), ellipsis, actionText) } - return fmt.Sprintf("%s. See more at %s #%s", actionText, b.URL, b.BillID) + return fmt.Sprintf("%s See more at %s #%s", tweetContent, b.URL, b.BillID) } func (b *Bill) SetNextRun() { diff --git a/pkg/models/bill_test.go b/pkg/models/bill_test.go index 9580973..fd899ee 100644 --- a/pkg/models/bill_test.go +++ b/pkg/models/bill_test.go @@ -61,41 +61,56 @@ func TestSetNextRun(t *testing.T) { func TestCreateTweet(t *testing.T) { bill := Bill{ + Title: "Testing bill", Classification: "Ordinance", URL: "https://chicago.legistar.com", BillID: "O201011", Data: `[]`, } tweetEnd := "See more at https://chicago.legistar.com #O201011" - if bill.CreateTweet() != fmt.Sprintf("Ordinance O2010-11. %s", tweetEnd) { + if bill.CreateTweet() != fmt.Sprintf("O2010-11: Testing bill. %s", tweetEnd) { t.Errorf("Tweet with no actions is incorrect: %s", bill.CreateTweet()) } bill.Data = `[{"action": "fake"}]` - if bill.CreateTweet() != fmt.Sprintf("Ordinance O2010-11. %s", tweetEnd) { + if bill.CreateTweet() != fmt.Sprintf("O2010-11: Testing bill. %s", tweetEnd) { t.Errorf("Tweet with invalid action is incorrect: %s", bill.CreateTweet()) } bill.Data = `[{"action": "Introduced", "actor": "Chicago City Council"}]` - if bill.CreateTweet() != fmt.Sprintf("Ordinance O2010-11 was introduced in Chicago City Council. %s", tweetEnd) { + if bill.CreateTweet() != fmt.Sprintf("O2010-11: Testing bill was introduced in Chicago City Council. %s", tweetEnd) { t.Errorf("Tweet for introduction is incorrect: %s", bill.CreateTweet()) } bill.Data = `[{"action": "Referred"}]` - if bill.CreateTweet() != fmt.Sprintf("Ordinance O2010-11 was referred to committee. %s", tweetEnd) { + if bill.CreateTweet() != fmt.Sprintf("O2010-11: Testing bill was referred to committee. %s", tweetEnd) { t.Errorf("Tweet for referral with no entity is incorrect: %s", bill.CreateTweet()) } bill.Data = `[{"action": "Referred", "actor": "", "committee": "Test Committee"}]` - if bill.CreateTweet() != fmt.Sprintf("Ordinance O2010-11 was referred to the Test Committee. %s", tweetEnd) { + if bill.CreateTweet() != fmt.Sprintf("O2010-11: Testing bill was referred to the Test Committee. %s", tweetEnd) { t.Errorf("Tweet for referral with committee is incorrect: %s", bill.CreateTweet()) } bill.Data = `[{"action": "Recommended for Passage", "actor": "Test Committee"}]` - if bill.CreateTweet() != fmt.Sprintf("Ordinance O2010-11 was recommended to pass by the Test Committee. %s", tweetEnd) { + if bill.CreateTweet() != fmt.Sprintf("O2010-11: Testing bill was recommended to pass by the Test Committee. %s", tweetEnd) { t.Errorf("Tweet for committee passage is incorrect: %s", bill.CreateTweet()) } bill.Data = `[{"action": "Passed"}]` - if bill.CreateTweet() != fmt.Sprintf("Ordinance O2010-11 passed. %s", tweetEnd) { + if bill.CreateTweet() != fmt.Sprintf("O2010-11: Testing bill passed. %s", tweetEnd) { t.Errorf("Tweet for passage is incorrect: %s", bill.CreateTweet()) } bill.Data = `[{"action": "Passed"}, {"action": "Referred"}]` - if bill.CreateTweet() != fmt.Sprintf("Ordinance O2010-11 passed. %s", tweetEnd) { + if bill.CreateTweet() != fmt.Sprintf("O2010-11: Testing bill passed. %s", tweetEnd) { t.Errorf("Tweet for most recent action is incorrect: %s", bill.CreateTweet()) } + bill.Title = "" + if bill.CreateTweet() != fmt.Sprintf("O2010-11 passed. %s", tweetEnd) { + t.Errorf("Tweet for bill without title is incorrect: %s", bill.CreateTweet()) + } + bill.Data = `[{"action": "Recommended for Passage", "actor": "Committee on Ethics and Government Oversight"}]` + bill.Title = "Certification of city funding requirement for Laborers' and Retirement Board Employees Annuity and Benefit Fund of Chicago for tax year 2020, payment year 2021" + if bill.CreateTweet() != fmt.Sprintf("O2010-11: Certification of city funding requirement for Laborers' and Retirement Board Employees Annuity and Benefit Fund of Chicago for tax year 2020, pay... was recommended to pass by the Committee on Ethics and Government Oversight. %s", tweetEnd) { + t.Errorf("Clipped title tweet text with action is incorrect: %s", bill.CreateTweet()) + } + bill.Title = "Certification of city funding requirement for Laborers' and Retirement Board Employees Annuity and Benefit Fund of Chicago for tax year 2020, payment year 2021 " + bill.Data = `[]` + if bill.CreateTweet() != fmt.Sprintf("O2010-11: Certification of city funding requirement for Laborers' and Retirement Board Employees Annuity and Benefit Fund of Chicago for tax year 2020, payment year 2021... %s", tweetEnd) { + t.Errorf("Clipped title tweet text with no actions is incorrect: %s", bill.CreateTweet()) + } }