commit 6eda9b681078ee4c2502776abf46096ffedbeb90
parent dafe2435106c71c9888655cd4a761f4d2871acb4
Author: Russ Cox <rsc@golang.org>
Date: Mon, 24 Oct 2016 10:53:48 -0400
issuedb: add resync for pulling in old events
Diffstat:
| M | issuedb/dash.go | | | 171 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------- |
| M | issuedb/main.go | | | 139 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------- |
2 files changed, 243 insertions(+), 67 deletions(-)
diff --git a/issuedb/dash.go b/issuedb/dash.go
@@ -10,7 +10,9 @@ import (
"log"
"os"
"sort"
+ "strconv"
"strings"
+ "time"
)
type action struct {
@@ -42,41 +44,38 @@ type issueState struct {
waitingForInfo bool
}
-func dashActions() ([]action, int) {
+func dashActions(proj string) ([]action, int) {
var actions []action
var maxIssue int64
- var last int64
- for {
- var all []History
- if err := storage.Select(db, &all, "where RowID > ? order by RowID asc limit 100", last); err != nil {
- log.Fatal("sql: %v", err)
- }
- if len(all) == 0 {
- break
- }
- for _, h := range all {
- if maxIssue < h.Issue {
- maxIssue = h.Issue
- }
- switch h.Action {
- case "issue":
- actions = append(actions, action{h.Time, opCreate, h.Issue, ""})
- case "milestone?", "milestoned":
- if h.Text != "" {
- actions = append(actions, action{h.Time, opMilestone, h.Issue, h.Text})
- }
- case "demilestoned":
- actions = append(actions, action{h.Time, opDemilestone, h.Issue, h.Text})
- case "close?", "closed":
- actions = append(actions, action{h.Time, opClose, h.Issue, ""})
- case "reopened":
- actions = append(actions, action{h.Time, opReopen, h.Issue, ""})
- case "labeled":
- actions = append(actions, action{h.Time, opLabel, h.Issue, h.Text})
- case "unlabeled":
- actions = append(actions, action{h.Time, opUnlabel, h.Issue, h.Text})
+ rows, err := db.Query("select * from History where Project = ? order by Time", proj)
+ if err != nil {
+ log.Fatal("sql: %v", err)
+ }
+ for rows.Next() {
+ var h History
+ if err := rows.Scan(&h.URL, &h.Project, &h.Issue, &h.Time, &h.Who, &h.Action, &h.Text); err != nil {
+ log.Fatal("sql scan History: %v", err)
+ }
+ if maxIssue < h.Issue {
+ maxIssue = h.Issue
+ }
+ switch h.Action {
+ case "issue":
+ actions = append(actions, action{h.Time, opCreate, h.Issue, ""})
+ case "milestone?", "milestoned":
+ if h.Text != "" {
+ actions = append(actions, action{h.Time, opMilestone, h.Issue, h.Text})
}
- last = h.RowID
+ case "demilestoned":
+ actions = append(actions, action{h.Time, opDemilestone, h.Issue, h.Text})
+ case "close?", "closed":
+ actions = append(actions, action{h.Time, opClose, h.Issue, ""})
+ case "reopened":
+ actions = append(actions, action{h.Time, opReopen, h.Issue, ""})
+ case "labeled":
+ actions = append(actions, action{h.Time, opLabel, h.Issue, h.Text})
+ case "unlabeled":
+ actions = append(actions, action{h.Time, opUnlabel, h.Issue, h.Text})
}
}
sort.Stable(actionsByTime(actions))
@@ -139,10 +138,11 @@ func plot(actions []action, maxIssue int, emit func([]issueState, string)) {
const minDate = "2016-04-01"
func dash() {
- actions, maxIssue := dashActions()
+ actions, maxIssue := dashActions("golang/go")
plotRelease(actions, maxIssue, "Go1.8")
plotRelease(actions, maxIssue, "Go1.9")
plotNeeds(actions, maxIssue)
+ plotActivity()
}
func plotRelease(actions []action, maxIssue int, release string) {
@@ -239,3 +239,108 @@ func plotNeeds(actions []action, maxIssue int) {
fmt.Fprintf(&buf, "\n];\n\n")
os.Stdout.Write(buf.Bytes())
}
+
+func plotActivity() {
+ rows, err := db.Query("select Who, count(*) from History where Time >= '2016-04-05' group by Who")
+ if err != nil {
+ log.Fatalf("sql activity: %v", err)
+ }
+ totalWho := map[string]int{}
+ for rows.Next() {
+ var who string
+ var count int
+ if err := rows.Scan(&who, &count); err != nil {
+ log.Fatal("sql scan counts: %v", err)
+ }
+ totalWho[who] += count
+ }
+
+ var allWho []string
+ for who := range totalWho {
+ allWho = append(allWho, who)
+ }
+ sort.Slice(allWho, func(i, j int) bool {
+ ti := totalWho[allWho[i]]
+ tj := totalWho[allWho[j]]
+ if ti != tj {
+ return ti > tj
+ }
+ return allWho[i] < allWho[j]
+ })
+
+ if len(allWho) > 40 {
+ allWho = allWho[:40]
+ }
+ plotActivityCounts("GithubActivityData", "", allWho)
+ for _, action := range []string{"assigned", "closed", "comment", "labeled", "mentioned", "milestoned", "renamed", "subscribed"} {
+ plotActivityCounts("GithubActivityData_"+action, " and Action = '"+action+"'", allWho)
+ }
+}
+
+type weekActivity struct {
+ week string
+ count map[string]int
+}
+
+func plotActivityCounts(name, cond string, allWho []string) {
+ rows, err := db.Query("select strftime('%Y-%W', Time) as Week, Who, count(*) as N from History where Time >= '2016-08-01'" + cond + " group by Week, Who order by Week, Who")
+ if err != nil {
+ log.Fatalf("sql activity counts: %v", err)
+ }
+ thisWeek := ""
+ var weeks []weekActivity
+ for rows.Next() {
+ var count int
+ var week, who string
+ if err := rows.Scan(&week, &who, &count); err != nil {
+ log.Fatalf("sql scan activity: %v", err)
+ }
+ if thisWeek != week {
+ weeks = append(weeks, weekActivity{week: week, count: map[string]int{}})
+ thisWeek = week
+ }
+ w := &weeks[len(weeks)-1]
+ w.count[who] += count
+ }
+
+ var buf bytes.Buffer
+ fmt.Fprintf(&buf, "var %s = ", name)
+ printActivity(&buf, allWho, weeks)
+ os.Stdout.Write(buf.Bytes())
+}
+
+func printActivity(buf *bytes.Buffer, allWho []string, weeks []weekActivity) {
+ fmt.Fprintf(buf, "[\n")
+ fmt.Fprintf(buf, " ['Date'")
+ for _, who := range allWho {
+ fmt.Fprintf(buf, ", '%s'", who)
+ }
+ fmt.Fprintf(buf, "],\n")
+ for _, w := range weeks {
+ fmt.Fprintf(buf, " [%s", weekToDate(w.week))
+ for _, who := range allWho {
+ fmt.Fprintf(buf, ", %d", w.count[who])
+ }
+ fmt.Fprintf(buf, "],\n")
+ }
+ fmt.Fprintf(buf, "];\n\n")
+}
+
+func weekToDate(w string) string {
+ y, err := strconv.Atoi(w[:4])
+ if err != nil {
+ log.Fatalf("bad week %s", w)
+ }
+ ww, err := strconv.Atoi(w[5:])
+ if err != nil {
+ log.Fatalf("bad week %s", w)
+ }
+ now := time.Date(y, time.January, 1, 12, 0, 0, 0, time.UTC)
+ if ww > 0 {
+ for now.Weekday() != time.Monday {
+ now = now.AddDate(0, 0, 1)
+ }
+ now = now.AddDate(0, 0, (ww-1)*7)
+ }
+ return fmt.Sprintf("myDate('%s')", now.Format(time.RFC3339)[:10])
+}
diff --git a/issuedb/main.go b/issuedb/main.go
@@ -39,19 +39,20 @@ type ProjectSync struct {
}
type RawJSON struct {
- RowID int64 `dbstore:",rowid"`
+ URL string `dbstore:",key"`
Project string
+ Issue int64
Type string
JSON []byte `dbstore:",blob"`
}
type History struct {
- RowID int64 `dbstore:",rowid"`
+ URL string `dbstore:",key"`
Project string
Issue int64
Time string
Who string
- Action string
+ Action string `dbstore:",key"`
Text string
}
@@ -70,6 +71,7 @@ Commands are:
init <clientid> <clientsecret> (initialize new database)
add <owner/repo> (add new repository)
sync (sync repositories)
+ resync (full resync to catch very old events)
The default database is $HOME/githubissue.db.
`)
@@ -156,13 +158,13 @@ func main() {
}
return
- case "sync":
+ case "sync", "resync":
var projects []ProjectSync
if err := storage.Select(db, &projects, ""); err != nil {
log.Fatalf("reading projects: %v", err)
}
for _, proj := range projects {
- doSync(&proj)
+ doSync(&proj, args[0] == "resync")
}
case "refill":
@@ -173,11 +175,14 @@ func main() {
}
}
-func doSync(proj *ProjectSync) {
+func doSync(proj *ProjectSync, resync bool) {
println("WOULD SYNC", proj.Name)
syncIssueComments(proj)
syncIssues(proj)
- syncIssueEvents(proj)
+ syncIssueEvents(proj, 0)
+ if resync {
+ syncIssueEventsByIssue(proj)
+ }
}
func syncIssueComments(proj *ProjectSync) {
@@ -203,7 +208,7 @@ func downloadByDate(proj *ProjectSync, api string, since *string, sinceName stri
if api == "/issues/comments" {
delete(values, "per_page")
}
- if *since != "" {
+ if since != nil && *since != "" {
values.Set("since", *since)
}
urlStr := "https://api.github.com/repos/" + proj.Name + api + "?" + values.Encode()
@@ -217,7 +222,10 @@ func downloadByDate(proj *ProjectSync, api string, since *string, sinceName stri
var last string
for _, m := range all {
var meta struct {
- Updated string `json:"updated_at"`
+ URL string
+ Updated string `json:"updated_at"`
+ Number int64 // for /issues feed
+ IssueURL string `json:"issue_url"` // for /issues/comments feed
}
if err := json.Unmarshal(m, &meta); err != nil {
return fmt.Errorf("parsing message: %v", err)
@@ -228,16 +236,32 @@ func downloadByDate(proj *ProjectSync, api string, since *string, sinceName stri
last = meta.Updated
var raw RawJSON
+ raw.URL = meta.URL
raw.Project = proj.Name
+ switch api {
+ default:
+ log.Fatal("downloadByDate: unknown API: %v", api)
+ case "/issues":
+ raw.Issue = meta.Number
+ case "/issues/comments":
+ i := strings.LastIndex(meta.IssueURL, "/")
+ n, err := strconv.ParseInt(meta.IssueURL[i+1:], 10, 64)
+ if err != nil {
+ log.Fatal("cannot find issue number in /issues/comments API: %v", urlStr)
+ }
+ raw.Issue = n
+ }
raw.Type = api
raw.JSON = m
if err := storage.Insert(tx, &raw); err != nil {
return fmt.Errorf("writing JSON to database: %v", err)
}
}
- *since = last
- if err := storage.Write(tx, proj, sinceName); err != nil {
- return fmt.Errorf("updating database metadata: %v", err)
+ if since != nil {
+ *since = last
+ if err := storage.Write(tx, proj, sinceName); err != nil {
+ return fmt.Errorf("updating database metadata: %v", err)
+ }
}
if err := tx.Commit(); err != nil {
return err
@@ -250,7 +274,7 @@ func downloadByDate(proj *ProjectSync, api string, since *string, sinceName stri
}
}
-func syncIssueEvents(proj *ProjectSync) {
+func syncIssueEvents(proj *ProjectSync, id int) {
tx, err := db.Begin()
if err != nil {
log.Fatalf("starting db transaction: %v", err)
@@ -263,7 +287,10 @@ func syncIssueEvents(proj *ProjectSync) {
"page": {"1"},
"per_page": {"100"},
}
- const api = "/issues/events"
+ var api = "/issues/events"
+ if id > 0 {
+ api = fmt.Sprintf("/issues/%d/events", id)
+ }
urlStr := "https://api.github.com/repos/" + proj.Name + api + "?" + values.Encode()
var (
firstID int64
@@ -273,7 +300,11 @@ func syncIssueEvents(proj *ProjectSync) {
err = downloadPages(urlStr, proj.EventETag, func(resp *http.Response, all []json.RawMessage) error {
for _, m := range all {
var meta struct {
- ID int64 `json:"id"`
+ ID int64 `json:"id"`
+ URL string `json:"url"`
+ Issue struct {
+ Number int64
+ }
}
if err := json.Unmarshal(m, &meta); err != nil {
return fmt.Errorf("parsing message: %v", err)
@@ -286,13 +317,19 @@ func syncIssueEvents(proj *ProjectSync) {
firstID = meta.ID
firstETag = resp.Header.Get("Etag")
}
- if proj.EventID != 0 && meta.ID <= proj.EventID {
+ if id == 0 && proj.EventID != 0 && meta.ID <= proj.EventID {
return done
}
var raw RawJSON
+ raw.URL = meta.URL
raw.Project = proj.Name
- raw.Type = api
+ raw.Type = "/issues/events"
+ if id > 0 {
+ raw.Issue = int64(id)
+ } else {
+ raw.Issue = meta.Issue.Number
+ }
raw.JSON = m
if err := storage.Insert(tx, &raw); err != nil {
return fmt.Errorf("writing JSON to database: %v", err)
@@ -304,10 +341,13 @@ func syncIssueEvents(proj *ProjectSync) {
err = nil
}
if err != nil {
+ if strings.Contains(err.Error(), "304 Not Modified") {
+ return
+ }
log.Fatalf("syncing events: %v", err)
}
- if firstID != 0 {
+ if id == 0 && firstID != 0 {
proj.EventID = firstID
proj.EventETag = firstETag
if err := storage.Write(tx, proj, "EventID", "EventETag"); err != nil {
@@ -320,6 +360,30 @@ func syncIssueEvents(proj *ProjectSync) {
}
}
+func syncIssueEventsByIssue(proj *ProjectSync) {
+ rows, err := db.Query("select URL from RawJSON where Type = ? group by URL", "/issues")
+ if err != nil {
+ log.Fatal(err)
+ }
+ var ids []int
+ for rows.Next() {
+ var url string
+ if err := rows.Scan(&url); err != nil {
+ log.Fatal(err)
+ }
+ i := strings.LastIndex(url, "/")
+ id, err := strconv.Atoi(url[i+1:])
+ if err != nil {
+ log.Fatal(url, err)
+ }
+ ids = append(ids, id)
+ }
+ for _, id := range ids {
+ println("ID", id)
+ syncIssueEvents(proj, id)
+ }
+}
+
func downloadPages(url, etag string, do func(*http.Response, []json.RawMessage) error) error {
nfail := 0
for n := 0; url != ""; n++ {
@@ -343,6 +407,17 @@ func downloadPages(url, etag string, do func(*http.Response, []json.RawMessage)
return fmt.Errorf("reading body: %v", err)
}
if resp.StatusCode != 200 {
+ if resp.StatusCode == 403 {
+ if resp.Header.Get("X-Ratelimit-Remaining") == "0" {
+ n, _ := strconv.Atoi(resp.Header.Get("X-Ratelimit-Reset"))
+ if n > 0 {
+ t := time.Unix(int64(n), 0)
+ println("RATELIMIT", t.String())
+ time.Sleep(t.Sub(time.Now()) + 1*time.Minute)
+ goto again
+ }
+ }
+ }
if resp.StatusCode == 500 {
nfail++
if nfail < 2 {
@@ -418,6 +493,8 @@ func js(x interface{}) string {
}
type ghIssueEvent struct {
+ // NOTE: Issue field is not present when downloading for a specific issue,
+ // only in the master feed for the whole repo. So do not add it here.
Actor struct {
Login string `json:"login"`
} `json:"actor"`
@@ -426,11 +503,8 @@ type ghIssueEvent struct {
Name string `json:"name"`
} `json:"label"`
CreatedAt string `json:"created_at"`
- Issue struct {
- Number int64 `json:"number"`
- } `json:"issue"`
- CommitID string `json:"commit_id"`
- Assignee struct {
+ CommitID string `json:"commit_id"`
+ Assignee struct {
Login string `json:"login"`
} `json:"assignee"`
Milestone struct {
@@ -475,22 +549,22 @@ func refill() {
if _, err := db.Exec("delete from History"); err != nil {
log.Fatal(err)
}
- var last int64
+ last := ""
for {
var all []RawJSON
- if err := storage.Select(db, &all, "where RowID > ? order by RowID asc limit 100", last); err != nil {
+ if err := storage.Select(db, &all, "where URL > ? order by URL asc limit 100", last); err != nil {
log.Fatal("sql: %v", err)
}
if len(all) == 0 {
break
}
- println("GOT", len(all), all[0].RowID, all[len(all)-1].RowID)
+ println("GOT", len(all), all[0].URL, all[0].Type, all[len(all)-1].URL, all[len(all)-1].Type)
tx, err := db.Begin()
if err != nil {
log.Fatal(err)
}
for _, m := range all {
- last = m.RowID
+ last = m.URL
switch m.Type {
default:
println("TYPE", m.Type)
@@ -501,9 +575,9 @@ func refill() {
continue
}
var h History
- h.RowID = m.RowID * 10
+ h.URL = m.URL
h.Project = m.Project
- h.Issue = ev.Issue.Number
+ h.Issue = m.Issue
h.Time = ev.CreatedAt
h.Who = ev.Actor.Login
h.Action = ev.Event
@@ -549,7 +623,7 @@ func refill() {
continue
}
var h History
- h.RowID = m.RowID * 10
+ h.URL = m.URL
h.Project = m.Project
h.Issue = n
h.Time = ev.UpdatedAt
@@ -573,7 +647,7 @@ func refill() {
continue
}
var h History
- h.RowID = m.RowID * 10
+ h.URL = m.URL
h.Project = m.Project
h.Issue = n
h.Time = ev.CreatedAt // best we can do
@@ -588,7 +662,6 @@ func refill() {
}
if ev.Assignee.Login != "" {
- h.RowID++
h.Action = "assign?"
h.Text = ev.Assignee.Login
if err := storage.Insert(tx, &h); err != nil {
@@ -596,7 +669,6 @@ func refill() {
}
}
if ev.Milestone.Title != "" {
- h.RowID++
h.Action = "milestone?"
h.Text = ev.Assignee.Login
if err := storage.Insert(tx, &h); err != nil {
@@ -604,7 +676,6 @@ func refill() {
}
}
if ev.State != "open" {
- h.RowID++
h.Action = "close?"
h.Text = ""
if err := storage.Insert(tx, &h); err != nil {
