commit 543888eeaf89c38d330d82a92d8336bfac1b5b1d
parent 01e19ce0ac606b69f74a551707aca0cca5c8c00f
Author: Russ Cox <rsc@golang.org>
Date: Wed, 19 Jun 2019 15:51:44 -0400
issuedb: revise database sync strategy yet again
Now just store the raw JSON and never try to distill in the database.
Let programs do what they want with the raw JSON.
Add todo generator using raw JSON.
Drop dashboard generator.
Diffstat:
| D | issuedb/dash.go | | | 346 | ------------------------------------------------------------------------------- |
| M | issuedb/main.go | | | 273 | ++++++++++++++++++++++++++++++++----------------------------------------------- |
| A | issuedb/todo.go | | | 305 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
3 files changed, 416 insertions(+), 508 deletions(-)
diff --git a/issuedb/dash.go b/issuedb/dash.go
@@ -1,346 +0,0 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
- "bytes"
- "fmt"
- "log"
- "os"
- "sort"
- "strconv"
- "strings"
- "time"
-)
-
-type action struct {
- time string
- op int
- number int64
- text string
-}
-
-const (
- _ = iota
- opCreate
- opMilestone
- opDemilestone
- opClose
- opReopen
- opLabel
- opUnlabel
-)
-
-type issueState struct {
- createTime string
- closeTime string
- milestone string
- needsInvestigation bool
- needsFix bool
- needsDecision bool
- blocked bool
- waitingForInfo bool
-}
-
-func dashActions(proj string) ([]action, int) {
- var actions []action
- var maxIssue int64
- rows, err := db.Query("select * from History where Project = ? order by Time", proj)
- if err != nil {
- log.Fatal("sql: %v", err)
- }
- for rows.Next() {
- var h History
- if err := rows.Scan(&h.URL, &h.Project, &h.Issue, &h.Time, &h.Who, &h.Action, &h.Text); err != nil {
- log.Fatal("sql scan History: %v", err)
- }
- if maxIssue < h.Issue {
- maxIssue = h.Issue
- }
- switch h.Action {
- case "issue":
- actions = append(actions, action{h.Time, opCreate, h.Issue, ""})
- case "milestone?", "milestoned":
- if h.Text != "" {
- actions = append(actions, action{h.Time, opMilestone, h.Issue, h.Text})
- }
- case "demilestoned":
- actions = append(actions, action{h.Time, opDemilestone, h.Issue, h.Text})
- case "close?", "closed":
- actions = append(actions, action{h.Time, opClose, h.Issue, ""})
- case "reopened":
- actions = append(actions, action{h.Time, opReopen, h.Issue, ""})
- case "labeled":
- actions = append(actions, action{h.Time, opLabel, h.Issue, h.Text})
- case "unlabeled":
- actions = append(actions, action{h.Time, opUnlabel, h.Issue, h.Text})
- }
- }
- sort.Stable(actionsByTime(actions))
- return actions, int(maxIssue)
-}
-
-type actionsByTime []action
-
-func (x actionsByTime) Len() int { return len(x) }
-func (x actionsByTime) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
-func (x actionsByTime) Less(i, j int) bool { return x[i].time < x[j].time }
-
-func plot(actions []action, maxIssue int, emit func([]issueState, string)) {
- var lastTime string
- state := make([]issueState, maxIssue+1)
- for _, a := range actions {
- thisTime := a.time[:10]
- if thisTime != lastTime {
- if lastTime != "" {
- emit(state, lastTime)
- }
- lastTime = thisTime
- }
- s := &state[a.number]
- switch a.op {
- case opCreate:
- s.createTime = a.time
- case opMilestone:
- s.milestone = a.text
- case opDemilestone:
- if s.milestone == a.text {
- s.milestone = ""
- }
- case opClose:
- s.closeTime = a.time
- case opReopen:
- s.closeTime = ""
- case opLabel, opUnlabel:
- var setting *bool
- switch a.text {
- case "NeedsInvestigation":
- setting = &s.needsInvestigation
- case "NeedsFix":
- setting = &s.needsFix
- case "NeedsDecision":
- setting = &s.needsDecision
- case "WaitingForInfo":
- setting = &s.waitingForInfo
- }
- if setting != nil {
- *setting = a.op == opLabel
- }
- }
- }
- if lastTime != "" {
- emit(state, lastTime)
- }
-}
-
-const minDate = "2016-04-01"
-
-func dash() {
- actions, maxIssue := dashActions("golang/go")
- plotRelease(actions, maxIssue, "Go1.8")
- plotRelease(actions, maxIssue, "Go1.9")
- plotNeeds(actions, maxIssue)
- plotActivity()
-}
-
-func plotRelease(actions []action, maxIssue int, release string) {
- releaseEarly := release + "Early"
- releaseMaybe := release + "Maybe"
-
- var buf bytes.Buffer
- fmt.Fprintf(&buf, "var %sData = [", strings.Replace(release, ".", "", -1))
- fmt.Fprintf(&buf, " ['Date', 'No Milestone', '%s', '%s', '%s']", releaseEarly, release, releaseMaybe)
- plot(actions, maxIssue, func(issues []issueState, time string) {
- if time < minDate {
- return
- }
- var numNone, numRelease, numReleaseEarly, numReleaseMaybe int
- for id := range issues {
- issue := &issues[id]
- if issue.createTime == "" || issue.closeTime != "" {
- continue
- }
- switch issue.milestone {
- case "":
- if time == "2016-10-05" {
- println("NONE", id)
- }
- numNone++
- case release:
- numRelease++
- case releaseEarly:
- numReleaseEarly++
- case releaseMaybe:
- numReleaseMaybe++
- }
- }
- fmt.Fprintf(&buf, ",\n [myDate(\"%s\"), %d, %d, %d, %d]", time, numNone, numReleaseEarly, numRelease, numReleaseMaybe)
- })
- fmt.Fprintf(&buf, "\n];\n\n")
- os.Stdout.Write(buf.Bytes())
-}
-
-func plotNeeds(actions []action, maxIssue int) {
- var buf bytes.Buffer
- fmt.Fprintf(&buf, "var TriageData = [")
- fmt.Fprintf(&buf, " ['Date', 'Triage', 'NeedsInvestigation', 'NeedsInvestigation+Waiting', 'NeedsInvestigation+Blocked', 'NeedsDecision', 'NeedsDecision+Waiting', 'NeedsDecision+Blocked', 'NeedsFix', 'NeedsFix+Waiting', 'NeedsFix+Blocked']")
- plot(actions, maxIssue, func(issues []issueState, time string) {
- if time < minDate {
- return
- }
- const (
- triage = iota
- needsInvestigation
- needsInvestigationWaitingForInfo
- needsInvestigationBlocked
- needsDecision
- needsDecisionWaitingForInfo
- needsDecisionBlocked
- needsFix
- needsFixWaitingForInfo
- needsFixBlocked
- maxCount
- )
- var count [maxCount]int
- for id := range issues {
- issue := &issues[id]
- if issue.createTime == "" || issue.closeTime != "" {
- continue
- }
- if issue.milestone != "" && !strings.HasPrefix(issue.milestone, "Go1.8") {
- continue
- }
- ix := triage
- switch {
- case issue.needsInvestigation:
- ix = needsInvestigation
- case issue.needsDecision:
- ix = needsDecision
- case issue.needsFix:
- ix = needsFix
- }
- if ix != triage {
- if issue.waitingForInfo {
- ix += 1
- } else if issue.blocked {
- ix += 2
- }
- }
- count[ix]++
- }
- fmt.Fprintf(&buf, ",\n [myDate(\"%s\")", time)
- for _, x := range count {
- fmt.Fprintf(&buf, ", %d", x)
- }
- fmt.Fprintf(&buf, "]")
- })
- fmt.Fprintf(&buf, "\n];\n\n")
- os.Stdout.Write(buf.Bytes())
-}
-
-func plotActivity() {
- rows, err := db.Query("select Who, count(*) from History where Time >= '2016-04-05' group by Who")
- if err != nil {
- log.Fatalf("sql activity: %v", err)
- }
- totalWho := map[string]int{}
- for rows.Next() {
- var who string
- var count int
- if err := rows.Scan(&who, &count); err != nil {
- log.Fatal("sql scan counts: %v", err)
- }
- totalWho[who] += count
- }
-
- var allWho []string
- for who := range totalWho {
- allWho = append(allWho, who)
- }
- sort.Slice(allWho, func(i, j int) bool {
- ti := totalWho[allWho[i]]
- tj := totalWho[allWho[j]]
- if ti != tj {
- return ti > tj
- }
- return allWho[i] < allWho[j]
- })
-
- if len(allWho) > 40 {
- allWho = allWho[:40]
- }
- plotActivityCounts("GithubActivityData", "", allWho)
- for _, action := range []string{"assigned", "closed", "comment", "labeled", "mentioned", "milestoned", "renamed", "subscribed"} {
- plotActivityCounts("GithubActivityData_"+action, " and Action = '"+action+"'", allWho)
- }
-}
-
-type weekActivity struct {
- week string
- count map[string]int
-}
-
-func plotActivityCounts(name, cond string, allWho []string) {
- rows, err := db.Query("select strftime('%Y-%W', Time) as Week, Who, count(*) as N from History where Time >= '2016-08-01'" + cond + " group by Week, Who order by Week, Who")
- if err != nil {
- log.Fatalf("sql activity counts: %v", err)
- }
- thisWeek := ""
- var weeks []weekActivity
- for rows.Next() {
- var count int
- var week, who string
- if err := rows.Scan(&week, &who, &count); err != nil {
- log.Fatalf("sql scan activity: %v", err)
- }
- if thisWeek != week {
- weeks = append(weeks, weekActivity{week: week, count: map[string]int{}})
- thisWeek = week
- }
- w := &weeks[len(weeks)-1]
- w.count[who] += count
- }
-
- var buf bytes.Buffer
- fmt.Fprintf(&buf, "var %s = ", name)
- printActivity(&buf, allWho, weeks)
- os.Stdout.Write(buf.Bytes())
-}
-
-func printActivity(buf *bytes.Buffer, allWho []string, weeks []weekActivity) {
- fmt.Fprintf(buf, "[\n")
- fmt.Fprintf(buf, " ['Date'")
- for _, who := range allWho {
- fmt.Fprintf(buf, ", '%s'", who)
- }
- fmt.Fprintf(buf, "],\n")
- for _, w := range weeks {
- fmt.Fprintf(buf, " [%s", weekToDate(w.week))
- for _, who := range allWho {
- fmt.Fprintf(buf, ", %d", w.count[who])
- }
- fmt.Fprintf(buf, "],\n")
- }
- fmt.Fprintf(buf, "];\n\n")
-}
-
-func weekToDate(w string) string {
- y, err := strconv.Atoi(w[:4])
- if err != nil {
- log.Fatalf("bad week %s", w)
- }
- ww, err := strconv.Atoi(w[5:])
- if err != nil {
- log.Fatalf("bad week %s", w)
- }
- now := time.Date(y, time.January, 1, 12, 0, 0, 0, time.UTC)
- if ww > 0 {
- for now.Weekday() != time.Monday {
- now = now.AddDate(0, 0, 1)
- }
- now = now.AddDate(0, 0, (ww-1)*7)
- }
- return fmt.Sprintf("myDate('%s')", now.Format(time.RFC3339)[:10])
-}
diff --git a/issuedb/main.go b/issuedb/main.go
@@ -1,3 +1,7 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
package main
import (
@@ -44,16 +48,7 @@ type RawJSON struct {
Issue int64
Type string
JSON []byte `dbstore:",blob"`
-}
-
-type History struct {
- URL string `dbstore:",key"`
- Project string
- Issue int64
Time string
- Who string
- Action string `dbstore:",key"`
- Text string
}
var (
@@ -85,7 +80,6 @@ func main() {
storage.Register(new(Auth))
storage.Register(new(ProjectSync))
storage.Register(new(RawJSON))
- storage.Register(new(History))
flag.Usage = usage
flag.Parse()
@@ -164,24 +158,62 @@ func main() {
log.Fatalf("reading projects: %v", err)
}
for _, proj := range projects {
- doSync(&proj, args[0] == "resync")
+ if match(proj.Name, args[1:]) {
+ doSync(&proj, args[0] == "resync")
+ }
+ }
+ for _, arg := range args[1:] {
+ if arg != didArg {
+ log.Printf("unknown project: %s", arg)
+ }
+ }
+
+ case "retime":
+ retime()
+
+ case "todo":
+ var projects []ProjectSync
+ if err := storage.Select(db, &projects, ""); err != nil {
+ log.Fatalf("reading projects: %v", err)
+ }
+ for _, proj := range projects {
+ if match(proj.Name, args[1:]) {
+ todo(&proj)
+ }
+ }
+ for _, arg := range args[1:] {
+ if arg != didArg {
+ log.Printf("unknown project: %s", arg)
+ }
}
+ }
+}
- case "refill":
- refill()
+const didArg = "\x00"
- case "dash":
- dash()
+func match(name string, args []string) bool {
+ if len(args) == 0 {
+ return true
+ }
+ ok := false
+ for i, arg := range args {
+ if name == arg {
+ args[i] = didArg
+ ok = true
+ }
}
+ return ok
}
func doSync(proj *ProjectSync, resync bool) {
println("WOULD SYNC", proj.Name)
- syncIssueComments(proj)
syncIssues(proj)
- syncIssueEvents(proj, 0)
+ syncIssueComments(proj)
if resync {
+ syncIssueEvents(proj, 0, true)
syncIssueEventsByIssue(proj)
+ } else {
+ syncIssueEvents(proj, 0, false)
}
}
@@ -222,10 +254,11 @@ func downloadByDate(proj *ProjectSync, api string, since *string, sinceName stri
var last string
for _, m := range all {
var meta struct {
- URL string
- Updated string `json:"updated_at"`
- Number int64 // for /issues feed
- IssueURL string `json:"issue_url"` // for /issues/comments feed
+ URL string
+ Updated string `json:"updated_at"`
+ Number int64 // for /issues feed
+ IssueURL string `json:"issue_url"` // for /issues/comments feed
+ CreatedAt string `json:"created_at"`
}
if err := json.Unmarshal(m, &meta); err != nil {
return fmt.Errorf("parsing message: %v", err)
@@ -253,6 +286,7 @@ func downloadByDate(proj *ProjectSync, api string, since *string, sinceName stri
}
raw.Type = api
raw.JSON = m
+ raw.Time = meta.CreatedAt
if err := storage.Insert(tx, &raw); err != nil {
return fmt.Errorf("writing JSON to database: %v", err)
}
@@ -274,7 +308,7 @@ func downloadByDate(proj *ProjectSync, api string, since *string, sinceName stri
}
}
-func syncIssueEvents(proj *ProjectSync, id int) {
+func syncIssueEvents(proj *ProjectSync, id int, short bool) {
tx, err := db.Begin()
if err != nil {
log.Fatalf("starting db transaction: %v", err)
@@ -317,7 +351,7 @@ func syncIssueEvents(proj *ProjectSync, id int) {
firstID = meta.ID
firstETag = resp.Header.Get("Etag")
}
- if id == 0 && proj.EventID != 0 && meta.ID <= proj.EventID {
+ if id == 0 && (proj.EventID != 0 && meta.ID <= proj.EventID || short) {
return done
}
@@ -366,12 +400,19 @@ func syncIssueEventsByIssue(proj *ProjectSync) {
log.Fatal(err)
}
var ids []int
+ suffix := "repos/" + proj.Name + "/issues/"
for rows.Next() {
var url string
if err := rows.Scan(&url); err != nil {
log.Fatal(err)
}
i := strings.LastIndex(url, "/")
+ if !strings.HasSuffix(url[:i+1], suffix) {
+ continue
+ }
+ if url[i+1:] < "30140" {
+ continue
+ }
id, err := strconv.Atoi(url[i+1:])
if err != nil {
log.Fatal(url, err)
@@ -380,7 +421,7 @@ func syncIssueEventsByIssue(proj *ProjectSync) {
}
for _, id := range ids {
println("ID", id)
- syncIssueEvents(proj, id)
+ syncIssueEvents(proj, id, false)
}
}
@@ -400,7 +441,7 @@ func downloadPages(url, etag string, do func(*http.Response, []json.RawMessage)
if err != nil {
return err
}
- println("RESP:", js(resp.Header))
+ //println("RESP:", js(resp.Header))
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
@@ -418,9 +459,10 @@ func downloadPages(url, etag string, do func(*http.Response, []json.RawMessage)
}
}
}
- if resp.StatusCode == 500 {
+ if resp.StatusCode == 500 || resp.StatusCode == 502 {
nfail++
if nfail < 2 {
+ println("REPEAT:", resp.Status, string(data))
time.Sleep(time.Duration(nfail) * 2 * time.Second)
goto again
}
@@ -498,15 +540,19 @@ type ghIssueEvent struct {
Actor struct {
Login string `json:"login"`
} `json:"actor"`
- Event string `json:"event"`
- Label struct {
+ Event string `json:"event"`
+ Labels []struct {
Name string `json:"name"`
- } `json:"label"`
- CreatedAt string `json:"created_at"`
- CommitID string `json:"commit_id"`
- Assignee struct {
+ } `json:"labels"`
+ LockReason string `json:"lock_reason"`
+ CreatedAt string `json:"created_at"`
+ CommitID string `json:"commit_id"`
+ Assigner struct {
Login string `json:"login"`
- } `json:"assignee"`
+ } `json:"assigner"`
+ Assignees []struct {
+ Login string `json:"login"`
+ } `json:"assignees"`
Milestone struct {
Title string `json:"title"`
} `json:"milestone"`
@@ -518,6 +564,7 @@ type ghIssueEvent struct {
type ghIssueComment struct {
IssueURL string `json:"issue_url"`
+ HTMLURL string `json:"html_url"`
User struct {
Login string `json:"login"`
} `json:"user"`
@@ -527,32 +574,36 @@ type ghIssueComment struct {
}
type ghIssue struct {
- URL string `json:"url"`
- User struct {
+ URL string `json:"url"`
+ HTMLURL string `json:"html_url"`
+ User struct {
Login string `json:"login"`
} `json:"user"`
Title string `json:"title"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
+ ClosedAt string `json:"closed_at"`
Body string `json:"body"`
- Assignee struct {
+ Assignees []struct {
Login string `json:"login"`
- } `json:"assignee"`
+ } `json:"assignees"`
Milestone struct {
Title string `json:"title"`
} `json:"milestone"`
- State string `json:"state"`
- PullRequest *struct{} `json:"pull_request"`
+ State string `json:"state"`
+ PullRequest *struct{} `json:"pull_request"`
+ Locked bool
+ ActiveLockReason string `json:"active_lock_reason"`
+ Labels []struct {
+ Name string `json:"name"`
+ } `json:"labels"`
}
-func refill() {
- if _, err := db.Exec("delete from History"); err != nil {
- log.Fatal(err)
- }
+func retime() {
last := ""
for {
var all []RawJSON
- if err := storage.Select(db, &all, "where URL > ? order by URL asc limit 100", last); err != nil {
+ if err := storage.Select(db, &all, "where URL > ? and Time = ? order by URL asc limit 100", last, ""); err != nil {
log.Fatal("sql: %v", err)
}
if len(all) == 0 {
@@ -564,125 +615,23 @@ func refill() {
log.Fatal(err)
}
for _, m := range all {
- last = m.URL
- switch m.Type {
- default:
- println("TYPE", m.Type)
- case "/issues/events":
- var ev ghIssueEvent
- if err := json.Unmarshal(m.JSON, &ev); err != nil {
- log.Printf("unmarshal: %v\n%s", err, m.JSON)
- continue
- }
- var h History
- h.URL = m.URL
- h.Project = m.Project
- h.Issue = m.Issue
- h.Time = ev.CreatedAt
- h.Who = ev.Actor.Login
- h.Action = ev.Event
- expectText := true
- switch ev.Event {
- default:
- log.Printf("unknown event: %s\n%s", ev.Event, m.JSON)
- expectText = false
- case "subscribed", "unsubscribed", "reopened", "locked", "unlocked", "head_ref_deleted", "head_ref_restored", "mentioned":
- // ok
- expectText = false
- case "closed", "merged", "referenced":
- h.Text = ev.CommitID
- expectText = ev.Event == "merged"
- case "assigned", "unassigned":
- h.Text = ev.Assignee.Login
- case "labeled", "unlabeled":
- h.Text = ev.Label.Name
- case "milestoned", "demilestoned":
- h.Text = ev.Milestone.Title
- case "renamed":
- if ev.Rename.From != "" {
- h.Text = ev.Rename.From + " → " + ev.Rename.To
- }
- }
- if expectText && h.Text == "" {
- log.Printf("missing text: %s\n%s", ev.Event, m.JSON)
- }
- if err := storage.Insert(tx, &h); err != nil {
- log.Fatal(err)
- }
-
- case "/issues/comments":
- var ev ghIssueComment
- if err := json.Unmarshal(m.JSON, &ev); err != nil {
- log.Printf("unmarshal: %v\n%s", err, m.JSON)
- continue
- }
- i := strings.LastIndex(ev.IssueURL, "/")
- n, err := strconv.ParseInt(ev.IssueURL[i+1:], 10, 64)
- if err != nil {
- log.Printf("bad issue comment:\n%s", m.JSON)
- continue
- }
- var h History
- h.URL = m.URL
- h.Project = m.Project
- h.Issue = n
- h.Time = ev.UpdatedAt
- h.Who = ev.User.Login
- h.Action = "comment"
- h.Text = ev.Body
- if err := storage.Insert(tx, &h); err != nil {
- log.Fatal(err)
- }
-
- case "/issues":
- var ev ghIssue
- if err := json.Unmarshal(m.JSON, &ev); err != nil {
- log.Printf("unmarshal: %v\n%s", err, m.JSON)
- continue
- }
- i := strings.LastIndex(ev.URL, "/")
- n, err := strconv.ParseInt(ev.URL[i+1:], 10, 64)
- if err != nil {
- log.Printf("bad issue:\n%s", m.JSON)
- continue
- }
- var h History
- h.URL = m.URL
- h.Project = m.Project
- h.Issue = n
- h.Time = ev.CreatedAt // best we can do
- h.Who = ev.User.Login
- h.Action = "issue"
- if ev.PullRequest != nil {
- h.Action = "pullrequest"
- }
- h.Text = ev.Body
- if err := storage.Insert(tx, &h); err != nil {
- log.Fatal(err)
- }
-
- if ev.Assignee.Login != "" {
- h.Action = "assign?"
- h.Text = ev.Assignee.Login
- if err := storage.Insert(tx, &h); err != nil {
- log.Fatal(err)
- }
- }
- if ev.Milestone.Title != "" {
- h.Action = "milestone?"
- h.Text = ev.Assignee.Login
- if err := storage.Insert(tx, &h); err != nil {
- log.Fatal(err)
- }
- }
- if ev.State != "open" {
- h.Action = "close?"
- h.Text = ""
- if err := storage.Insert(tx, &h); err != nil {
- log.Fatal(err)
- }
- }
+ var meta struct {
+ CreatedAt string `json:"created_at"`
+ }
+ if err := json.Unmarshal(m.JSON, &meta); err != nil {
+ log.Fatal(err)
}
+ if meta.CreatedAt == "" {
+ log.Fatalf("missing created_at: %s", m.JSON)
+ }
+ tm, err := time.Parse(time.RFC3339, meta.CreatedAt)
+ if err != nil {
+ log.Fatalf("parse: %v", err)
+ }
+ if _, err := tx.Exec("update RawJSON set Time = ? where URL = ?", tm.UTC().Format(time.RFC3339Nano), m.URL); err != nil {
+ log.Fatal(err)
+ }
+ last = m.URL
}
if err := tx.Commit(); err != nil {
log.Fatal(err)
diff --git a/issuedb/todo.go b/issuedb/todo.go
@@ -0,0 +1,305 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "crypto/sha256"
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "rsc.io/todo/task"
+)
+
+type ghItem struct {
+ Type string
+ URL string
+ Time time.Time
+ Issue ghIssue
+ Event ghIssueEvent
+ Comment ghIssueComment
+}
+
+const timeFormat = "2006-01-02 15:04:05 -0700"
+
+func todo(proj *ProjectSync) {
+ println("#", proj.Name)
+ root := filepath.Join(os.Getenv("HOME"), "todo/github", filepath.Base(proj.Name))
+ data, _ := ioutil.ReadFile(filepath.Join(root, "synctime"))
+ var syncTime time.Time
+ if len(data) > 0 {
+ t, err := time.Parse(time.RFC3339, string(data))
+ if err != nil {
+ log.Fatalf("parsing %s: %v", filepath.Join(root, "synctime"), err)
+ }
+ syncTime = t
+ }
+
+ l := task.OpenList(root)
+
+ // Start 10 minutes back just in case there is time skew in some way on GitHub.
+ // (If this is not good enough, we can always impose our own sequence numbering
+ // in the RawJSON table.)
+ startTime := syncTime.Add(-10 * time.Minute)
+ endTime := syncTime
+ process(proj, startTime, func(proj *ProjectSync, issue int64, items []*ghItem) {
+ fmt.Fprintf(os.Stderr, "%v#%v\n", proj.Name, issue)
+ if end := items[len(items)-1].Time; endTime.Before(end) {
+ endTime = end
+ }
+ todoIssue(l, proj, issue, items)
+ })
+
+ if err := ioutil.WriteFile(filepath.Join(root, "synctime"), []byte(endTime.Local().Format(time.RFC3339)), 0666); err != nil {
+ log.Fatal(err)
+ }
+}
+
+func todoIssue(l *task.List, proj *ProjectSync, issue int64, items []*ghItem) {
+ id := fmt.Sprint(issue)
+ t, err := l.Read(id)
+ var last time.Time
+ if err != nil {
+ if items[0].Type != "/issues" {
+ log.Printf("sync: missing creation for %v/%v", proj.Name, issue)
+ return
+ }
+ it := &items[0].Issue
+ last = items[0].Time
+ hdr := map[string]string{
+ "url": it.HTMLURL,
+ "author": it.User.Login,
+ "title": it.Title,
+ "updated": last.Format(timeFormat),
+ }
+ syncHdr(hdr, hdr, it)
+ t, err = l.Create(id, items[0].Time.Local(), hdr, []byte(bodyText(it.User.Login, "reported", it.Body)))
+ if err != nil {
+ log.Fatal(err)
+ }
+ items = items[1:]
+ } else {
+ last, err = time.Parse(timeFormat, t.Header("updated"))
+ if err != nil {
+ log.Fatalf("sync: bad updated time in %v", issue)
+ }
+ }
+
+ haveEID := make(map[string]bool)
+ for _, eid := range t.EIDs() {
+ haveEID[eid] = true
+ }
+
+ for _, it := range items {
+ if last.Before(it.Time) {
+ last = it.Time
+ }
+ h := sha256.Sum256([]byte(it.URL))
+ eid := fmt.Sprintf("%x", h)[:8]
+ if haveEID[eid] {
+ continue
+ }
+
+ switch it.Type {
+ default:
+ log.Fatalf("unexpected type %s", it.Type)
+ case "/issues":
+ continue
+ case "/issues/events":
+ ev := &it.Event
+ hdr := map[string]string{
+ "#id": eid,
+ "updated": last.Local().Format(timeFormat),
+ }
+ what := "@" + ev.Actor.Login + " " + ev.Event
+ switch ev.Event {
+ case "closed", "merged", "referenced":
+ what += ": " + "https://github.com/" + proj.Name + "/commit/" + ev.CommitID
+ if ev.Event == "closed" || ev.Event == "merged" {
+ hdr["closed"] = it.Time.Local().Format(time.RFC3339)
+ }
+ case "assigned", "unassigned":
+ var list []string
+ for _, who := range ev.Assignees {
+ list = append(list, who.Login)
+ }
+ what += ": " + strings.Join(list, ", ")
+ if ev.Event == "assigned" {
+ hdr["assign"] = addList(t.Header("assign"), list)
+ } else {
+ hdr["assign"] = deleteList(t.Header("assign"), list)
+ }
+ case "labeled", "unlabeled":
+ var list []string
+ for _, lab := range ev.Labels {
+ list = append(list, lab.Name)
+ }
+ what += ": " + strings.Join(list, ", ")
+ if ev.Event == "labeled" {
+ hdr["label"] = addList(t.Header("label"), list)
+ } else {
+ hdr["label"] = deleteList(t.Header("label"), list)
+ }
+ case "milestoned":
+ what += ": " + ev.Milestone.Title
+ hdr["milestone"] = ev.Milestone.Title
+ case "demilestoned":
+ hdr["milestone"] = ""
+ case "renamed":
+ what += ":\n\t" + ev.Rename.From + " →\n\t" + ev.Rename.To
+ }
+ if err := l.Write(t, it.Time.Local(), hdr, []byte(what)); err != nil {
+ log.Fatal(err)
+ }
+ case "/issues/comments":
+ com := &it.Comment
+ hdr := map[string]string{
+ "#id": eid,
+ "#url": com.HTMLURL,
+ "updated": last.Local().Format(timeFormat),
+ }
+ if err := l.Write(t, it.Time.Local(), hdr, []byte(bodyText(com.User.Login, "commented", com.Body))); err != nil {
+ log.Fatal(err)
+ }
+ }
+ }
+}
+
+func addList(old string, add []string) string {
+ have := make(map[string]bool)
+ for _, name := range strings.Split(old, ", ") {
+ have[name] = true
+ }
+ for _, name := range add {
+ if !have[name] {
+ old += ", " + name
+ have[name] = true
+ }
+ }
+ return old
+}
+
+func deleteList(old string, del []string) string {
+ drop := make(map[string]bool)
+ for _, name := range del {
+ drop[name] = true
+ }
+ var list []string
+ for _, name := range strings.Split(old, ", ") {
+ if name != "" && !drop[name] {
+ list = append(list, name)
+ }
+ }
+ return strings.Join(list, ", ")
+}
+
+func syncHdr(old, hdr map[string]string, it *ghIssue) {
+ pr := ""
+ if it.PullRequest != nil {
+ pr = "pr"
+ }
+ if old["pr"] != pr {
+ hdr["pr"] = pr
+ }
+ if old["milestone"] != it.Milestone.Title {
+ hdr["milestone"] = it.Milestone.Title
+ }
+ locked := ""
+ if it.Locked {
+ locked := it.ActiveLockReason
+ if locked == "" {
+ locked = "locked"
+ }
+ }
+ if old["locked"] != locked {
+ hdr["locked"] = locked
+ }
+ closed := ""
+ if it.ClosedAt != "" {
+ closed = it.ClosedAt
+ }
+ if old["closed"] != closed {
+ hdr["closed"] = closed
+ }
+ var list []string
+ for _, who := range it.Assignees {
+ list = append(list, who.Login)
+ }
+ all := strings.Join(list, ", ")
+ if old["assign"] != all {
+ hdr["assign"] = all
+ }
+ list = nil
+ for _, lab := range it.Labels {
+ list = append(list, lab.Name)
+ }
+ all = strings.Join(list, ", ")
+ if old["label"] != all {
+ hdr["label"] = all
+ }
+}
+
+func process(proj *ProjectSync, since time.Time, do func(proj *ProjectSync, issue int64, item []*ghItem)) {
+ rows, err := db.Query("select * from RawJSON where Project = ? and Time >= ? order by Issue, Time, Type", proj.Name, since.UTC().Format(time.RFC3339))
+ if err != nil {
+ log.Fatalf("sql: %v", err)
+ }
+
+ var items []*ghItem
+ var lastIssue int64
+ for rows.Next() {
+ var raw RawJSON
+ if err := rows.Scan(&raw.URL, &raw.Project, &raw.Issue, &raw.Type, &raw.JSON, &raw.Time); err != nil {
+ log.Fatal("sql scan RawJSON: %v", err)
+ }
+ if raw.Issue != lastIssue {
+ if len(items) > 0 {
+ do(proj, lastIssue, items)
+ }
+ items = items[:0]
+ lastIssue = raw.Issue
+ }
+
+ var ev ghIssueEvent
+ var com ghIssueComment
+ var issue ghIssue
+ switch raw.Type {
+ default:
+ log.Fatalf("unknown type %s", raw.Type)
+ case "/issues/comments":
+ err = json.Unmarshal(raw.JSON, &com)
+ case "/issues/events":
+ err = json.Unmarshal(raw.JSON, &ev)
+ case "/issues":
+ err = json.Unmarshal(raw.JSON, &issue)
+ }
+ if err != nil {
+ log.Fatalf("unmarshal: %v", err)
+ }
+ tm, err := time.Parse(time.RFC3339, raw.Time)
+ if err != nil {
+ log.Fatalf("parse time: %v", err)
+ }
+
+ items = append(items, &ghItem{Type: raw.Type, URL: raw.URL, Time: tm, Issue: issue, Event: ev, Comment: com})
+ }
+ if len(items) > 0 {
+ do(proj, lastIssue, items)
+ }
+}
+
+func bodyText(who, verb, data string) []byte {
+ body := "@" + who + " " + verb + ":\n"
+ b := strings.Replace(data, "\r\n", "\n", -1)
+ b = strings.TrimRight(b, "\n")
+ b = strings.Replace(b, "\n", "\n\t", -1)
+ body += "\n\t" + b
+ return []byte(body)
+}
