Skip to content

Commit b25b732

Browse files
youencheneyouenchene
committed
refactor: Simplify PR and review CSV generation logic
- Consolidated PR and review collection into unified processing and writing functions. - Updated `PullRequest` and `PullRequestReview` structs to include `Org` and `Repo` fields for streamlined data handling. - Removed redundant `Append` functions in favor of new `WritePullRequests` and `WritePullRequestReviews`. - Enhanced maintainability by refactoring unnecessary file handling logic.
1 parent 468d0d0 commit b25b732

4 files changed

Lines changed: 41 additions & 95 deletions

File tree

‎command/import/import.go‎

Lines changed: 24 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -261,12 +261,12 @@ func Run(args []string) error {
261261
}
262262

263263
// New: fetch PRs and reviews and write to unified CSVs
264-
// Prepare unified PR file: remove any previous run to avoid duplicates
265264
prUnifiedPath := "data/pr.csv"
266-
_ = os.Remove(prUnifiedPath)
267-
// Also reset reviews file for a clean run
268265
rvUnifiedPath := "data/pr_review.csv"
269-
_ = os.Remove(rvUnifiedPath)
266+
267+
var allPRs []gh.PullRequest
268+
var allReviews []gh.PullRequestReview
269+
270270
for _, r := range repos {
271271
if *repoFilter != "" && !allowedRepos[r.Name] {
272272
continue
@@ -277,11 +277,14 @@ func Run(args []string) error {
277277
slog.Warn("phase.prs.fetch.error", "owner", r.Owner.Login, "repo", r.Name, "error", err)
278278
continue
279279
}
280-
// Append to unified PR CSV with repo column
281-
if err := ccsv.AppendPullRequests(prUnifiedPath, *org, r.Name, prs); err != nil {
282-
slog.Warn("phase.prs.csv.error", "repo", r.Name, "error", err)
280+
// Collect all PRs
281+
for i := range prs {
282+
prs[i].Org = *org
283+
prs[i].Repo = r.Name
283284
}
284-
// For each PR, fetch reviews and append to unified pr_review.csv
285+
allPRs = append(allPRs, prs...)
286+
287+
// For each PR, fetch reviews and collect them
285288
for _, pr := range prs {
286289
reviews, err := ghc.ListAllPullRequestReviews(ctx, r.Owner.Login, r.Name, pr.Number)
287290
if err != nil {
@@ -291,12 +294,23 @@ func Run(args []string) error {
291294
if len(reviews) == 0 {
292295
continue
293296
}
294-
if err := ccsv.AppendPullRequestReviews(rvUnifiedPath, *org, r.Name, pr.Number, reviews); err != nil {
295-
slog.Warn("phase.pr.reviews.csv.error", "repo", r.Name, "pr", pr.Number, "error", err)
297+
// Collect all reviews
298+
for i := range reviews {
299+
reviews[i].Org = *org
300+
reviews[i].Repo = r.Name
301+
reviews[i].PullRequestNumber = pr.Number
296302
}
303+
allReviews = append(allReviews, reviews...)
297304
}
298305
}
299306

307+
// Write all collected PRs and reviews at once
308+
if err := ccsv.WritePullRequests(prUnifiedPath, allPRs); err != nil {
309+
slog.Warn("phase.prs.csv.error", "error", err)
310+
}
311+
if err := ccsv.WritePullRequestReviews(rvUnifiedPath, allReviews); err != nil {
312+
slog.Warn("phase.pr.reviews.csv.error", "error", err)
313+
}
300314
slog.Info("import.done", "reports", len(reports))
301315
return nil
302316
}

‎connectors/csv/pr.go‎

Lines changed: 8 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111

1212
// WritePullRequestCSV writes a complete CSV snapshot of PRs for a repository.
1313
// Headers: org, repo, number, title, url, state, created_at, closed_at, merged_at, creator
14-
func WritePullRequestCSV(path string, org string, repo string, prs []gh.PullRequest) error {
14+
func WritePullRequests(path string, prs []gh.PullRequest) error {
1515
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
1616
return err
1717
}
@@ -39,103 +39,35 @@ func WritePullRequestCSV(path string, org string, repo string, prs []gh.PullRequ
3939
if pr.User != nil {
4040
creator = pr.User.Login
4141
}
42-
row := []string{org, repo, strconv.Itoa(pr.Number), pr.Title, pr.HTMLURL, pr.State, created, closed, merged, creator}
42+
row := []string{pr.Org, pr.Repo, strconv.Itoa(pr.Number), pr.Title, pr.HTMLURL, pr.State, created, closed, merged, creator}
4343
if err := w.Write(row); err != nil {
4444
return err
4545
}
4646
}
4747
return w.Error()
4848
}
4949

50-
// AppendPullRequests appends a slice of PRs to a single CSV, writing header if the file does not exist.
51-
// Headers: org, repo, number, title, url, state, created_at, closed_at, merged_at, creator
52-
func AppendPullRequests(path string, org string, repo string, prs []gh.PullRequest) error {
53-
if len(prs) == 0 {
54-
return nil
55-
}
50+
func WritePullRequestReviews(path string, reviews []gh.PullRequestReview) error {
5651
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
5752
return err
5853
}
59-
var f *os.File
60-
var err error
61-
if _, statErr := os.Stat(path); statErr == nil {
62-
f, err = os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0o644)
63-
if err != nil {
64-
return err
65-
}
66-
} else {
67-
f, err = os.Create(path)
68-
if err != nil {
69-
return err
70-
}
71-
w := csv.NewWriter(f)
72-
if err := w.Write([]string{"org", "repo", "number", "title", "url", "state", "created_at", "closed_at", "merged_at", "creator"}); err != nil {
73-
_ = f.Close()
74-
return err
75-
}
76-
w.Flush()
54+
f, err := os.Create(path)
55+
if err != nil {
56+
return err
7757
}
7858
defer f.Close()
7959
w := csv.NewWriter(f)
8060
defer w.Flush()
81-
for _, pr := range prs {
82-
created := pr.CreatedAt.UTC().Format(time.RFC3339)
83-
closed := ""
84-
if pr.ClosedAt != nil {
85-
closed = pr.ClosedAt.UTC().Format(time.RFC3339)
86-
}
87-
merged := ""
88-
if pr.MergedAt != nil {
89-
merged = pr.MergedAt.UTC().Format(time.RFC3339)
90-
}
91-
creator := ""
92-
if pr.User != nil {
93-
creator = pr.User.Login
94-
}
95-
row := []string{org, repo, strconv.Itoa(pr.Number), pr.Title, pr.HTMLURL, pr.State, created, closed, merged, creator}
96-
if err := w.Write(row); err != nil {
97-
return err
98-
}
99-
}
100-
return w.Error()
101-
}
102-
103-
// AppendPullRequestReviews appends a batch of reviews for a PR to a CSV file.
104-
// Headers: org, repo, number, state, submitted_at, user
105-
func AppendPullRequestReviews(path string, org string, repo string, number int, reviews []gh.PullRequestReview) error {
106-
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
61+
if err := w.Write([]string{"org", "repo", "number", "state", "submitted_at", "user"}); err != nil {
10762
return err
10863
}
109-
// open file and write header if creating
110-
var f *os.File
111-
var err error
112-
if _, statErr := os.Stat(path); statErr == nil {
113-
f, err = os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0o644)
114-
if err != nil {
115-
return err
116-
}
117-
} else {
118-
f, err = os.Create(path)
119-
if err != nil {
120-
return err
121-
}
122-
w := csv.NewWriter(f)
123-
if err := w.Write([]string{"org", "repo", "number", "state", "submitted_at", "user"}); err != nil {
124-
_ = f.Close()
125-
return err
126-
}
127-
w.Flush()
128-
}
129-
defer f.Close()
130-
w := csv.NewWriter(f)
131-
defer w.Flush()
13264
for _, rv := range reviews {
13365
sub := rv.SubmittedAt.UTC().Format(time.RFC3339)
13466
user := ""
13567
if rv.User != nil {
13668
user = rv.User.Login
13769
}
138-
row := []string{org, repo, strconv.Itoa(number), rv.State, sub, user}
70+
row := []string{rv.Org, rv.Repo, strconv.Itoa(rv.PullRequestNumber), rv.State, sub, user}
13971
if err := w.Write(row); err != nil {
14072
return err
14173
}

‎domain/github/github.go‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ type Label struct {
3737

3838
// PullRequest represents a GitHub pull request (subset of fields we need)
3939
type PullRequest struct {
40+
Org string `json:"org"`
41+
Repo string `json:"repo"`
4042
Number int `json:"number"`
4143
Title string `json:"title"`
4244
State string `json:"state"`
@@ -50,9 +52,12 @@ type PullRequest struct {
5052

5153
// PullRequestReview represents a review on a PR
5254
type PullRequestReview struct {
53-
State string `json:"state"` // APPROVED|CHANGES_REQUESTED|COMMENTED|DISMISSED
54-
SubmittedAt time.Time `json:"submitted_at"`
55-
User *User `json:"user"`
55+
Org string `json:"org"`
56+
Repo string `json:"repo"`
57+
PullRequestNumber int `json:"pull_request_number"`
58+
State string `json:"state"` // APPROVED|CHANGES_REQUESTED|COMMENTED|DISMISSED
59+
SubmittedAt time.Time `json:"submitted_at"`
60+
User *User `json:"user"`
5661
}
5762

5863
// TimelineEvent captures various events, including project card movements

‎go.sum‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,8 @@ github.com/labstack/echo/v4 v4.11.4 h1:vDZmA+qNeh1pd/cCkEicDMrjtrnMGQ1QFI9gWN1zG
44
github.com/labstack/echo/v4 v4.11.4/go.mod h1:noh7EvLwqDsmh/X/HWKPUl1AjzJrhyptRyEbQJfxen8=
55
github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0=
66
github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU=
7-
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
8-
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
97
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
108
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
11-
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
129
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
1310
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
1411
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -25,14 +22,12 @@ golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
2522
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
2623
golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
2724
golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
28-
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
2925
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
30-
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
31-
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
3226
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
3327
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
3428
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
3529
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
30+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
3631
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
3732
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
3833
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

0 commit comments

Comments
 (0)