Activity | Brain Phrye

Added some activity graphs to this ( linked off my about page ). Currently they just look at activity on a set of gitlab servers I use. I’m going to try and add github later. And then maybe raid some other data sources.

Just a quick explanation of how I did it.

First I wrote a program to connect to gitlab servers and grab activity events. The meat of it is this:

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
import (
        // [... std modules deleted ...]
	"github.com/xanzy/go-gitlab"
)

var (
	dataDir    string
)

// ActivityYear tracks activity by month-day in a year.
type ActivityYear struct {
	file   string
	Update string            `json:"update"`
	Stats  map[string]uint16 `json:"stats"`
}

var activity map[string]*ActivityYear
var updates map[string]string

func init() {
	flag.StringVar(&dataDir, "data", "../../data/activity", "Data directory.")
}

func main() {
	flag.Parse()
	os.MkdirAll(dataDir, 0755)

	git := gitlab.NewClient(nil, os.Getenv("GITLAB_TOKEN"))
	git.SetBaseURL(fmt.Sprintf("%s/api/v4", os.Getenv("GITLAB_URL")))
	dataPrefix := strings.ReplaceAll(os.Getenv("GITLAB_URL"), "https://", "")
	dataPrefix = strings.ReplaceAll(dataPrefix, "http://", "")
	dataPrefix = strings.ReplaceAll(dataPrefix, "/", "")
	dataPrefix = strings.ReplaceAll(dataPrefix, ".", "_")

	opts := &gitlab.ListContributionEventsOptions{
		ListOptions: gitlab.ListOptions{
			PerPage: 20,
			Page:    1,
		},
	}
	updates = make(map[string]string)
	activity = make(map[string]*ActivityYear)
	fmt.Printf("Connecting to %s...", os.Getenv("GITLAB_URL"))
	for {
		events, resp, err := git.Events.ListCurrentUserContributionEvents(opts)
		if err != nil {
			fmt.Printf("%s\n%v\n", err, resp)
			os.Exit(1)
		}
		for _, event := range events {
			year := time.Time(*event.CreatedAt).Format("2006")
			monthDay := time.Time(*event.CreatedAt).Format("01-02")
			if activity[year] == nil {
				var actYear ActivityYear
				dataFile := path.Join(dataDir,
					fmt.Sprintf("%s_%s", dataPrefix,
						time.Time(*event.CreatedAt).Format("2006.json")))
				dataBytes, err := ioutil.ReadFile(dataFile)
				if err != nil {
					actYear.Stats = make(map[string]uint16)
				} else {
					json.Unmarshal(dataBytes, &actYear)
				}
				actYear.file = dataFile
				activity[year] = &actYear
				fmt.Printf("\nChecking %s...", year)
			}
			if updates[year] == "" {
				updates[year] = event.CreatedAt.String()
			}
			if activity[year].Update >= event.CreatedAt.String() {
				goto WriteOut
			}
			activity[year].Stats[monthDay]++
			fmt.Printf(".")
		}
		if resp.CurrentPage >= resp.TotalPages {
			break
		}

		opts.Page = resp.NextPage
	}

WriteOut:
	fmt.Printf("\nWriting...")
	for year := range activity {
		fmt.Printf("%s...", year)
		activity[year].Update = updates[year]
		dataBytes, err := json.Marshal(activity[year])
		if err != nil {
			fmt.Printf("\n%s\n", err)
			os.Exit(1)
		}
		err = ioutil.WriteFile(activity[year].file, dataBytes, 0644)
		if err != nil {
			fmt.Printf("\n%s\n", err)
			os.Exit(1)
		}
	}
	fmt.Println("")
}

Yes, it even includes a goto - the horrors. It’s cleaner than any other way I can think of to skip out when I’ve got all the data I need.

Other modules that fetch activity will generate the same JSON structure. These all then get pulled together with a summary module also written in Go:

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
import (
        // [... std modules deleted ...]
	"github.com/jinzhu/now"
)

var (
	dataDir string
)

// ActivityYear tracks activity by month-day in a year.
type ActivityYear struct {
	file   string
	year   int
	Update string            `json:"update"`
	Stats  map[string]uint16 `json:"stats"`
}

// Day is a day's entry.
type Day struct {
	Stat int32  `json:"stat"`
	Date string `json:"date"`
}

// CalendarYear is 7 row by 53/54 column array representing a year.
type CalendarYear struct {
	Days [][]Day `json:"days"`
}

func init() {
	flag.StringVar(&dataDir, "data", "../../data/activity", "Data directory.")
}

func calendarJSON(activity *ActivityYear) ([]byte, error) {
	var calendar CalendarYear
	calendar.Days = make([][]Day, 7)

	y, err := now.Parse(fmt.Sprintf("%d", (activity.year)))
	if err != nil {
		fmt.Printf("Failed to parse year: %s", y)
		os.Exit(1)
	}

	now.WeekStartDay = time.Monday
	firstMonday := now.New(y).BeginningOfWeek()
	weeks := 0
	for d := firstMonday; d.Year() <= activity.year; d = d.AddDate(0, 0, 7) {
		weeks++
	}
	for i := 0; i < 7; i++ {
		d := firstMonday.AddDate(0, 0, i)
		calendar.Days[i] = make([]Day, weeks)
		for week := 0; week < weeks; week++ {
			calendar.Days[i][week].Date = d.Format("2006-01-02")
			if d.Year() == activity.year {
				calendar.Days[i][week].Stat = int32(activity.Stats[d.Format("01-02")])
			} else {
				calendar.Days[i][week].Stat = -1
			}
			d = d.AddDate(0, 0, 7)
		}
	}
	return json.Marshal(calendar)
}

func main() {
	flag.Parse()

	for year := 2016; year <= time.Now().Year(); year++ {
		datafiles, err := filepath.Glob(fmt.Sprintf("%s/*_%d.json", dataDir, year))
		if err != nil {
			fmt.Printf("Error globbing files: %s\n", err)
			os.Exit(1)
		}
		var summary ActivityYear
		summary.year = year
		summary.file = fmt.Sprintf("%s/sum%d.json", dataDir, year)
		summary.Stats = make(map[string]uint16)
		fmt.Printf("Processing...")
		for _, datafile := range datafiles {
			var service ActivityYear
			fmt.Printf("%s...", filepath.Base(datafile))
			dataBytes, err := ioutil.ReadFile(datafile)
			if err != nil {
				fmt.Printf("Error processing %s: %s\n", filepath.Base(datafile), err)
			} else {
				json.Unmarshal(dataBytes, &service)
			}
			if service.Update > summary.Update {
				summary.Update = service.Update
			}
			for monthDay, actions := range service.Stats {
				summary.Stats[monthDay] += actions
			}
		}
		fmt.Println()
		summaryJSON, err := calendarJSON(&summary)
		if err != nil {
			fmt.Printf("Error marshalling %d: %s\n", year, err)
			os.Exit(1)
		}
		ioutil.WriteFile(summary.file, summaryJSON, 0644)
	}
}

Now every year exists in a data/activity/sumYYYY.json file. All that’s needed is to display the data in the about page. This is done with hugo shortcode:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
<style>
th { width: 10px; }
td { width: 10px; height: 10px; }

.c0 { background-color: rgb(237, 237, 237); }
.c1 { background-color: rgb(172, 213, 242); }
.c2 { background-color: rgb(127, 168, 201); }
.c3 { background-color: rgb(82, 123, 160); }
.c4 { background-color: rgb(37, 78, 119); }
.c5 { background-color: rgb(32, 73, 114); }
.c6 { background-color: rgb(27, 68, 109); }
.c7 { background-color: rgb(22, 63, 104); }
.c8 { background-color: rgb(17, 58, 99); }
.c9 { background-color: rgb(12, 53, 94); }
</style>

{{- $activity := .Site.Data.activity -}}
{{- range (slice 2016 2017 2018 2019) -}}
{{ . }}:
{{ with (index $activity (printf "sum%d" .)) -}}
<table class="activity">
  {{- range .days -}}<tr>{{- range . -}}
    <td {{ if ge .stat 0 -}}
      title="{{ .date }}{{ if gt .stat 0 }}: {{ .stat }}{{ end }}"
      {{- end }} class="c{{ math.Ceil (div .stat 10) }}"></td>
  {{- end -}}</tr>{{- end -}}
</table>
{{- end -}}
{{- end -}}

There are some issues though. It seems that gitlab has some sort of limit to the activities it will track. My personal gitlab server only goes back 18 months. The gitlab.com server has activity going back to 2016 (clearly not much) so not sure what setting I need to do to extend mine.

Obviously if I make more than 100 actions in a day I’ll blow past my colour range - which, in another issue, is kind of lacking variation past 50 contributions. That seems like a lot but if my history on my personal server went back further it would include a bunch of stuff I did in digitising documents with git-annex (which was an annoying dead-end).