LineageOS/android_development
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge "Complete functionality to add the first seen timestamp to non-upstreamed commits"

Browse Source
This commit is contained in:
Scott Lobdell
2018-05-16 18:15:21 +00:00
committed by Android (Google) Code Review
parent 260d98d6a5 ceb0b6a273
commit 18b23bf143
14 changed files with 165 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tools/repo_diff/service/repodiff/Makefile
View File

@@ -13,7 +13,7 @@ DOCKER_REPLICA_COUNT="1"
DOCKER_CANONICAL_ID=$(DOCKER_CONTAINER_REGISTRY)/$(GOOGLE_PROJECT_ID)/$(DOCKER_IMAGE_NAME):$(DOCKER_TAG_NAME)
PORT_HTTP="80"
GCE_ZONE="us-west1-a"
GCE_ZONE="us-west1-b"
GCE_IMAGE_PROJECT="ubuntu-os-cloud"
GCE_IMAGE_FAMILY="ubuntu-1604-lts"
@@ -123,8 +123,9 @@ deploy:
--service-account $(SERVICE_ACCOUNT) \
2>/dev/null || true
@echo "Hackily waiting a bit for instance to start up"
@sleep 30
./tools/clear_service_account_keys.py $(SERVICE_ACCOUNT)
# TODO(slobdell) need to add a mechanism to block until startup script has completed
@sleep 60
./tools/clear_service_account_keys.py $(SERVICE_ACCOUNT) 2>/dev/null || true
gcloud iam service-accounts keys create $(TMP_CREDENTIAL_FNAME) --iam-account $(SERVICE_ACCOUNT)
$(RUN_COMMAND_REMOTE) 'mkdir -p /tmp/scripts'
$(SCP_TO_HOST) remote_scripts/* "$(REMOTE_MACHINE_NAME)":/tmp/scripts/

20
tools/repo_diff/service/repodiff/controllers/contract.go Normal file
View File

@@ -0,0 +1,20 @@
package controllers
import (
ent "repodiff/entities"
"repodiff/repositories"
)
type Committer interface {
InsertCommitRows(commitRows []ent.AnalyzedCommitRow) error
GetFirstSeenTimestamp(commitHashes []string, nullTimestamp ent.RepoTimestamp) (map[string]ent.RepoTimestamp, error)
GetMostRecentCommits() ([]ent.AnalyzedCommitRow, error)
}
func MaybeNullObjectCommitRepository(target ent.MappedDiffTarget) Committer {
c, err := repositories.NewCommitRepository(target)
if err != nil {
return repositories.NewNullObject(err)
}
return c
}

21
tools/repo_diff/service/repodiff/controllers/denormalize.go
View File

@@ -4,6 +4,7 @@ import (
e "repodiff/entities"
"repodiff/interactors"
"repodiff/repositories"
"repodiff/utils"
)
func DenormalizeData(config e.ApplicationConfig) error {
@@ -125,5 +126,23 @@ func denormalizeCommitRows(target e.DiffTarget) error {
if err != nil {
return err
}
return denormalizeRepo.DenormalizeToRecentCommits(commitRows)
commitToTimestamp, err := MaybeNullObjectCommitRepository(
mappedTarget,
).GetFirstSeenTimestamp(
extractCommitHashes(commitRows),
utils.TimestampSeconds(),
)
if err != nil {
return err
}
return denormalizeRepo.DenormalizeToRecentCommits(commitRows, commitToTimestamp)
}
func extractCommitHashes(commitRows []e.AnalyzedCommitRow) []string {
hashes := make([]string, len(commitRows))
for i, row := range commitRows {
hashes[i] = row.Commit
}
return hashes
}

30
tools/repo_diff/service/repodiff/controllers/differential.go
View File

@@ -164,9 +164,11 @@ func TransferScriptOutputToDownstream(
return err
}
analyzedDiffRows, analyzedCommitRows := interactors.ApplyApplicationMutations(
diffRows,
commitRows,
manifestFileGroup,
interactors.AppProcessingParameters{
DiffRows: diffRows,
CommitRows: commitRows,
Manifests: manifestFileGroup,
},
)
return persistEntities(target, analyzedDiffRows, analyzedCommitRows)
}
@@ -220,11 +222,11 @@ func persistEntities(target ent.DiffTarget, diffRows []ent.AnalyzedDiffRow, comm
return errors.Wrap(err, "Error persisting diff rows")
}
err = persistCommitRowsDownstream(mappedTarget, commitRows)
if err != nil {
return errors.Wrap(err, "Error persist commit rows")
}
return nil
return MaybeNullObjectCommitRepository(
mappedTarget,
).InsertCommitRows(
commitRows,
)
}
func csvFileToDiffRows(csvFile string) ([]ent.DiffRow, error) {
@@ -288,15 +290,3 @@ func persistDiffRowsDownstream(mappedTarget ent.MappedDiffTarget, diffRows []ent
}
return nil
}
func persistCommitRowsDownstream(mappedTarget ent.MappedDiffTarget, commitRows []ent.AnalyzedCommitRow) error {
c, err := repositories.NewCommitRepository(mappedTarget)
if err != nil {
return errors.Wrap(err, "Error instantiating a new commit repository")
}
err = c.InsertCommitRows(commitRows)
if err != nil {
return errors.Wrap(err, "Error inserting rows from controller")
}
return nil
}

18
tools/repo_diff/service/repodiff/interactors/application.go
View File

@@ -5,10 +5,20 @@ import (
ent "repodiff/entities"
)
func ApplyApplicationMutations(diffRows []ent.DiffRow, commitRows []ent.CommitRow, manifests *ent.ManifestFileGroup) ([]ent.AnalyzedDiffRow, []ent.AnalyzedCommitRow) {
projectNameToType := ProjectNamesToType(manifests)
return diffRowsToAnalyzed(diffRows, projectNameToType),
commitRowsToAnalyzed(commitRows, projectNameToType)
// AppProcessingParameters defines all possible inputs that are necessary
// prior to applying any application business logic. Any outputs should
// be derived from purely deterministic means; As such, the interactors
// package should be 100% testable and free of any error return types
type AppProcessingParameters struct {
DiffRows []ent.DiffRow
CommitRows []ent.CommitRow
Manifests *ent.ManifestFileGroup
}
func ApplyApplicationMutations(p AppProcessingParameters) ([]ent.AnalyzedDiffRow, []ent.AnalyzedCommitRow) {
projectNameToType := ProjectNamesToType(p.Manifests)
return diffRowsToAnalyzed(p.DiffRows, projectNameToType),
commitRowsToAnalyzed(p.CommitRows, projectNameToType)
}
func commitRowsToAnalyzed(commitRows []ent.CommitRow, projectNameToType TypeMap) []ent.AnalyzedCommitRow {

8
tools/repo_diff/service/repodiff/mappers/mappers.go
View File

@@ -87,7 +87,7 @@ func diffRowToDenormalizedCols(d e.AnalyzedDiffRow, rowIndex int) []interface{}
}
}
func commitRowToDenormalizedCols(commitRow e.AnalyzedCommitRow, rowIndex int) []interface{} {
func commitRowToDenormalizedCols(commitRow e.AnalyzedCommitRow, firstSeen e.RepoTimestamp, rowIndex int) []interface{} {
return []interface{}{
rowIndex,
commitRow.Commit,
@@ -96,6 +96,7 @@ func commitRowToDenormalizedCols(commitRow e.AnalyzedCommitRow, rowIndex int) []
commitRow.Subject,
GetAuthorTechArea(commitRow.Author),
constants.ProjectTypeToDisplay[commitRow.Type],
utils.TimestampToDataStudioDatetime(firstSeen),
}
}
@@ -155,11 +156,12 @@ func DiffRowsToDenormalizedCols(diffRows []e.AnalyzedDiffRow) [][]interface{} {
return rows
}
func CommitRowsToDenormalizedCols(commitRows []e.AnalyzedCommitRow) [][]interface{} {
func CommitRowsToDenormalizedCols(commitRows []e.AnalyzedCommitRow, commitToTimestamp map[string]e.RepoTimestamp) [][]interface{} {
rows := make([][]interface{}, len(commitRows))
for i, commitRow := range commitRows {
rows[i] = commitRowToDenormalizedCols(
commitRow,
commitToTimestamp[commitRow.Commit],
i,
)
}
@@ -171,7 +173,7 @@ func DiffRowsToAggregateChangesOverTime(diffRows []e.AnalyzedDiffRow) [][]interf
return nil
}
cols := []interface{}{
utils.TimestampToDatastudioDatetime(e.RepoTimestamp(diffRows[0].DBInsertTimestamp)),
utils.TimestampToDataStudioDatetime(e.RepoTimestamp(diffRows[0].DBInsertTimestamp)),
getSumOfAttribute(
diffRows,
func(d e.AnalyzedDiffRow) int {

53
tools/repo_diff/service/repodiff/repositories/commit.go
View File

@@ -15,6 +15,20 @@ import (
"repodiff/utils"
)
type NullCommit struct {
originalErr error
}
func (n NullCommit) InsertCommitRows(commitRows []e.AnalyzedCommitRow) error {
return n.originalErr
}
func (n NullCommit) GetFirstSeenTimestamp(commitHashes []string, nullTimestamp e.RepoTimestamp) (map[string]e.RepoTimestamp, error) {
return nil, n.originalErr
}
func (n NullCommit) GetMostRecentCommits() ([]e.AnalyzedCommitRow, error) {
return nil, n.originalErr
}
type Commit struct {
db *sql.DB
target e.MappedDiffTarget
@@ -135,7 +149,7 @@ func (c Commit) GetMostRecentCommits() ([]e.AnalyzedCommitRow, error) {
return commitRows, nil
}
func (c Commit) GetFirstSeenTimestamp(commitHashes []string) (map[string]e.RepoTimestamp, error) {
func (c Commit) GetFirstSeenTimestamp(commitHashes []string, nullTimestamp e.RepoTimestamp) (map[string]e.RepoTimestamp, error) {
if len(commitHashes) == 0 {
return map[string]e.RepoTimestamp{}, nil
}
@@ -165,18 +179,21 @@ func (c Commit) GetFirstSeenTimestamp(commitHashes []string) (map[string]e.RepoT
FROM project_commit
WHERE upstream_target_id = ?
AND downstream_target_id = ?
AND commit_ IN(?)
AND commit_ IN(?`+strings.Repeat(",?", len(commitHashes)-1)+`)
GROUP BY commit_
`,
c.target.UpstreamTarget,
c.target.DownstreamTarget,
strings.Join(commitHashes, ", "),
append(
[]interface{}{
c.target.UpstreamTarget,
c.target.DownstreamTarget,
},
asInterfaceSlice(commitHashes)...,
)...,
)
if err := interactors.AnyError(errSelect, errMapping); err != nil {
return nil, err
} else if len(commitToTimestamp) != len(commitHashes) {
return nil, errors.New("Not all input commit hashes exist")
}
mutateEmptyValues(commitToTimestamp, commitHashes, nullTimestamp)
return commitToTimestamp, nil
}
@@ -188,3 +205,25 @@ func NewCommitRepository(target e.MappedDiffTarget) (Commit, error) {
timestampGenerator: utils.TimestampSeconds,
}, errors.Wrap(err, "Could not establish a database connection")
}
func NewNullObject(originalErr error) NullCommit {
return NullCommit{
originalErr: originalErr,
}
}
func mutateEmptyValues(existing map[string]e.RepoTimestamp, shouldExist []string, defaultValue e.RepoTimestamp) {
for _, key := range shouldExist {
if _, ok := existing[key]; !ok {
existing[key] = defaultValue
}
}
}
func asInterfaceSlice(strings []string) []interface{} {
casted := make([]interface{}, len(strings))
for i, s := range strings {
casted[i] = s
}
return casted
}

17
tools/repo_diff/service/repodiff/repositories/commit_test.go
View File

@@ -88,7 +88,8 @@ func TestGetFirstSeenTimestamp(t *testing.T) {
commitHashes := []string{
"61d5e61b6b6dfbf52d0d433759da964db31cc106",
}
commitToTimestamp, err := c.GetFirstSeenTimestamp(commitHashes)
nullTimestamp := ent.RepoTimestamp(0)
commitToTimestamp, err := c.GetFirstSeenTimestamp(commitHashes, nullTimestamp)
assert.Equal(t, nil, err, "Error should be nil")
assert.Equal(t, len(commitHashes), len(commitToTimestamp), "Length of returned values")
assert.Equal(t, oldFakeTimestamp, commitToTimestamp["61d5e61b6b6dfbf52d0d433759da964db31cc106"], "Expected returned timestamp")
@@ -96,20 +97,26 @@ func TestGetFirstSeenTimestamp(t *testing.T) {
func TestGetFirstSeenTimestampEmpty(t *testing.T) {
c, _ := repositories.NewCommitRepository(fakeMappedTarget)
commitToTimestamp, err := c.GetFirstSeenTimestamp([]string{})
nullTimestamp := ent.RepoTimestamp(0)
commitToTimestamp, err := c.GetFirstSeenTimestamp([]string{}, nullTimestamp)
assert.Equal(t, nil, err, "Error should be nil")
assert.Equal(t, 0, len(commitToTimestamp), "Length of returned values")
}
func TestGetFirstSeenTimestampMutateReturned(t *testing.T) {
c, _ := repositories.NewCommitRepository(fakeMappedTarget)
commitToTimestamp, _ := c.GetFirstSeenTimestamp([]string{})
nullTimestamp := ent.RepoTimestamp(0)
commitToTimestamp, _ := c.GetFirstSeenTimestamp([]string{}, nullTimestamp)
commitToTimestamp["some_key"] = ent.RepoTimestamp(0)
}
func TestGetFirstSeenTimestampNonExistent(t *testing.T) {
c, _ := repositories.NewCommitRepository(fakeMappedTarget)
nonExistentHash := "ae8e745ba09f61ddfa46ed6bba54c4bd07b2e93b"
_, err := c.GetFirstSeenTimestamp([]string{nonExistentHash})
assert.NotEqual(t, nil, err, "Error should be generated")
nullTimestamp := ent.RepoTimestamp(123)
nonExistentHashes := []string{nonExistentHash}
commitToTimestamp, err := c.GetFirstSeenTimestamp(nonExistentHashes, nullTimestamp)
assert.Equal(t, nil, err, "Error should not be generated")
assert.Equal(t, len(nonExistentHashes), len(commitToTimestamp), "Fetched results should match the length of the input")
assert.Equal(t, nullTimestamp, commitToTimestamp[nonExistentHash], "Populated value should equal the input null timestamp")
}

8
tools/repo_diff/service/repodiff/repositories/denormalizer.go
View File

@@ -238,7 +238,7 @@ func (s ScopedDenormalizer) DenormalizeToChangesOverTime(diffRows []ent.Analyzed
)
}
func (s ScopedDenormalizer) DenormalizeToRecentCommits(commitRows []ent.AnalyzedCommitRow) error {
func (s ScopedDenormalizer) DenormalizeToRecentCommits(commitRows []ent.AnalyzedCommitRow, commitToTimestamp map[string]ent.RepoTimestamp) error {
table := "denormalized_view_recent_commit"
if err := s.deleteExistingView(table); err != nil {
return err
@@ -257,20 +257,22 @@ func (s ScopedDenormalizer) DenormalizeToRecentCommits(commitRows []ent.Analyzed
subject,
tech_area,
project_type,
first_seen_datastudio_datetime,
upstream_url,
upstream_branch,
downstream_url,
downstream_branch
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
table,
),
s.rowsWithScopedIndices(
mappers.CommitRowsToDenormalizedCols(commitRows),
mappers.CommitRowsToDenormalizedCols(commitRows, commitToTimestamp),
),
),
errorMessageForTable(table),
)
}
func (s ScopedDenormalizer) deleteExistingView(tableName string) error {
_, err := s.db.Exec(
fmt.Sprintf(

27
tools/repo_diff/service/repodiff/repositories/denormalizer_test.go
View File

@@ -12,6 +12,8 @@ import (
"repodiff/repositories"
)
const arbitraryTimestamp = e.RepoTimestamp(1525978906)
var fakeTarget = e.DiffTarget{
Upstream: e.Project{
URL: "https://keystone-qcom.googlesource.com/platform/manifest",
@@ -83,7 +85,12 @@ func TestDenormalizeToRecentCommits(t *testing.T) {
fixtures := []e.AnalyzedCommitRow{
fixture,
}
err := d.DenormalizeToRecentCommits(fixtures)
err := d.DenormalizeToRecentCommits(
fixtures,
map[string]e.RepoTimestamp{
"61d5e61b6b6dfbf52d0d433759da964db31cc106": arbitraryTimestamp,
},
)
assert.Equal(t, nil, err, "Error should be nil")
assert.Equal(t, len(fixtures), getRowCountAtTable(tableName), "Rows should be inserted")
}
@@ -133,7 +140,14 @@ func TestDenormalizeToTopCommitter(t *testing.T) {
scopedD, _ := repositories.NewScopedDenormalizerRepository(fakeTarget, fakeMappedTarget)
err = scopedD.DenormalizeToRecentCommits(fakeCommitRows)
err = scopedD.DenormalizeToRecentCommits(
fakeCommitRows,
map[string]e.RepoTimestamp{
"540eecd728a407e4b31a38f4ea9416dea7d05c0c": arbitraryTimestamp,
"ea999655a8af4b7d6a8033d1c864ca87617d0ede": arbitraryTimestamp,
"4cc9725c953f57f8abe63b729e26125feac1be4e": arbitraryTimestamp,
},
)
assert.Equal(t, nil, err, "Error should be nil")
assert.Equal(t, 3, getRowCountAtTable("denormalized_view_recent_commit"), "Rows should be inserted")
@@ -187,7 +201,14 @@ func TestDenormalizeToTopTechArea(t *testing.T) {
scopedD, _ := repositories.NewScopedDenormalizerRepository(fakeTarget, fakeMappedTarget)
err = scopedD.DenormalizeToRecentCommits(fakeCommitRows)
err = scopedD.DenormalizeToRecentCommits(
fakeCommitRows,
map[string]e.RepoTimestamp{
"540eecd728a407e4b31a38f4ea9416dea7d05c0c": arbitraryTimestamp,
"ea999655a8af4b7d6a8033d1c864ca87617d0ede": arbitraryTimestamp,
"4cc9725c953f57f8abe63b729e26125feac1be4e": arbitraryTimestamp,
},
)
assert.Equal(t, nil, err, "Error should be nil")
assert.Equal(t, 3, getRowCountAtTable("denormalized_view_recent_commit"), "Rows should be inserted")

1
tools/repo_diff/service/repodiff/tools/migrations/2018_05_09__16_51_57_downgrade.sql Normal file
View File

@@ -0,0 +1 @@
ALTER TABLE denormalized_view_recent_commit DROP COLUMN first_seen_datastudio_datetime;

1
tools/repo_diff/service/repodiff/tools/migrations/2018_05_09__16_51_57_upgrade.sql Normal file
View File

@@ -0,0 +1 @@
ALTER TABLE denormalized_view_recent_commit ADD first_seen_datastudio_datetime CHAR(10) NOT NULL DEFAULT "2099050923";

2
tools/repo_diff/service/repodiff/utils/time.go
View File

@@ -17,7 +17,7 @@ func TimestampToDate(timestamp ent.RepoTimestamp) string {
}
// Formats a timestamp into a datetime acceptable for MySQL
func TimestampToDatastudioDatetime(timestamp ent.RepoTimestamp) string {
func TimestampToDataStudioDatetime(timestamp ent.RepoTimestamp) string {
asTime := t.Unix(int64(timestamp), 0)
return fmt.Sprintf(
"%04d%02d%02d%02d",

4
tools/repo_diff/service/repodiff/utils/time_test.go
View File

@@ -20,7 +20,7 @@ func TestTimestampToDate(t *testing.T) {
assert.Equal(t, "2018-02-22", TimestampToDate(timestamp), "Date conversion")
}
func TestTimestampToDatastudioDatetime(t *testing.T) {
func TestTimestampToDataStudioDatetime(t *testing.T) {
var timestamp ent.RepoTimestamp = 1519322647
assert.Equal(t, "2018022210", TimestampToDatastudioDatetime(timestamp), "Datetime conversion")
assert.Equal(t, "2018022210", TimestampToDataStudioDatetime(timestamp), "Datetime conversion")
}