updater/projects: bulk optimize the DB queries

Signed-off-by: Arthur Zamarin <arthurzam@gentoo.org>

1 files changed, 22 insertions, 36 deletions

diff --git a/pkg/portage/projects/update.go b/pkg/portage/projects/update.go
index 2480d74..0b556b1 100644
--- a/pkg/portage/projects/update.go
+++ b/pkg/portage/projects/update.go
@@ -12,9 +12,8 @@ import (
 	"time"
 )
 
-// UpdatePkgCheckResults will update the database table that contains all pkgcheck results
+// UpdateProjects will update the database table that contains all projects
 func UpdateProjects() {
-
 	database.Connect()
 	defer database.DBCon.Close()
 
@@ -22,59 +21,46 @@ func UpdateProjects() {
 		log.SetOutput(ioutil.Discard)
 	}
 
-	// get the pkg check results from qa-reports.gentoo.org
+	// get projects from api.gentoo.org
 	projectList, err := parseProjectList()
 	if err != nil {
-		logger.Error.Println("Error while parsing project list. Aborting...")
+		logger.Error.Println("Error while parsing project list", err)
+		return
 	}
 
-	// clean up the database
-	deleteAllProjects()
-
-	// insert new project list
-	for _, project := range projectList.Projects {
-		database.DBCon.Model(&project).Insert()
+	var members []*models.MaintainerToProject
+	for _, project := range projectList {
 		for _, member := range project.Members {
-			database.DBCon.Model(&models.MaintainerToProject{
+			members = append(members, &models.MaintainerToProject{
 				Id:              member.Email + "-" + project.Email,
 				MaintainerEmail: member.Email,
 				ProjectEmail:    project.Email,
-			}).Insert()
+			})
 		}
 	}
 
+	// clean up the database
+	database.TruncateTable[models.Project]("email")
+	database.TruncateTable[models.MaintainerToProject]("id")
+
+	// insert new project list
+	database.DBCon.Model(&projectList).Insert()
+	database.DBCon.Model(&members).Insert()
+
 	updateStatus()
 }
 
-// parseQAReport gets the xml from qa-reports.gentoo.org and parses it
-func parseProjectList() (models.ProjectList, error) {
+// parseProjectList gets the xml from api.gentoo.org and parses it
+func parseProjectList() ([]models.Project, error) {
 	resp, err := http.Get("https://api.gentoo.org/metastructure/projects.xml")
 	if err != nil {
-		return models.ProjectList{}, err
+		return nil, err
 	}
 	defer resp.Body.Close()
-	xmlData, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return models.ProjectList{}, err
-	}
-	var projectList models.ProjectList
-	xml.Unmarshal(xmlData, &projectList)
-	return projectList, err
-}
 
-// deleteAllOutdated deletes all entries in the outdated table
-func deleteAllProjects() {
-	var allProjects []*models.Project
-	database.DBCon.Model(&allProjects).Select()
-	for _, project := range allProjects {
-		database.DBCon.Model(project).WherePK().Delete()
-	}
-
-	var allMaintainerToProjects []*models.MaintainerToProject
-	database.DBCon.Model(&allMaintainerToProjects).Select()
-	for _, maintainerToProject := range allMaintainerToProjects {
-		database.DBCon.Model(maintainerToProject).WherePK().Delete()
-	}
+	var projectList models.ProjectList
+	err = xml.NewDecoder(resp.Body).Decode(&projectList)
+	return projectList.Projects, err
 }
 
 func updateStatus() {