Identify GCI users in repository information

This begins to identify users in several meta repository stats,
including stargazers, watchers and forks.

bust-cache

Closes #100

lib/queries/github_repo_info.graphql

@@ -0,0 +1,28 @@
+query($org: String!) {
+  organization(login: $org) {
+    repositories(first: 100) {
+      nodes {
+        watchers(first: 100) {
+          nodes {
+            login
+            name
+          }
+        }
+        stargazers(last: 100) {
+          nodes {
+            login
+            name
+          }
+        }
+        forks(last: 100) {
+          nodes {
+            owner {
+              login
+            }
+            createdAt
+          }
+        }
+      }
+    }
+  }
+}

lib/queries/index.js

@@ -0,0 +1,3 @@
+const { loadQuery } = require('../utils')
+
+module.exports.GITHUB_REPO_INFO_QUERY = loadQuery('github_repo_info')

lib/scrape.js

@@ -1,4 +1,5 @@
 const fetch = require('node-fetch')
+const GraphQL = require('graphql-client')
 const chattie = require('chattie')
 const fs = require('fs')
 const json2yaml = require('json2yaml')
@@ -7,10 +8,14 @@ const validUsername = require('valid-github-username')
 const wdk = require('wikidata-sdk')
 const cheerio = require('cheerio')
 
+const { GITHUB_REPO_INFO_QUERY } = require('./queries')
+const { getLatestCommitMessage } = require('./utils')
+
 const GH_BASE = 'https://github.com'
 const GH_USER_BASE = `${GH_BASE}/users`
 const GH_ORG_BASE = `${GH_BASE}/orgs`
 const GH_API_BASE = 'https://api.github.com'
+const GH_GQL_BASE = 'https://api.github.com/graphql'
 const GCI_API_BASE = 'https://codein.withgoogle.com/api'
 
 const MIN_SEARCH_SCORE = 10
@@ -44,6 +49,13 @@ const GH_API_OPTIONS = {
     : {},
 }
 
+const GH_GQL_OPTIONS = {
+  url: GH_GQL_BASE,
+  headers: process.env.GITHUB_TOKEN
+    ? { Authorization: `bearer ${process.env.GITHUB_TOKEN}` }
+    : {},
+}
+
 const GH_WEB_OPTIONS = {
   headers: {
     Accept: 'text/html',
@@ -55,6 +67,11 @@ const GH_WEB_OPTIONS = {
   compress: false,
 }
 
+const client = GraphQL(GH_GQL_OPTIONS)
+
+let COMPETITION_OPEN
+let BUST_GITHUB_CACHE
+
 let existingData = []
 try {
   existingData = JSON.parse(
@@ -86,6 +103,71 @@ async function fetchLeaders(id) {
   return leaders
 }
 
+let repositoryInfo = {}
+async function fetchRepositoryInfo(org) {
+  if (repositoryInfo[org]) return repositoryInfo[org]
+
+  const { data } = await client.query(GITHUB_REPO_INFO_QUERY, { org })
+
+  if (data) {
+    const info = data.organization.repositories.nodes.map(node => ({
+      watchers: node.watchers.nodes,
+      stargazers: node.stargazers.nodes,
+      forks: node.forks.nodes,
+    }))
+
+    repositoryInfo[org] = info
+
+    return info
+  } else {
+    return []
+  }
+}
+
+async function getGitHubUserFromRepoInfo(org, displayName, shortName) {
+  let repos = []
+  try {
+    repos = await fetchRepositoryInfo(org)
+  } catch (e) {
+    console.error(`Could not fetch repository info for ${org}...`)
+  }
+
+  let logins = []
+  let names = {}
+
+  repos.forEach(repo => {
+    logins = logins
+      .concat(repo.watchers.map(u => u.login.toLowerCase()))
+      .concat(repo.stargazers.map(u => u.login.toLowerCase()))
+      .concat(
+        repo.forks
+          .map(u => {
+            const createdAt = new Date(u.createdAt)
+            if (createdAt.getTime() > COMPETITION_OPEN.getTime()) {
+              return u.owner.login.toLowerCase()
+            }
+          })
+          .filter(login => login)
+      )
+
+    repo.watchers.forEach(watcher => {
+      if (watcher.name) {
+        names[watcher.name.toLowerCase()] = watcher.login
+      }
+    })
+  })
+
+  logins = logins.filter((item, pos, self) => self.indexOf(item) === pos)
+
+  if (logins.includes(shortName.toLowerCase())) {
+    return shortName
+  }
+
+  if (names[displayName.toLowerCase()]) {
+    return names[displayName.toLowerCase()]
+  }
+}
+
 async function checkGitHubUserExists(user) {
   const res = await fetch(`${GH_BASE}/${user}`)
   return res.status === 200
@@ -286,25 +368,43 @@ async function findGitHubUser(displayName, org) {
 
   const shortName = validUsername(displayName)
 
-  const username = await findGitHubUserInOrg(displayName, org)
-  if (username) return username
+  let userInOrg
+  try {
+    userInOrg = await findGitHubUserInOrg(displayName, org)
+  } catch (e) {
+    console.error(`Failed to find user ${displayName} in org ${org}...`)
+  }
+  if (userInOrg) {
+    console.log(`${displayName}: ${userInOrg} (method: user in org)`)
+    return userInOrg
+  }
 
   let user
   try {
     user = await getGitHubUser(shortName)
   } catch (e) {
-    console.error(`Got error when finding user ${shortName}...`)
+    console.error(`Failed to find user ${shortName}...`)
   }
-  if (!user) return
 
-  const login = user.login
+  if (!user) {
+    const userFromRepo = await getGitHubUserFromRepoInfo(
+      org,
+      displayName,
+      shortName
+    )
 
-  const { competition_open_starts } = await fetchProgram()
+    if (!userFromRepo) {
+      return
+    }
+
+    user = userFromRepo
+  }
+
+  const login = user.login
 
   const updatedTime = new Date(user.updated_at)
-  const openTime = new Date(competition_open_starts)
 
-  if (updatedTime.getTime() - openTime.getTime() < 0) return
+  if (updatedTime.getTime() - COMPETITION_OPEN.getTime() < 0) return
 
   let orgs = []
   try {
@@ -315,7 +415,9 @@ async function findGitHubUser(displayName, org) {
   } catch (e) {
     console.error('Could not fetch user history...')
   }
+
   if (orgs.includes(org)) {
+    console.log(`${displayName}: ${user.login} (method: found user from name)`)
     return user.login
   }
 }
@@ -345,7 +447,8 @@ async function freshenUserGitHubCache(user, existingUser, organization) {
       existingUser &&
       existingUser.github_updated &&
       existingUser.github_account
-    )
+    ) ||
+    BUST_GITHUB_CACHE
   ) {
     return {
       login: await findGitHubUser(user.display_name, organization),
@@ -400,6 +503,8 @@ async function fetchOrgsWithData() {
   const orgWiki = await Promise.all(fetchingWiki)
 
   const fetchingAll = orgs.map(async (org, index) => {
+    await fetchRepositoryInfo(orgGitHub[index])
+
     const existingOrg = existingData.find(existing => existing.id === org.id)
     const fetchingUsers = orgLeaders[index].map(async user => {
       let existingUser
@@ -445,6 +550,16 @@ async function fetchDates() {
 }
 
 ;(async () => {
+  const { competition_open_starts } = await fetchProgram()
+  COMPETITION_OPEN = new Date(competition_open_starts)
+
+  const { stdout } = await getLatestCommitMessage()
+  BUST_GITHUB_CACHE = stdout.toLowerCase().includes('bust-cache')
+
+  if (BUST_GITHUB_CACHE) {
+    console.log('Busting cache...')
+  }
+
   const orgs = await fetchOrgsWithData()
   const dates = await fetchDates()
 

lib/utils.js

@@ -0,0 +1,8 @@
+const fs = require('fs')
+const util = require('util')
+const exec = util.promisify(require('child_process').exec)
+
+module.exports.getLatestCommitMessage = () => exec('git log -1 --pretty=%B')
+
+module.exports.loadQuery = name =>
+  fs.readFileSync(`${__dirname}/queries/${name}.graphql`).toString()

package.json

@@ -26,6 +26,7 @@
     "feed-read-parser": "^0.0.6",
     "find-rss": "^1.6.4",
     "glob": "^7.1.2",
+    "graphql-client": "^2.0.0",
     "jquery": "^3.2.1",
     "jquery.i18n": "git+https://github.com/wikimedia/jquery.i18n.git",
     "json2yaml": "^1.1.0",