From 6395be479ad26602c82212685fd048d23d12afff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Odd=20Str=C3=A5b=C3=B8?= Date: Fri, 17 Oct 2025 18:55:11 +0200 Subject: [PATCH 01/12] add recent languages debug print --- source/plugins/languages/analyzer/recent.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/source/plugins/languages/analyzer/recent.mjs b/source/plugins/languages/analyzer/recent.mjs index 927c91a7189..38eee939f20 100644 --- a/source/plugins/languages/analyzer/recent.mjs +++ b/source/plugins/languages/analyzer/recent.mjs @@ -61,6 +61,7 @@ export class RecentAnalyzer extends Analyzer { this.results.latest = Math.round((new Date().getTime() - new Date(commits.slice(-1).shift()?.created_at).getTime()) / (1000 * 60 * 60 * 24)) this.results.commits = commits.length + this.debug(JSON.stringify(commits)); //Retrieve edited files and filter edited lines (those starting with +/-) from patches this.debug("fetching patches") const patches = [ From c32878ca41906dba3e10fc9f19b2b12bc4298cd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Odd=20Str=C3=A5b=C3=B8?= Date: Fri, 17 Oct 2025 19:11:02 +0200 Subject: [PATCH 02/12] Install xz-utils --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3ff5209a48a..a744b8ac9bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ RUN chmod +x /metrics/source/app/action/index.mjs \ && apt-get install -y curl unzip \ && curl -fsSL https://deno.land/x/install/install.sh | DENO_INSTALL=/usr/local sh \ # Install ruby to support github licensed gem - && apt-get install -y ruby-full git g++ cmake pkg-config libssl-dev \ + && apt-get install -y ruby-full git g++ cmake pkg-config libssl-dev xz-utils \ && gem install licensed \ # Install python for node-gyp && apt-get install -y python3 \ From de58e40391d57e531a9e7330f7935d3637f85efd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Odd=20Str=C3=A5b=C3=B8?= Date: Fri, 17 Oct 2025 19:31:04 +0200 Subject: [PATCH 03/12] loop over items. --- source/plugins/languages/analyzer/recent.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/plugins/languages/analyzer/recent.mjs b/source/plugins/languages/analyzer/recent.mjs index 38eee939f20..781999b0f83 100644 --- a/source/plugins/languages/analyzer/recent.mjs +++ b/source/plugins/languages/analyzer/recent.mjs @@ -61,7 +61,10 @@ export class RecentAnalyzer extends Analyzer { this.results.latest = Math.round((new Date().getTime() - new Date(commits.slice(-1).shift()?.created_at).getTime()) / (1000 * 60 * 60 * 24)) this.results.commits = commits.length - this.debug(JSON.stringify(commits)); + commits.forEach(commit => { + this.debug(JSON.stringify(commit)); + }); + //Retrieve edited files and filter edited lines (those starting with +/-) from patches this.debug("fetching patches") const patches = [ From 3a765e7bd9d8663e3f5786b2984b0627ff7245b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Odd=20Str=C3=A5b=C3=B8?= Date: Fri, 17 Oct 2025 23:16:16 +0200 Subject: [PATCH 04/12] Attempt implementing new API pattern --- source/plugins/languages/analyzer/recent.mjs | 50 +++++++++++++++++--- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/source/plugins/languages/analyzer/recent.mjs b/source/plugins/languages/analyzer/recent.mjs index 781999b0f83..a62912778ab 100644 --- a/source/plugins/languages/analyzer/recent.mjs +++ b/source/plugins/languages/analyzer/recent.mjs @@ -30,7 +30,7 @@ export class RecentAnalyzer extends Analyzer { async patches() { //Fetch commits from recent activity this.debug(`fetching patches from last ${this.days || ""} days up to ${this.load || "∞"} events`) - const commits = [], pages = Math.ceil((this.load || Infinity) / 100) + const pages = Math.ceil((this.load || Infinity) / 100) if (this.context.mode === "repository") { try { const {data: {default_branch: branch}} = await this.rest.repos.get(this.context) @@ -42,10 +42,11 @@ export class RecentAnalyzer extends Analyzer { this.debug(`failed to get default branch for ${this.context.owner}/${this.context.repo} (${error})`) } } + const events = [] try { for (let page = 1; page <= pages; page++) { this.debug(`fetching events page ${page}`) - commits.push( + events.push( ...(await (this.context.mode === "repository" ? this.rest.activity.listRepoEvents(this.context) : this.rest.activity.listEventsForAuthenticatedUser({username: this.login, per_page: 100, page}))).data .filter(({type, payload}) => (type === "PushEvent") && ((this.context.mode !== "repository") || ((this.context.mode === "repository") && (payload?.ref?.includes?.(`refs/heads/${this.context.branch}`))))) .filter(({actor}) => (this.account === "organization") || (this.context.mode === "repository") ? true : !filters.text(actor.login, [this.login], {debug: false})) @@ -57,6 +58,43 @@ export class RecentAnalyzer extends Analyzer { catch { this.debug("no more page to load") } + this.debug(`fetched ${events.length} events`) + + const wanted = new Map(); + events.forEach(event => { + var key = `${event.repo.name}@${event.payload.ref}` + var item = wanted.get(key) ?? { commits: [] } + item.repo = event.repo.name + item.ref = event.payload.ref + item.commits.push(event.payload.before) + item.commits.push(event.payload.head) + wanted.set(key, item) + }); + + const commits = [] + for ([key, item] of wanted) { + try { + for (let page = 1; page <= pages; page++) { + this.debug(`fetching commits page ${page}`) + commits.push( + ...(await this.rest.request(`https://api.github.com/repos/${item.repo}/git/commits?sha=${item.ref}&per_page=20&page=${page}`)).data + .map(x => { item.commits = item.commits.filter(c => c != x.sha); return x }) + .filter(({ committer }) => (this.account === "organization") || (this.context.mode === "repository") ? true : !filters.text(committer.login, [this.login], { debug: false })) + .filter(({ commit }) => ((!this.days) || (new Date(commit.committer.date) > new Date(Date.now() - this.days * 24 * 60 * 60 * 1000)))), + ) + if (item.commits < 1) { + this.debug("found expected commits") + break + } + } + } + catch { + this.debug("no more page to load") + } + } + + + this.debug(`fetched ${commits.length} commits`) this.results.latest = Math.round((new Date().getTime() - new Date(commits.slice(-1).shift()?.created_at).getTime()) / (1000 * 60 * 60 * 24)) this.results.commits = commits.length @@ -108,15 +146,15 @@ export class RecentAnalyzer extends Analyzer { } /**Run linguist against a commit and compute edited lines and bytes*/ - async linguist(_, {commit, cache: {languages}}) { - const cache = {files: {}, languages} - const result = {total: 0, files: 0, missed: {lines: 0, bytes: 0}, lines: {}, stats: {}, languages: {}} + async linguist(_, { commit, cache: { languages } }) { + const cache = { files: {}, languages } + const result = { total: 0, files: 0, missed: { lines: 0, bytes: 0 }, lines: {}, stats: {}, languages: {} } const edited = new Set() for (const edition of commit.editions) { edited.add(edition.path) //Guess file language with linguist - const {files: {results: files}, languages: {results: languages}, unknown} = await linguist(edition.path, {fileContent: edition.patch}) + const { files: { results: files }, languages: { results: languages }, unknown } = await linguist(edition.path, { fileContent: edition.patch }) Object.assign(cache.files, files) Object.assign(cache.languages, languages) if (!(edition.path in cache.files)) From c0e6e165f155a8a43cd0d141136884422d94111e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Odd=20Str=C3=A5b=C3=B8?= Date: Fri, 17 Oct 2025 23:31:06 +0200 Subject: [PATCH 05/12] Fix for loop --- source/plugins/languages/analyzer/recent.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/plugins/languages/analyzer/recent.mjs b/source/plugins/languages/analyzer/recent.mjs index a62912778ab..9e400969036 100644 --- a/source/plugins/languages/analyzer/recent.mjs +++ b/source/plugins/languages/analyzer/recent.mjs @@ -72,7 +72,7 @@ export class RecentAnalyzer extends Analyzer { }); const commits = [] - for ([key, item] of wanted) { + for (const item of wanted.values()) { try { for (let page = 1; page <= pages; page++) { this.debug(`fetching commits page ${page}`) From 141a2752bc1800fe00f75a58afd5998a01aefe66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Odd=20Str=C3=A5b=C3=B8?= Date: Fri, 17 Oct 2025 23:51:57 +0200 Subject: [PATCH 06/12] Dump commits URL --- source/plugins/languages/analyzer/recent.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/source/plugins/languages/analyzer/recent.mjs b/source/plugins/languages/analyzer/recent.mjs index 9e400969036..4b2a9d52c4f 100644 --- a/source/plugins/languages/analyzer/recent.mjs +++ b/source/plugins/languages/analyzer/recent.mjs @@ -76,6 +76,7 @@ export class RecentAnalyzer extends Analyzer { try { for (let page = 1; page <= pages; page++) { this.debug(`fetching commits page ${page}`) + this.debug(`https://api.github.com/repos/${item.repo}/git/commits?sha=${item.ref}&per_page=20&page=${page}`) commits.push( ...(await this.rest.request(`https://api.github.com/repos/${item.repo}/git/commits?sha=${item.ref}&per_page=20&page=${page}`)).data .map(x => { item.commits = item.commits.filter(c => c != x.sha); return x }) From 0c9e5096337ff036c37f91157b822c0a7eb6dc44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Odd=20Str=C3=A5b=C3=B8?= Date: Fri, 17 Oct 2025 23:56:27 +0200 Subject: [PATCH 07/12] Restructure dockerfile to allow partial caching --- Dockerfile | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index a744b8ac9bb..44f26b4f9ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,8 @@ # Base image -FROM node:20-bookworm-slim - -# Copy repository -COPY . /metrics -WORKDIR /metrics +FROM node:20-bookworm-slim AS base # Setup -RUN chmod +x /metrics/source/app/action/index.mjs \ +RUN \ # Install latest chrome dev package, fonts to support major charsets and skip chromium download on puppeteer install # Based on https://github.com/GoogleChrome/puppeteer/blob/master/docs/troubleshooting.md#running-puppeteer-in-docker && apt-get update \ @@ -24,7 +20,16 @@ RUN chmod +x /metrics/source/app/action/index.mjs \ # Install python for node-gyp && apt-get install -y python3 \ # Clean apt/lists - && rm -rf /var/lib/apt/lists/* \ + && rm -rf /var/lib/apt/lists/* + + +FROM base + +# Copy repository +COPY . /metrics +WORKDIR /metrics + +RUN chmod +x /metrics/source/app/action/index.mjs\ # Install node modules and rebuild indexes && npm ci \ && npm run build From bbc181a4b837c1cad2fb67fcfeeca3cf3c881042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Odd=20Str=C3=A5b=C3=B8?= Date: Sat, 18 Oct 2025 00:05:49 +0200 Subject: [PATCH 08/12] Fix commits API URL --- source/plugins/languages/analyzer/recent.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/plugins/languages/analyzer/recent.mjs b/source/plugins/languages/analyzer/recent.mjs index 4b2a9d52c4f..b090bbe4b69 100644 --- a/source/plugins/languages/analyzer/recent.mjs +++ b/source/plugins/languages/analyzer/recent.mjs @@ -76,9 +76,9 @@ export class RecentAnalyzer extends Analyzer { try { for (let page = 1; page <= pages; page++) { this.debug(`fetching commits page ${page}`) - this.debug(`https://api.github.com/repos/${item.repo}/git/commits?sha=${item.ref}&per_page=20&page=${page}`) + this.debug(`https://api.github.com/repos/${item.repo}/commits?sha=${item.ref}&per_page=20&page=${page}`) commits.push( - ...(await this.rest.request(`https://api.github.com/repos/${item.repo}/git/commits?sha=${item.ref}&per_page=20&page=${page}`)).data + ...(await this.rest.request(`https://api.github.com/repos/${item.repo}/commits?sha=${item.ref}&per_page=20&page=${page}`)).data .map(x => { item.commits = item.commits.filter(c => c != x.sha); return x }) .filter(({ committer }) => (this.account === "organization") || (this.context.mode === "repository") ? true : !filters.text(committer.login, [this.login], { debug: false })) .filter(({ commit }) => ((!this.days) || (new Date(commit.committer.date) > new Date(Date.now() - this.days * 24 * 60 * 60 * 1000)))), From f6a53a19f5cef67e26e0a2404197782f5c2f09f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Odd=20Str=C3=A5b=C3=B8?= Date: Sat, 18 Oct 2025 00:08:08 +0200 Subject: [PATCH 09/12] Fix dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 44f26b4f9ba..1d1648e8b7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM node:20-bookworm-slim AS base # Setup -RUN \ +RUN echo -n \ # Install latest chrome dev package, fonts to support major charsets and skip chromium download on puppeteer install # Based on https://github.com/GoogleChrome/puppeteer/blob/master/docs/troubleshooting.md#running-puppeteer-in-docker && apt-get update \ From 51b6572c3102107dc869a220968402defb013145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Odd=20Str=C3=A5b=C3=B8?= Date: Sat, 18 Oct 2025 00:46:12 +0200 Subject: [PATCH 10/12] Fix patch fetching --- source/plugins/languages/analyzer/recent.mjs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/source/plugins/languages/analyzer/recent.mjs b/source/plugins/languages/analyzer/recent.mjs index b090bbe4b69..69950db4f4d 100644 --- a/source/plugins/languages/analyzer/recent.mjs +++ b/source/plugins/languages/analyzer/recent.mjs @@ -100,16 +100,11 @@ export class RecentAnalyzer extends Analyzer { this.results.latest = Math.round((new Date().getTime() - new Date(commits.slice(-1).shift()?.created_at).getTime()) / (1000 * 60 * 60 * 24)) this.results.commits = commits.length - commits.forEach(commit => { - this.debug(JSON.stringify(commit)); - }); - //Retrieve edited files and filter edited lines (those starting with +/-) from patches this.debug("fetching patches") const patches = [ ...await Promise.allSettled( commits - .flatMap(({payload}) => payload.commits) .filter(({committer}) => filters.text(committer?.email, this.authoring, {debug: false})) .map(commit => commit.url) .map(async commit => (await this.rest.request(commit)).data), @@ -118,9 +113,9 @@ export class RecentAnalyzer extends Analyzer { .filter(({status}) => status === "fulfilled") .map(({value}) => value) .filter(({parents}) => parents.length <= 1) - .map(({sha, commit: {message, committer}, verification, files}) => ({ + .map(({ sha, commit: { message, author }, verification, files }) => ({ sha, - name: `${message} (authored by ${committer.name} on ${committer.date})`, + name: `${message} (authored by ${author.name} on ${author.date})`, verified: verification?.verified ?? null, editions: files.map(({filename, patch = ""}) => { const edition = { From 582a522ffd4e9efb2178c956fccea610f418cd08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Odd=20Str=C3=A5b=C3=B8?= Date: Sat, 18 Oct 2025 01:13:06 +0200 Subject: [PATCH 11/12] Fix NaN days in recently used languages --- source/plugins/languages/analyzer/recent.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/plugins/languages/analyzer/recent.mjs b/source/plugins/languages/analyzer/recent.mjs index 69950db4f4d..20603e3c935 100644 --- a/source/plugins/languages/analyzer/recent.mjs +++ b/source/plugins/languages/analyzer/recent.mjs @@ -97,7 +97,7 @@ export class RecentAnalyzer extends Analyzer { this.debug(`fetched ${commits.length} commits`) - this.results.latest = Math.round((new Date().getTime() - new Date(commits.slice(-1).shift()?.created_at).getTime()) / (1000 * 60 * 60 * 24)) + this.results.latest = Math.round((new Date().getTime() - new Date(events.slice(-1).shift()?.created_at).getTime()) / (1000 * 60 * 60 * 24)) this.results.commits = commits.length //Retrieve edited files and filter edited lines (those starting with +/-) from patches From b79aff3dcef1c90b518d92eb029d2c8942c3337d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Odd=20Str=C3=A5b=C3=B8?= Date: Sat, 18 Oct 2025 01:59:15 +0200 Subject: [PATCH 12/12] Fix linter complaints --- source/plugins/languages/analyzer/recent.mjs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/source/plugins/languages/analyzer/recent.mjs b/source/plugins/languages/analyzer/recent.mjs index 20603e3c935..6eb371b58ef 100644 --- a/source/plugins/languages/analyzer/recent.mjs +++ b/source/plugins/languages/analyzer/recent.mjs @@ -60,16 +60,16 @@ export class RecentAnalyzer extends Analyzer { } this.debug(`fetched ${events.length} events`) - const wanted = new Map(); + const wanted = new Map() events.forEach(event => { - var key = `${event.repo.name}@${event.payload.ref}` - var item = wanted.get(key) ?? { commits: [] } + let key = `${event.repo.name}@${event.payload.ref}` + let item = wanted.get(key) ?? { commits: [] } item.repo = event.repo.name item.ref = event.payload.ref item.commits.push(event.payload.before) item.commits.push(event.payload.head) wanted.set(key, item) - }); + }) const commits = [] for (const item of wanted.values()) { @@ -79,7 +79,10 @@ export class RecentAnalyzer extends Analyzer { this.debug(`https://api.github.com/repos/${item.repo}/commits?sha=${item.ref}&per_page=20&page=${page}`) commits.push( ...(await this.rest.request(`https://api.github.com/repos/${item.repo}/commits?sha=${item.ref}&per_page=20&page=${page}`)).data - .map(x => { item.commits = item.commits.filter(c => c != x.sha); return x }) + .map(x => { + item.commits = item.commits.filter(c => c !== x.sha) + return x + }) .filter(({ committer }) => (this.account === "organization") || (this.context.mode === "repository") ? true : !filters.text(committer.login, [this.login], { debug: false })) .filter(({ commit }) => ((!this.days) || (new Date(commit.committer.date) > new Date(Date.now() - this.days * 24 * 60 * 60 * 1000)))), ) @@ -94,8 +97,6 @@ export class RecentAnalyzer extends Analyzer { } } - - this.debug(`fetched ${commits.length} commits`) this.results.latest = Math.round((new Date().getTime() - new Date(events.slice(-1).shift()?.created_at).getTime()) / (1000 * 60 * 60 * 24)) this.results.commits = commits.length