Refactor GitLab integration (#1116)

.env.example

@@ -1,3 +1,8 @@
 OTA_ENGINE_SENDINBLUE_API_KEY='xkeysib-3f51c…'
 OTA_ENGINE_SMTP_PASSWORD='password'
+
+# If both GitHub and GitLab tokens are defined, GitHub takes precedence for dataset publishing
 OTA_ENGINE_GITHUB_TOKEN=ghp_XXXXXXXXX
+
+OTA_ENGINE_GITLAB_TOKEN=XXXXXXXXXX
+OTA_ENGINE_GITLAB_RELEASES_TOKEN=XXXXXXXXXX

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased [minor]
+
+> Development of this release was supported by the [European Union](https://commission.europa.eu/) and the [French Ministry for Foreign Affairs](https://www.diplomatie.gouv.fr/fr/politique-etrangere-de-la-france/diplomatie-numerique/) through its ministerial [State Startups incubator](https://beta.gouv.fr/startups/open-terms-archive.html) under the aegis of the Ambassador for Digital Affairs.
+
+### Added
+
+- Add support for GitLab for issue reporting
+- Add support for GitLab Releases for publishing datasets
+
 ## 2.5.0 - 2024-10-29
 
 _Full changeset and discussions: [#1115](https://github.com/OpenTermsArchive/engine/pull/1115)._

scripts/dataset/publish/github/index.js

@@ -0,0 +1,36 @@
+import fsApi from 'fs';
+import path from 'path';
+import url from 'url';
+
+import config from 'config';
+import { Octokit } from 'octokit';
+
+import * as readme from '../../assets/README.template.js';
+
+export default async function publish({ archivePath, releaseDate, stats }) {
+  const octokit = new Octokit({ auth: process.env.OTA_ENGINE_GITHUB_TOKEN });
+
+  const [ owner, repo ] = url.parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')).pathname.split('/').filter(component => component);
+
+  const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
+
+  const { data: { upload_url: uploadUrl, html_url: releaseUrl } } = await octokit.rest.repos.createRelease({
+    owner,
+    repo,
+    tag_name: tagName,
+    name: readme.title({ releaseDate }),
+    body: readme.body(stats),
+  });
+
+  await octokit.rest.repos.uploadReleaseAsset({
+    data: fsApi.readFileSync(archivePath),
+    headers: {
+      'content-type': 'application/zip',
+      'content-length': fsApi.statSync(archivePath).size,
+    },
+    name: path.basename(archivePath),
+    url: uploadUrl,
+  });
+
+  return releaseUrl;
+}

scripts/dataset/publish/gitlab/index.js

@@ -0,0 +1,133 @@
+import fsApi from 'fs';
+import path from 'path';
+
+import config from 'config';
+import dotenv from 'dotenv';
+import FormData from 'form-data';
+import nodeFetch from 'node-fetch';
+
+import GitLab from '../../../../src/reporter/gitlab/index.js';
+import * as readme from '../../assets/README.template.js';
+import logger from '../../logger/index.js';
+
+dotenv.config();
+
+export default async function publish({
+  archivePath,
+  releaseDate,
+  stats,
+}) {
+  let projectId = null;
+  const gitlabAPIUrl = config.get('@opentermsarchive/engine.dataset.apiBaseURL');
+
+  const [ owner, repo ] = new URL(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL'))
+    .pathname
+    .split('/')
+    .filter(Boolean);
+  const commonParams = { owner, repo };
+
+  try {
+    const repositoryPath = `${commonParams.owner}/${commonParams.repo}`;
+
+    const options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
+
+    options.method = 'GET';
+    options.headers = {
+      'Content-Type': 'application/json',
+      ...options.headers,
+    };
+
+    const response = await nodeFetch(
+      `${gitlabAPIUrl}/projects/${encodeURIComponent(repositoryPath)}`,
+      options,
+    );
+    const res = await response.json();
+
+    projectId = res.id;
+  } catch (error) {
+    logger.error(`Error while obtaining projectId: ${error}`);
+    projectId = null;
+  }
+
+  const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
+
+  try {
+    let options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
+
+    options.method = 'POST';
+    options.body = {
+      ref: 'main',
+      tag_name: tagName,
+      name: readme.title({ releaseDate }),
+      description: readme.body(stats),
+    };
+    options.headers = {
+      'Content-Type': 'application/json',
+      ...options.headers,
+    };
+
+    options.body = JSON.stringify(options.body);
+
+    const releaseResponse = await nodeFetch(
+      `${gitlabAPIUrl}/projects/${projectId}/releases`,
+      options,
+    );
+    const releaseRes = await releaseResponse.json();
+
+    const releaseId = releaseRes.commit.id;
+
+    logger.info(`Created release with releaseId: ${releaseId}`);
+
+    // Upload the package
+    options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
+    options.method = 'PUT';
+    options.body = fsApi.createReadStream(archivePath);
+
+    // restrict characters to the ones allowed by GitLab APIs
+    const packageName = config.get('@opentermsarchive/engine.dataset.title').replace(/[^a-zA-Z0-9.\-_]/g, '-');
+    const packageVersion = tagName.replace(/[^a-zA-Z0-9.\-_]/g, '-');
+    const packageFileName = archivePath.replace(/[^a-zA-Z0-9.\-_/]/g, '-');
+
+    logger.debug(`packageName: ${packageName}, packageVersion: ${packageVersion} packageFileName: ${packageFileName}`);
+
+    const packageResponse = await nodeFetch(
+      `${gitlabAPIUrl}/projects/${projectId}/packages/generic/${packageName}/${packageVersion}/${packageFileName}?status=default&select=package_file`,
+      options,
+    );
+    const packageRes = await packageResponse.json();
+
+    const packageFilesId = packageRes.id;
+
+    logger.debug(`package file id: ${packageFilesId}`);
+
+    // use the package id to build the download url for the release
+    const publishedPackageUrl = `${config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')}/-/package_files/${packageFilesId}/download`;
+
+    // Create the release and link the package
+    const formData = new FormData();
+
+    formData.append('name', archivePath);
+    formData.append('url', publishedPackageUrl);
+    formData.append('file', fsApi.createReadStream(archivePath), { filename: path.basename(archivePath) });
+
+    options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
+    options.method = 'POST';
+    options.headers = {
+      ...formData.getHeaders(),
+      ...options.headers,
+    };
+    options.body = formData;
+
+    const uploadResponse = await nodeFetch(
+      `${gitlabAPIUrl}/projects/${projectId}/releases/${tagName}/assets/links`,
+      options,
+    );
+    const uploadRes = await uploadResponse.json();
+    const releaseUrl = uploadRes.direct_asset_url;
+
+    return releaseUrl;
+  } catch (error) {
+    logger.error('Failed to create release or upload ZIP file:', error);
+    throw error;
+  }
+}

scripts/dataset/publish/index.js

@@ -1,36 +1,15 @@
-import fsApi from 'fs';
-import path from 'path';
-import url from 'url';
+import publishGitHub from './github/index.js';
+import publishGitLab from './gitlab/index.js';
 
-import config from 'config';
-import { Octokit } from 'octokit';
+export default function publishRelease({ archivePath, releaseDate, stats }) {
+  // If both GitHub and GitLab tokens are defined, GitHub takes precedence
+  if (process.env.OTA_ENGINE_GITHUB_TOKEN) {
+    return publishGitHub({ archivePath, releaseDate, stats });
+  }
 
-import * as readme from '../assets/README.template.js';
+  if (process.env.OTA_ENGINE_GITLAB_TOKEN) {
+    return publishGitLab({ archivePath, releaseDate, stats });
+  }
 
-export default async function publish({ archivePath, releaseDate, stats }) {
-  const octokit = new Octokit({ auth: process.env.OTA_ENGINE_GITHUB_TOKEN });
-
-  const [ owner, repo ] = url.parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')).pathname.split('/').filter(component => component);
-
-  const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
-
-  const { data: { upload_url: uploadUrl, html_url: releaseUrl } } = await octokit.rest.repos.createRelease({
-    owner,
-    repo,
-    tag_name: tagName,
-    name: readme.title({ releaseDate }),
-    body: readme.body(stats),
-  });
-
-  await octokit.rest.repos.uploadReleaseAsset({
-    data: fsApi.readFileSync(archivePath),
-    headers: {
-      'content-type': 'application/zip',
-      'content-length': fsApi.statSync(archivePath).size,
-    },
-    name: path.basename(archivePath),
-    url: uploadUrl,
-  });
-
-  return releaseUrl;
+  throw new Error('No GitHub nor GitLab token found in environment variables (OTA_ENGINE_GITHUB_TOKEN or OTA_ENGINE_GITLAB_TOKEN). Cannot publish the dataset without authentication.');
 }

src/index.js

@@ -55,21 +55,17 @@ export default async function track({ services, types, extractOnly, schedule })
     logger.warn('Environment variable "OTA_ENGINE_SENDINBLUE_API_KEY" was not found; the Notifier module will be ignored');
   }
 
-  if (process.env.OTA_ENGINE_GITHUB_TOKEN) {
-    if (config.has('@opentermsarchive/engine.reporter.githubIssues.repositories.declarations')) {
-      try {
-        const reporter = new Reporter(config.get('@opentermsarchive/engine.reporter'));
-
-        await reporter.initialize();
-        archivist.attach(reporter);
-      } catch (error) {
-        logger.error('Cannot instantiate the Reporter module; it will be ignored:', error);
-      }
-    } else {
-      logger.warn('Configuration key "reporter.githubIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created');
+  if (process.env.OTA_ENGINE_GITHUB_TOKEN || process.env.OTA_ENGINE_GITLAB_TOKEN) {
+    try {
+      const reporter = new Reporter(config.get('@opentermsarchive/engine.reporter'));
+
+      await reporter.initialize();
+      archivist.attach(reporter);
+    } catch (error) {
+      logger.error('Cannot instantiate the Reporter module; it will be ignored:', error);
     }
   } else {
-    logger.warn('Environment variable "OTA_ENGINE_GITHUB_TOKEN" was not found; the Reporter module will be ignored');
+    logger.warn('Environment variable with token for GitHub or GitLab was not found; the Reporter module will be ignored');
   }
 
   if (!schedule) {

src/reporter/factory.js

@@ -0,0 +1,13 @@
+import GitHub from './github/index.js';
+import GitLab from './gitlab/index.js';
+
+export function createReporter(config) {
+  switch (config.type) {
+  case 'github':
+    return new GitHub(config.repositories.declarations);
+  case 'gitlab':
+    return new GitLab(config.repositories.declarations, config.baseURL, config.apiBaseURL);
+  default:
+    throw new Error(`Unsupported reporter type: ${config.type}`);
+  }
+}

src/reporter/github.js → src/reporter/github/index.js

@@ -2,7 +2,7 @@ import { createRequire } from 'module';
 
 import { Octokit } from 'octokit';
 
-import logger from '../logger/index.js';
+import logger from '../../logger/index.js';
 
 const require = createRequire(import.meta.url);
 
@@ -14,7 +14,7 @@ export default class GitHub {
   static ISSUE_STATE_ALL = 'all';
 
   constructor(repository) {
-    const { version } = require('../../package.json');
+    const { version } = require('../../../package.json');
 
     this.octokit = new Octokit({
       auth: process.env.OTA_ENGINE_GITHUB_TOKEN,
@@ -198,4 +198,16 @@ export default class GitHub {
       logger.error(`Failed to update issue "${title}": ${error.stack}`);
     }
   }
+
+  generateDeclarationURL(serviceName) {
+    return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/declarations/${encodeURIComponent(serviceName)}.json`;
+  }
+
+  generateVersionURL(serviceName, termsType) {
+    return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}.md`;
+  }
+
+  generateSnapshotsBaseUrl(serviceName, termsType) {
+    return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}`;
+  }
 }

src/reporter/github.test.js → src/reporter/github/index.test.js

@@ -3,7 +3,7 @@ import { createRequire } from 'module';
 import { expect } from 'chai';
 import nock from 'nock';
 
-import GitHub from './github.js';
+import GitHub from './index.js';
 
 const require = createRequire(import.meta.url);
 

src/reporter/labels.test.js → src/reporter/github/labels.test.js

@@ -2,7 +2,7 @@ import { createRequire } from 'module';
 
 import chai from 'chai';
 
-import { MANAGED_BY_OTA_MARKER } from './github.js';
+import { MANAGED_BY_OTA_MARKER } from './index.js';
 
 const require = createRequire(import.meta.url);