From e2fb8c2f9d126e763e4f0c0ffba158f2d0c5c17a Mon Sep 17 00:00:00 2001 From: Damien Arrachequesne Date: Mon, 22 Jan 2024 08:33:21 +0100 Subject: [PATCH 01/10] fix: exclude offline nodes when calling serverCount() Related: https://github.com/socketio/socket.io-mongo-adapter/issues/22 --- lib/index.ts | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/lib/index.ts b/lib/index.ts index 2f54a8d..7b36fa5 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -617,6 +617,13 @@ export class MongoAdapter extends Adapter { } public serverCount(): Promise { + this.nodesMap.forEach((lastSeen, uid) => { + const nodeSeemsDown = Date.now() - lastSeen > this.heartbeatTimeout; + if (nodeSeemsDown) { + debug("node %s seems down", uid); + this.nodesMap.delete(uid); + } + }); return Promise.resolve(1 + this.nodesMap.size); } @@ -671,20 +678,9 @@ export class MongoAdapter extends Adapter { }).catch(onPublishError); } - private getExpectedResponseCount() { - this.nodesMap.forEach((lastSeen, uid) => { - const nodeSeemsDown = Date.now() - lastSeen > this.heartbeatTimeout; - if (nodeSeemsDown) { - debug("node %s seems down", uid); - this.nodesMap.delete(uid); - } - }); - return this.nodesMap.size; - } - async fetchSockets(opts: BroadcastOptions): Promise { const localSockets = await super.fetchSockets(opts); - const expectedResponseCount = this.getExpectedResponseCount(); + const expectedResponseCount = (await this.serverCount()) - 1; if (opts.flags?.local || expectedResponseCount === 0) { return localSockets; @@ -745,7 +741,7 @@ export class MongoAdapter extends Adapter { private async serverSideEmitWithAck(packet: any[]) { const ack = packet.pop(); - const expectedResponseCount = this.getExpectedResponseCount(); + const expectedResponseCount = (await this.serverCount()) - 1; debug( 'waiting for %d responses to "serverSideEmit" request', From 0c80f7fd1da772cc54971fd93a1fa93f0c5e47d0 Mon Sep 17 00:00:00 2001 From: bohdankovt <103412497+bohdankovt@users.noreply.github.com> Date: Tue, 23 Jan 2024 15:35:35 +0200 Subject: [PATCH 02/10] fix: add support for AWS DocumentDB (#21) Providing an `undefined` value for the `resumeAfter` option would throw on DocumentDB: > MongoServerError: BSON field '$changeStream.resumeAfter' is the wrong type 'null', expected type 'object' Reference: https://docs.aws.amazon.com/documentdb/latest/developerguide/change_streams.html --- lib/index.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/index.ts b/lib/index.ts index 7b36fa5..e80ba42 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -7,7 +7,7 @@ import { } from "socket.io-adapter"; import { randomBytes } from "crypto"; import { ObjectId, MongoServerError, WithId, Document } from "mongodb"; -import type { Collection, ChangeStream, ResumeToken } from "mongodb"; +import type { Collection, ChangeStream, ChangeStreamOptions } from "mongodb"; const randomId = () => randomBytes(8).toString("hex"); const debug = require("debug")("socket.io-mongo-adapter"); @@ -161,7 +161,7 @@ export function createAdapter( let isClosed = false; let adapters = new Map(); let changeStream: ChangeStream; - let resumeToken: ResumeToken; + let changeStreamOpts: ChangeStreamOptions = {}; const initChangeStream = () => { if (isClosed || (changeStream && !changeStream.closed)) { @@ -178,14 +178,12 @@ export function createAdapter( }, }, ], - { - resumeAfter: resumeToken, - } + changeStreamOpts ); changeStream.on("change", (event: any) => { if (event.operationType === "insert") { - resumeToken = changeStream.resumeToken; + changeStreamOpts.resumeAfter = changeStream.resumeToken; adapters.get(event.fullDocument?.nsp)?.onEvent(event); } }); @@ -197,7 +195,7 @@ export function createAdapter( !err.hasErrorLabel("ResumableChangeStreamError") ) { // the resume token was not found in the oplog - resumeToken = null; + changeStreamOpts = {}; } }); From d3fa03874038ed9ec011d8795ac7dc6d840f4abe Mon Sep 17 00:00:00 2001 From: Damien Arrachequesne Date: Tue, 23 Jan 2024 15:56:09 +0100 Subject: [PATCH 03/10] fix: ensure CSR works with a capped collection The findOneAndDelete() operation is not authorized on a capped collection, so the connection state recovery would always fail when fetching the session. > MongoServerError: cannot remove from a capped collection Related: https://github.com/socketio/socket.io-mongo-adapter/issues/20 --- lib/index.ts | 64 +++- test/connection-state-recovery.ts | 482 +++++++++++++++++++++++------- 2 files changed, 422 insertions(+), 124 deletions(-) diff --git a/lib/index.ts b/lib/index.ts index e80ba42..090684c 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -806,28 +806,22 @@ export class MongoAdapter extends Adapter { try { results = await Promise.all([ // could use a sparse index on [data.pid] (only index the documents whose type is EventType.SESSION) - this.mongoCollection.findOneAndDelete({ - type: EventType.SESSION, - "data.pid": pid, - }), + this.findSession(pid), this.mongoCollection.findOne({ type: EventType.BROADCAST, _id: eventOffset, }), ]); } catch (e) { + debug("error while fetching session: %s", (e as Error).message); return Promise.reject("error while fetching session"); } - const result = (results[0]?.ok - ? results[0].value // mongodb@5 - : results[0]) as unknown as WithId; // mongodb@6 - - if (!result || !results[1]) { + if (!results[0] || !results[1]) { return Promise.reject("session or offset not found"); } - const session = result.data; + const session = results[0].data; // could use a sparse index on [_id, nsp, data.opts.rooms, data.opts.except] (only index the documents whose type is EventType.BROADCAST) const cursor = this.mongoCollection.find({ @@ -889,4 +883,54 @@ export class MongoAdapter extends Adapter { return session; } + + private findSession( + pid: PrivateSessionId + ): Promise | undefined> { + const isCollectionCapped = !this.addCreatedAtField; + if (isCollectionCapped) { + return this.mongoCollection + .findOne( + { + type: EventType.SESSION, + "data.pid": pid, + }, + { + sort: { + _id: -1, + }, + } + ) + .then((result) => { + if (!result) { + debug("session not found"); + return; + } + + if (result.data.sid) { + debug("session found, adding tombstone"); + + // since the collection is capped, we cannot remove documents from it, so we add a tombstone to prevent recovering the same session twice + // note: we could also have used two distinct collections, one for the events (capped) and the other for the sessions (not capped, with a TTL) + const TOMBSTONE_SESSION = { pid, tombstone: true }; + this.persistSession(TOMBSTONE_SESSION); + + return result; + } else { + debug("tombstone session found"); + } + }); + } else { + return this.mongoCollection + .findOneAndDelete({ + type: EventType.SESSION, + "data.pid": pid, + }) + .then((result) => { + return result?.ok && result.value + ? result.value // mongodb@5 + : (result as unknown as WithId); // mongodb@6 + }); + } + } } diff --git a/test/connection-state-recovery.ts b/test/connection-state-recovery.ts index 42f0d18..57fb411 100644 --- a/test/connection-state-recovery.ts +++ b/test/connection-state-recovery.ts @@ -9,168 +9,422 @@ import { AddressInfo } from "net"; const NODES_COUNT = 3; describe("connection state recovery", () => { - let servers: Server[], ports: number[], mongoClient: MongoClient; - - beforeEach(async () => { - servers = []; - ports = []; - - mongoClient = new MongoClient( - "mongodb://localhost:27017/?replicaSet=rs0&directConnection=true" - ); - await mongoClient.connect(); - - const collection = mongoClient.db("test").collection("events"); - - return new Promise((resolve) => { - for (let i = 1; i <= NODES_COUNT; i++) { - const httpServer = createServer(); - const io = new Server(httpServer, { - pingInterval: 1500, - pingTimeout: 1600, - connectionStateRecovery: { - maxDisconnectionDuration: 5000, - }, - adapter: createAdapter(collection), - }); - httpServer.listen(async () => { - const port = (httpServer.address() as AddressInfo).port; - - servers.push(io); - ports.push(port); - - if (servers.length === NODES_COUNT) { - resolve(); - } + describe("with a capped collection", () => { + let servers: Server[], ports: number[], mongoClient: MongoClient; + + beforeEach(async () => { + servers = []; + ports = []; + + mongoClient = new MongoClient( + "mongodb://localhost:27017/?replicaSet=rs0&directConnection=true" + ); + await mongoClient.connect(); + + try { + await mongoClient.db("test").createCollection("events-capped", { + capped: true, + size: 1e6, }); + } catch (e) { + // collection already exists } - }); - }); - afterEach(async () => { - servers.forEach((server) => { - // @ts-ignore - server.httpServer.close(); - server.of("/").adapter.close(); - server.of("/foo").adapter.close(); + const collection = mongoClient.db("test").collection("events-capped"); + + return new Promise((resolve) => { + for (let i = 1; i <= NODES_COUNT; i++) { + const httpServer = createServer(); + const io = new Server(httpServer, { + pingInterval: 1500, + pingTimeout: 1600, + connectionStateRecovery: { + maxDisconnectionDuration: 5000, + }, + adapter: createAdapter(collection), + }); + httpServer.listen(async () => { + const port = (httpServer.address() as AddressInfo).port; + + servers.push(io); + ports.push(port); + + if (servers.length === NODES_COUNT) { + resolve(); + } + }); + } + }); }); - await mongoClient.close(); - }); - it("should restore the session", (done) => { - const socket = ioc(`http://localhost:${ports[0]}`, { - reconnectionDelay: 20, + afterEach(async () => { + servers.forEach((server) => { + // @ts-ignore + server.httpServer.close(); + server.of("/").adapter.close(); + server.of("/foo").adapter.close(); + }); + await mongoClient.close(); }); - let initialId: string; + it("should restore the session", (done) => { + const socket = ioc(`http://localhost:${ports[0]}`, { + reconnectionDelay: 20, + }); - socket.once("connect", () => { - expect(socket.recovered).to.eql(false); - initialId = socket.id; + let initialId: string; - servers[0].emit("init"); - }); + socket.once("connect", () => { + expect(socket.recovered).to.eql(false); + initialId = socket.id; - socket.on("init", () => { - // under the hood, the client saves the offset of this packet, so now we force the reconnection - socket.io.engine.close(); + servers[0].emit("init"); + }); + + socket.on("init", () => { + // under the hood, the client saves the offset of this packet, so now we force the reconnection + socket.io.engine.close(); - socket.on("connect", () => { - expect(socket.recovered).to.eql(true); - expect(socket.id).to.eql(initialId); + socket.on("connect", () => { + expect(socket.recovered).to.eql(true); + expect(socket.id).to.eql(initialId); - socket.disconnect(); - done(); + socket.disconnect(); + done(); + }); }); }); - }); - it("should restore any missed packets", (done) => { - const socket = ioc(`http://localhost:${ports[0]}`, { - reconnectionDelay: 20, - }); + it("should restore any missed packets", (done) => { + const socket = ioc(`http://localhost:${ports[0]}`, { + reconnectionDelay: 20, + }); + + servers[0].once("connection", (socket) => { + socket.join("room1"); + + socket.on("disconnect", () => { + // let's send some packets while the client is disconnected + socket.emit("myEvent", 1); + servers[0].emit("myEvent", 2); + servers[0].to("room1").emit("myEvent", 3); + + // those packets should not be received by the client upon reconnection (room mismatch) + servers[0].to("room2").emit("myEvent", 4); + servers[0].except("room1").emit("myEvent", 5); + servers[0].of("/foo").emit("myEvent", 6); + }); + }); + + socket.once("connect", () => { + servers[1].emit("init"); + }); + + socket.on("init", () => { + // under the hood, the client saves the offset of this packet, so now we force the reconnection + socket.io.engine.close(); + + socket.on("connect", () => { + expect(socket.recovered).to.eql(true); - servers[0].once("connection", (socket) => { - socket.join("room1"); + setTimeout(() => { + expect(events).to.eql([1, 2, 3]); - socket.on("disconnect", () => { - // let's send some packets while the client is disconnected - socket.emit("myEvent", 1); - servers[0].emit("myEvent", 2); - servers[0].to("room1").emit("myEvent", 3); + socket.disconnect(); + done(); + }, 50); + }); + }); - // those packets should not be received by the client upon reconnection (room mismatch) - servers[0].to("room2").emit("myEvent", 4); - servers[0].except("room1").emit("myEvent", 5); - servers[0].of("/foo").emit("myEvent", 6); + const events: number[] = []; + + socket.on("myEvent", (val) => { + events.push(val); }); }); - socket.once("connect", () => { - servers[1].emit("init"); + it("should restore the session only once", (done) => { + const socket = ioc(`http://localhost:${ports[0]}`, { + reconnectionDelay: 20, + }); + + let initialId: string; + + socket.once("connect", () => { + expect(socket.recovered).to.eql(false); + initialId = socket.id; + + servers[0].emit("init"); + }); + + socket.once("init", () => { + // under the hood, the client saves the offset of this packet, so now we force the reconnection + socket.io.engine.close(); + + socket.once("connect", () => { + expect(socket.recovered).to.eql(true); + expect(socket.id).to.eql(initialId); + + // unlike above, manual disconnection is not recoverable + socket.disconnect().connect(); + socket.once("connect", () => { + expect(socket.recovered).to.eql(false); + + socket.disconnect(); + done(); + }); + }); + }); }); - socket.on("init", () => { - // under the hood, the client saves the offset of this packet, so now we force the reconnection - socket.io.engine.close(); + it("should fail to restore an unknown session (invalid session ID)", (done) => { + const socket = ioc(`http://localhost:${ports[0]}`, { + reconnectionDelay: 20, + }); - socket.on("connect", () => { - expect(socket.recovered).to.eql(true); + socket.once("connect", () => { + // @ts-ignore + socket._pid = "abc"; + // @ts-ignore + socket._lastOffset = "507f191e810c19729de860ea"; + // force reconnection + socket.io.engine.close(); - setTimeout(() => { - expect(events).to.eql([1, 2, 3]); + socket.on("connect", () => { + expect(socket.recovered).to.eql(false); socket.disconnect(); done(); - }, 50); + }); }); }); - const events: number[] = []; + it("should fail to restore an unknown session (invalid offset)", (done) => { + const socket = ioc(`http://localhost:${ports[0]}`, { + reconnectionDelay: 20, + upgrade: false, + }); - socket.on("myEvent", (val) => { - events.push(val); + socket.once("connect", () => { + // @ts-ignore + socket._lastOffset = "abc"; + socket.io.engine.close(); + + socket.on("connect", () => { + expect(socket.recovered).to.eql(false); + + socket.disconnect(); + done(); + }); + }); }); }); - it("should fail to restore an unknown session (invalid session ID)", (done) => { - const socket = ioc(`http://localhost:${ports[0]}`, { - reconnectionDelay: 20, + describe("with a TTL index", () => { + let servers: Server[], ports: number[], mongoClient: MongoClient; + + beforeEach(async () => { + servers = []; + ports = []; + + mongoClient = new MongoClient( + "mongodb://localhost:27017/?replicaSet=rs0&directConnection=true" + ); + await mongoClient.connect(); + + const collection = mongoClient.db("test").collection("events-ttl"); + + await collection.createIndex( + { createdAt: 1 }, + { expireAfterSeconds: 3600, background: true } + ); + + return new Promise((resolve) => { + for (let i = 1; i <= NODES_COUNT; i++) { + const httpServer = createServer(); + const io = new Server(httpServer, { + pingInterval: 1500, + pingTimeout: 1600, + connectionStateRecovery: { + maxDisconnectionDuration: 5000, + }, + adapter: createAdapter(collection, { + addCreatedAtField: true, + }), + }); + httpServer.listen(async () => { + const port = (httpServer.address() as AddressInfo).port; + + servers.push(io); + ports.push(port); + + if (servers.length === NODES_COUNT) { + resolve(); + } + }); + } + }); }); - socket.once("connect", () => { - // @ts-ignore - socket._pid = "abc"; - // @ts-ignore - socket._lastOffset = "507f191e810c19729de860ea"; - // force reconnection - socket.io.engine.close(); + afterEach(async () => { + servers.forEach((server) => { + // @ts-ignore + server.httpServer.close(); + server.of("/").adapter.close(); + server.of("/foo").adapter.close(); + }); + await mongoClient.close(); + }); - socket.on("connect", () => { + it("should restore the session", (done) => { + const socket = ioc(`http://localhost:${ports[0]}`, { + reconnectionDelay: 20, + }); + + let initialId: string; + + socket.once("connect", () => { expect(socket.recovered).to.eql(false); + initialId = socket.id; + + servers[0].emit("init"); + }); - socket.disconnect(); - done(); + socket.on("init", () => { + // under the hood, the client saves the offset of this packet, so now we force the reconnection + socket.io.engine.close(); + + socket.on("connect", () => { + expect(socket.recovered).to.eql(true); + expect(socket.id).to.eql(initialId); + + socket.disconnect(); + done(); + }); }); }); - }); - it("should fail to restore an unknown session (invalid offset)", (done) => { - const socket = ioc(`http://localhost:${ports[0]}`, { - reconnectionDelay: 20, - upgrade: false, + it("should restore any missed packets", (done) => { + const socket = ioc(`http://localhost:${ports[0]}`, { + reconnectionDelay: 20, + }); + + servers[0].once("connection", (socket) => { + socket.join("room1"); + + socket.on("disconnect", () => { + // let's send some packets while the client is disconnected + socket.emit("myEvent", 1); + servers[0].emit("myEvent", 2); + servers[0].to("room1").emit("myEvent", 3); + + // those packets should not be received by the client upon reconnection (room mismatch) + servers[0].to("room2").emit("myEvent", 4); + servers[0].except("room1").emit("myEvent", 5); + servers[0].of("/foo").emit("myEvent", 6); + }); + }); + + socket.once("connect", () => { + servers[1].emit("init"); + }); + + socket.on("init", () => { + // under the hood, the client saves the offset of this packet, so now we force the reconnection + socket.io.engine.close(); + + socket.on("connect", () => { + expect(socket.recovered).to.eql(true); + + setTimeout(() => { + expect(events).to.eql([1, 2, 3]); + + socket.disconnect(); + done(); + }, 50); + }); + }); + + const events: number[] = []; + + socket.on("myEvent", (val) => { + events.push(val); + }); }); - socket.once("connect", () => { - // @ts-ignore - socket._lastOffset = "abc"; - socket.io.engine.close(); + it("should restore the session only once", (done) => { + const socket = ioc(`http://localhost:${ports[0]}`, { + reconnectionDelay: 20, + }); - socket.on("connect", () => { + let initialId: string; + + socket.once("connect", () => { expect(socket.recovered).to.eql(false); + initialId = socket.id; + + servers[0].emit("init"); + }); + + socket.once("init", () => { + // under the hood, the client saves the offset of this packet, so now we force the reconnection + socket.io.engine.close(); + + socket.once("connect", () => { + expect(socket.recovered).to.eql(true); + expect(socket.id).to.eql(initialId); + + // unlike above, manual disconnection is not recoverable + socket.disconnect().connect(); + socket.once("connect", () => { + expect(socket.recovered).to.eql(false); + + socket.disconnect(); + done(); + }); + }); + }); + }); + + it("should fail to restore an unknown session (invalid session ID)", (done) => { + const socket = ioc(`http://localhost:${ports[0]}`, { + reconnectionDelay: 20, + }); + + socket.once("connect", () => { + // @ts-ignore + socket._pid = "abc"; + // @ts-ignore + socket._lastOffset = "507f191e810c19729de860ea"; + // force reconnection + socket.io.engine.close(); + + socket.on("connect", () => { + expect(socket.recovered).to.eql(false); - socket.disconnect(); - done(); + socket.disconnect(); + done(); + }); + }); + }); + + it("should fail to restore an unknown session (invalid offset)", (done) => { + const socket = ioc(`http://localhost:${ports[0]}`, { + reconnectionDelay: 20, + upgrade: false, + }); + + socket.once("connect", () => { + // @ts-ignore + socket._lastOffset = "abc"; + socket.io.engine.close(); + + socket.on("connect", () => { + expect(socket.recovered).to.eql(false); + + socket.disconnect(); + done(); + }); }); }); }); From b80c9bb1c95e5c03147a43729180ee0a28f8aca9 Mon Sep 17 00:00:00 2001 From: Damien Arrachequesne Date: Tue, 23 Jan 2024 16:40:42 +0100 Subject: [PATCH 04/10] chore(release): 0.3.2 Diff: https://github.com/socketio/socket.io-mongo-adapter/compare/0.3.1...0.3.2 --- CHANGELOG.md | 12 ++++++++++++ package.json | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b741f9..580d8bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # History +- [0.3.2](#032-2024-01-23) (Jan 2024) - [0.3.1](#031-2024-01-10) (Jan 2024) - [0.3.0](#030-2023-02-23) (Feb 2023) - [0.2.1](#021-2022-05-03) (May 2022) @@ -8,6 +9,17 @@ # Release notes +## [0.3.2](https://github.com/socketio/socket.io-mongo-adapter/compare/0.3.1...0.3.2) (2024-01-23) + + +### Bug Fixes + +* add support for AWS DocumentDB ([#21](https://github.com/socketio/socket.io-mongo-adapter/issues/21)) ([0c80f7f](https://github.com/socketio/socket.io-mongo-adapter/commit/0c80f7fd1da772cc54971fd93a1fa93f0c5e47d0)) +* ensure CSR works with a capped collection ([d3fa038](https://github.com/socketio/socket.io-mongo-adapter/commit/d3fa03874038ed9ec011d8795ac7dc6d840f4abe)) +* exclude offline nodes when calling serverCount() ([e2fb8c2](https://github.com/socketio/socket.io-mongo-adapter/commit/e2fb8c2f9d126e763e4f0c0ffba158f2d0c5c17a)) + + + ## [0.3.1](https://github.com/socketio/socket.io-mongo-adapter/compare/0.3.0...0.3.1) (2024-01-10) diff --git a/package.json b/package.json index e6e4a99..470b908 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@socket.io/mongo-adapter", - "version": "0.3.1", + "version": "0.3.2", "description": "The Socket.IO MongoDB adapter, allowing to broadcast events between several Socket.IO servers", "license": "MIT", "repository": { From 5980a7a8ffbd5daeb17f8c19b8e8460688101795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=ABl=20Luijmes?= Date: Tue, 18 Jun 2024 15:32:59 +0200 Subject: [PATCH 05/10] chore(deps): upgrade socket.io-adapter There is a CVE in a dependency of the adapter: ws https://github.com/advisories/GHSA-3h5v-q93c-6h6q --- package.json | 4 +- yarn.lock | 967 ++++++++++++++++++++++++++++----------------------- 2 files changed, 533 insertions(+), 438 deletions(-) diff --git a/package.json b/package.json index 7270974..255c2b9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@gynzy/mongo-adapter", - "version": "0.3.1-gynzy-1", + "version": "0.3.1-gynzy-2", "description": "The Socket.IO MongoDB adapter, allowing to broadcast events between several Socket.IO servers", "license": "MIT", "repository": { @@ -23,7 +23,7 @@ "mongodb": "*" }, "peerDependencies": { - "socket.io-adapter": "^2.5.2" + "socket.io-adapter": "^2.5.4" }, "devDependencies": { "@types/expect.js": "^0.3.29", diff --git a/yarn.lock b/yarn.lock index 3220fad..55e7188 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,179 +2,195 @@ # yarn lockfile v1 -"@babel/code-frame@^7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.10.4.tgz#168da1a36e90da68ae8d49c0f1b48c7c6249213a" - integrity sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg== +"@ampproject/remapping@^2.2.0": + version "2.3.0" + resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.3.0.tgz#ed441b6fa600072520ce18b43d2c8cc8caecc7f4" + integrity sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw== + dependencies: + "@jridgewell/gen-mapping" "^0.3.5" + "@jridgewell/trace-mapping" "^0.3.24" + +"@babel/code-frame@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.24.7.tgz#882fd9e09e8ee324e496bd040401c6f046ef4465" + integrity sha512-BcYH1CVJBO9tvyIZ2jVeXgSIMvGZ2FDRvDdOIVQyuklNKSsx+eppDEBq/g47Ayw+RqNFE+URvOShmf+f/qwAlA== dependencies: - "@babel/highlight" "^7.10.4" + "@babel/highlight" "^7.24.7" + picocolors "^1.0.0" + +"@babel/compat-data@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.24.7.tgz#d23bbea508c3883ba8251fb4164982c36ea577ed" + integrity sha512-qJzAIcv03PyaWqxRgO4mSU3lihncDT296vnyuE2O8uA4w3UHWI4S3hgeZd1L8W1Bft40w9JxJ2b412iDUFFRhw== "@babel/core@^7.7.5": - version "7.12.3" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.12.3.tgz#1b436884e1e3bff6fb1328dc02b208759de92ad8" - integrity sha512-0qXcZYKZp3/6N2jKYVxZv0aNCsxTSVCiK72DTiTYZAu7sjg73W0/aynWjMbiGd87EQL4WyA8reiJVh92AVla9g== - dependencies: - "@babel/code-frame" "^7.10.4" - "@babel/generator" "^7.12.1" - "@babel/helper-module-transforms" "^7.12.1" - "@babel/helpers" "^7.12.1" - "@babel/parser" "^7.12.3" - "@babel/template" "^7.10.4" - "@babel/traverse" "^7.12.1" - "@babel/types" "^7.12.1" - convert-source-map "^1.7.0" + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.24.7.tgz#b676450141e0b52a3d43bc91da86aa608f950ac4" + integrity sha512-nykK+LEK86ahTkX/3TgauT0ikKoNCfKHEaZYTUVupJdTLzGNvrblu4u6fa7DhZONAltdf8e662t/abY8idrd/g== + dependencies: + "@ampproject/remapping" "^2.2.0" + "@babel/code-frame" "^7.24.7" + "@babel/generator" "^7.24.7" + "@babel/helper-compilation-targets" "^7.24.7" + "@babel/helper-module-transforms" "^7.24.7" + "@babel/helpers" "^7.24.7" + "@babel/parser" "^7.24.7" + "@babel/template" "^7.24.7" + "@babel/traverse" "^7.24.7" + "@babel/types" "^7.24.7" + convert-source-map "^2.0.0" debug "^4.1.0" - gensync "^1.0.0-beta.1" - json5 "^2.1.2" - lodash "^4.17.19" - resolve "^1.3.2" - semver "^5.4.1" - source-map "^0.5.0" - -"@babel/generator@^7.12.1", "@babel/generator@^7.12.5": - version "7.12.5" - resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.12.5.tgz#a2c50de5c8b6d708ab95be5e6053936c1884a4de" - integrity sha512-m16TQQJ8hPt7E+OS/XVQg/7U184MLXtvuGbCdA7na61vha+ImkyyNM/9DDA0unYCVZn3ZOhng+qz48/KBOT96A== - dependencies: - "@babel/types" "^7.12.5" + gensync "^1.0.0-beta.2" + json5 "^2.2.3" + semver "^6.3.1" + +"@babel/generator@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.24.7.tgz#1654d01de20ad66b4b4d99c135471bc654c55e6d" + integrity sha512-oipXieGC3i45Y1A41t4tAqpnEZWgB/lC6Ehh6+rOviR5XWpTtMmLN+fGjz9vOiNRt0p6RtO6DtD0pdU3vpqdSA== + dependencies: + "@babel/types" "^7.24.7" + "@jridgewell/gen-mapping" "^0.3.5" + "@jridgewell/trace-mapping" "^0.3.25" jsesc "^2.5.1" - source-map "^0.5.0" - -"@babel/helper-function-name@^7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.10.4.tgz#d2d3b20c59ad8c47112fa7d2a94bc09d5ef82f1a" - integrity sha512-YdaSyz1n8gY44EmN7x44zBn9zQ1Ry2Y+3GTA+3vH6Mizke1Vw0aWDM66FOYEPw8//qKkmqOckrGgTYa+6sceqQ== - dependencies: - "@babel/helper-get-function-arity" "^7.10.4" - "@babel/template" "^7.10.4" - "@babel/types" "^7.10.4" - -"@babel/helper-get-function-arity@^7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.10.4.tgz#98c1cbea0e2332f33f9a4661b8ce1505b2c19ba2" - integrity sha512-EkN3YDB+SRDgiIUnNgcmiD361ti+AVbL3f3Henf6dqqUyr5dMsorno0lJWJuLhDhkI5sYEpgj6y9kB8AOU1I2A== - dependencies: - "@babel/types" "^7.10.4" - -"@babel/helper-member-expression-to-functions@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.12.1.tgz#fba0f2fcff3fba00e6ecb664bb5e6e26e2d6165c" - integrity sha512-k0CIe3tXUKTRSoEx1LQEPFU9vRQfqHtl+kf8eNnDqb4AUJEy5pz6aIiog+YWtVm2jpggjS1laH68bPsR+KWWPQ== - dependencies: - "@babel/types" "^7.12.1" - -"@babel/helper-module-imports@^7.12.1": - version "7.12.5" - resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.12.5.tgz#1bfc0229f794988f76ed0a4d4e90860850b54dfb" - integrity sha512-SR713Ogqg6++uexFRORf/+nPXMmWIn80TALu0uaFb+iQIUoR7bOC7zBWyzBs5b3tBBJXuyD0cRu1F15GyzjOWA== - dependencies: - "@babel/types" "^7.12.5" - -"@babel/helper-module-transforms@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.12.1.tgz#7954fec71f5b32c48e4b303b437c34453fd7247c" - integrity sha512-QQzehgFAZ2bbISiCpmVGfiGux8YVFXQ0abBic2Envhej22DVXV9nCFaS5hIQbkyo1AdGb+gNME2TSh3hYJVV/w== - dependencies: - "@babel/helper-module-imports" "^7.12.1" - "@babel/helper-replace-supers" "^7.12.1" - "@babel/helper-simple-access" "^7.12.1" - "@babel/helper-split-export-declaration" "^7.11.0" - "@babel/helper-validator-identifier" "^7.10.4" - "@babel/template" "^7.10.4" - "@babel/traverse" "^7.12.1" - "@babel/types" "^7.12.1" - lodash "^4.17.19" - -"@babel/helper-optimise-call-expression@^7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.10.4.tgz#50dc96413d594f995a77905905b05893cd779673" - integrity sha512-n3UGKY4VXwXThEiKrgRAoVPBMqeoPgHVqiHZOanAJCG9nQUL2pLRQirUzl0ioKclHGpGqRgIOkgcIJaIWLpygg== - dependencies: - "@babel/types" "^7.10.4" - -"@babel/helper-replace-supers@^7.12.1": - version "7.12.5" - resolved "https://registry.yarnpkg.com/@babel/helper-replace-supers/-/helper-replace-supers-7.12.5.tgz#f009a17543bbbbce16b06206ae73b63d3fca68d9" - integrity sha512-5YILoed0ZyIpF4gKcpZitEnXEJ9UoDRki1Ey6xz46rxOzfNMAhVIJMoune1hmPVxh40LRv1+oafz7UsWX+vyWA== - dependencies: - "@babel/helper-member-expression-to-functions" "^7.12.1" - "@babel/helper-optimise-call-expression" "^7.10.4" - "@babel/traverse" "^7.12.5" - "@babel/types" "^7.12.5" - -"@babel/helper-simple-access@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.12.1.tgz#32427e5aa61547d38eb1e6eaf5fd1426fdad9136" - integrity sha512-OxBp7pMrjVewSSC8fXDFrHrBcJATOOFssZwv16F3/6Xtc138GHybBfPbm9kfiqQHKhYQrlamWILwlDCeyMFEaA== - dependencies: - "@babel/types" "^7.12.1" - -"@babel/helper-split-export-declaration@^7.11.0": - version "7.11.0" - resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.11.0.tgz#f8a491244acf6a676158ac42072911ba83ad099f" - integrity sha512-74Vejvp6mHkGE+m+k5vHY93FX2cAtrw1zXrZXRlG4l410Nm9PxfEiVTn1PjDPV5SnmieiueY4AFg2xqhNFuuZg== - dependencies: - "@babel/types" "^7.11.0" - -"@babel/helper-validator-identifier@^7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.4.tgz#a78c7a7251e01f616512d31b10adcf52ada5e0d2" - integrity sha512-3U9y+43hz7ZM+rzG24Qe2mufW5KhvFg/NhnNph+i9mgCtdTCtMJuI1TMkrIUiK7Ix4PYlRF9I5dhqaLYA/ADXw== - -"@babel/helpers@^7.12.1": - version "7.12.5" - resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.12.5.tgz#1a1ba4a768d9b58310eda516c449913fe647116e" - integrity sha512-lgKGMQlKqA8meJqKsW6rUnc4MdUk35Ln0ATDqdM1a/UpARODdI4j5Y5lVfUScnSNkJcdCRAaWkspykNoFg9sJA== - dependencies: - "@babel/template" "^7.10.4" - "@babel/traverse" "^7.12.5" - "@babel/types" "^7.12.5" - -"@babel/highlight@^7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.10.4.tgz#7d1bdfd65753538fabe6c38596cdb76d9ac60143" - integrity sha512-i6rgnR/YgPEQzZZnbTHHuZdlE8qyoBNalD6F+q4vAFlcMEcqmkoG+mPqJYJCo63qPf74+Y1UZsl3l6f7/RIkmA== - dependencies: - "@babel/helper-validator-identifier" "^7.10.4" - chalk "^2.0.0" - js-tokens "^4.0.0" -"@babel/parser@^7.10.4", "@babel/parser@^7.12.3", "@babel/parser@^7.12.5": - version "7.12.5" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.12.5.tgz#b4af32ddd473c0bfa643bd7ff0728b8e71b81ea0" - integrity sha512-FVM6RZQ0mn2KCf1VUED7KepYeUWoVShczewOCfm3nzoBybaih51h+sYVVGthW9M6lPByEPTQf+xm27PBdlpwmQ== - -"@babel/template@^7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.10.4.tgz#3251996c4200ebc71d1a8fc405fba940f36ba278" - integrity sha512-ZCjD27cGJFUB6nmCB1Enki3r+L5kJveX9pq1SvAUKoICy6CZ9yD8xO086YXdYhvNjBdnekm4ZnaP5yC8Cs/1tA== - dependencies: - "@babel/code-frame" "^7.10.4" - "@babel/parser" "^7.10.4" - "@babel/types" "^7.10.4" - -"@babel/traverse@^7.12.1", "@babel/traverse@^7.12.5": - version "7.12.5" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.12.5.tgz#78a0c68c8e8a35e4cacfd31db8bb303d5606f095" - integrity sha512-xa15FbQnias7z9a62LwYAA5SZZPkHIXpd42C6uW68o8uTuua96FHZy1y61Va5P/i83FAAcMpW8+A/QayntzuqA== - dependencies: - "@babel/code-frame" "^7.10.4" - "@babel/generator" "^7.12.5" - "@babel/helper-function-name" "^7.10.4" - "@babel/helper-split-export-declaration" "^7.11.0" - "@babel/parser" "^7.12.5" - "@babel/types" "^7.12.5" - debug "^4.1.0" +"@babel/helper-compilation-targets@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.24.7.tgz#4eb6c4a80d6ffeac25ab8cd9a21b5dfa48d503a9" + integrity sha512-ctSdRHBi20qWOfy27RUb4Fhp07KSJ3sXcuSvTrXrc4aG8NSYDo1ici3Vhg9bg69y5bj0Mr1lh0aeEgTvc12rMg== + dependencies: + "@babel/compat-data" "^7.24.7" + "@babel/helper-validator-option" "^7.24.7" + browserslist "^4.22.2" + lru-cache "^5.1.1" + semver "^6.3.1" + +"@babel/helper-environment-visitor@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.24.7.tgz#4b31ba9551d1f90781ba83491dd59cf9b269f7d9" + integrity sha512-DoiN84+4Gnd0ncbBOM9AZENV4a5ZiL39HYMyZJGZ/AZEykHYdJw0wW3kdcsh9/Kn+BRXHLkkklZ51ecPKmI1CQ== + dependencies: + "@babel/types" "^7.24.7" + +"@babel/helper-function-name@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.24.7.tgz#75f1e1725742f39ac6584ee0b16d94513da38dd2" + integrity sha512-FyoJTsj/PEUWu1/TYRiXTIHc8lbw+TDYkZuoE43opPS5TrI7MyONBE1oNvfguEXAD9yhQRrVBnXdXzSLQl9XnA== + dependencies: + "@babel/template" "^7.24.7" + "@babel/types" "^7.24.7" + +"@babel/helper-hoist-variables@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.24.7.tgz#b4ede1cde2fd89436397f30dc9376ee06b0f25ee" + integrity sha512-MJJwhkoGy5c4ehfoRyrJ/owKeMl19U54h27YYftT0o2teQ3FJ3nQUf/I3LlJsX4l3qlw7WRXUmiyajvHXoTubQ== + dependencies: + "@babel/types" "^7.24.7" + +"@babel/helper-module-imports@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.24.7.tgz#f2f980392de5b84c3328fc71d38bd81bbb83042b" + integrity sha512-8AyH3C+74cgCVVXow/myrynrAGv+nTVg5vKu2nZph9x7RcRwzmh0VFallJuFTZ9mx6u4eSdXZfcOzSqTUm0HCA== + dependencies: + "@babel/traverse" "^7.24.7" + "@babel/types" "^7.24.7" + +"@babel/helper-module-transforms@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.24.7.tgz#31b6c9a2930679498db65b685b1698bfd6c7daf8" + integrity sha512-1fuJEwIrp+97rM4RWdO+qrRsZlAeL1lQJoPqtCYWv0NL115XM93hIH4CSRln2w52SqvmY5hqdtauB6QFCDiZNQ== + dependencies: + "@babel/helper-environment-visitor" "^7.24.7" + "@babel/helper-module-imports" "^7.24.7" + "@babel/helper-simple-access" "^7.24.7" + "@babel/helper-split-export-declaration" "^7.24.7" + "@babel/helper-validator-identifier" "^7.24.7" + +"@babel/helper-simple-access@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.24.7.tgz#bcade8da3aec8ed16b9c4953b74e506b51b5edb3" + integrity sha512-zBAIvbCMh5Ts+b86r/CjU+4XGYIs+R1j951gxI3KmmxBMhCg4oQMsv6ZXQ64XOm/cvzfU1FmoCyt6+owc5QMYg== + dependencies: + "@babel/traverse" "^7.24.7" + "@babel/types" "^7.24.7" + +"@babel/helper-split-export-declaration@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.24.7.tgz#83949436890e07fa3d6873c61a96e3bbf692d856" + integrity sha512-oy5V7pD+UvfkEATUKvIjvIAH/xCzfsFVw7ygW2SI6NClZzquT+mwdTfgfdbUiceh6iQO0CHtCPsyze/MZ2YbAA== + dependencies: + "@babel/types" "^7.24.7" + +"@babel/helper-string-parser@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.24.7.tgz#4d2d0f14820ede3b9807ea5fc36dfc8cd7da07f2" + integrity sha512-7MbVt6xrwFQbunH2DNQsAP5sTGxfqQtErvBIvIMi6EQnbgUOuVYanvREcmFrOPhoXBrTtjhhP+lW+o5UfK+tDg== + +"@babel/helper-validator-identifier@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.7.tgz#75b889cfaf9e35c2aaf42cf0d72c8e91719251db" + integrity sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w== + +"@babel/helper-validator-option@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.24.7.tgz#24c3bb77c7a425d1742eec8fb433b5a1b38e62f6" + integrity sha512-yy1/KvjhV/ZCL+SM7hBrvnZJ3ZuT9OuZgIJAGpPEToANvc3iM6iDvBnRjtElWibHU6n8/LPR/EjX9EtIEYO3pw== + +"@babel/helpers@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.24.7.tgz#aa2ccda29f62185acb5d42fb4a3a1b1082107416" + integrity sha512-NlmJJtvcw72yRJRcnCmGvSi+3jDEg8qFu3z0AFoymmzLx5ERVWyzd9kVXr7Th9/8yIJi2Zc6av4Tqz3wFs8QWg== + dependencies: + "@babel/template" "^7.24.7" + "@babel/types" "^7.24.7" + +"@babel/highlight@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.24.7.tgz#a05ab1df134b286558aae0ed41e6c5f731bf409d" + integrity sha512-EStJpq4OuY8xYfhGVXngigBJRWxftKX9ksiGDnmlY3o7B/V7KIAc9X4oiK87uPJSc/vs5L869bem5fhZa8caZw== + dependencies: + "@babel/helper-validator-identifier" "^7.24.7" + chalk "^2.4.2" + js-tokens "^4.0.0" + picocolors "^1.0.0" + +"@babel/parser@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.24.7.tgz#9a5226f92f0c5c8ead550b750f5608e766c8ce85" + integrity sha512-9uUYRm6OqQrCqQdG1iCBwBPZgN8ciDBro2nIOFaiRz1/BCxaI7CNvQbDHvsArAC7Tw9Hda/B3U+6ui9u4HWXPw== + +"@babel/template@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.24.7.tgz#02efcee317d0609d2c07117cb70ef8fb17ab7315" + integrity sha512-jYqfPrU9JTF0PmPy1tLYHW4Mp4KlgxJD9l2nP9fD6yT/ICi554DmrWBAEYpIelzjHf1msDP3PxJIRt/nFNfBig== + dependencies: + "@babel/code-frame" "^7.24.7" + "@babel/parser" "^7.24.7" + "@babel/types" "^7.24.7" + +"@babel/traverse@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.24.7.tgz#de2b900163fa741721ba382163fe46a936c40cf5" + integrity sha512-yb65Ed5S/QAcewNPh0nZczy9JdYXkkAbIsEo+P7BE7yO3txAY30Y/oPa3QkQ5It3xVG2kpKMg9MsdxZaO31uKA== + dependencies: + "@babel/code-frame" "^7.24.7" + "@babel/generator" "^7.24.7" + "@babel/helper-environment-visitor" "^7.24.7" + "@babel/helper-function-name" "^7.24.7" + "@babel/helper-hoist-variables" "^7.24.7" + "@babel/helper-split-export-declaration" "^7.24.7" + "@babel/parser" "^7.24.7" + "@babel/types" "^7.24.7" + debug "^4.3.1" globals "^11.1.0" - lodash "^4.17.19" -"@babel/types@^7.10.4", "@babel/types@^7.11.0", "@babel/types@^7.12.1", "@babel/types@^7.12.5": - version "7.12.6" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.12.6.tgz#ae0e55ef1cce1fbc881cd26f8234eb3e657edc96" - integrity sha512-hwyjw6GvjBLiyy3W0YQf0Z5Zf4NpYejUnKFcfcUhZCSffoBBp30w6wP2Wn6pk31jMYZvcOrB/1b7cGXvEoKogA== +"@babel/types@^7.24.7": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.24.7.tgz#6027fe12bc1aa724cd32ab113fb7f1988f1f66f2" + integrity sha512-XEFXSlxiG5td2EJRe8vOmRbaXVgfcBlszKujvVmWIK/UpywWljQCfzAv3RQCGujWQ1RD4YYWEAqDXfuJiy8f5Q== dependencies: - "@babel/helper-validator-identifier" "^7.10.4" - lodash "^4.17.19" + "@babel/helper-string-parser" "^7.24.7" + "@babel/helper-validator-identifier" "^7.24.7" to-fast-properties "^2.0.0" "@cspotcode/source-map-support@^0.8.0": @@ -196,19 +212,33 @@ resolve-from "^5.0.0" "@istanbuljs/schema@^0.1.2": - version "0.1.2" - resolved "https://registry.yarnpkg.com/@istanbuljs/schema/-/schema-0.1.2.tgz#26520bf09abe4a5644cd5414e37125a8954241dd" - integrity sha512-tsAQNx32a8CoFhjhijUIhI4kccIAgmGhy8LZMZgGfmXcpMbPRUqn5LWmgRttILi6yeGmBJd2xsPkFMs0PzgPCw== + version "0.1.3" + resolved "https://registry.yarnpkg.com/@istanbuljs/schema/-/schema-0.1.3.tgz#e45e384e4b8ec16bce2fd903af78450f6bf7ec98" + integrity sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA== -"@jridgewell/resolve-uri@^3.0.3": - version "3.1.0" - resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78" - integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w== +"@jridgewell/gen-mapping@^0.3.5": + version "0.3.5" + resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz#dcce6aff74bdf6dad1a95802b69b04a2fcb1fb36" + integrity sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg== + dependencies: + "@jridgewell/set-array" "^1.2.1" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/trace-mapping" "^0.3.24" -"@jridgewell/sourcemap-codec@^1.4.10": - version "1.4.14" - resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24" - integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw== +"@jridgewell/resolve-uri@^3.0.3", "@jridgewell/resolve-uri@^3.1.0": + version "3.1.2" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz#7a0ee601f60f99a20c7c7c5ff0c80388c1189bd6" + integrity sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw== + +"@jridgewell/set-array@^1.2.1": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.2.1.tgz#558fb6472ed16a4c850b889530e6b36438c49280" + integrity sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A== + +"@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@^1.4.14": + version "1.4.15" + resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32" + integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg== "@jridgewell/trace-mapping@0.3.9": version "0.3.9" @@ -218,22 +248,30 @@ "@jridgewell/resolve-uri" "^3.0.3" "@jridgewell/sourcemap-codec" "^1.4.10" -"@mongodb-js/saslprep@^1.1.0": - version "1.1.4" - resolved "https://registry.yarnpkg.com/@mongodb-js/saslprep/-/saslprep-1.1.4.tgz#24ec1c4915a65f5c506bb88c081731450d91bb1c" - integrity sha512-8zJ8N1x51xo9hwPh6AWnKdLGEC5N3lDa6kms1YHmFBoRhTpJR6HG8wWk0td1MVCu9cD4YBrvjZEtd5Obw0Fbnw== +"@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.25": + version "0.3.25" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz#15f190e98895f3fc23276ee14bc76b675c2e50f0" + integrity sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ== + dependencies: + "@jridgewell/resolve-uri" "^3.1.0" + "@jridgewell/sourcemap-codec" "^1.4.14" + +"@mongodb-js/saslprep@^1.1.5": + version "1.1.7" + resolved "https://registry.yarnpkg.com/@mongodb-js/saslprep/-/saslprep-1.1.7.tgz#d1700facfd6916c50c2c88fd6d48d363a56c702f" + integrity sha512-dCHW/oEX0KJ4NjDULBo3JiOaK5+6axtpBbS+ao2ZInoAL9/YRQLhXzSNAFz7hP4nzLkIqsfYAK/PDE3+XHny0Q== dependencies: sparse-bitfield "^3.0.3" "@socket.io/component-emitter@~3.1.0": - version "3.1.0" - resolved "https://registry.yarnpkg.com/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz#96116f2a912e0c02817345b3c10751069920d553" - integrity sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg== + version "3.1.2" + resolved "https://registry.yarnpkg.com/@socket.io/component-emitter/-/component-emitter-3.1.2.tgz#821f8442f4175d8f0467b9daf26e3a18e2d02af2" + integrity sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA== "@tsconfig/node10@^1.0.7": - version "1.0.9" - resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.9.tgz#df4907fc07a886922637b15e02d4cebc4c0021b2" - integrity sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA== + version "1.0.11" + resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.11.tgz#6ee46400685f130e278128c7b38b7e031ff5b2f2" + integrity sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw== "@tsconfig/node12@^1.0.7": version "1.0.11" @@ -246,9 +284,9 @@ integrity sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow== "@tsconfig/node16@^1.0.2": - version "1.0.3" - resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.3.tgz#472eaab5f15c1ffdd7f8628bd4c4f753995ec79e" - integrity sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ== + version "1.0.4" + resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.4.tgz#0b92dcc0cc1c81f6f306a381f28e31b1a56536e9" + integrity sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA== "@types/cookie@^0.4.1": version "0.4.1" @@ -256,36 +294,43 @@ integrity sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q== "@types/cors@^2.8.12": - version "2.8.13" - resolved "https://registry.yarnpkg.com/@types/cors/-/cors-2.8.13.tgz#b8ade22ba455a1b8cb3b5d3f35910fd204f84f94" - integrity sha512-RG8AStHlUiV5ysZQKq97copd2UmVYw3/pRMLefISZ3S1hK104Cwm7iLQ3fTKx+lsUH2CE8FlLaYeEA2LSeqYUA== + version "2.8.17" + resolved "https://registry.yarnpkg.com/@types/cors/-/cors-2.8.17.tgz#5d718a5e494a8166f569d986794e49c48b216b2b" + integrity sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA== dependencies: "@types/node" "*" "@types/expect.js@^0.3.29": - version "0.3.29" - resolved "https://registry.yarnpkg.com/@types/expect.js/-/expect.js-0.3.29.tgz#28dd359155b84b8ecb094afc3f4b74c3222dca3b" - integrity sha512-zLlr7lW52PKk7GAMJc2v8zaVJUgkrOJBa+6/aGbzq/TYsrqrNT719kkf/98lrgCiV+VO/uwJM0E328NGtcB5yQ== + version "0.3.32" + resolved "https://registry.yarnpkg.com/@types/expect.js/-/expect.js-0.3.32.tgz#ef70862d413f4e1fbc0bbc01c660d60089ac4d2c" + integrity sha512-vUK0KSPtQTeANmOfiqsNNA/8hJ0xz8gOyB0ZhYRtoYOZBtZYir7ujNGr6GKw2hJAjltW0ocCNIGn9YxIXTT99Q== "@types/mocha@^8.2.1": - version "8.2.1" - resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-8.2.1.tgz#f3f3ae4590c5386fc7c543aae9b78d4cf30ffee9" - integrity sha512-NysN+bNqj6E0Hv4CTGWSlPzMW6vTKjDpOteycDkV4IWBsO+PU48JonrPzV9ODjiI2XrjmA05KInLgF5ivZ/YGQ== + version "8.2.3" + resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-8.2.3.tgz#bbeb55fbc73f28ea6de601fbfa4613f58d785323" + integrity sha512-ekGvFhFgrc2zYQoX4JeZPmVzZxw6Dtllga7iGHzfbYIYkAMUx/sAFP2GdFpLff+vdHXu5fl7WX9AT+TtqYcsyw== + +"@types/node@*", "@types/node@>=10.0.0": + version "20.14.5" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.14.5.tgz#fe35e3022ebe58b8f201580eb24e1fcfc0f2487d" + integrity sha512-aoRR+fJkZT2l0aGOJhuA8frnCSoNX6W7U2mpNq63+BxBIj5BQFt8rHy627kijCmm63ijdSdwvGgpUsU6MBsZZA== + dependencies: + undici-types "~5.26.4" -"@types/node@*", "@types/node@>=10.0.0", "@types/node@^14.14.7": - version "14.14.7" - resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.7.tgz#8ea1e8f8eae2430cf440564b98c6dfce1ec5945d" - integrity sha512-Zw1vhUSQZYw+7u5dAwNbIA9TuTotpzY/OF7sJM9FqPOF3SPjKnxrjoTktXDZgUjybf4cWVBP7O8wvKdSaGHweg== +"@types/node@^14.14.7": + version "14.18.63" + resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.63.tgz#1788fa8da838dbb5f9ea994b834278205db6ca2b" + integrity sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ== "@types/webidl-conversions@*": - version "7.0.0" - resolved "https://registry.yarnpkg.com/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz#2b8e60e33906459219aa587e9d1a612ae994cfe7" - integrity sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog== + version "7.0.3" + resolved "https://registry.yarnpkg.com/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz#1306dbfa53768bcbcfc95a1c8cde367975581859" + integrity sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA== "@types/whatwg-url@^11.0.2": - version "11.0.4" - resolved "https://registry.yarnpkg.com/@types/whatwg-url/-/whatwg-url-11.0.4.tgz#ffed0dc8d89d91f62e3f368fcbda222a487c4f63" - integrity sha512-lXCmTWSHJvf0TRSO58nm978b8HJ/EdsSsEKLd3ODHFjo+3VGAyyTp4v50nWvwtzBxSMQrVOK7tcuN0zGPLICMw== + version "11.0.5" + resolved "https://registry.yarnpkg.com/@types/whatwg-url/-/whatwg-url-11.0.5.tgz#aaa2546e60f0c99209ca13360c32c78caf2c409f" + integrity sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ== dependencies: "@types/webidl-conversions" "*" @@ -298,14 +343,16 @@ accepts@~1.3.4: negotiator "0.6.3" acorn-walk@^8.1.1: - version "8.2.0" - resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1" - integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== + version "8.3.3" + resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.3.3.tgz#9caeac29eefaa0c41e3d4c65137de4d6f34df43e" + integrity sha512-MxXdReSRhGO7VlFe1bRG/oI7/mdLV9B9JJT0N8vZOhF7gFRR5l3M8W9G8JxmKV+JC5mGqJ0QvqfSOLsCPa4nUw== + dependencies: + acorn "^8.11.0" -acorn@^8.4.1: - version "8.8.1" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.1.tgz#0a3f9cbecc4ec3bea6f0a80b66ae8dd2da250b73" - integrity sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA== +acorn@^8.11.0, acorn@^8.4.1: + version "8.12.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.12.0.tgz#1627bfa2e058148036133b8d9b51a700663c294c" + integrity sha512-RTvkC4w+KNXrM39/lWCUaG0IbRkWdCv7W/IOW9oU6SawyxulvkQy5HQPVTKxEjczcUvapcrw3cFx/60VN/NRNw== aggregate-error@^3.0.0: version "3.1.0" @@ -320,7 +367,7 @@ ansi-colors@4.1.1: resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.1.tgz#cbb9ae256bf750af1eab344f229aa27fe94ba348" integrity sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA== -ansi-regex@^5.0.0: +ansi-regex@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== @@ -377,9 +424,9 @@ argparse@^2.0.1: integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== balanced-match@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" - integrity sha512-9Y0g0Q8rmSt+H33DfKv7FOc3v+iRI+o1lbzt8jGcIosYW37IIW/2XVYq5NPdmaD5NQ59Nk26Kl/vZbwW9Fr8vg== + version "1.0.2" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" + integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== base64id@2.0.0, base64id@~2.0.0: version "2.0.0" @@ -387,9 +434,9 @@ base64id@2.0.0, base64id@~2.0.0: integrity sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog== binary-extensions@^2.0.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" - integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== + version "2.3.0" + resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522" + integrity sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw== brace-expansion@^1.1.7: version "1.1.11" @@ -407,21 +454,31 @@ brace-expansion@^2.0.1: balanced-match "^1.0.0" braces@~3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" - integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== + version "3.0.3" + resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" + integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== dependencies: - fill-range "^7.0.1" + fill-range "^7.1.1" browser-stdout@1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.1.tgz#baa559ee14ced73452229bad7326467c61fabd60" integrity sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw== -bson@^6.2.0: - version "6.2.0" - resolved "https://registry.yarnpkg.com/bson/-/bson-6.2.0.tgz#4b6acafc266ba18eeee111373c2699304a9ba0a3" - integrity sha512-ID1cI+7bazPDyL9wYy9GaQ8gEEohWvcUl/Yf0dIdutJxnmInEEyCsb4awy/OiBfall7zBA179Pahi3vCdFze3Q== +browserslist@^4.22.2: + version "4.23.1" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.23.1.tgz#ce4af0534b3d37db5c1a4ca98b9080f985041e96" + integrity sha512-TUfofFo/KsK/bWZ9TWQ5O26tsWW4Uhmt8IYklbnUa70udB6P2wA7w7o4PY4muaEPBQaAX+CEnmmIA41NVHtPVw== + dependencies: + caniuse-lite "^1.0.30001629" + electron-to-chromium "^1.4.796" + node-releases "^2.0.14" + update-browserslist-db "^1.0.16" + +bson@^6.7.0: + version "6.7.0" + resolved "https://registry.yarnpkg.com/bson/-/bson-6.7.0.tgz#51973b132cdc424c8372fda3cb43e3e3e2ae2227" + integrity sha512-w2IquM5mYzYZv6rs3uN2DZTOBe2a0zXLj53TGDqwF4l6Sz/XsISrisXOJihArF9+BZ6Cq/GjVht7Sjfmri7ytQ== caching-transform@^4.0.0: version "4.0.0" @@ -439,11 +496,16 @@ camelcase@^5.0.0, camelcase@^5.3.1: integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg== camelcase@^6.0.0: - version "6.2.0" - resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.2.0.tgz#924af881c9d525ac9d87f40d964e5cea982a1809" - integrity sha512-c7wVvbw3f37nuobQNtgsgG9POC9qMbNuMQmTCqZv23b6MIz0fcYpBiOlv9gEN/hdLdnZTDQhg6e9Dq5M1vKvfg== + version "6.3.0" + resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" + integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== -chalk@^2.0.0: +caniuse-lite@^1.0.30001629: + version "1.0.30001636" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001636.tgz#b15f52d2bdb95fad32c2f53c0b68032b85188a78" + integrity sha512-bMg2vmr8XBsbL6Lr0UHXy/21m84FTxDLWn2FSqMd5PrlbMxwJlQnC2YWYxVgp66PZE+BBNF2jYQUBKCo1FDeZg== + +chalk@^2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== @@ -533,11 +595,14 @@ concat-map@0.0.1: integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== convert-source-map@^1.7.0: - version "1.7.0" - resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.7.0.tgz#17a2cb882d7f77d3490585e2ce6c524424a3a442" - integrity sha512-4FJkXzKXEDB1snCFZlLP4gpC3JILicCpGbzG9f9G7tGqGCzETQ2hWPrcinA9oU4wtf2biUaEH5065UnMeR33oA== - dependencies: - safe-buffer "~5.1.1" + version "1.9.0" + resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.9.0.tgz#7faae62353fb4213366d0ca98358d22e8368b05f" + integrity sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A== + +convert-source-map@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-2.0.0.tgz#4b560f649fc4e918dd0ab75cf4961e8bc882d82a" + integrity sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg== cookie@~0.4.1: version "0.4.2" @@ -557,7 +622,7 @@ create-require@^1.1.0: resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== -cross-spawn@^7.0.0: +cross-spawn@^7.0.0, cross-spawn@^7.0.3: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== @@ -566,13 +631,20 @@ cross-spawn@^7.0.0: shebang-command "^2.0.0" which "^2.0.1" -debug@4.3.4, debug@^4.1.0, debug@^4.1.1, debug@~4.3.1, debug@~4.3.2: +debug@4.3.4: version "4.3.4" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== dependencies: ms "2.1.2" +debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@~4.3.1, debug@~4.3.2, debug@~4.3.4: + version "4.3.5" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.5.tgz#e83444eceb9fedd4a1da56d671ae2446a01a6e1e" + integrity sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg== + dependencies: + ms "2.1.2" + decamelize@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -584,9 +656,9 @@ decamelize@^4.0.0: integrity sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ== default-require-extensions@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/default-require-extensions/-/default-require-extensions-3.0.0.tgz#e03f93aac9b2b6443fc52e5e4a37b3ad9ad8df96" - integrity sha512-ek6DpXq/SCpvjhpFsLFRVtIxJCRw6fUR42lYMVZuUMK7n8eMz4Uh5clckdBjEpLhn/gEBZo7hDJnJcwdKLKQjg== + version "3.0.1" + resolved "https://registry.yarnpkg.com/default-require-extensions/-/default-require-extensions-3.0.1.tgz#bfae00feeaeada68c2ae256c62540f60b80625bd" + integrity sha512-eXTJmRbm2TIt9MgWTsOH1wEuhew6XGZcMeGKCtLedIg/NCsg1iBePXkceTdK4Fii7pzmN9tGsZhKzZ4h7O/fxw== dependencies: strip-bom "^4.0.0" @@ -600,31 +672,36 @@ diff@^4.0.1: resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== +electron-to-chromium@^1.4.796: + version "1.4.805" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.805.tgz#1d526e384c20944a3c68f618f9774edc384c4733" + integrity sha512-8W4UJwX/w9T0QSzINJckTKG6CYpAUTqsaWcWIsdud3I1FYJcMgW9QqT1/4CBff/pP/TihWh13OmiyY8neto6vw== + emoji-regex@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== -engine.io-client@~6.4.0: - version "6.4.0" - resolved "https://registry.yarnpkg.com/engine.io-client/-/engine.io-client-6.4.0.tgz#88cd3082609ca86d7d3c12f0e746d12db4f47c91" - integrity sha512-GyKPDyoEha+XZ7iEqam49vz6auPnNJ9ZBfy89f+rMMas8AuiMWOZ9PVzu8xb9ZC6rafUqiGHSCfu22ih66E+1g== +engine.io-client@~6.5.2: + version "6.5.4" + resolved "https://registry.yarnpkg.com/engine.io-client/-/engine.io-client-6.5.4.tgz#b8bc71ed3f25d0d51d587729262486b4b33bd0d0" + integrity sha512-GeZeeRjpD2qf49cZQ0Wvh/8NJNfeXkXXcoGh+F77oEAgo9gUHwT1fCRxSNU+YEEaysOJTnsFHmM5oAcPy4ntvQ== dependencies: "@socket.io/component-emitter" "~3.1.0" debug "~4.3.1" - engine.io-parser "~5.0.3" - ws "~8.11.0" + engine.io-parser "~5.2.1" + ws "~8.17.1" xmlhttprequest-ssl "~2.0.0" -engine.io-parser@~5.0.3: - version "5.0.6" - resolved "https://registry.yarnpkg.com/engine.io-parser/-/engine.io-parser-5.0.6.tgz#7811244af173e157295dec9b2718dfe42a64ef45" - integrity sha512-tjuoZDMAdEhVnSFleYPCtdL2GXwVTGtNjoeJd9IhIG3C1xs9uwxqRNEu5WpnDZCaozwVlK/nuQhpodhXSIMaxw== +engine.io-parser@~5.2.1: + version "5.2.2" + resolved "https://registry.yarnpkg.com/engine.io-parser/-/engine.io-parser-5.2.2.tgz#37b48e2d23116919a3453738c5720455e64e1c49" + integrity sha512-RcyUFKA93/CXH20l4SoVvzZfrSDMOTUS3bWVpTt2FuFP+XYrL8i8oonHP7WInRyVHXh0n/ORtoeiE1os+8qkSw== -engine.io@~6.4.1: - version "6.4.1" - resolved "https://registry.yarnpkg.com/engine.io/-/engine.io-6.4.1.tgz#8056b4526a88e779f9c280d820422d4e3eeaaae5" - integrity sha512-JFYQurD/nbsA5BSPmbaOSLa3tSVj8L6o4srSwXXY3NqE+gGUNmmPTbhn8tjzcCtSqhFgIeqef81ngny8JM25hw== +engine.io@~6.5.2: + version "6.5.5" + resolved "https://registry.yarnpkg.com/engine.io/-/engine.io-6.5.5.tgz#430b80d8840caab91a50e9e23cb551455195fc93" + integrity sha512-C5Pn8Wk+1vKBoHghJODM63yk8MvrO9EWZUfkAt5HAqIgPE4/8FF0PEGHXtEd40l223+cE5ABWuPzm38PHFXfMA== dependencies: "@types/cookie" "^0.4.1" "@types/cors" "^2.8.12" @@ -634,18 +711,18 @@ engine.io@~6.4.1: cookie "~0.4.1" cors "~2.8.5" debug "~4.3.1" - engine.io-parser "~5.0.3" - ws "~8.11.0" + engine.io-parser "~5.2.1" + ws "~8.17.1" es6-error@^4.0.1: version "4.1.1" resolved "https://registry.yarnpkg.com/es6-error/-/es6-error-4.1.1.tgz#9e3af407459deed47e9a91f9b885a84eb05c561d" integrity sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg== -escalade@^3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" - integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw== +escalade@^3.1.1, escalade@^3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.2.tgz#54076e9ab29ea5bf3d8f1ed62acffbb88272df27" + integrity sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA== escape-string-regexp@4.0.0: version "4.0.0" @@ -667,17 +744,17 @@ expect.js@0.3.1: resolved "https://registry.yarnpkg.com/expect.js/-/expect.js-0.3.1.tgz#b0a59a0d2eff5437544ebf0ceaa6015841d09b5b" integrity sha512-okDF/FAPEul1ZFLae4hrgpIqAeapoo5TRdcg/lD0iN9S3GWrBFIJwNezGH1DMtIz+RxU4RrFmMq7WUUvDg3J6A== -fill-range@^7.0.1: - version "7.0.1" - resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40" - integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ== +fill-range@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292" + integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg== dependencies: to-regex-range "^5.0.1" find-cache-dir@^3.2.0: - version "3.3.1" - resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-3.3.1.tgz#89b33fad4a4670daa94f855f7fbe31d6d84fe880" - integrity sha512-t2GDMt3oGC/v+BMwzmllWDuJF/xcDtE5j/fCGbqDD7OLuJkj0cfh1YSA5VKPvwMeLFLNDBkwOKZ2X85jGLVftQ== + version "3.3.2" + resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-3.3.2.tgz#b30c5b6eff0730731aea9bbd9dbecbd80256d64b" + integrity sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig== dependencies: commondir "^1.0.1" make-dir "^3.0.2" @@ -723,16 +800,11 @@ fs.realpath@^1.0.0: integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== fsevents@~2.3.2: - version "2.3.2" - resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" - integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== - -function-bind@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" - integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== + version "2.3.3" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" + integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== -gensync@^1.0.0-beta.1: +gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" integrity sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg== @@ -754,15 +826,26 @@ glob-parent@~5.1.2: dependencies: is-glob "^4.0.1" -glob@7.2.0, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6: - version "7.2.0" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023" - integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q== +glob@8.1.0: + version "8.1.0" + resolved "https://registry.yarnpkg.com/glob/-/glob-8.1.0.tgz#d388f656593ef708ee3e34640fdfb99a9fd1c33e" + integrity sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ== dependencies: fs.realpath "^1.0.0" inflight "^1.0.4" inherits "2" - minimatch "^3.0.4" + minimatch "^5.0.1" + once "^1.3.0" + +glob@^7.1.3, glob@^7.1.4, glob@^7.1.6: + version "7.2.3" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" + integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.1.1" once "^1.3.0" path-is-absolute "^1.0.0" @@ -772,9 +855,9 @@ globals@^11.1.0: integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== graceful-fs@^4.1.15: - version "4.2.4" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.4.tgz#2256bde14d3632958c465ebc96dc467ca07a29fb" - integrity sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw== + version "4.2.11" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" + integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== has-flag@^3.0.0: version "3.0.0" @@ -786,13 +869,6 @@ has-flag@^4.0.0: resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== -has@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" - integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw== - dependencies: - function-bind "^1.1.1" - hasha@^5.0.0: version "5.2.2" resolved "https://registry.yarnpkg.com/hasha/-/hasha-5.2.2.tgz#a48477989b3b327aea3c04f53096d816d97522a1" @@ -841,13 +917,6 @@ is-binary-path@~2.1.0: dependencies: binary-extensions "^2.0.0" -is-core-module@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.1.0.tgz#a4cc031d9b1aca63eecbd18a650e13cb4eeab946" - integrity sha512-YcV7BgVMRFRua2FqQzKtTDMz8iCuLEyGKjr70q8Zm1yy2qKcurbFEd79PAdHV77oL3NrAaOVQIbMmiHQCHB7ZA== - dependencies: - has "^1.0.3" - is-extglob@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" @@ -876,9 +945,9 @@ is-plain-obj@^2.1.0: integrity sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA== is-stream@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.0.tgz#bde9c32680d6fae04129d6ac9d921ce7815f78e3" - integrity sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw== + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" + integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== is-typedarray@^1.0.0: version "1.0.0" @@ -900,10 +969,10 @@ isexe@^2.0.0: resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw== -istanbul-lib-coverage@^3.0.0, istanbul-lib-coverage@^3.0.0-alpha.1: - version "3.0.0" - resolved "https://registry.yarnpkg.com/istanbul-lib-coverage/-/istanbul-lib-coverage-3.0.0.tgz#f5944a37c70b550b02a78a5c3b2055b280cec8ec" - integrity sha512-UiUIqxMgRDET6eR+o5HbfRYP1l0hqkWOs7vNxC/mggutCMUIhWMm8gAHb8tHlyfD3/l6rlgNA5cKdDzEAf6hEg== +istanbul-lib-coverage@^3.0.0, istanbul-lib-coverage@^3.2.0: + version "3.2.2" + resolved "https://registry.yarnpkg.com/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz#2d166c4b0644d43a39f04bf6c2edd1e585f31756" + integrity sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg== istanbul-lib-hook@^3.0.0: version "3.0.0" @@ -923,40 +992,39 @@ istanbul-lib-instrument@^4.0.0: semver "^6.3.0" istanbul-lib-processinfo@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/istanbul-lib-processinfo/-/istanbul-lib-processinfo-2.0.2.tgz#e1426514662244b2f25df728e8fd1ba35fe53b9c" - integrity sha512-kOwpa7z9hme+IBPZMzQ5vdQj8srYgAtaRqeI48NGmAQ+/5yKiHLV0QbYqQpxsdEF0+w14SoB8YbnHKcXE2KnYw== + version "2.0.3" + resolved "https://registry.yarnpkg.com/istanbul-lib-processinfo/-/istanbul-lib-processinfo-2.0.3.tgz#366d454cd0dcb7eb6e0e419378e60072c8626169" + integrity sha512-NkwHbo3E00oybX6NGJi6ar0B29vxyvNwoC7eJ4G4Yq28UfY758Hgn/heV8VRFhevPED4LXfFz0DQ8z/0kw9zMg== dependencies: archy "^1.0.0" - cross-spawn "^7.0.0" - istanbul-lib-coverage "^3.0.0-alpha.1" - make-dir "^3.0.0" + cross-spawn "^7.0.3" + istanbul-lib-coverage "^3.2.0" p-map "^3.0.0" rimraf "^3.0.0" - uuid "^3.3.3" + uuid "^8.3.2" istanbul-lib-report@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz#7518fe52ea44de372f460a76b5ecda9ffb73d8a6" - integrity sha512-wcdi+uAKzfiGT2abPpKZ0hSU1rGQjUQnLvtY5MpQ7QCTahD3VODhcu4wcfY1YtkGaDD5yuydOLINXsfbus9ROw== + version "3.0.1" + resolved "https://registry.yarnpkg.com/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz#908305bac9a5bd175ac6a74489eafd0fc2445a7d" + integrity sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw== dependencies: istanbul-lib-coverage "^3.0.0" - make-dir "^3.0.0" + make-dir "^4.0.0" supports-color "^7.1.0" istanbul-lib-source-maps@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.0.tgz#75743ce6d96bb86dc7ee4352cf6366a23f0b1ad9" - integrity sha512-c16LpFRkR8vQXyHZ5nLpY35JZtzj1PQY1iZmesUbf1FZHbIupcWfjgOXBY9YHkLEQ6puz1u4Dgj6qmU/DisrZg== + version "4.0.1" + resolved "https://registry.yarnpkg.com/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz#895f3a709fcfba34c6de5a42939022f3e4358551" + integrity sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw== dependencies: debug "^4.1.1" istanbul-lib-coverage "^3.0.0" source-map "^0.6.1" istanbul-reports@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/istanbul-reports/-/istanbul-reports-3.0.2.tgz#d593210e5000683750cb09fc0644e4b6e27fd53b" - integrity sha512-9tZvz7AiR3PEDNGiV9vIouQ/EAcqMXFmkcA1CDFTwOB98OZVDL0PH9glHotf5Ugp6GCOTypfzGWI/OqjWNCRUw== + version "3.1.7" + resolved "https://registry.yarnpkg.com/istanbul-reports/-/istanbul-reports-3.1.7.tgz#daed12b9e1dca518e15c056e1e537e741280fa0b" + integrity sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g== dependencies: html-escaper "^2.0.0" istanbul-lib-report "^3.0.0" @@ -974,9 +1042,9 @@ js-yaml@4.1.0: argparse "^2.0.1" js-yaml@^3.13.1: - version "3.14.0" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.0.tgz#a7a34170f26a21bb162424d8adacb4113a69e482" - integrity sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A== + version "3.14.1" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.1.tgz#dae812fdb3825fa306609a8717383c50c36a0537" + integrity sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g== dependencies: argparse "^1.0.7" esprima "^4.0.0" @@ -986,7 +1054,7 @@ jsesc@^2.5.1: resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== -json5@^2.1.2: +json5@^2.2.3: version "2.2.3" resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== @@ -1010,11 +1078,6 @@ lodash.flattendeep@^4.4.0: resolved "https://registry.yarnpkg.com/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz#fb030917f86a3134e5bc9bec0d69e0013ddfedb2" integrity sha512-uHaJFihxmJcEX3kT4I23ABqKKalJ/zDrDg0lsFtc1h+3uw49SIJ5beyhx5ExVRti3AvKoOJngIj7xz3oylPdWQ== -lodash@^4.17.19: - version "4.17.21" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" - integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== - log-symbols@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-4.1.0.tgz#3fbdbb95b4683ac9fc785111e792e558d4abd503" @@ -1023,6 +1086,13 @@ log-symbols@4.1.0: chalk "^4.1.0" is-unicode-supported "^0.1.0" +lru-cache@^5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920" + integrity sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w== + dependencies: + yallist "^3.0.2" + make-dir@^3.0.0, make-dir@^3.0.2: version "3.1.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" @@ -1030,6 +1100,13 @@ make-dir@^3.0.0, make-dir@^3.0.2: dependencies: semver "^6.0.0" +make-dir@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-4.0.0.tgz#c3c2307a771277cd9638305f915c29ae741b614e" + integrity sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw== + dependencies: + semver "^7.5.3" + make-error@^1.1.1: version "1.3.6" resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" @@ -1059,17 +1136,24 @@ minimatch@5.0.1: dependencies: brace-expansion "^2.0.1" -minimatch@^3.0.4: +minimatch@^3.0.4, minimatch@^3.1.1: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== dependencies: brace-expansion "^1.1.7" +minimatch@^5.0.1: + version "5.1.6" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.1.6.tgz#1cfcb8cf5522ea69952cd2af95ae09477f122a96" + integrity sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g== + dependencies: + brace-expansion "^2.0.1" + mocha@^10.2.0: - version "10.2.0" - resolved "https://registry.yarnpkg.com/mocha/-/mocha-10.2.0.tgz#1fd4a7c32ba5ac372e03a17eef435bd00e5c68b8" - integrity sha512-IDY7fl/BecMwFHzoqF2sg/SHHANeBoMMXFlS9r0OXKDssYE1M5O43wUY/9BVPeIvfH2zmEbBfseqN9gBQZzXkg== + version "10.4.0" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-10.4.0.tgz#ed03db96ee9cfc6d20c56f8e2af07b961dbae261" + integrity sha512-eqhGB8JKapEYcC4ytX/xrzKforgEc3j1pGlAXVy3eRwrtAy5/nIfT1SvgGzfN0XZZxeLq0aQWkOUAmqIJiv+bA== dependencies: ansi-colors "4.1.1" browser-stdout "1.3.1" @@ -1078,13 +1162,12 @@ mocha@^10.2.0: diff "5.0.0" escape-string-regexp "4.0.0" find-up "5.0.0" - glob "7.2.0" + glob "8.1.0" he "1.2.0" js-yaml "4.1.0" log-symbols "4.1.0" minimatch "5.0.1" ms "2.1.3" - nanoid "3.3.3" serialize-javascript "6.0.0" strip-json-comments "3.1.1" supports-color "8.1.1" @@ -1094,20 +1177,20 @@ mocha@^10.2.0: yargs-unparser "2.0.0" mongodb-connection-string-url@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.0.tgz#b4f87f92fd8593f3b9365f592515a06d304a1e9c" - integrity sha512-t1Vf+m1I5hC2M5RJx/7AtxgABy1cZmIPQRMXw+gEIPn/cZNF3Oiy+l0UIypUwVB5trcWHq3crg2g3uAR9aAwsQ== + version "3.0.1" + resolved "https://registry.yarnpkg.com/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.1.tgz#c13e6ac284ae401752ebafdb8cd7f16c6723b141" + integrity sha512-XqMGwRX0Lgn05TDB4PyG2h2kKO/FfWJyCzYQbIhXUxz7ETt0I/FqHjUeqj37irJ+Dl1ZtU82uYyj14u2XsZKfg== dependencies: "@types/whatwg-url" "^11.0.2" whatwg-url "^13.0.0" mongodb@*: - version "6.3.0" - resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-6.3.0.tgz#ec9993b19f7ed2ea715b903fcac6171c9d1d38ca" - integrity sha512-tt0KuGjGtLUhLoU263+xvQmPHEGTw5LbcNC73EoFRYgSHwZt5tsoJC110hDyO1kjQzpgNrpdcSza9PknWN4LrA== + version "6.7.0" + resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-6.7.0.tgz#f86e51e6530e6a2ca4a99d7cfdf6f409223ac199" + integrity sha512-TMKyHdtMcO0fYBNORiYdmM25ijsHs+Njs963r4Tro4OQZzqYigAzYQouwWRg4OIaiLRUEGUh/1UAcH5lxdSLIA== dependencies: - "@mongodb-js/saslprep" "^1.1.0" - bson "^6.2.0" + "@mongodb-js/saslprep" "^1.1.5" + bson "^6.7.0" mongodb-connection-string-url "^3.0.0" ms@2.1.2: @@ -1120,11 +1203,6 @@ ms@2.1.3: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== -nanoid@3.3.3: - version "3.3.3" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.3.tgz#fd8e8b7aa761fe807dba2d1b98fb7241bb724a25" - integrity sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w== - negotiator@0.6.3: version "0.6.3" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" @@ -1137,6 +1215,11 @@ node-preload@^0.2.1: dependencies: process-on-spawn "^1.0.0" +node-releases@^2.0.14: + version "2.0.14" + resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.14.tgz#2ffb053bceb8b2be8495ece1ab6ce600c4461b0b" + integrity sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw== + normalize-path@^3.0.0, normalize-path@~3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65" @@ -1252,10 +1335,10 @@ path-key@^3.1.0: resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== -path-parse@^1.0.6: - version "1.0.7" - resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" - integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== +picocolors@^1.0.0, picocolors@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1" + integrity sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew== picomatch@^2.0.4, picomatch@^2.2.1: version "2.3.1" @@ -1270,9 +1353,9 @@ pkg-dir@^4.1.0: find-up "^4.0.0" prettier@^2.1.2: - version "2.8.4" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.8.4.tgz#34dd2595629bfbb79d344ac4a91ff948694463c3" - integrity sha512-vIS4Rlc2FNh0BySk3Wkd6xmwxB0FpOndW5fisM5H8hsZSxU2VWVB5CWIkIjWvrHjIhxk2g3bfMKM87zNTrZddw== + version "2.8.8" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.8.8.tgz#e8c5d7e98a4305ffe3de2e1fc4aca1a71c28b1da" + integrity sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q== process-on-spawn@^1.0.0: version "1.0.0" @@ -1322,14 +1405,6 @@ resolve-from@^5.0.0: resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-5.0.0.tgz#c35225843df8f776df21c57557bc087e9dfdfc69" integrity sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw== -resolve@^1.3.2: - version "1.19.0" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.19.0.tgz#1af5bf630409734a067cae29318aac7fa29a267c" - integrity sha512-rArEXAgsBG4UgRGcynxWIWKFvh/XZCcS8UJdHhwy91zwAvCZIbcs+vAbflgBnNjYMs/i/i+/Ux6IZhML1yPvxg== - dependencies: - is-core-module "^2.1.0" - path-parse "^1.0.6" - rimraf@^3.0.0: version "3.0.2" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" @@ -1337,20 +1412,20 @@ rimraf@^3.0.0: dependencies: glob "^7.1.3" -safe-buffer@^5.1.0, safe-buffer@~5.1.1: - version "5.1.2" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" - integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== +safe-buffer@^5.1.0: + version "5.2.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" + integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== -semver@^5.4.1: - version "5.7.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" - integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== +semver@^6.0.0, semver@^6.3.0, semver@^6.3.1: + version "6.3.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== -semver@^6.0.0, semver@^6.3.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" - integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== +semver@^7.5.3: + version "7.6.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.2.tgz#1e3b34759f896e8f14d6134732ce798aeb0c6e13" + integrity sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w== serialize-javascript@6.0.0: version "6.0.0" @@ -1377,51 +1452,48 @@ shebang-regex@^3.0.0: integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== signal-exit@^3.0.2: - version "3.0.3" - resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" - integrity sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA== + version "3.0.7" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" + integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== socket.io-adapter@~2.5.2: - version "2.5.2" - resolved "https://registry.yarnpkg.com/socket.io-adapter/-/socket.io-adapter-2.5.2.tgz#5de9477c9182fdc171cd8c8364b9a8894ec75d12" - integrity sha512-87C3LO/NOMc+eMcpcxUBebGjkpMDkNBS9tf7KJqcDsmL936EChtVva71Dw2q4tQcuVC+hAUy4an2NO/sYXmwRA== + version "2.5.5" + resolved "https://registry.yarnpkg.com/socket.io-adapter/-/socket.io-adapter-2.5.5.tgz#c7a1f9c703d7756844751b6ff9abfc1780664082" + integrity sha512-eLDQas5dzPgOWCk9GuuJC2lBqItuhKI4uxGgo9aIV7MYbk2h9Q6uULEh8WBzThoI7l+qU9Ast9fVUmkqPP9wYg== dependencies: - ws "~8.11.0" + debug "~4.3.4" + ws "~8.17.1" socket.io-client@^4.6.1: - version "4.6.1" - resolved "https://registry.yarnpkg.com/socket.io-client/-/socket.io-client-4.6.1.tgz#80d97d5eb0feca448a0fb6d69a7b222d3d547eab" - integrity sha512-5UswCV6hpaRsNg5kkEHVcbBIXEYoVbMQaHJBXJCyEQ+CiFPV1NIOY0XOFWG4XR4GZcB8Kn6AsRs/9cy9TbqVMQ== + version "4.7.5" + resolved "https://registry.yarnpkg.com/socket.io-client/-/socket.io-client-4.7.5.tgz#919be76916989758bdc20eec63f7ee0ae45c05b7" + integrity sha512-sJ/tqHOCe7Z50JCBCXrsY3I2k03iOiUe+tj1OmKeD2lXPiGH/RUCdTZFoqVyN7l1MnpIzPrGtLcijffmeouNlQ== dependencies: "@socket.io/component-emitter" "~3.1.0" debug "~4.3.2" - engine.io-client "~6.4.0" - socket.io-parser "~4.2.1" + engine.io-client "~6.5.2" + socket.io-parser "~4.2.4" -socket.io-parser@~4.2.1: - version "4.2.2" - resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.2.2.tgz#1dd384019e25b7a3d374877f492ab34f2ad0d206" - integrity sha512-DJtziuKypFkMMHCm2uIshOYC7QaylbtzQwiMYDuCKy3OPkjLzu4B2vAhTlqipRHHzrI0NJeBAizTK7X+6m1jVw== +socket.io-parser@~4.2.4: + version "4.2.4" + resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.2.4.tgz#c806966cf7270601e47469ddeec30fbdfda44c83" + integrity sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew== dependencies: "@socket.io/component-emitter" "~3.1.0" debug "~4.3.1" socket.io@^4.6.1: - version "4.6.1" - resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-4.6.1.tgz#62ec117e5fce0692fa50498da9347cfb52c3bc70" - integrity sha512-KMcaAi4l/8+xEjkRICl6ak8ySoxsYG+gG6/XfRCPJPQ/haCRIJBTL4wIl8YCsmtaBovcAXGLOShyVWQ/FG8GZA== + version "4.7.5" + resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-4.7.5.tgz#56eb2d976aef9d1445f373a62d781a41c7add8f8" + integrity sha512-DmeAkF6cwM9jSfmp6Dr/5/mfMwb5Z5qRrSXLpo3Fq5SqyU8CMF15jIN4ZhfSwu35ksM1qmHZDQ/DK5XTccSTvA== dependencies: accepts "~1.3.4" base64id "~2.0.0" + cors "~2.8.5" debug "~4.3.2" - engine.io "~6.4.1" + engine.io "~6.5.2" socket.io-adapter "~2.5.2" - socket.io-parser "~4.2.1" - -source-map@^0.5.0: - version "0.5.7" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc" - integrity sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ== + socket.io-parser "~4.2.4" source-map@^0.6.1: version "0.6.1" @@ -1453,20 +1525,20 @@ sprintf-js@~1.0.2: integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g== string-width@^4.1.0, string-width@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.0.tgz#952182c46cc7b2c313d1596e623992bd163b72b5" - integrity sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg== + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== dependencies: emoji-regex "^8.0.0" is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.0" + strip-ansi "^6.0.1" -strip-ansi@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532" - integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w== +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== dependencies: - ansi-regex "^5.0.0" + ansi-regex "^5.0.1" strip-bom@^4.0.0: version "4.0.0" @@ -1528,9 +1600,9 @@ tr46@^4.1.1: punycode "^2.3.0" ts-node@^10.9.1: - version "10.9.1" - resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.9.1.tgz#e73de9102958af9e1f0b168a6ff320e25adcff4b" - integrity sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw== + version "10.9.2" + resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.9.2.tgz#70f021c9e185bccdca820e26dc413805c101c71f" + integrity sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ== dependencies: "@cspotcode/source-map-support" "^0.8.0" "@tsconfig/node10" "^1.0.7" @@ -1559,14 +1631,27 @@ typedarray-to-buffer@^3.1.5: is-typedarray "^1.0.0" typescript@^4.9.4: - version "4.9.4" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.4.tgz#a2a3d2756c079abda241d75f149df9d561091e78" - integrity sha512-Uz+dTXYzxXXbsFpM86Wh3dKCxrQqUcVMxwU54orwlJjOpO3ao8L7j5lH+dWfTwgCwIuM9GQ2kvVotzYJMXTBZg== + version "4.9.5" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.5.tgz#095979f9bcc0d09da324d58d03ce8f8374cbe65a" + integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g== + +undici-types@~5.26.4: + version "5.26.5" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" + integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== + +update-browserslist-db@^1.0.16: + version "1.0.16" + resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.16.tgz#f6d489ed90fb2f07d67784eb3f53d7891f736356" + integrity sha512-KVbTxlBYlckhF5wgfyZXTWnMn7MMZjMu9XG8bPlliUOP9ThaF4QnhP8qrjrH7DRzHfSk0oQv1wToW+iA5GajEQ== + dependencies: + escalade "^3.1.2" + picocolors "^1.0.1" -uuid@^3.3.3: - version "3.4.0" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" - integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== +uuid@^8.3.2: + version "8.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" + integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== v8-compile-cache-lib@^3.0.1: version "3.0.1" @@ -1592,9 +1677,9 @@ whatwg-url@^13.0.0: webidl-conversions "^7.0.0" which-module@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" - integrity sha512-B+enWhmw6cjfVC7kS8Pj9pCrKSc5txArRyaYGe088shv/FGWH+0Rjx/xPgtsWfsUtS27FkP697E4DDhgrgoc0Q== + version "2.0.1" + resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.1.tgz#776b1fe35d90aebe99e8ac15eb24093389a4a409" + integrity sha512-iBdZ57RDvnOR9AGBhML2vFZf7h8vmBjhoaZqODJBFWHVtKkDmKuHai3cx5PgVMrX5YDNp27AofYbAwctSS+vhQ== which@^2.0.1: version "2.0.2" @@ -1641,10 +1726,10 @@ write-file-atomic@^3.0.0: signal-exit "^3.0.2" typedarray-to-buffer "^3.1.5" -ws@~8.11.0: - version "8.11.0" - resolved "https://registry.yarnpkg.com/ws/-/ws-8.11.0.tgz#6a0d36b8edfd9f96d8b25683db2f8d7de6e8e143" - integrity sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg== +ws@~8.17.1: + version "8.17.1" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.17.1.tgz#9293da530bb548febc95371d90f9c878727d919b" + integrity sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ== xmlhttprequest-ssl@~2.0.0: version "2.0.0" @@ -1652,16 +1737,21 @@ xmlhttprequest-ssl@~2.0.0: integrity sha512-QKxVRxiRACQcVuQEYFsI1hhkrMlrXHPegbbd1yn9UHOmRxY+si12nQYzri3vbzt8VdTTRviqcKxcyllFas5z2A== y18n@^4.0.0: - version "4.0.1" - resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.1.tgz#8db2b83c31c5d75099bb890b23f3094891e247d4" - integrity sha512-wNcy4NvjMYL8gogWWYAO7ZFWFfHcbdbE57tZO8e4cbpj8tfUcwrwqSl3ad8HxpYWCdXcJUCeKKZS62Av1affwQ== + version "4.0.3" + resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.3.tgz#b5f259c82cd6e336921efd7bfd8bf560de9eeedf" + integrity sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ== y18n@^5.0.5: version "5.0.8" resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== -yargs-parser@20.2.4, yargs-parser@^20.2.2: +yallist@^3.0.2: + version "3.1.1" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" + integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== + +yargs-parser@20.2.4: version "20.2.4" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.4.tgz#b42890f14566796f85ae8e3a25290d205f154a54" integrity sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA== @@ -1674,6 +1764,11 @@ yargs-parser@^18.1.2: camelcase "^5.0.0" decamelize "^1.2.0" +yargs-parser@^20.2.2: + version "20.2.9" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee" + integrity sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w== + yargs-unparser@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/yargs-unparser/-/yargs-unparser-2.0.0.tgz#f131f9226911ae5d9ad38c432fe809366c2325eb" From ab15f6bf3f6b2590bd47b6679c1abe2910239d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=ABl=20Luijmes?= Date: Tue, 18 Jun 2024 16:20:55 +0200 Subject: [PATCH 06/10] fix: typing in tests --- test/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/index.ts b/test/index.ts index c626992..71aae36 100644 --- a/test/index.ts +++ b/test/index.ts @@ -477,12 +477,12 @@ describe("@socket.io/mongodb-adapter", () => { clientSockets[2].on("test3", partialDone); await mongoClient.connect(); - servers[0].to(clientSockets[1].id).emit("test2"); + servers[0].to(clientSockets[1].id as string).emit("test2"); await sleep(500); - servers[1].to(clientSockets[2].id).emit("test3"); - servers[2].to(clientSockets[0].id).emit("test1"); + servers[1].to(clientSockets[2].id as string).emit("test3"); + servers[2].to(clientSockets[0].id as string).emit("test1"); }); }); From ef874e19510d298b1c055ab7ec0564846556f5f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=ABl=20Luijmes?= Date: Tue, 18 Jun 2024 16:55:26 +0200 Subject: [PATCH 07/10] fix(build): use prepack -> build --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 641ee9b..94c021d 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "test": "npm run format:check && tsc && nyc mocha --require ts-node/register test/index.ts", "format:check": "prettier --parser typescript --check 'lib/**/*.ts' 'test/**/*.ts'", "format:fix": "prettier --parser typescript --write 'lib/**/*.ts' 'test/**/*.ts'", - "prepack": "tsc" + "build": "tsc" }, "dependencies": { "debug": "~4.3.1", From 51fa660268f45d6a83d984990d304fea25547ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=ABl=20Luijmes?= Date: Tue, 18 Jun 2024 16:55:39 +0200 Subject: [PATCH 08/10] chore: update jsonnet --- .github/jsonnet/GIT_VERSION | 2 +- .github/jsonnet/README.md | 2 +- .github/jsonnet/base.jsonnet | 14 +- .github/jsonnet/buckets.jsonnet | 152 +++++++++ .github/jsonnet/cache.jsonnet | 128 ++++++++ .github/jsonnet/clusters.jsonnet | 41 +-- .github/jsonnet/complete-workflows.jsonnet | 18 +- .github/jsonnet/databases.jsonnet | 18 +- .github/jsonnet/deployment.jsonnet | 159 ++++++---- .github/jsonnet/docker.jsonnet | 12 +- .github/jsonnet/helm.jsonnet | 91 +++--- .github/jsonnet/images.jsonnet | 13 +- .github/jsonnet/index.jsonnet | 6 +- .github/jsonnet/misc.jsonnet | 141 +++++---- .github/jsonnet/mongo.jsonnet | 212 ++++++++++--- .github/jsonnet/newrelic.jsonnet | 16 +- .github/jsonnet/notifications.jsonnet | 21 +- .github/jsonnet/pnpm.jsonnet | 15 + .github/jsonnet/pubsub.jsonnet | 16 +- .github/jsonnet/pulumi.jsonnet | 352 ++++++++++++++++----- .github/jsonnet/ruby.jsonnet | 101 +++--- .github/jsonnet/services.jsonnet | 25 +- .github/jsonnet/yarn.jsonnet | 135 ++++---- .github/workflows/misc.yml | 28 -- .github/workflows/pr.yml | 94 ------ .github/workflows/publish-prod.yml | 74 ----- 26 files changed, 1204 insertions(+), 682 deletions(-) create mode 100644 .github/jsonnet/buckets.jsonnet create mode 100644 .github/jsonnet/cache.jsonnet create mode 100644 .github/jsonnet/pnpm.jsonnet delete mode 100644 .github/workflows/misc.yml delete mode 100644 .github/workflows/pr.yml delete mode 100644 .github/workflows/publish-prod.yml diff --git a/.github/jsonnet/GIT_VERSION b/.github/jsonnet/GIT_VERSION index 1724762..8273035 100644 --- a/.github/jsonnet/GIT_VERSION +++ b/.github/jsonnet/GIT_VERSION @@ -1 +1 @@ -4187fef119638c2f8453bf4fd3d6da5641e4ffee +e4a3467b90e35c3dadf461f338838837bf02d2f0 diff --git a/.github/jsonnet/README.md b/.github/jsonnet/README.md index b719864..805e8f6 100644 --- a/.github/jsonnet/README.md +++ b/.github/jsonnet/README.md @@ -1,2 +1,2 @@ These files come from https://www.github.com/gynzy/lib-jsonnet/ -Do not update here, but extend the libraries upstream. \ No newline at end of file +Do not update here, but extend the libraries upstream. diff --git a/.github/jsonnet/base.jsonnet b/.github/jsonnet/base.jsonnet index 2b4875e..0b85fa8 100644 --- a/.github/jsonnet/base.jsonnet +++ b/.github/jsonnet/base.jsonnet @@ -1,3 +1,6 @@ +local images = import 'images.jsonnet'; +local misc = import 'misc.jsonnet'; + { pipeline(name, jobs, event=['pull_request'], permissions=null, concurrency=null):: { [name + '.yml']: @@ -14,7 +17,7 @@ name, timeoutMinutes=30, runsOn=null, - image=$.default_job_image, + image=images.default_job_image, steps=[], ifClause=null, needs=null, @@ -36,7 +39,7 @@ { container: { image: image, - } + (if useCredentials then { credentials: { username: '_json_key', password: $.secret('docker_gcr_io') } } else {}), + } + (if useCredentials then { credentials: { username: '_json_key', password: misc.secret('docker_gcr_io') } } else {}), } ) + { @@ -49,7 +52,7 @@ (if permissions == null then {} else { permissions: permissions }) + (if concurrency == null then {} else { concurrency: concurrency }) + (if continueOnError == null then {} else { 'continue-on-error': continueOnError }) + - (if env == null then {} else { env: env }) + (if env == null then {} else { env: env }), }, ghExternalJob( @@ -65,7 +68,7 @@ } else {}), }, - step(name, run, env=null, workingDirectory=null, ifClause=null, id=null, continueOnError=null):: + step(name, run, env=null, workingDirectory=null, ifClause=null, id=null, continueOnError=null, shell=null):: [ { name: name, @@ -74,7 +77,8 @@ + (if env != null then { env: env } else {}) + (if ifClause != null then { 'if': ifClause } else {}) + (if id != null then { id: id } else {}) - + (if continueOnError == null then {} else { 'continue-on-error': continueOnError }), + + (if continueOnError == null then {} else { 'continue-on-error': continueOnError }) + + (if shell == null then {} else { 'shell': shell }), ], action(name, uses, env=null, with=null, id=null, ifClause=null, continueOnError=null):: diff --git a/.github/jsonnet/buckets.jsonnet b/.github/jsonnet/buckets.jsonnet new file mode 100644 index 0000000..b2298d9 --- /dev/null +++ b/.github/jsonnet/buckets.jsonnet @@ -0,0 +1,152 @@ +{ + // Uploads all files in the source folder to the destination bucket, including compression and TTL headers. + // + // Warnings: + // - remote/destination files not included in the source will be DELETED recursively if pruneRemote is true! + // - the files in the source directory will be modified. Do not attempt to use this directory after running this command. + // - must be run with bash shell. + // + // Parameters: + // sourcePath: The source directory to upload. Can be a local folder of a path in a bucket, depending on sourceBucket. Required. + // sourceBucket: The source bucket. If null, the sourcePath is a local directory. Defaults to null. + // destinationBucket: The destination bucket. Required. + // destinationPath: The destination directory in the bucket. Required. + // + // pruneRemote: If true, all files in the destination bucket that are not in the source will be deleted. Can only be used with destinationPath containing 'pr-'. + // + // compressFileExtentions: A list of file extentions that will be compressed. Set to an empty list to disable compression. + // compressJobs: The number of parallel gzip compression jobs. Use 4 for arc-runner-2 and 16 for arc-runner-16. Defaults to 4. + // + // lowTTLfiles: A list of files, or a single regex, that will be uploaded with a low TTL. Use this for files that are not fingerprinted. + // + // lowTTL: The TTL for lowTTLfiles. Defaults to 60 seconds. + // lowTTLStaleWhileRevalidate: The stale-while-revalidate value for lowTTLfiles. Defaults to 60 seconds. + // lowTTLHeader: The Cache-Control header for lowTTLfiles. This is generated from lowTTL and lowTTLStaleWhileRevalidate. + // + // highTTL: The TTL for all other files. Defaults to 1 week. + // highTTLStaleWhileRevalidate: The stale-while-revalidate value for all other files. Defaults to 1 day. + // highTTLHeader: The Cache-Control header for all other files. This is generated from highTTL and highTTLStaleWhileRevalidate. + // + // additionalHeaders: Additional headers to add to all uploaded files. This should be an array of strings. + uploadFilesToBucketCommand( + sourcePath, + sourceBucket=null, + destinationBucket, + destinationPath, + pruneRemote=false, + compressFileExtentions=['css', 'svg', 'html', 'json', 'js', 'xml', 'txt', 'map'], + compressJobs=4, + lowTTLfiles=[], + lowTTL=60, + lowTTLStaleWhileRevalidate=60, + lowTTLHeader='Cache-Control: public, max-age=' + lowTTL + (if lowTTLStaleWhileRevalidate == 0 then '' else ', stale-while-revalidate=' + lowTTLStaleWhileRevalidate), + highTTL=604800, // 1 week + highTTLStaleWhileRevalidate=86400, // 1 day + highTTLHeader='Cache-Control: public, max-age=' + highTTL + (if highTTLStaleWhileRevalidate == 0 then '' else ', stale-while-revalidate=' + highTTLStaleWhileRevalidate), + additionalHeaders=[], + ):: + // if this function is called with remote pruning, destination must contain pr- + assert !pruneRemote || std.length(std.findSubstr('/pr-', destinationPath)) > 0; + + local hasLowTTLfiles = (std.isArray(lowTTLfiles) && std.length(lowTTLfiles) > 0) || (std.isString(lowTTLfiles) && lowTTLfiles != ''); + local lowTTLfilesRegex = if std.isArray(lowTTLfiles) then '(' + std.strReplace(std.join('|', lowTTLfiles), '.', '\\.') + ')' else lowTTLfiles; + local highTTLfilesRegex = '(?!' + lowTTLfilesRegex + ').*'; + + local hasCompressedFiles = (std.isArray(compressFileExtentions) && std.length(compressFileExtentions) > 0) || (std.isString(compressFileExtentions) && compressFileExtentions != ''); + local compressedFilesRegex = '(' + std.join('|', std.map(function(ext) '((.*(\\.|/))?' + ext + ')', compressFileExtentions)) + ')'; + local uncompressedFilesRegex = '(?!' + compressedFilesRegex + ').*'; + + local compressionHeader = 'Content-Encoding: gzip'; + + + local rsyncCommand = function(name, excludeRegexes, headers) + local excludeRegex = if std.length(excludeRegexes) == 0 then null else '^((' + std.join(')|(', excludeRegexes) + '))$'; + + 'echo "Uploading ' + name + ' files"\n' + + 'gsutil -m ' + std.join(' ', std.map(function(header) '-h "' + header + '" ', headers + additionalHeaders)) + 'rsync -r -c' + + (if excludeRegex == null then '' else ' -x "' + excludeRegex + '"') + + (if pruneRemote then ' -d' else '') + + (if sourceBucket == null then ' ./' else ' gs://' + sourceBucket + '/' + sourcePath + '/') + + ' gs://' + destinationBucket + '/' + destinationPath + '/;\n' + + 'echo "Uploading ' + name + ' files completed"; echo\n' + + '\n'; + + 'set -e -o pipefail;\n' + + (if sourceBucket == null then 'cd ' + sourcePath + ';\n' else '') + + '\n' + + + + if hasCompressedFiles then + ( + if sourceBucket == null then + 'echo "Compressing files in parallel before uploading"\n' + + '{\n' + + " for file in `find . -type f -regextype posix-egrep -regex '" + compressedFilesRegex + "' | sed --expression 's/\\.\\///g'`; do\n" + + ' echo "gzip -9 $file; mv $file.gz $file"\n' + + ' done\n' + + '} | parallel --halt now,fail=1 -j ' + compressJobs + '\n' + + 'echo "Compressing files in parallel completed"\n' + + '\n' + else '' + ) + + + if hasLowTTLfiles then + rsyncCommand( + 'highTTL compressed', + excludeRegexes=[lowTTLfilesRegex, uncompressedFilesRegex], + headers=[highTTLHeader, compressionHeader], + ) + + rsyncCommand( + 'highTTL uncompressed', + excludeRegexes=[lowTTLfilesRegex, compressedFilesRegex], + headers=[highTTLHeader], + ) + + + rsyncCommand( + 'lowTTL compressed', + excludeRegexes=[highTTLfilesRegex, uncompressedFilesRegex], + headers=[lowTTLHeader, compressionHeader], + ) + + rsyncCommand( + 'lowTTL uncompressed', + excludeRegexes=[highTTLfilesRegex, compressedFilesRegex], + headers=[lowTTLHeader], + ) + + + else // no lowTTL files, with compression + rsyncCommand( + 'compressed', + excludeRegexes=[uncompressedFilesRegex], + headers=[highTTLHeader, compressionHeader], + ) + + rsyncCommand( + 'uncompressed', + excludeRegexes=[compressedFilesRegex], + headers=[highTTLHeader], + ) + + + else // no compression + if hasLowTTLfiles then + rsyncCommand( + 'highTTL', + excludeRegexes=[lowTTLfilesRegex], + headers=[highTTLHeader], + ) + + + rsyncCommand( + 'lowTTL', + excludeRegexes=[highTTLfilesRegex], + headers=[lowTTLHeader], + ) + + + else // no lowTTL files, no compression + rsyncCommand( + 'all', + excludeRegexes=[], + headers=[highTTLHeader], + ), + +} diff --git a/.github/jsonnet/cache.jsonnet b/.github/jsonnet/cache.jsonnet new file mode 100644 index 0000000..9d85bef --- /dev/null +++ b/.github/jsonnet/cache.jsonnet @@ -0,0 +1,128 @@ +local base = import 'base.jsonnet'; + +{ + // Fetch a cache from the cache server. + // This is a generic function that can be used to fetch any cache. It is advised to wrap this function + // in a more specific function that fetches a specific cache, setting the cacheName and folders parameters. + // + // To be paired with the uploadCache function. + // + // Parameters: + // cacheName: The name of the cache to fetch. The name of the repository is usually a good option. Required. + // backupCacheName: The name of a backup cache to fetch if the main cache fails. Default is null. + // folders: A list of folders that are in the cache. These will be deleted if the download fails. Can be an empty list if additionalCleanupCommands are used. + // additionalCleanupCommands: A list of additional commands to run if the download fails. Default is an empty list. + // ifClause: An optional if clause to conditionally run this step. Default is null. + // workingDirectory: The working directory for this step. Default is null. + // retry: Whether to retry the download if it fails. Default is true. + // continueWithoutCache: Whether to continue if the cache is not found. Default is true. + fetchCache( + cacheName, + backupCacheName=null, + folders=[], + version='v1', + backupCacheVersion=version, + additionalCleanupCommands=[], + ifClause=null, + workingDirectory=null, + retry=true, + continueWithoutCache=true, + ):: + assert std.length(folders) > 0 || std.length(additionalCleanupCommands) > 0; + + local downloadCommand(cacheName, version, nextSteps, indent = '') = + indent + 'wget -q -O - "https://storage.googleapis.com/files-gynzy-com-test/ci-cache/' + cacheName + '-' + version + '.tar.zst" | unzstd | tar xf -\n' + + indent + 'if [ $? -ne 0 ]; then\n' + + indent + ' echo "Cache download failed, cleanup up partial downloads"\n' + + (if std.length(folders) > 0 then indent + ' rm -rf ' + std.join(' ', folders) + '\n' else '') + + std.join(' ', std.map(function(cmd) indent + ' ' + cmd + '\n', additionalCleanupCommands)) + + indent + ' echo "Cleanup complete"; echo\n\n' + + nextSteps + + indent + 'fi\n'; + + local downloadCommandWithRetry(cacheName, version, nextSteps, indent = '') = + downloadCommand( + cacheName, + version, + if retry then + indent + ' echo "Retrying download..."\n' + + downloadCommand(cacheName, version, nextSteps, indent + ' ') + else + nextSteps, + indent, + ); + + local backupIndent = (if retry then ' ' else ' '); + + local downloadFailedCommand = backupIndent + 'echo "Cache download failed :( ' + (if continueWithoutCache then 'Continuing without cache"' else 'Aborting"; exit 1') + '\n'; + + base.step( + 'download ' + cacheName + ' cache', + run= + 'set +e;\n' + + 'command -v zstd || { apt update && apt install -y zstd; }\n' + + 'echo "Downloading cache"\n' + + downloadCommandWithRetry( + cacheName, + version, + if backupCacheName != null then + backupIndent + 'echo "Downloading backup cache"\n' + + downloadCommandWithRetry(backupCacheName, backupCacheVersion, backupIndent + downloadFailedCommand, indent=backupIndent) + else + downloadFailedCommand, + ), + ifClause=ifClause, + workingDirectory=workingDirectory, + ), + + // Uploads a cache to the cache server. + // This is a generic function that can be used to upload any cache. It is advised to wrap this function + // in a more specific function that uploads a specific cache, setting the cacheName and folders parameters. + // + // To be paired with the fetchCache function. + // + // Parameters: + // cacheName: The name of the cache to upload. The name of the repository is usually a good option. Required. + // folders: A list of folders to include in the cache. Required unless tarCommand is given. + // compressionLevel: The compression level to use for zstd. Default is 10. + // tarCommand: The command to run to create the tar file. Default is 'tar -c ' + std.join(' ', folders). + uploadCache( + cacheName, + folders=null, + version='v1', + compressionLevel=10, + tarCommand='tar -c ' + std.join(' ', folders), + ):: + local cacheBucketPath = function(temp=false) + 'gs://files-gynzy-com-test/ci-cache/' + cacheName + '-' + version + '.tar.zst' + (if temp then '.tmp' else ''); + + base.step( + 'upload-gatsby-cache', + run= + 'set -e\n' + + '\n' + + 'command -v zstd || { apt update && apt install -y zstd; }\n' + + '\n' + + 'echo "Create and upload cache"\n' + + tarCommand + ' | zstdmt -' + compressionLevel + ' | gsutil cp - "' + cacheBucketPath(temp=true) + '"\n' + + 'gsutil mv "' + cacheBucketPath(temp=true) + '" "' + cacheBucketPath(temp=false) + '"\n' + + + 'echo "Upload finished"\n' + ), + + // Removes a cache from the cache server. + // This is a generic function that can be used to remove any cache. It is advised to wrap this function + // in a more specific function that removes a specific cache, setting the cacheName parameter. + // + // Parameters: + // cacheName: The name of the cache to remove. The name of the repository is usually a good option. Required. + // version: The version of the cache to remove. Default is 'v1'. + removeCache(cacheName, version='v1'):: + base.step( + 'remove ' + cacheName + ' cache', + run= + 'set +e;\n' + + 'gsutil rm "gs://files-gynzy-com-test/ci-cache/' + cacheName + '-' + version + '.tar.zst"\n' + + 'echo "Cache removed"\n' + ), +} diff --git a/.github/jsonnet/clusters.jsonnet b/.github/jsonnet/clusters.jsonnet index 3ebe0e7..10c9b65 100644 --- a/.github/jsonnet/clusters.jsonnet +++ b/.github/jsonnet/clusters.jsonnet @@ -1,24 +1,27 @@ +local misc = import 'misc.jsonnet'; + { - clusters: { - test: { - project: 'gynzy-test-project', - name: 'test', - zone: 'europe-west4-b', - secret: '${{ secrets.GCE_NEW_TEST_JSON }}', - }, + test: { + project: 'gynzy-test-project', + name: 'test', + zone: 'europe-west4-b', + secret: misc.secret('GCE_NEW_TEST_JSON'), + jobNodeSelectorType: 'preemptible', + }, - prod: { - project: 'unicorn-985', - name: 'prod-europe-west4', - zone: 'europe-west4', - secret: '${{ secrets.GCE_JSON }}', - }, + prod: { + project: 'unicorn-985', + name: 'prod-europe-west4', + zone: 'europe-west4', + secret: misc.secret('GCE_JSON'), + jobNodeSelectorType: 'worker', + }, - 'gynzy-intern': { - project: 'gynzy-intern', - name: 'gynzy-intern', - zone: 'europe-west4', - secret: '${{ secrets.CONTINUOUS_DEPLOYMENT_GCE_JSON }}', - }, + 'gynzy-intern': { + project: 'gynzy-intern', + name: 'gynzy-intern', + zone: 'europe-west4', + secret: misc.secret('CONTINUOUS_DEPLOYMENT_GCE_JSON'), + jobNodeSelectorType: 'worker', }, } diff --git a/.github/jsonnet/complete-workflows.jsonnet b/.github/jsonnet/complete-workflows.jsonnet index d688931..096278e 100644 --- a/.github/jsonnet/complete-workflows.jsonnet +++ b/.github/jsonnet/complete-workflows.jsonnet @@ -1,3 +1,7 @@ +local base = import 'base.jsonnet'; +local misc = import 'misc.jsonnet'; +local yarn = import 'yarn.jsonnet'; + { /* @param {string[]} repositories - The repositories to publish to @@ -9,21 +13,21 @@ workflowJavascriptPackage(repositories=['gynzy'], isPublicFork=true, checkVersionBump=true, testJob=null, branch='main'):: local runsOn = (if isPublicFork then 'ubuntu-latest' else null); - $.pipeline( + base.pipeline( 'misc', - [$.verifyJsonnet(fetch_upstream=false, runsOn=runsOn)], + [misc.verifyJsonnet(fetch_upstream=false, runsOn=runsOn)], ) + - $.pipeline( + base.pipeline( 'publish-prod', [ - $.yarnPublishJob(repositories=repositories, runsOn=runsOn), + yarn.yarnPublishJob(repositories=repositories, runsOn=runsOn), ], - event={ push: { branches: [ branch ] } }, + event={ push: { branches: [branch] } }, ) + - $.pipeline( + base.pipeline( 'pr', [ - $.yarnPublishPreviewJob(repositories=repositories, runsOn=runsOn, checkVersionBump=checkVersionBump), + yarn.yarnPublishPreviewJob(repositories=repositories, runsOn=runsOn, checkVersionBump=checkVersionBump), ] + (if testJob != null then [testJob] diff --git a/.github/jsonnet/databases.jsonnet b/.github/jsonnet/databases.jsonnet index baa7755..c048cbf 100644 --- a/.github/jsonnet/databases.jsonnet +++ b/.github/jsonnet/databases.jsonnet @@ -1,3 +1,6 @@ +local base = import 'base.jsonnet'; +local images = import 'images.jsonnet'; + { database_servers: { test: { @@ -26,13 +29,6 @@ region: 'europe-west4', project: 'unicorn-985', }, - 'eu-w4-metrics-production': { - type: 'mysql', - server: 'eu-w4-metrics-production', - region: 'europe-west4', - project: 'unicorn-985', - lifecycle: 'deprecated', - }, 'gynzy-test': { type: 'mysql', server: 'gynzy-test', @@ -97,9 +93,9 @@ // delete database by setting it to null and calling prune afterwards local pluginOptions = std.prune(mysqlActionOptions { task: 'clone', database: null }); - $.action( + base.action( 'copy-database', - $.mysql_action_image, + images.mysql_action_image, with=pluginOptions ), @@ -110,9 +106,9 @@ // delete database by setting it to null and calling prune afterwards local pluginOptions = std.prune(mysqlActionOptions { task: 'remove', database: null }); - $.action( + base.action( 'delete-database', - $.mysql_action_image, + images.mysql_action_image, with=pluginOptions ), } diff --git a/.github/jsonnet/deployment.jsonnet b/.github/jsonnet/deployment.jsonnet index 60de82c..d6171cb 100644 --- a/.github/jsonnet/deployment.jsonnet +++ b/.github/jsonnet/deployment.jsonnet @@ -1,8 +1,13 @@ +local base = import 'base.jsonnet'; +local images = import 'images.jsonnet'; +local misc = import 'misc.jsonnet'; +local notifications = import 'notifications.jsonnet'; + { _assertMergeShaIsLatestCommit(branch, sha='${{ github.sha }}', repository='${{ github.repository }}'):: [ - $.step('install jq curl', 'apk add --no-cache jq curl'), - $.step( + base.step('install jq curl', 'apk add --no-cache jq curl'), + base.step( 'assert merge sha is latest commit', ||| HEAD_SHA=$(curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/${GITHUB_REPOSITORY}/branches/${TARGET_BRANCH} | jq -r .commit.sha); @@ -35,78 +40,112 @@ * - PR B is merged into the default branch * - now github closes both PRs and without this additional sanity check, both would create a deploy event * + * For more complex deployment scenarios, use the branchMergeDeploymentEventHook instead + * * params: * deployToTest {boolean} - if true, a deployment event is also created for the test environment * prodBranch {string|null} - the branch to deploy to production. defaults to the default branch of the repository. but can be set to a differring release branch * testBranch {string|null} - the branch to deploy to test. defaults to the default branch of the repository. but can be set to a differring test branch * extraDeployTargets {string[]} - deploy targets to create deployment events for. defaults to ['production']. these targets will triger based on the configured prodBranch * runsOn {string|null} - the name of the runner to run this job on. defaults to null, which will later on means the default self hosted runner will be used + * notifyOnTestDeploy {boolean} - if true, a slack message is sent when a test deployment is created */ - masterMergeDeploymentEventHook(deployToTest=false, prodBranch=null, testBranch=null, deployTargets=['production'], runsOn=null):: - $.pipeline( + masterMergeDeploymentEventHook(deployToTest=false, prodBranch=null, testBranch=null, deployTargets=['production'], runsOn=null, notifyOnTestDeploy=false):: + local branches = [ + { + branch: (if prodBranch != null then prodBranch else '_default_'), + deployments: deployTargets, + notifyOnDeploy: true, + }, + ] + (if deployToTest then [ + { + branch: (if testBranch != null then testBranch else '_default_'), + deployments: ['test'], + notifyOnDeploy: notifyOnTestDeploy, + }, + ] else []); + + self.branchMergeDeploymentEventHook(branches, runsOn=runsOn), + + + /* + * Creates a production deployment event on pr-close if the following conditions are met: + * - the pr is merged + * - the pr is not merged by the virko user + * - the pr is merged into the default branch + * - the merge sha is the latest commit on the default branch. + * this prevents a deployment from beeing created in a specific edge case: + * - PR A is merged into PR B + * - PR B is merged into the default branch + * - now github closes both PRs and without this additional sanity check, both would create a deploy event + * + * params: + * branches {{branch: string, deployments: string[], notifyOnDeploy: boolean}[]} - an array of the branches to create deployment events for. + * Each branch object has the following properties: + * branch {string} - the branch to which the PR has to be merged into. If '_default_' is used, the default branch of the repository is used. + * deployments {string[]} - the environments to deploy to. e.g. ['production', 'test'] + * notifyOnDeploy {boolean} - if true, a slack message is sent when a deployment is created + * runsOn {string|null} - the name of the runner to run this job on. defaults to null, which will later on means the default self hosted runner will be used + */ + branchMergeDeploymentEventHook(branches, runsOn=null):: + base.pipeline( 'create-merge-deployment', [ - $.ghJob( - 'create-merge-deployment-prod', - useCredentials=false, - runsOn=runsOn, - permissions={ deployments: 'write', contents: 'read' }, - ifClause="${{ github.actor != 'gynzy-virko' && github.event.pull_request.merged == true}}", - steps=$._assertMergeShaIsLatestCommit(branch=(if prodBranch != null then prodBranch else '${{ github.event.pull_request.base.repo.default_branch }}')) + - std.map( - function(deploymentTarget) - $.action( - 'publish-deploy-' + deploymentTarget + '-event', - 'chrnorm/deployment-action@v2', - ifClause='${{ github.event.pull_request.base.ref == ' + (if prodBranch != null then "'" + prodBranch + "'" else 'github.event.pull_request.base.repo.default_branch') + " && steps.assert-merge-sha-is-latest-commit.outputs.CREATE_DEPLOY_EVENT == 'true' }}", - with={ - token: $.secret('VIRKO_GITHUB_TOKEN'), - environment: deploymentTarget, - 'auto-merge': 'false', - ref: '${{ github.event.pull_request.head.sha }}', - description: 'Auto deploy ' + deploymentTarget + ' on PR merge. pr: ${{ github.event.number }} ref: ${{ github.event.pull_request.head.sha }}', - payload: '{ "pr" : ${{ github.event.number }}, "branch": "${{ github.head_ref }}", "base_ref": "${{ github.event.pull_request.base.sha }}", "head_ref": "${{ github.event.pull_request.head.sha }}" }', - } - ), - deployTargets, - ) + - [ - $.sendSlackMessage( - message='Deploy to prod of pr: ${{ github.event.number }} with title: ${{ github.event.pull_request.title }} branch: ${{ github.head_ref }} started!', - ifClause='${{ github.event.pull_request.base.ref == ' + (if prodBranch != null then "'" + prodBranch + "'" else 'github.event.pull_request.base.repo.default_branch') + " && steps.assert-merge-sha-is-latest-commit.outputs.CREATE_DEPLOY_EVENT == 'true' }}", + ( + local branchName = if branch.branch == '_default_' then '${{ github.event.pull_request.base.repo.default_branch }}' else branch.branch; + local branchNameForJob = if branch.branch == '_default_' then 'default-branch' else branch.branch; + local branchNameInExpression = if branch.branch == '_default_' then 'github.event.pull_request.base.repo.default_branch' else "'" + branch.branch + "'"; + + local ifClause = '${{ github.event.pull_request.base.ref == ' + branchNameInExpression + " && steps.assert-merge-sha-is-latest-commit.outputs.CREATE_DEPLOY_EVENT == 'true' }}"; + + base.ghJob( + 'create-merge-deployment-' + branchNameForJob + '-to-' + std.join('-', branch.deployments), + useCredentials=false, + runsOn=runsOn, + permissions={ deployments: 'write', contents: 'read' }, + ifClause="${{ github.actor != 'gynzy-virko' && github.event.pull_request.merged == true}}", + steps=self._assertMergeShaIsLatestCommit(branch=branchName) + + std.map( + function(deploymentTarget) + base.action( + 'publish-deploy-' + deploymentTarget + '-event', + 'chrnorm/deployment-action@v2', + ifClause=ifClause, + with={ + token: misc.secret('VIRKO_GITHUB_TOKEN'), + environment: deploymentTarget, + 'auto-merge': 'false', + ref: '${{ github.event.pull_request.head.sha }}', + description: 'Auto deploy ' + deploymentTarget + ' on PR merge. pr: ${{ github.event.number }} ref: ${{ github.event.pull_request.head.sha }}', + payload: '{ "pr" : ${{ github.event.number }}, "branch": "${{ github.head_ref }}", "base_ref": "${{ github.event.pull_request.base.sha }}", "head_ref": "${{ github.event.pull_request.head.sha }}" }', + } + ), + branch.deployments, + ) + + ( + if branch.notifyOnDeploy then + [ + notifications.sendSlackMessage( + message='Deploy to ' + std.join(' and ', branch.deployments) + ' of started!\nTitle: ${{ github.event.pull_request.title }}\nBranch: ${{ github.head_ref }}', + ifClause=ifClause, + ), + ] + else [] ), - ], - ), - ] + - (if deployToTest == true && std.member(deployTargets, 'test') == false then - [ - $.ghJob( - 'create-merge-deployment-test', - useCredentials=false, - permissions={ deployments: 'write', contents: 'read' }, - ifClause="${{ github.actor != 'gynzy-virko' && github.event.pull_request.merged == true}}", - steps=$._assertMergeShaIsLatestCommit(branch=(if testBranch != null then testBranch else '${{ github.event.pull_request.base.repo.default_branch }}')) + - [ - $.action( - 'publish-deploy-test-event', - 'chrnorm/deployment-action@v2', - ifClause='${{ github.event.pull_request.base.ref == ' + (if testBranch != null then "'" + testBranch + "'" else 'github.event.pull_request.base.repo.default_branch') + " && steps.assert-merge-sha-is-latest-commit.outputs.CREATE_DEPLOY_EVENT == 'true' }}", - with={ - token: $.secret('VIRKO_GITHUB_TOKEN'), - environment: 'test', - 'auto-merge': 'false', - ref: '${{ github.event.pull_request.head.sha }}', - description: 'Auto deploy test on PR merge. pr: ${{ github.event.number }} ref: ${{ github.event.pull_request.head.sha }}', - payload: '{ "pr" : ${{ github.event.number }}, "branch": "${{ github.head_ref }}", "base_ref": "${{ github.event.pull_request.base.sha }}", "head_ref": "${{ github.event.pull_request.head.sha }}" }', - } - ), - ], - ), - ] else []), + ) + ) + for branch in branches + ], event={ pull_request: { types: ['closed'], }, }, ), + + /* + * Generate a github ifClause for the provided deployment targets: + */ + deploymentTargets(targets):: + '${{ ' + std.join(' || ', std.map(function(target) "github.event.deployment.environment == '" + target + "'", targets)) + ' }}', } diff --git a/.github/jsonnet/docker.jsonnet b/.github/jsonnet/docker.jsonnet index d33a9c7..a23ddc1 100644 --- a/.github/jsonnet/docker.jsonnet +++ b/.github/jsonnet/docker.jsonnet @@ -1,3 +1,7 @@ +local base = import 'base.jsonnet'; +local images = import 'images.jsonnet'; +local misc = import 'misc.jsonnet'; + { buildDocker( imageName, @@ -9,12 +13,12 @@ registry='eu.gcr.io', project='unicorn-985', ):: - $.action( - 'build-docker', - $.docker_action_image, + base.action( + 'build-docker ' + imageName, + images.docker_action_image, with={ context: context, - gcloud_service_key: $.secret('docker_gcr_io_base64'), + gcloud_service_key: misc.secret('docker_gcr_io_base64'), image_name: imageName, image_tag: imageTag, project_id: project, diff --git a/.github/jsonnet/helm.jsonnet b/.github/jsonnet/helm.jsonnet index 8b7a871..8e033f8 100644 --- a/.github/jsonnet/helm.jsonnet +++ b/.github/jsonnet/helm.jsonnet @@ -1,8 +1,15 @@ +local base = import 'base.jsonnet'; +local clusters = import 'clusters.jsonnet'; +local databases = import 'databases.jsonnet'; +local images = import 'images.jsonnet'; +local misc = import 'misc.jsonnet'; +local services = import 'services.jsonnet'; + { deployHelm(cluster, release, values, chartPath, delete=false, useHelm3=true, title=null, ifClause=null, ttl=null, namespace='default'):: - $.action( + base.action( (if title == null then if delete then 'delete-helm' else 'deploy-helm' else title), - $.helm_action_image, + images.helm_action_image, with={ clusterProject: cluster.project, clusterLocation: cluster.zone, @@ -27,10 +34,10 @@ helmPath='./helm/' + serviceName, deploymentName=serviceName + '-prod', ifClause=null, - cluster=$.clusters.prod, + cluster=clusters.prod, namespace='default', ):: - $.deployHelm( + self.deployHelm( cluster, deploymentName, { @@ -52,17 +59,17 @@ options={}, helmPath='./helm/' + serviceName, deploymentName=serviceName + '-prod', - image=$.default_job_image, + image=images.default_job_image, useCredentials=false, ):: - $.ghJob( + base.ghJob( 'deploy-prod', ifClause="${{ github.event.deployment.environment == 'production' }}", image=image, useCredentials=useCredentials, steps=[ - $.checkout(), - $.helmDeployProd(serviceName, options, helmPath, deploymentName), + misc.checkout(), + self.helmDeployProd(serviceName, options, helmPath, deploymentName), ], ), @@ -71,10 +78,10 @@ options={}, helmPath='./helm/' + serviceName, deploymentName=serviceName + '-master', - cluster=$.clusters.test, + cluster=clusters.test, namespace='default', ):: - $.deployHelm( + self.deployHelm( cluster, deploymentName, { @@ -95,17 +102,17 @@ options={}, helmPath='./helm/' + serviceName, deploymentName=serviceName + '-master', - image=$.default_job_image, + image=images.default_job_image, useCredentials=false, ):: - $.ghJob( + base.ghJob( 'deploy-test', ifClause="${{ github.event.deployment.environment == 'test' }}", image=image, useCredentials=useCredentials, steps=[ - $.checkout(), - $.helmDeployTest(serviceName, options, helmPath, deploymentName), + misc.checkout(), + self.helmDeployTest(serviceName, options, helmPath, deploymentName), ], ), @@ -114,10 +121,10 @@ options={}, helmPath='./helm/' + serviceName, deploymentName=serviceName + '-pr-${{ github.event.number }}', - cluster=$.clusters.test, + cluster=clusters.test, namespace='default', ):: - $.deployHelm( + self.deployHelm( cluster, deploymentName, { @@ -138,16 +145,16 @@ options={}, helmPath='./helm/' + serviceName, deploymentName=serviceName + '-pr-${{ github.event.number }}', - image=$.default_job_image, + image=images.default_job_image, useCredentials=false, ):: - $.ghJob( + base.ghJob( 'deploy-pr', image=image, useCredentials=useCredentials, steps=[ - $.checkout(), - $.helmDeployPR(serviceName, options, helmPath, deploymentName), + misc.checkout(), + self.helmDeployPR(serviceName, options, helmPath, deploymentName), ], ), @@ -156,10 +163,10 @@ options={}, helmPath='./helm/' + serviceName, deploymentName=serviceName + '-pr-${{ github.event.number }}', - cluster=$.clusters.test, + cluster=clusters.test, namespace='default', ):: - $.deployHelm( + self.deployHelm( cluster, deploymentName, options, @@ -176,16 +183,16 @@ deploymentName=serviceName + '-pr-${{ github.event.number }}', mysqlDeleteOptions={ enabled: false }, ):: - $.ghJob( + base.ghJob( 'helm-delete-pr', - image=$.default_job_image, + image=images.default_job_image, useCredentials=false, steps=[ - $.checkout(), - $.helmDeletePr(serviceName, options, helmPath, deploymentName), + misc.checkout(), + self.helmDeletePr(serviceName, options, helmPath, deploymentName), ] + - (if mysqlDeleteOptions.enabled then [$.deleteDatabase(mysqlDeleteOptions)] else []), - services=(if mysqlDeleteOptions.enabled then { 'cloudsql-proxy': $.cloudsql_proxy_service(mysqlDeleteOptions.database) } else null), + (if mysqlDeleteOptions.enabled then [databases.deleteDatabase(mysqlDeleteOptions)] else []), + services=(if mysqlDeleteOptions.enabled then { 'cloudsql-proxy': services.cloudsql_proxy_service(mysqlDeleteOptions.database) } else null), ), helmDeletePRPipeline( @@ -194,10 +201,10 @@ helmPath='./helm/' + serviceName, deploymentName=serviceName + '-pr-${{ github.event.number }}', ):: - $.pipeline( + base.pipeline( 'close-pr', [ - $.helmDeletePRJob(serviceName, options, helmPath, deploymentName), + self.helmDeletePRJob(serviceName, options, helmPath, deploymentName), ], event={ pull_request: { @@ -213,8 +220,8 @@ deploymentName=serviceName + '-canary', ifClause=null, ):: - $.deployHelm( - $.clusters.prod, + self.deployHelm( + clusters.prod, deploymentName, { identifier: 'prod', @@ -234,17 +241,17 @@ options={}, helmPath='./helm/' + serviceName + '-canary', deploymentName=serviceName + '-canary', - image=$.default_job_image, + image=images.default_job_image, useCredentials=false, ):: - $.ghJob( + base.ghJob( 'deploy-canary', image=image, useCredentials=useCredentials, ifClause="${{ github.event.deployment.environment == 'canary' }}", steps=[ - $.checkout(), - $.helmDeployCanary(serviceName, options, helmPath, deploymentName), + misc.checkout(), + self.helmDeployCanary(serviceName, options, helmPath, deploymentName), ], ), @@ -255,8 +262,8 @@ deploymentName=serviceName + '-canary', ifClause=null, ):: - $.deployHelm( - $.clusters.prod, + self.deployHelm( + clusters.prod, deploymentName, { identifier: 'prod', @@ -277,14 +284,14 @@ helmPath='./helm/' + serviceName + '-canary', deploymentName=serviceName + '-canary', ):: - $.ghJob( + base.ghJob( 'kill-canary', ifClause="${{ github.event.deployment.environment == 'kill-canary' || github.event.deployment.environment == 'production' }}", - image=$.default_job_image, + image=images.default_job_image, useCredentials=false, steps=[ - $.checkout(), - $.helmKillCanary(serviceName, options, helmPath, deploymentName), + misc.checkout(), + self.helmKillCanary(serviceName, options, helmPath, deploymentName), ], ), } diff --git a/.github/jsonnet/images.jsonnet b/.github/jsonnet/images.jsonnet index fbb9b3d..f7c6492 100644 --- a/.github/jsonnet/images.jsonnet +++ b/.github/jsonnet/images.jsonnet @@ -1,17 +1,20 @@ { jsonnet_bin_image: 'eu.gcr.io/unicorn-985/docker-images_jsonnet:v1', helm_action_image: 'docker://europe-docker.pkg.dev/gynzy-test-project/public-images/helm-action:v2', - mysql_action_image: 'docker://europe-docker.pkg.dev/gynzy-test-project/public-images/mysql-action:v1', + mysql_action_image: 'docker://europe-docker.pkg.dev/gynzy-test-project/public-images/docker-images_mysql-cloner-action:v1', docker_action_image: 'docker://europe-docker.pkg.dev/gynzy-test-project/public-images/push-to-gcr-github-action:v1', - default_job_image: 'alpine:3.18.3', + default_job_image: 'alpine:3.20.0', default_mysql57_image: 'eu.gcr.io/unicorn-985/docker-images_mysql57_utf8mb4:v1', default_mysql8_image: 'eu.gcr.io/unicorn-985/docker-images_mysql8_utf8mb4:v1', default_cloudsql_image: 'eu.gcr.io/unicorn-985/docker-images_cloudsql-sidecar:v1', default_redis_image: 'redis:5.0.6', default_unicorns_image: 'node:18.15', default_pubsub_image: 'messagebird/gcloud-pubsub-emulator:latest', - default_backend_nest_image: 'node:18.15', + default_backend_nest_image: 'node:20.12.1', default_mongodb_image: 'eu.gcr.io/unicorn-985/docker-images_mongo6-replicated:v1', - mongo_action_image: 'docker://europe-docker.pkg.dev/gynzy-test-project/public-images/action-mongo-cloner:v1', - default_python_image: 'python:3.11.4', + mongo_job_image: 'europe-docker.pkg.dev/gynzy-test-project/public-images/docker-images_mongo-cloner-job:v1', + default_python_image: 'python:3.12.1', + default_pulumi_node_image: 'node:18', + job_poster_image: 'europe-docker.pkg.dev/gynzy-test-project/public-images/docker-images_job-poster:v1', + newrelic_deployment_marker_image: 'newrelic/deployment-marker-action@v2.5.0', } diff --git a/.github/jsonnet/index.jsonnet b/.github/jsonnet/index.jsonnet index c26c847..2a17154 100644 --- a/.github/jsonnet/index.jsonnet +++ b/.github/jsonnet/index.jsonnet @@ -1,5 +1,5 @@ (import 'base.jsonnet') + -(import 'clusters.jsonnet') + +{ clusters: import 'clusters.jsonnet' } + (import 'databases.jsonnet') + (import 'docker.jsonnet') + (import 'helm.jsonnet') + @@ -15,4 +15,6 @@ (import 'deployment.jsonnet') + (import 'notifications.jsonnet') + (import 'complete-workflows.jsonnet') + -{} +{ pnpm: import 'pnpm.jsonnet' } + +{ cache: import 'cache.jsonnet' } + +{ buckets: import 'buckets.jsonnet' } diff --git a/.github/jsonnet/misc.jsonnet b/.github/jsonnet/misc.jsonnet index 5d97734..93e2cf3 100644 --- a/.github/jsonnet/misc.jsonnet +++ b/.github/jsonnet/misc.jsonnet @@ -1,44 +1,57 @@ +local base = import 'base.jsonnet'; +local images = import 'images.jsonnet'; + { checkout(ifClause=null, fullClone=false, ref=null):: local with = (if fullClone then { 'fetch-depth': 0 } else {}) + (if ref != null then { ref: ref } else {}); - $.action( + base.action( 'Check out repository code', 'actions/checkout@v3', with=with, ifClause=ifClause - ), + ) + + base.step('git safe directory', "command -v git && git config --global --add safe.directory '*' || true"), lint(service):: - $.step('lint-' + service, - './node_modules/.bin/eslint "./packages/' + service + '/{app,lib,tests,config,addon}/**/*.js" --quiet'), + base.step('lint-' + service, + './node_modules/.bin/eslint "./packages/' + service + '/{app,lib,tests,config,addon}/**/*.js" --quiet'), lintAll():: - $.step('lint', 'yarn lint'), + base.step('lint', 'yarn lint'), verifyGoodFences():: - $.step('verify-good-fences', 'yarn run gf'), + base.step('verify-good-fences', 'yarn run gf'), improvedAudit():: - $.step('audit', 'yarn improved-audit'), + base.step('audit', 'yarn improved-audit'), + + verifyJsonnetWorkflow():: + base.pipeline( + 'misc', + [ + self.verifyJsonnet(fetch_upstream=false), + ], + event='pull_request', + ), verifyJsonnet(fetch_upstream=true, runsOn=null):: - $.ghJob( + base.ghJob( 'verify-jsonnet-gh-actions', runsOn=runsOn, - image=$.jsonnet_bin_image, + image=images.jsonnet_bin_image, steps=[ - $.checkout(ref='${{ github.event.pull_request.head.sha }}'), - $.step('remove-workflows', 'rm -f .github/workflows/*'), + self.checkout(ref='${{ github.event.pull_request.head.sha }}'), + base.step('remove-workflows', 'rm -f .github/workflows/*'), ] + ( - if fetch_upstream then [$.step('fetch latest lib-jsonnet', - ' rm -rf .github/jsonnet/;\n mkdir .github/jsonnet/;\n cd .github;\n curl https://files.gynzy.net/lib-jsonnet/v1/jsonnet-prod.tar.gz | tar xvzf -;\n ')] else [] + if fetch_upstream then [base.step('fetch latest lib-jsonnet', + ' rm -rf .github/jsonnet/;\n mkdir .github/jsonnet/;\n cd .github;\n curl https://files.gynzy.net/lib-jsonnet/v1/jsonnet-prod.tar.gz | tar xvzf -;\n ')] else [] ) + [ - $.step('generate-workflows', 'jsonnet -m .github/workflows/ -S .github.jsonnet;'), - $.step('git workaround', 'git config --global --add safe.directory $PWD'), - $.step('check-jsonnet-diff', 'git diff --exit-code'), - $.step( + base.step('generate-workflows', 'jsonnet -m .github/workflows/ -S .github.jsonnet;'), + base.step('git workaround', 'git config --global --add safe.directory $PWD'), + base.step('check-jsonnet-diff', 'git diff --exit-code'), + base.step( 'possible-causes-for-error', 'echo "Possible causes: \n' + '1. You updated jsonnet files, but did not regenerate the workflows. \n' + @@ -59,16 +72,16 @@ titleUpdateAction='prefix', otherOptions={}, ):: - $.pipeline( + base.pipeline( 'update-pr-description', event={ pull_request: { types: ['opened'] }, }, jobs=[ - $.ghJob( + base.ghJob( 'update-pr-description', steps=[ - $.action( + base.action( 'update-pr-description', 'gynzy/pr-update-action@v2', with={ @@ -98,7 +111,7 @@ '${{ secrets.' + secretName + ' }}', pollUrlForContent(url, expectedContent, name='verify-deploy', attempts='100', interval='2000', ifClause=null):: - $.action( + base.action( name, 'gynzy/wait-for-http-content@v1', with={ @@ -111,16 +124,16 @@ ), cleanupOldBranchesPipelineCron():: - $.pipeline( + base.pipeline( 'purge-old-branches', [ - $.ghJob( + base.ghJob( 'purge-old-branches', useCredentials=false, steps=[ - $.step('setup', 'apk add git bash'), - $.checkout(), - $.action( + base.step('setup', 'apk add git bash'), + self.checkout(), + base.action( 'Run delete-old-branches-action', 'beatlabs/delete-old-branches-action@6e94df089372a619c01ae2c2f666bf474f890911', with={ @@ -144,35 +157,6 @@ }, ), - codiumAIPRAgent():: - $.pipeline( - 'codium-ai', - [ - $.ghJob( - 'pr_agent_job', - useCredentials=false, - ifClause='${{ github.event.pull_request.draft == false }}', - steps=[ - $.action( - 'PR Agent action step', - 'gynzy/pr-agent@712f0ff0c37b71c676398f73c6ea0198eb9cdd03', - continueOnError=true, - env={ - OPENAI_KEY: $.secret('OPENAI_KEY'), - GITHUB_TOKEN: $.secret('GITHUB_TOKEN'), - }, - ), - ] - ), - ], - event={ - pull_request: { - types: ['opened', 'reopened', 'ready_for_review'], - }, - issue_comment: {}, - } - ), - // Test if the changed files match the given glob patterns. // Can test for multiple pattern groups, and sets multiple outputs. // @@ -188,7 +172,7 @@ // Requires the 'pull-requests': 'read' permission // // Example: - // $.testForChangedFiles({ + // misc.testForChangedFiles({ // 'app': ['packages/*/app/**/*', 'package.json'], // 'lib': ['packages/*/lib/**/*'], // }) @@ -203,8 +187,8 @@ // See https://github.com/dorny/paths-filter for more information. testForChangedFiles(changedFiles, headRef=null, baseRef=null):: [ - $.step('git safe directory', 'git config --global --add safe.directory $PWD'), - $.action( + base.step('git safe directory', 'git config --global --add safe.directory $PWD'), + base.action( 'check-for-changes', uses='dorny/paths-filter@v2', id='changes', @@ -219,6 +203,15 @@ ), ], + // Wait for the given jobs to finish. + // Exits successfully if all jobs are successful, otherwise exits with an error. + // + // Parameters: + // name: the name of the github job + // jobs: a list of job names to wait for + // + // Returns: + // a job that waits for the given jobs to finish awaitJob(name, jobs):: local dependingJobs = std.flatMap( function(job) @@ -227,18 +220,18 @@ jobs ); [ - $.ghJob( + base.ghJob( 'await-' + name, ifClause='${{ always() }}', needs=dependingJobs, useCredentials=false, steps=[ - $.step( + base.step( 'success', 'exit 0', ifClause="${{ contains(join(needs.*.result, ','), 'success') }}" ), - $.step( + base.step( 'failure', 'exit 1', ifClause="${{ contains(join(needs.*.result, ','), 'failure') }}" @@ -246,4 +239,30 @@ ], ), ], + + // Post a job to a kubernetes cluster + // + // Parameters: + // name: the name of the github job + // job_name: the name of the job to be posted + // cluster: the cluster to post the job to. This should be an object from the clusters module + // image: the image to use for the job + // environment: a map of environment variables to pass to the job + // command: the command to run in the job (optional) + postJob(name, job_name, cluster, image, environment, command=''):: + base.action( + name, + 'docker://' + images.job_poster_image, + env={ + JOB_NAME: job_name, + IMAGE: image, + COMMAND: command, + ENVIRONMENT: std.join(' ', std.objectFields(environment)), + GCE_JSON: cluster.secret, + GKE_PROJECT: cluster.project, + GKE_ZONE: cluster.zone, + GKE_CLUSTER: cluster.name, + NODESELECTOR_TYPE: cluster.jobNodeSelectorType, + } + environment, + ), } diff --git a/.github/jsonnet/mongo.jsonnet b/.github/jsonnet/mongo.jsonnet index 6e3a933..efd6668 100644 --- a/.github/jsonnet/mongo.jsonnet +++ b/.github/jsonnet/mongo.jsonnet @@ -1,65 +1,203 @@ +local base = import 'base.jsonnet'; +local clusters = import 'clusters.jsonnet'; +local images = import 'images.jsonnet'; +local misc = import 'misc.jsonnet'; + +local testProjectId = '5da5889579358e19bf4b16ea'; +local prodProjectId = '5dde7f71a6f239a82fa155f4'; + +// clusterId can be found in the collections-page url: +// https://cloud.mongodb.com/v2/#/metrics/replicaSet//explorer + +local testProjectSettings = { + projectId: testProjectId, + gke_cluster: clusters.test, + CIUsername: 'github-actions', + CIPassword: misc.secret('MONGO_CI_PASSWORD_TEST'), + lifecycle: 'test', + type: 'mongodb', +}; +local prodProjectSettings = { + projectId: prodProjectId, + gke_cluster: clusters.prod, + CIUsername: 'github-actions', + CIPassword: misc.secret('MONGO_CI_PASSWORD_PROD'), + lifecycle: 'prod', + type: 'mongodb', +}; + { - mongo_servers: { - test: { - type: 'mongo', + // List of available MongoDB clusters. + mongo_clusters: { + test: testProjectSettings { name: 'test', connectionString: 'test-pri.kl1s6.gcp.mongodb.net', - gke_cluster: 'test', - gke_project: 'gynzy-test-project', - gke_zone: 'europe-west4-b', - password_secret: 'mongodb-pass-test', - gce_json: $.secret('gce_new_test_json'), - lifecycle: 'test', - projectId: '5da5889579358e19bf4b16ea' // test + clusterId: '5db8186b79358e06a806d134', }, - prod: { - type: 'mongo', + prod: prodProjectSettings { name: 'prod', connectionString: 'prod-pri.silfd.gcp.mongodb.net', - projectId: '5dde7f71a6f239a82fa155f4', // prod + clusterId: '5dde8398cf09a237555dda74', + description: 'This cluster contains data for the scores service. It was the first mongoCluster created, and unfortunately named "prod".', }, - 'board-prod': { - type: 'mongo', + 'board-prod': prodProjectSettings { name: 'board-prod', connectionString: 'board-prod-pri.silfd.mongodb.net', - projectId: '5dde7f71a6f239a82fa155f4', // prod + clusterId: '61b0820f1ddfbc0366c41ddc', }, - 'adaptive-learning-prod': { - type: 'mongo', + 'adaptive-learning-prod': prodProjectSettings { name: 'adaptive-learning-prod', connectionString: 'adaptive-learning-prod-pri.silfd.mongodb.net', - projectId: '5dde7f71a6f239a82fa155f4', // prod + clusterId: '6239cfa3b065b60fe7af89bb', }, - 'accounts-prod': { - type: 'mongo', + 'accounts-prod': prodProjectSettings { name: 'accounts-prod', connectionString: 'accounts-prod-pri.silfd.mongodb.net', - projectId: '5dde7f71a6f239a82fa155f4', // prod + clusterId: '6322ef3b3f5f105b67ecb904', }, - 'interaction-prod': { - type: 'mongo', + 'interaction-prod': prodProjectSettings { name: 'interaction-prod', connectionString: 'interaction-prod-pri.silfd.mongodb.net', - projectId: '5dde7f71a6f239a82fa155f4', // prod + clusterId: '63d7bfaf42209e2ca71398ce', }, }, + // TODO: remove + mongo_servers: self.mongo_clusters, + + // Generate a deeplink to the Atlas UI for a given cluster and database. + // + // If the database is null, the link will point to the cluster overview. + // Otherwise, it will point to the database explorer. + // + // Parameters: + // cluster: The MongoDB cluster. One of the objects from the mongo_servers list. + // database: The name of the database (optional). + // + // Returns: + // string The deeplink to the Atlas UI. + atlasDeeplink(mongoCluster, database=null):: + if database == null || mongoCluster.clusterId == null then + 'https://cloud.mongodb.com/v2/' + mongoCluster.projectId + '#clusters/detail/' + mongoCluster.name + else + 'https://cloud.mongodb.com/v2/' + mongoCluster.projectId + '#/metrics/replicaSet/' + mongoCluster.clusterId + '/explorer/' + database, - copyMongoDatabase(mongoActionsOptions):: - assert std.length(std.findSubstr('_pr_', mongoActionsOptions.MONGO_DST)) > 0; // target db gets deleted. must contain _pr_ + // Copy a MongoDB database to a new database. + // It does this by posting a job that runs the mongo-action image with the clone task. + // + // Parameters: + // service: The name of the service. + // mongoCluster: The MongoDB cluster to connect to. One of the objects from the mongo_servers list. + // testDatabase: The name of the source database. + // prDatabase: The name of the PR database. + // + // Returns: + // The job definition. + copyMongoDatabase(service, mongoCluster, testDatabase, prDatabase):: + assert std.length(std.findSubstr('_pr_', prDatabase)) > 0; // target db gets deleted. must contain _pr_ + + misc.postJob( + name='copy-mongo-db', + job_name='mongo-copy-' + service + '-${{ github.run_number }}-${{ github.run_attempt }}', + cluster=mongoCluster.gke_cluster, + image=images.mongo_job_image, + environment={ + TASK: 'clone', + MONGO_SRC: testDatabase, + MONGO_DST: prDatabase, + MONGO_HOST: mongoCluster.connectionString, + MONGO_USER: mongoCluster.CIUsername, + MONGO_PASS: mongoCluster.CIPassword, + }, + ), + + // Delete a MongoDB PR database. + // It does this by posting a job that runs the mongo-action image with the delete task. + // + // Parameters: + // service: The name of the service. + // mongoCluster: The MongoDB cluster to connect to. One of the objects from the mongo_servers list. + // prDatabase: The name of the PR database. + // + // Returns: + // The job definition. + deleteMongoPrDatabase(service, mongoCluster, prDatabase):: + assert std.length(std.findSubstr('_pr_', prDatabase)) > 0; // target db gets deleted. must contain _pr_ + + misc.postJob( + name='delete-mongo-db', + job_name='mongo-delete-' + service + '-${{ github.run_number }}-${{ github.run_attempt }}', + cluster=mongoCluster.gke_cluster, + image=images.mongo_job_image, + environment={ + TASK: 'delete', + MONGO_DST: prDatabase, + MONGO_HOST: mongoCluster.connectionString, + MONGO_USER: mongoCluster.CIUsername, + MONGO_PASS: mongoCluster.CIPassword, + }, + ), - $.action( - 'copy-mongo-db', - $.mongo_action_image, - env=mongoActionsOptions { TASK: 'clone' } + // Sync the indexes of a MongoDB database with the current codebase. + // + // Parameters: + // service: The name of the service. + // image: The name of the Docker image to use. + // mongoCluster: The MongoDB cluster to connect to. One of the objects from the mongo_servers list. + // database: The name of the database. + // + // Returns: + // The job definition. + mongoSyncIndexes(service, image, mongoCluster, database):: + misc.postJob( + name='sync-mongo-indexes-' + mongoCluster.name + '-' + database, + job_name='mongo-sync-' + service + '-${{ github.run_number }}-${{ github.run_attempt }}', + cluster=mongoCluster.gke_cluster, + image=image, + environment={ + SERVICE: service, + MONGO_SYNC_INDEXES: 'true', + MONGO_DB: database, + MONGO_HOST: mongoCluster.connectionString, + MONGO_USER: mongoCluster.CIUsername, + MONGO_PASS: mongoCluster.CIPassword, + IS_ATLAS_MONGO: 'true', + }, + command='docker/mongo-sync-indexes.sh', ), - deleteMongoPrDatabase(mongoActionsOptions):: - assert std.length(std.findSubstr('_pr_', mongoActionsOptions.MONGO_DST)) > 0; // target db gets deleted. must contain _pr_ + // Generate a diff of the indexes of a MongoDB database and the currect codebase. + // The diff is posted as a comment on the pull request. + // + // Parameters: + // service: The name of the service. + // image: The name of the Docker image to use. + // mongoCluster: The MongoDB cluster to connect to. One of the objects from the mongo_servers list. + // database: The name of the database. + // + // Returns: + // The job definition. + mongoDiffIndexes(service, image, mongoCluster, database):: + local mongoDBLink = self.atlasDeeplink(mongoCluster, database); - $.action( - 'delete-mongo-db', - $.mongo_action_image, - env=mongoActionsOptions { TASK: 'delete' } + misc.postJob( + name='diff-mongo-indexes-' + mongoCluster.name + '-' + database, + job_name='mongo-diff-' + service + '-${{ github.run_number }}-${{ github.run_attempt }}', + cluster=mongoCluster.gke_cluster, + image=image, + environment={ + SERVICE: service, + MONGO_DIFF_INDEXES: 'true', + GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}', + GITHUB_PR: '${{ github.event.number }}', + MONGO_DB: database, + MONGO_HOST: mongoCluster.connectionString, + MONGO_USER: mongoCluster.CIUsername, + MONGO_PASS: mongoCluster.CIPassword, + MONGO_CLUSTER: mongoCluster.name, + MONGO_DEEPLINK: mongoDBLink, + IS_ATLAS_MONGO: 'true', + }, + command='docker/mongo-sync-indexes.sh', ), } diff --git a/.github/jsonnet/newrelic.jsonnet b/.github/jsonnet/newrelic.jsonnet index d01dccf..a0b2e48 100644 --- a/.github/jsonnet/newrelic.jsonnet +++ b/.github/jsonnet/newrelic.jsonnet @@ -1,20 +1,24 @@ +local base = import 'base.jsonnet'; +local images = import 'images.jsonnet'; +local misc = import 'misc.jsonnet'; +local yarn = import 'yarn.jsonnet'; + { postReleaseToNewRelicJob( apps, - ):: - $.ghJob( + base.ghJob( 'post-newrelic-release', - image=$.default_backend_nest_image, + image=images.default_backend_nest_image, useCredentials=false, ifClause="${{ github.event.deployment.environment == 'production' }}", steps=[ - $.checkoutAndYarn(ref='${{ github.sha }}'), - $.step( + yarn.checkoutAndYarn(ref='${{ github.sha }}'), + base.step( 'post-newrelic-release', 'node .github/scripts/newrelic.js', env={ - NEWRELIC_API_KEY: $.secret('NEWRELIC_API_KEY'), + NEWRELIC_API_KEY: misc.secret('NEWRELIC_API_KEY'), NEWRELIC_APPS: std.join( ' ', std.flatMap( function(app) diff --git a/.github/jsonnet/notifications.jsonnet b/.github/jsonnet/notifications.jsonnet index 6cb0e7d..fc1dfa5 100644 --- a/.github/jsonnet/notifications.jsonnet +++ b/.github/jsonnet/notifications.jsonnet @@ -1,6 +1,9 @@ +local base = import 'base.jsonnet'; +local images = import 'images.jsonnet'; + { notifiyDeployFailure(channel='#dev-deployments', name='notify-failure', environment='production'):: - $.action( + base.action( name, 'act10ns/slack@v2', with={ @@ -13,7 +16,7 @@ ), sendSlackMessage(channel='#dev-deployments', stepName='sendSlackMessage', message=null, ifClause=null):: - $.action( + base.action( stepName, 'act10ns/slack@v2', with={ @@ -24,4 +27,18 @@ }, ifClause=ifClause, ), + + // This action is used to create a deployment marker in New Relic. + // GUID is the entity guid of the application in New Relic. It can be found by All Entities > (select service) > Metadata > Entity GUID + newrelicCreateDeploymentMarker(stepName='newrelic-deployment', entityGuid):: + base.action( + stepName, + images.newrelic_deployment_marker_image, + with={ + apiKey: $.secret('NEWRELIC_API_KEY'), + guid: entityGuid, + commit: '${{ github.sha }}', + version: '${{ github.sha }}', + }, + ), } diff --git a/.github/jsonnet/pnpm.jsonnet b/.github/jsonnet/pnpm.jsonnet new file mode 100644 index 0000000..cf25f94 --- /dev/null +++ b/.github/jsonnet/pnpm.jsonnet @@ -0,0 +1,15 @@ +local base = import 'base.jsonnet'; + +{ + install(args=[], with={}):: + base.action( + 'Install application code', + 'pnpm/action-setup@v2', + with={ + version: '^8.14.0', + run_install: ||| + - args: %(args)s + ||| % { args: args }, + } + ), +} diff --git a/.github/jsonnet/pubsub.jsonnet b/.github/jsonnet/pubsub.jsonnet index 702be96..9a6b445 100644 --- a/.github/jsonnet/pubsub.jsonnet +++ b/.github/jsonnet/pubsub.jsonnet @@ -1,15 +1,19 @@ +local base = import 'base.jsonnet'; +local misc = import 'misc.jsonnet'; +local yarn = import 'yarn.jsonnet'; + { deletePrPubsubSubscribersJob(needs=null):: - $.ghJob( + base.ghJob( 'delete-pubsub-pr-subscribers', useCredentials=false, image='google/cloud-sdk:alpine', steps=[ - $.configureGoogleAuth($.secret('GCE_NEW_TEST_JSON')), - $.step('install jq', 'apk add jq'), - $.step('show auth', 'gcloud auth list'), - $.step('wait for pod termination', 'sleep 60'), - $.step( + yarn.configureGoogleAuth(misc.secret('GCE_NEW_TEST_JSON')), + base.step('install jq', 'apk add jq'), + base.step('show auth', 'gcloud auth list'), + base.step('wait for pod termination', 'sleep 60'), + base.step( 'delete subscriptions', "\n gcloud --project gynzy-test-project pubsub subscriptions list --format json | jq -r '.[].name' | grep -- '-pr-${{ github.event.number }}' | xargs -r gcloud --project gynzy-test-project pubsub subscriptions delete" ), ], diff --git a/.github/jsonnet/pulumi.jsonnet b/.github/jsonnet/pulumi.jsonnet index 035061e..76c1c05 100644 --- a/.github/jsonnet/pulumi.jsonnet +++ b/.github/jsonnet/pulumi.jsonnet @@ -1,36 +1,90 @@ +local base = import 'base.jsonnet'; +local images = import 'images.jsonnet'; +local misc = import 'misc.jsonnet'; +local notifications = import 'notifications.jsonnet'; +local yarn = import 'yarn.jsonnet'; + +local pulumiSetupSteps = + base.action( + 'auth', + uses='google-github-actions/auth@v2', + id='auth', + with={ + credentials_json: misc.secret('PULUMI_SERVICE_ACCOUNT'), + } + ) + + base.action('setup-gcloud', uses='google-github-actions/setup-gcloud@v2') + + base.action('pulumi-cli-setup', 'pulumi/actions@v5') + + base.action('jsonnet-setup', 'kobtea/setup-jsonnet-action@v1'); + { - local pulumiSetupSteps = [ - $.action( - 'auth', - uses='google-github-actions/auth@v1', - id='auth', + pulumiPreview( + stack, + pulumiDir=null, + stepName='pulumi-preview-' + stack, + environmentVariables={}, + ):: + base.action( + name=stepName, + uses='pulumi/actions@v5', with={ - credentials_json: $.secret('PULUMI_SERVICE_ACCOUNT'), - } + command: 'preview', + 'stack-name': stack, + 'work-dir': pulumiDir, + 'comment-on-pr': true, + 'github-token': '${{ secrets.GITHUB_TOKEN }}', + upsert: true, + refresh: true, + }, + env={ + PULUMI_CONFIG_PASSPHRASE: '${{ secrets.PULUMI_CONFIG_PASSPHRASE }}', + } + environmentVariables, ), - $.action('setup-gcloud', uses='google-github-actions/setup-gcloud@v0'), - $.action('pulumi-cli-setup', 'pulumi/actions@v4'), - ], - pulumiPreview( + pulumiDeploy( stack, pulumiDir=null, - ): $.action( - 'pulumi-preview-' + stack, - uses='pulumi/actions@v4', - with={ - command: 'preview', - 'stack-name': stack, - 'work-dir': pulumiDir, - 'comment-on-pr': true, - 'github-token': '${{ secrets.GITHUB_TOKEN }}', - upsert: true, - refresh: true, - }, - env={ - PULUMI_CONFIG_PASSPHRASE: '${{ secrets.PULUMI_CONFIG_PASSPHRASE }}', - } - ), + stepName='pulumi-deploy-' + stack, + environmentVariables={}, + ):: + base.action( + name=stepName, + uses='pulumi/actions@v5', + with={ + command: 'up', + 'stack-name': stack, + 'work-dir': pulumiDir, + upsert: true, + refresh: true, + }, + env={ + PULUMI_CONFIG_PASSPHRASE: '${{ secrets.PULUMI_CONFIG_PASSPHRASE }}', + } + environmentVariables, + ), + + pulumiDestroy( + stack, + pulumiDir=null, + stepName='pulumi-destroy-' + stack, + environmentVariables={}, + ):: + // pulumi destroy is a destructive operation, so we only want to run it on stacks that contain pr- + assert std.length(std.findSubstr('pr-', stack)) > 0; + + base.action( + name=stepName, + uses='pulumi/actions@v5', + with={ + command: 'destroy', + remove: true, + 'stack-name': stack, + 'work-dir': pulumiDir, + refresh: true, + }, + env={ + PULUMI_CONFIG_PASSPHRASE: '${{ secrets.PULUMI_CONFIG_PASSPHRASE }}', + } + environmentVariables, + ), pulumiPreviewJob( stack, @@ -38,116 +92,252 @@ yarnDir=null, gitCloneRef='${{ github.event.pull_request.head.sha }}', cacheName=null, - ): $.ghJob( - 'pulumi-preview-' + stack, - image='node:18', - useCredentials=false, - steps=[$.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir)] + - pulumiSetupSteps + - [$.pulumiPreview(stack, pulumiDir=pulumiDir)], - ), + image=images.default_pulumi_node_image, + yarnNpmSource=null, + environmentVariables={}, + additionalSetupSteps=[], + ):: + base.ghJob( + 'pulumi-preview-' + stack, + image=image, + useCredentials=false, + steps=[ + yarn.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir, source=yarnNpmSource), + pulumiSetupSteps, + additionalSetupSteps, + self.pulumiPreview(stack, pulumiDir=pulumiDir, environmentVariables=environmentVariables), + ], + ), pulumiPreviewTestJob( stack='test', pulumiDir=null, yarnDir=null, + yarnNpmSource=null, gitCloneRef='${{ github.event.pull_request.head.sha }}', cacheName=null, - ): $.pulumiPreviewJob(stack, pulumiDir=pulumiDir, yarnDir=yarnDir, gitCloneRef=gitCloneRef, cacheName=cacheName), + image=images.default_pulumi_node_image, + environmentVariables={}, + additionalSetupSteps=[], + ):: + self.pulumiPreviewJob( + stack, + pulumiDir=pulumiDir, + yarnDir=yarnDir, + yarnNpmSource=yarnNpmSource, + gitCloneRef=gitCloneRef, + cacheName=cacheName, + image=image, + environmentVariables=environmentVariables, + additionalSetupSteps=additionalSetupSteps, + ), pulumiPreviewProdJob( stack='prod', pulumiDir=null, yarnDir=null, + yarnNpmSource=null, gitCloneRef='${{ github.event.pull_request.head.sha }}', cacheName=null, - ): $.pulumiPreviewJob(stack, pulumiDir=pulumiDir, yarnDir=yarnDir, gitCloneRef=gitCloneRef, cacheName=cacheName), + image=images.default_pulumi_node_image, + environmentVariables={}, + additionalSetupSteps=[], + ):: + self.pulumiPreviewJob( + stack, + pulumiDir=pulumiDir, + yarnDir=yarnDir, + yarnNpmSource=yarnNpmSource, + gitCloneRef=gitCloneRef, + cacheName=cacheName, + image=image, + environmentVariables=environmentVariables, + additionalSetupSteps=additionalSetupSteps, + ), pulumiPreviewTestAndProdJob( pulumiDir=null, yarnDir=null, + yarnNpmSource=null, gitCloneRef='${{ github.event.pull_request.head.sha }}', cacheName=null, - ): $.ghJob( - 'pulumi-preview', - image='node:18', - useCredentials=false, - steps=[$.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir)] + - pulumiSetupSteps + - [ - $.pulumiPreview('test', pulumiDir=pulumiDir), - $.pulumiPreview('prod', pulumiDir=pulumiDir), - ], - ), + image=images.default_pulumi_node_image, + productionStack='prod', + testStack='test', + environmentVariables={}, + additionalSetupSteps=[], + ):: + base.ghJob( + 'pulumi-preview', + image=image, + useCredentials=false, + steps=[ + yarn.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir, source=yarnNpmSource), + pulumiSetupSteps, + additionalSetupSteps, + self.pulumiPreview(testStack, pulumiDir=pulumiDir, environmentVariables=environmentVariables), + self.pulumiPreview(productionStack, pulumiDir=pulumiDir, environmentVariables=environmentVariables), + ], + ), pulumiDeployJob( stack, pulumiDir=null, yarnDir=null, + yarnNpmSource=null, gitCloneRef='${{ github.sha }}', cacheName=null, ifClause=null, - ): $.ghJob( - 'pulumi-deploy-' + stack, - ifClause=ifClause, - image='node:18', - useCredentials=false, - steps=[$.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir)] + - pulumiSetupSteps + - [ - $.action( - 'pulumi-deploy-' + stack, - uses='pulumi/actions@v4', - with={ - command: 'up', - 'stack-name': stack, - 'work-dir': pulumiDir, - upsert: true, - refresh: true, - }, - env={ - PULUMI_CONFIG_PASSPHRASE: '${{ secrets.PULUMI_CONFIG_PASSPHRASE }}', - } - ), - $.notifiyDeployFailure(), - ], - ), + image=images.default_pulumi_node_image, + jobName='pulumi-deploy-' + stack, + notifyOnFailure=true, + environmentVariables={}, + additionalSetupSteps=[], + ):: + base.ghJob( + name=jobName, + ifClause=ifClause, + image=image, + useCredentials=false, + steps=[ + yarn.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir, source=yarnNpmSource), + pulumiSetupSteps, + additionalSetupSteps, + self.pulumiDeploy(stack, pulumiDir=pulumiDir, stepName=jobName, environmentVariables=environmentVariables), + if notifyOnFailure then notifications.notifiyDeployFailure(environment=stack) else [], + ] + ), pulumiDeployTestJob( stack='test', pulumiDir=null, yarnDir=null, + yarnNpmSource=null, gitCloneRef='${{ github.sha }}', cacheName=null, + image=images.default_pulumi_node_image, ifClause="${{ github.event.deployment.environment == 'test' }}", - ): $.pulumiDeployJob(stack, pulumiDir=pulumiDir, yarnDir=yarnDir, gitCloneRef=gitCloneRef, cacheName=cacheName, ifClause=ifClause), + environmentVariables={}, + additionalSetupSteps=[], + ):: + self.pulumiDeployJob( + stack, + pulumiDir=pulumiDir, + yarnDir=yarnDir, + yarnNpmSource=yarnNpmSource, + gitCloneRef=gitCloneRef, + cacheName=cacheName, + ifClause=ifClause, + image=image, + environmentVariables=environmentVariables, + additionalSetupSteps=additionalSetupSteps, + ), pulumiDeployProdJob( stack='prod', pulumiDir=null, yarnDir=null, + yarnNpmSource=null, gitCloneRef='${{ github.sha }}', cacheName=null, + image=images.default_pulumi_node_image, ifClause="${{ github.event.deployment.environment == 'prod' || github.event.deployment.environment == 'production' }}", - ): $.pulumiDeployJob(stack, pulumiDir=pulumiDir, yarnDir=yarnDir, gitCloneRef=gitCloneRef, cacheName=cacheName, ifClause=ifClause), + environmentVariables={}, + additionalSetupSteps=[], + ):: + self.pulumiDeployJob( + stack, + pulumiDir=pulumiDir, + yarnDir=yarnDir, + yarnNpmSource=yarnNpmSource, + gitCloneRef=gitCloneRef, + cacheName=cacheName, + ifClause=ifClause, + image=image, + environmentVariables=environmentVariables, + additionalSetupSteps=additionalSetupSteps, + ), + + pulumiDestroyJob( + stack, + pulumiDir=null, + yarnDir=null, + yarnNpmSource=null, + gitCloneRef='${{ github.sha }}', + cacheName=null, + ifClause=null, + image=images.default_pulumi_node_image, + jobName='pulumi-destroy-' + stack, + notifyOnFailure=true, + environmentVariables={}, + additionalSetupSteps=[], + ):: + base.ghJob( + name=jobName, + ifClause=ifClause, + image=image, + useCredentials=false, + steps=[ + yarn.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir, source=yarnNpmSource), + pulumiSetupSteps, + additionalSetupSteps, + self.pulumiDestroy(stack, pulumiDir=pulumiDir, stepName=jobName, environmentVariables=environmentVariables), + if notifyOnFailure then notifications.notifiyDeployFailure(environment=stack) else [], + ], + ), + pulumiDefaultPipeline( pulumiDir='.', yarnDir=null, + yarnNpmSource=null, cacheName=null, deployTestWithProd=false, - ): - $.pipeline( + image=images.default_pulumi_node_image, + testStack='test', + productionStack='prod', + environmentVariables={}, + additionalSetupSteps=[], + ):: + base.pipeline( 'pulumi-preview', [ - $.pulumiPreviewTestAndProdJob(pulumiDir=pulumiDir, yarnDir=yarnDir, cacheName=cacheName), + self.pulumiPreviewTestAndProdJob( + pulumiDir=pulumiDir, + yarnDir=yarnDir, + yarnNpmSource=yarnNpmSource, + cacheName=cacheName, + image=image, + productionStack=productionStack, + testStack=testStack, + environmentVariables=environmentVariables, + additionalSetupSteps=additionalSetupSteps, + ), ], ) + - $.pipeline( + base.pipeline( 'pulumi-deploy', [ - $.pulumiDeployTestJob(pulumiDir=pulumiDir, yarnDir=yarnDir, cacheName=cacheName, ifClause=if deployTestWithProd then "${{ github.event.deployment.environment == 'test' || github.event.deployment.environment == 'prod' || github.event.deployment.environment == 'production' }}" else "${{ github.event.deployment.environment == 'test' }}"), - $.pulumiDeployProdJob(pulumiDir=pulumiDir, yarnDir=yarnDir, cacheName=cacheName), + self.pulumiDeployTestJob( + pulumiDir=pulumiDir, + yarnDir=yarnDir, + yarnNpmSource=yarnNpmSource, + cacheName=cacheName, + image=image, + environmentVariables=environmentVariables, + additionalSetupSteps=additionalSetupSteps, + ifClause=if deployTestWithProd then "${{ github.event.deployment.environment == 'test' || github.event.deployment.environment == 'prod' || github.event.deployment.environment == 'production' }}" else "${{ github.event.deployment.environment == 'test' }}" + ), + self.pulumiDeployProdJob( + pulumiDir=pulumiDir, + yarnDir=yarnDir, + yarnNpmSource=yarnNpmSource, + cacheName=cacheName, + image=image, + stack=productionStack, + environmentVariables=environmentVariables, + additionalSetupSteps=additionalSetupSteps, + ), ], event='deployment', ), diff --git a/.github/jsonnet/ruby.jsonnet b/.github/jsonnet/ruby.jsonnet index bafda85..e0cef8d 100644 --- a/.github/jsonnet/ruby.jsonnet +++ b/.github/jsonnet/ruby.jsonnet @@ -1,3 +1,11 @@ +local base = import 'base.jsonnet'; +local database = import 'databases.jsonnet'; +local docker = import 'docker.jsonnet'; +local helm = import 'helm.jsonnet'; +local misc = import 'misc.jsonnet'; +local notifications = import 'notifications.jsonnet'; +local services = import 'services.jsonnet'; + { rubyDeployPRPipeline( serviceName, @@ -17,7 +25,7 @@ database_name_source: serviceName, database_host: 'cloudsql-proxy', database_username: serviceName, - database_password: $.secret('database_password_test'), + database_password: misc.secret('database_password_test'), } + mysqlCloneOptions; local migrateOptionsWithDefaults = { @@ -25,34 +33,35 @@ RAILS_ENV: 'production', RAILS_DB_HOST: 'cloudsql-proxy', RAILS_DB_NAME: serviceName + '_pr_${{ github.event.number }}', - RAILS_DB_PASSWORD: $.secret('database_password_test'), + RAILS_DB_PASSWORD: misc.secret('database_password_test'), RAILS_DB_USER: serviceName, + SECRET_KEY_BASE: misc.secret('rails_secret_test'), } + migrateOptions; - $.pipeline( + base.pipeline( 'deploy-pr', [ - $.ghJob( + base.ghJob( 'deploy-pr', image=rubyImageName, steps=[ - $.checkout(ref='${{ github.event.pull_request.head.sha }}'), - $.setVerionFile(), + misc.checkout(ref='${{ github.event.pull_request.head.sha }}'), + self.setVerionFile(), ] + - (if mysqlCloneOptionsWithDefaults.enabled then [$.copyDatabase(mysqlCloneOptionsWithDefaults)] else []) + - (if migrateOptionsWithDefaults.enabled then $.rubyMigrate(migrateOptionsWithDefaults) else []) + + (if mysqlCloneOptionsWithDefaults.enabled then [database.copyDatabase(mysqlCloneOptionsWithDefaults)] else []) + + (if migrateOptionsWithDefaults.enabled then self.rubyMigrate(migrateOptionsWithDefaults) else []) + [ - $.buildDocker( + docker.buildDocker( dockerImageName, env={ - BUNDLE_GITHUB__COM: $.secret('BUNDLE_GITHUB__COM'), + BUNDLE_GITHUB__COM: misc.secret('BUNDLE_GITHUB__COM'), }, - build_args='BUNDLE_GITHUB__COM=' + $.secret('BUNDLE_GITHUB__COM'), + build_args='BUNDLE_GITHUB__COM=' + misc.secret('BUNDLE_GITHUB__COM'), ), - $.helmDeployPR(serviceName, helmDeployOptions), + helm.helmDeployPR(serviceName, helmDeployOptions), ], services={} + - (if mysqlCloneOptionsWithDefaults.enabled then { 'cloudsql-proxy': $.cloudsql_proxy_service(mysqlCloneOptionsWithDefaults.database) } else {}) + (if mysqlCloneOptionsWithDefaults.enabled then { 'cloudsql-proxy': services.cloudsql_proxy_service(mysqlCloneOptionsWithDefaults.database) } else {}) ), ], event='pull_request', @@ -60,18 +69,20 @@ rubyMigrate(migrateOptions):: local env = { - BUNDLE_GITHUB__COM: $.secret('BUNDLE_GITHUB__COM'), + BUNDLE_GITHUB__COM: misc.secret('BUNDLE_GITHUB__COM'), SSO_PUBLIC_KEY: '', RAILS_ENV: 'production', RAILS_DB_HOST: migrateOptions.RAILS_DB_HOST, RAILS_DB_NAME: migrateOptions.RAILS_DB_NAME, RAILS_DB_PASSWORD: migrateOptions.RAILS_DB_PASSWORD, RAILS_DB_USER: migrateOptions.RAILS_DB_USER, + SECRET_KEY_BASE: migrateOptions.SECRET_KEY_BASE, }; [ - $.step('bundle install', 'bundle install', env={ BUNDLE_GITHUB__COM: $.secret('BUNDLE_GITHUB__COM') }), - $.step('migrate-db', 'rails db:migrate;', env=env), + base.step('bundle install', 'bundle install', env={ BUNDLE_GITHUB__COM: misc.secret('BUNDLE_GITHUB__COM') }), + base.step('migrate-db', 'rails db:migrate;', env=env), + base.step('seed-db', 'rails db:seed;', env=env), ] , @@ -80,24 +91,24 @@ enableDatabase=false, generateCommands=null, extra_env={}, - services={ db: $.mysql57service(database='ci', password='ci', root_password='1234test', username='ci') }, + services={ db: services.mysql57service(database='ci', password='ci', root_password='1234test', username='ci') }, rubyImageName=null, ):: assert rubyImageName != null; - $.ghJob( + base.ghJob( 'apidocs', image=rubyImageName, ifClause="${{ github.event.deployment.environment == 'production' }}", steps=[ - $.checkout(), - $.step( + misc.checkout(), + base.step( 'generate', (if generateCommands != null then generateCommands else ' bundle config --delete without;\n bundle install;\n bundle exec rails db:test:prepare;\n bundle exec rails docs:generate;\n '), env={ RAILS_ENV: 'test', - GOOGLE_PRIVATE_KEY: $.secret('GOOGLE_PRIVATE_KEY'), - BUNDLE_GITHUB__COM: $.secret('BUNDLE_GITHUB__COM'), + GOOGLE_PRIVATE_KEY: misc.secret('GOOGLE_PRIVATE_KEY'), + BUNDLE_GITHUB__COM: misc.secret('BUNDLE_GITHUB__COM'), } + (if enableDatabase then { @@ -107,22 +118,22 @@ RAILS_DB_USER: 'ci', } else {}) + extra_env ), - $.action( + base.action( 'setup auth', - 'google-github-actions/auth@v1', + 'google-github-actions/auth@v2', with={ - credentials_json: $.secret('GCE_JSON'), + credentials_json: misc.secret('GCE_JSON'), }, id='auth', ), - $.action('setup-gcloud', 'google-github-actions/setup-gcloud@v0'), - $.step('deploy-api-docs', 'gsutil -m cp -r doc/api/** gs://apidocs.gynzy.com/' + serviceName + '/'), + base.action('setup-gcloud', 'google-github-actions/setup-gcloud@v2'), + base.step('deploy-api-docs', 'gsutil -m cp -r doc/api/** gs://apidocs.gynzy.com/' + serviceName + '/'), ], services=(if enableDatabase then services else null), ), setVerionFile(version='${{ github.event.pull_request.head.sha }}', file='VERSION'):: - $.step( + base.step( 'set-version', 'echo "' + version + '" > ' + file + ';\n echo "Generated version number:";\n cat ' + file + ';\n ' ), @@ -139,13 +150,13 @@ database_name_target: serviceName + '_pr_${{ github.event.number }}', database_host: 'cloudsql-proxy', database_username: serviceName, - database_password: $.secret('database_password_test'), + database_password: misc.secret('database_password_test'), } + mysqlDeleteOptions; - $.pipeline( + base.pipeline( 'close-pr', [ - $.helmDeletePRJob(serviceName, options, helmPath, deploymentName, mysqlDeleteOptionsWithDefaults), + helm.helmDeletePRJob(serviceName, options, helmPath, deploymentName, mysqlDeleteOptionsWithDefaults), ], event={ pull_request: { @@ -169,21 +180,22 @@ RAILS_ENV: 'production', RAILS_DB_HOST: 'cloudsql-proxy', RAILS_DB_NAME: serviceName, - RAILS_DB_PASSWORD: $.secret('database_password_test'), + RAILS_DB_PASSWORD: misc.secret('database_password_test'), RAILS_DB_USER: serviceName, + SECRET_KEY_BASE: misc.secret('rails_secret_test'), } + migrateOptions; - $.ghJob( + base.ghJob( 'deploy-test', ifClause="${{ github.event.deployment.environment == 'test' }}", image=image, useCredentials=useCredentials, steps= - [$.checkout()] + - (if migrateOptionsWithDefaults.enabled then $.rubyMigrate(migrateOptionsWithDefaults) else []) + - [$.helmDeployTest(serviceName, options, helmPath, deploymentName)], + [misc.checkout()] + + (if migrateOptionsWithDefaults.enabled then self.rubyMigrate(migrateOptionsWithDefaults) else []) + + [helm.helmDeployTest(serviceName, options, helmPath, deploymentName)], services={} + - (if migrateOptionsWithDefaults.enabled then { 'cloudsql-proxy': $.cloudsql_proxy_service(migrateOptionsWithDefaults.database) } else {}) + (if migrateOptionsWithDefaults.enabled then { 'cloudsql-proxy': services.cloudsql_proxy_service(migrateOptionsWithDefaults.database) } else {}) ), rubyDeployProdJob( @@ -201,21 +213,20 @@ RAILS_ENV: 'production', RAILS_DB_HOST: 'cloudsql-proxy', RAILS_DB_NAME: serviceName, - RAILS_DB_PASSWORD: $.secret('database_password_production'), + RAILS_DB_PASSWORD: misc.secret('database_password_production'), RAILS_DB_USER: serviceName, + SECRET_KEY_BASE: misc.secret('rails_secret_production'), } + migrateOptions; - $.ghJob( + base.ghJob( 'deploy-prod', ifClause="${{ github.event.deployment.environment == 'production' }}", image=image, useCredentials=useCredentials, - steps=[$.checkout()] + - (if migrateOptionsWithDefaults.enabled then $.rubyMigrate(migrateOptionsWithDefaults) else []) + - [$.helmDeployProd(serviceName, options, helmPath, deploymentName)] + [$.notifiyDeployFailure()], + steps=[misc.checkout()] + + (if migrateOptionsWithDefaults.enabled then self.rubyMigrate(migrateOptionsWithDefaults) else []) + + [helm.helmDeployProd(serviceName, options, helmPath, deploymentName)] + [notifications.notifiyDeployFailure()], services={} + - (if migrateOptionsWithDefaults.enabled then { 'cloudsql-proxy': $.cloudsql_proxy_service(migrateOptionsWithDefaults.database) } else {}) + (if migrateOptionsWithDefaults.enabled then { 'cloudsql-proxy': services.cloudsql_proxy_service(migrateOptionsWithDefaults.database) } else {}) ), - - } diff --git a/.github/jsonnet/services.jsonnet b/.github/jsonnet/services.jsonnet index 35fbb63..f1fd3ae 100644 --- a/.github/jsonnet/services.jsonnet +++ b/.github/jsonnet/services.jsonnet @@ -1,10 +1,13 @@ +local images = import 'images.jsonnet'; +local misc = import 'misc.jsonnet'; + { mysql57service(database=null, password=null, root_password=null, username=null, port='3306'):: { - image: $.default_mysql57_image, + image: images.default_mysql57_image, credentials: { username: '_json_key', - password: $.secret('docker_gcr_io'), + password: misc.secret('docker_gcr_io'), }, env: { MYSQL_DATABASE: database, @@ -19,10 +22,10 @@ mysql8service(database=null, password=null, root_password=null, username=null, port='3306'):: { - image: $.default_mysql8_image, + image: images.default_mysql8_image, credentials: { username: '_json_key', - password: $.secret('docker_gcr_io'), + password: misc.secret('docker_gcr_io'), }, env: { MYSQL_DATABASE: database, @@ -37,27 +40,27 @@ cloudsql_proxy_service(database):: { - image: $.default_cloudsql_image, + image: images.default_cloudsql_image, credentials: { username: '_json_key', - password: $.secret('docker_gcr_io'), + password: misc.secret('docker_gcr_io'), }, env: { GOOGLE_PROJECT: database.project, CLOUDSQL_ZONE: database.region, CLOUDSQL_INSTANCE: database.server, - SERVICE_JSON: $.secret('GCE_JSON'), + SERVICE_JSON: misc.secret('GCE_JSON'), }, ports: ['3306:3306'], }, redis_service():: { - image: $.default_redis_image, + image: images.default_redis_image, ports: ['6379:6379'], }, pubsub_service():: { - image: $.default_pubsub_image, + image: images.default_pubsub_image, ports: ['8681:8681'], }, @@ -68,11 +71,11 @@ password='therootpass', ):: { [name]: { - image: $.default_mongodb_image, + image: images.default_mongodb_image, ports: ['27017:27017'], credentials: { username: '_json_key', - password: $.secret('docker_gcr_io'), + password: misc.secret('docker_gcr_io'), }, env: { MONGO_INITDB_ROOT_USERNAME: username, diff --git a/.github/jsonnet/yarn.jsonnet b/.github/jsonnet/yarn.jsonnet index 752cca5..a688544 100644 --- a/.github/jsonnet/yarn.jsonnet +++ b/.github/jsonnet/yarn.jsonnet @@ -1,16 +1,21 @@ +local base = import 'base.jsonnet'; +local cache = import 'cache.jsonnet'; +local images = import 'images.jsonnet'; +local misc = import 'misc.jsonnet'; + { yarn(ifClause=null, prod=false, workingDirectory=null):: - $.step( + base.step( 'yarn' + (if prod then '-prod' else ''), run='yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline' + (if prod then ' --prod' else '') + ' || yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline' + (if prod then ' --prod' else ''), ifClause=ifClause, workingDirectory=workingDirectory ), - setNpmToken(ifClause=null, workingDirectory=null):: $.setGynzyNpmToken(ifClause=ifClause, workingDirectory=workingDirectory), + setNpmToken(ifClause=null, workingDirectory=null):: self.setGynzyNpmToken(ifClause=ifClause, workingDirectory=workingDirectory), setGynzyNpmToken(ifClause=null, workingDirectory=null):: - $.step( + base.step( 'set gynzy npm_token', run= ||| @@ -21,14 +26,14 @@ EOF |||, env={ - NPM_TOKEN: $.secret('npm_token'), + NPM_TOKEN: misc.secret('npm_token'), }, ifClause=ifClause, workingDirectory=workingDirectory, ), setGithubNpmToken(ifClause=null, workingDirectory=null):: - $.step( + base.step( 'set github npm_token', run= ||| @@ -39,83 +44,53 @@ EOF |||, env={ - NODE_AUTH_TOKEN: $.secret('GITHUB_TOKEN'), + NODE_AUTH_TOKEN: misc.secret('GITHUB_TOKEN'), }, ifClause=ifClause, workingDirectory=workingDirectory, ), - checkoutAndYarn(cacheName=null, ifClause=null, fullClone=false, ref=null, prod=false, workingDirectory=null):: - $.checkout(ifClause=ifClause, fullClone=fullClone, ref=ref) + - $.setGynzyNpmToken(ifClause=ifClause, workingDirectory=workingDirectory) + - (if cacheName == null then [] else $.fetchYarnCache(cacheName, ifClause=ifClause, workingDirectory=workingDirectory)) + - $.yarn(ifClause=ifClause, prod=prod, workingDirectory=workingDirectory), - - fetchYarnCache(cacheName, ifClause=null, workingDirectory=null):: $.step( - 'download yarn cache', - run= - ||| - set +e; - echo "Downloading yarn cache && node_modules" - wget -q -O - "https://storage.googleapis.com/files-gynzy-com-test/yarn-cache/${CACHE_NAME}.tar.gz" | tar xfz - - if [ $? -ne 0 ]; then - # download failed. cleanup node_modules because it can contain a yarn integrity file but not have all the data as specified - echo "Cache download failed, cleanup up caches and run yarn without cache" - find . -type d -name 'node_modules' | xargs rm -rf - rm -rf .yarncache - echo "Cleanup complete" - else - echo "Finished downloading yarn cache && node_modules" - fi - |||, - ifClause=ifClause, - workingDirectory=workingDirectory, - env={ - CACHE_NAME: cacheName, - }, - ), + checkoutAndYarn(cacheName=null, ifClause=null, fullClone=false, ref=null, prod=false, workingDirectory=null, source='gynzy'):: + misc.checkout(ifClause=ifClause, fullClone=fullClone, ref=ref) + + (if source == 'gynzy' then self.setGynzyNpmToken(ifClause=ifClause, workingDirectory=workingDirectory) else []) + + (if source == 'github' then self.setGithubNpmToken(ifClause=ifClause, workingDirectory=workingDirectory) else []) + + (if cacheName == null then [] else self.fetchYarnCache(cacheName, ifClause=ifClause, workingDirectory=workingDirectory)) + + self.yarn(ifClause=ifClause, prod=prod, workingDirectory=workingDirectory), + + fetchYarnCache(cacheName, ifClause=null, workingDirectory=null):: + cache.fetchCache( + cacheName=cacheName, + folders=['.yarncache'], + additionalCleanupCommands=["find . -type d -name 'node_modules' | xargs rm -rf"], + ifClause=ifClause, + workingDirectory=workingDirectory + ), updateYarnCachePipeline(cacheName, appsDir='packages', image=null, useCredentials=null):: - $.pipeline( + base.pipeline( 'update-yarn-cache', [ - $.ghJob( + base.ghJob( 'update-yarn-cache', image=image, useCredentials=useCredentials, ifClause="${{ github.event.deployment.environment == 'production' || github.event.deployment.environment == 'prod' }}", steps=[ - $.checkout() + - $.setGynzyNpmToken() + - $.yarn(), - $.action( + misc.checkout() + + self.setGynzyNpmToken() + + self.yarn(), + base.action( 'setup auth', - 'google-github-actions/auth@v1', + 'google-github-actions/auth@v2', with={ - credentials_json: $.secret('SERVICE_JSON'), + credentials_json: misc.secret('SERVICE_JSON'), }, id='auth', ), - $.action('setup-gcloud', 'google-github-actions/setup-gcloud@v0'), - $.step( - 'upload-yarn-cache', - ||| - set -e - - echo "Creating cache archive" - # v1 - ls "${APPS_DIR}/*/node_modules" -1 -d 2>/dev/null | xargs tar -czf "${CACHE_NAME}.tar.gz" .yarncache node_modules - - echo "Upload cache" - gsutil cp "${CACHE_NAME}.tar.gz" "gs://files-gynzy-com-test/yarn-cache/${CACHE_NAME}.tar.gz.tmp" - gsutil mv "gs://files-gynzy-com-test/yarn-cache/${CACHE_NAME}.tar.gz.tmp" "gs://files-gynzy-com-test/yarn-cache/${CACHE_NAME}.tar.gz" - - echo "Upload finished" - |||, - env={ - CACHE_NAME: cacheName, - APPS_DIR: appsDir, - } + base.action('setup-gcloud', 'google-github-actions/setup-gcloud@v2'), + cache.uploadCache( + cacheName=cacheName, + tarCommand='ls "' + appsDir + '/*/node_modules" -1 -d 2>/dev/null | xargs tar -c .yarncache node_modules', ), ], ), @@ -123,7 +98,7 @@ event='deployment', ), - configureGoogleAuth(secret):: $.step( + configureGoogleAuth(secret):: base.step( 'activate google service account', run= ||| @@ -136,7 +111,7 @@ ), yarnPublish(isPr=true, ifClause=null):: - $.step( + base.step( 'publish', ||| bash -c 'set -xeo pipefail; @@ -175,8 +150,8 @@ yarnPublishToRepositories(isPr, repositories, ifClause=null):: (std.flatMap(function(repository) - if repository == 'gynzy' then [$.setGynzyNpmToken(ifClause=ifClause), $.yarnPublish(isPr=isPr, ifClause=ifClause)] - else if repository == 'github' then [$.setGithubNpmToken(ifClause=ifClause), $.yarnPublish(isPr=isPr, ifClause=ifClause)] + if repository == 'gynzy' then [self.setGynzyNpmToken(ifClause=ifClause), self.yarnPublish(isPr=isPr, ifClause=ifClause)] + else if repository == 'github' then [self.setGithubNpmToken(ifClause=ifClause), self.yarnPublish(isPr=isPr, ifClause=ifClause)] else error 'Unknown repository type given.', repositories)), @@ -185,30 +160,30 @@ image='node:18', useCredentials=false, gitCloneRef='${{ github.event.pull_request.head.sha }}', - buildSteps=[$.step('build', 'yarn build')], + buildSteps=[base.step('build', 'yarn build')], checkVersionBump=true, repositories=['gynzy'], onChangedFiles=false, changedFilesHeadRef=null, changedFilesBaseRef=null, runsOn=null, - ): + ):: local ifClause = (if onChangedFiles != false then "steps.changes.outputs.package == 'true'" else null); - $.ghJob( + base.ghJob( 'yarn-publish-preview', runsOn=runsOn, image='node:18', useCredentials=false, steps= - [$.checkoutAndYarn(ref=gitCloneRef, fullClone=false)] + - (if onChangedFiles != false then $.testForChangedFiles({ package: onChangedFiles }, headRef=changedFilesHeadRef, baseRef=changedFilesBaseRef) else []) + + [self.checkoutAndYarn(ref=gitCloneRef, fullClone=false)] + + (if onChangedFiles != false then misc.testForChangedFiles({ package: onChangedFiles }, headRef=changedFilesHeadRef, baseRef=changedFilesBaseRef) else []) + (if checkVersionBump then [ - $.action('check-version-bump', uses='del-systems/check-if-version-bumped@v1', with={ + base.action('check-version-bump', uses='del-systems/check-if-version-bumped@v1', with={ token: '${{ github.token }}', }, ifClause=ifClause), ] else []) + (if onChangedFiles != false then std.map(function(step) std.map(function(s) s { 'if': ifClause }, step), buildSteps) else buildSteps) + - $.yarnPublishToRepositories(isPr=true, repositories=repositories, ifClause=ifClause), + self.yarnPublishToRepositories(isPr=true, repositories=repositories, ifClause=ifClause), permissions={ packages: 'write', contents: 'read', 'pull-requests': 'read' }, ), @@ -216,25 +191,25 @@ image='node:18', useCredentials=false, gitCloneRef='${{ github.sha }}', - buildSteps=[$.step('build', 'yarn build')], + buildSteps=[base.step('build', 'yarn build')], repositories=['gynzy'], onChangedFiles=false, changedFilesHeadRef=null, changedFilesBaseRef=null, ifClause=null, runsOn=null, - ): + ):: local stepIfClause = (if onChangedFiles != false then "steps.changes.outputs.package == 'true'" else null); - $.ghJob( + base.ghJob( 'yarn-publish', image='node:18', runsOn=runsOn, useCredentials=false, steps= - [$.checkoutAndYarn(ref=gitCloneRef, fullClone=false)] + - (if onChangedFiles != false then $.testForChangedFiles({ package: onChangedFiles }, headRef=changedFilesHeadRef, baseRef=changedFilesBaseRef) else []) + + [misc.checkoutAndYarn(ref=gitCloneRef, fullClone=false)] + + (if onChangedFiles != false then misc.testForChangedFiles({ package: onChangedFiles }, headRef=changedFilesHeadRef, baseRef=changedFilesBaseRef) else []) + (if onChangedFiles != false then std.map(function(step) std.map(function(s) s { 'if': stepIfClause }, step), buildSteps) else buildSteps) + - $.yarnPublishToRepositories(isPr=false, repositories=repositories, ifClause=stepIfClause), + self.yarnPublishToRepositories(isPr=false, repositories=repositories, ifClause=stepIfClause), permissions={ packages: 'write', contents: 'read', 'pull-requests': 'read' }, ifClause=ifClause, ), diff --git a/.github/workflows/misc.yml b/.github/workflows/misc.yml deleted file mode 100644 index 174a45d..0000000 --- a/.github/workflows/misc.yml +++ /dev/null @@ -1,28 +0,0 @@ -"jobs": - "verify-jsonnet-gh-actions": - "container": - "credentials": - "password": "${{ secrets.docker_gcr_io }}" - "username": "_json_key" - "image": "eu.gcr.io/unicorn-985/docker-images_jsonnet:v1" - "runs-on": "ubuntu-latest" - "steps": - - "name": "Check out repository code" - "uses": "actions/checkout@v3" - "with": - "ref": "${{ github.event.pull_request.head.sha }}" - - "name": "remove-workflows" - "run": "rm -f .github/workflows/*" - - "name": "generate-workflows" - "run": "jsonnet -m .github/workflows/ -S .github.jsonnet;" - - "name": "git workaround" - "run": "git config --global --add safe.directory $PWD" - - "name": "check-jsonnet-diff" - "run": "git diff --exit-code" - - "if": "failure()" - "name": "possible-causes-for-error" - "run": "echo \"Possible causes: \n1. You updated jsonnet files, but did not regenerate the workflows. \nTo fix, run 'yarn github:generate' locally and commit the changes. If this helps, check if your pre-commit hooks work.\n2. You used the wrong jsonnet binary. In this case, the newlines at the end of the files differ.\nTo fix, install the go binary. On mac, run 'brew uninstall jsonnet && brew install jsonnet-go'\"" - "timeout-minutes": 30 -"name": "misc" -"on": -- "pull_request" \ No newline at end of file diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml deleted file mode 100644 index 5774dd6..0000000 --- a/.github/workflows/pr.yml +++ /dev/null @@ -1,94 +0,0 @@ -"jobs": - "test": - "runs-on": "ubuntu-latest" - "steps": - - "name": "Check out repository code" - "uses": "actions/checkout@v3" - "with": - "ref": "${{ github.event.pull_request.head.ref }}" - - "name": "setup node" - "uses": "actions/setup-node@v3" - "with": - "node-version": 18 - - "name": "Start MongoDB" - "uses": "supercharge/mongodb-github-action@v1" - "with": - "mongodb-replica-set": "rs0" - "mongodb-version": 5 - - "name": "yarn" - "run": "yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline || yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline" - - "name": "test" - "run": "yarn test" - "timeout-minutes": 30 - "yarn-publish-preview": - "container": - "image": "node:18" - "permissions": - "contents": "read" - "packages": "write" - "pull-requests": "read" - "runs-on": "ubuntu-latest" - "steps": - - "name": "Check out repository code" - "uses": "actions/checkout@v3" - "with": - "ref": "${{ github.event.pull_request.head.sha }}" - - "env": - "NPM_TOKEN": "${{ secrets.npm_token }}" - "name": "set gynzy npm_token" - "run": | - cat < .npmrc - registry=https://npm.gynzy.net/ - always-auth="true" - "//npm.gynzy.net/:_authToken"="${NPM_TOKEN}" - EOF - - "name": "yarn" - "run": "yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline || yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline" - - "name": "build" - "run": "yarn build" - - "env": - "NPM_TOKEN": "${{ secrets.npm_token }}" - "name": "set gynzy npm_token" - "run": | - cat < .npmrc - registry=https://npm.gynzy.net/ - always-auth="true" - "//npm.gynzy.net/:_authToken"="${NPM_TOKEN}" - EOF - - "env": - "PR_NUMBER": "${{ github.event.number }}" - "name": "publish" - "run": | - bash -c 'set -xeo pipefail; - - cp package.json package.json.bak; - - VERSION=$(yarn version --non-interactive 2>/dev/null | grep "Current version" | grep -o -P '[0-9a-zA-Z_.-]+$' ); - if [[ ! -z "${PR_NUMBER}" ]]; then - echo "Setting tag/version for pr build."; - TAG=pr-$PR_NUMBER; - PUBLISHVERSION="$VERSION-pr$PR_NUMBER.$GITHUB_RUN_NUMBER"; - elif [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then - if [[ "${GITHUB_REF_NAME}" != "${VERSION}" ]]; then - echo "Tag version does not match package version. They should match. Exiting"; - exit 1; - fi - echo "Setting tag/version for release/tag build."; - PUBLISHVERSION=$VERSION; - TAG="latest"; - elif [[ "${GITHUB_REF_TYPE}" == "branch" && ( "${GITHUB_REF_NAME}" == "main" || "${GITHUB_REF_NAME}" == "master" ) ]] || [[ "${GITHUB_EVENT_NAME}" == "deployment" ]]; then - echo "Setting tag/version for release/tag build."; - PUBLISHVERSION=$VERSION; - TAG="latest"; - else - exit 1 - fi - - yarn publish --non-interactive --no-git-tag-version --tag "$TAG" --new-version "$PUBLISHVERSION"; - - mv package.json.bak package.json; - '; - "timeout-minutes": 30 -"name": "pr" -"on": -- "pull_request" \ No newline at end of file diff --git a/.github/workflows/publish-prod.yml b/.github/workflows/publish-prod.yml deleted file mode 100644 index d31f5bb..0000000 --- a/.github/workflows/publish-prod.yml +++ /dev/null @@ -1,74 +0,0 @@ -"jobs": - "yarn-publish": - "container": - "image": "node:18" - "permissions": - "contents": "read" - "packages": "write" - "pull-requests": "read" - "runs-on": "ubuntu-latest" - "steps": - - "name": "Check out repository code" - "uses": "actions/checkout@v3" - "with": - "ref": "${{ github.sha }}" - - "env": - "NPM_TOKEN": "${{ secrets.npm_token }}" - "name": "set gynzy npm_token" - "run": | - cat < .npmrc - registry=https://npm.gynzy.net/ - always-auth="true" - "//npm.gynzy.net/:_authToken"="${NPM_TOKEN}" - EOF - - "name": "yarn" - "run": "yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline || yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline" - - "name": "build" - "run": "yarn build" - - "env": - "NPM_TOKEN": "${{ secrets.npm_token }}" - "name": "set gynzy npm_token" - "run": | - cat < .npmrc - registry=https://npm.gynzy.net/ - always-auth="true" - "//npm.gynzy.net/:_authToken"="${NPM_TOKEN}" - EOF - - "env": {} - "name": "publish" - "run": | - bash -c 'set -xeo pipefail; - - cp package.json package.json.bak; - - VERSION=$(yarn version --non-interactive 2>/dev/null | grep "Current version" | grep -o -P '[0-9a-zA-Z_.-]+$' ); - if [[ ! -z "${PR_NUMBER}" ]]; then - echo "Setting tag/version for pr build."; - TAG=pr-$PR_NUMBER; - PUBLISHVERSION="$VERSION-pr$PR_NUMBER.$GITHUB_RUN_NUMBER"; - elif [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then - if [[ "${GITHUB_REF_NAME}" != "${VERSION}" ]]; then - echo "Tag version does not match package version. They should match. Exiting"; - exit 1; - fi - echo "Setting tag/version for release/tag build."; - PUBLISHVERSION=$VERSION; - TAG="latest"; - elif [[ "${GITHUB_REF_TYPE}" == "branch" && ( "${GITHUB_REF_NAME}" == "main" || "${GITHUB_REF_NAME}" == "master" ) ]] || [[ "${GITHUB_EVENT_NAME}" == "deployment" ]]; then - echo "Setting tag/version for release/tag build."; - PUBLISHVERSION=$VERSION; - TAG="latest"; - else - exit 1 - fi - - yarn publish --non-interactive --no-git-tag-version --tag "$TAG" --new-version "$PUBLISHVERSION"; - - mv package.json.bak package.json; - '; - "timeout-minutes": 30 -"name": "publish-prod" -"on": - "push": - "branches": - - "main" \ No newline at end of file From 46c4a93f8918109ff48097dfea016dcea43cd3ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=ABl=20Luijmes?= Date: Tue, 18 Jun 2024 17:05:18 +0200 Subject: [PATCH 09/10] chore: trigger ci From 51bf561b71cac3393d93aeb0d7f3cb82268edf89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=ABl=20Luijmes?= Date: Tue, 18 Jun 2024 17:07:10 +0200 Subject: [PATCH 10/10] Revert "chore: update jsonnet" This reverts commit 51fa660268f45d6a83d984990d304fea25547ad2. --- .github/jsonnet/GIT_VERSION | 2 +- .github/jsonnet/README.md | 2 +- .github/jsonnet/base.jsonnet | 14 +- .github/jsonnet/buckets.jsonnet | 152 --------- .github/jsonnet/cache.jsonnet | 128 -------- .github/jsonnet/clusters.jsonnet | 41 ++- .github/jsonnet/complete-workflows.jsonnet | 18 +- .github/jsonnet/databases.jsonnet | 18 +- .github/jsonnet/deployment.jsonnet | 159 ++++------ .github/jsonnet/docker.jsonnet | 12 +- .github/jsonnet/helm.jsonnet | 91 +++--- .github/jsonnet/images.jsonnet | 13 +- .github/jsonnet/index.jsonnet | 6 +- .github/jsonnet/misc.jsonnet | 141 ++++----- .github/jsonnet/mongo.jsonnet | 212 +++---------- .github/jsonnet/newrelic.jsonnet | 16 +- .github/jsonnet/notifications.jsonnet | 21 +- .github/jsonnet/pnpm.jsonnet | 15 - .github/jsonnet/pubsub.jsonnet | 16 +- .github/jsonnet/pulumi.jsonnet | 352 +++++---------------- .github/jsonnet/ruby.jsonnet | 101 +++--- .github/jsonnet/services.jsonnet | 25 +- .github/jsonnet/yarn.jsonnet | 135 ++++---- .github/workflows/misc.yml | 28 ++ .github/workflows/pr.yml | 94 ++++++ .github/workflows/publish-prod.yml | 74 +++++ 26 files changed, 682 insertions(+), 1204 deletions(-) delete mode 100644 .github/jsonnet/buckets.jsonnet delete mode 100644 .github/jsonnet/cache.jsonnet delete mode 100644 .github/jsonnet/pnpm.jsonnet create mode 100644 .github/workflows/misc.yml create mode 100644 .github/workflows/pr.yml create mode 100644 .github/workflows/publish-prod.yml diff --git a/.github/jsonnet/GIT_VERSION b/.github/jsonnet/GIT_VERSION index 8273035..1724762 100644 --- a/.github/jsonnet/GIT_VERSION +++ b/.github/jsonnet/GIT_VERSION @@ -1 +1 @@ -e4a3467b90e35c3dadf461f338838837bf02d2f0 +4187fef119638c2f8453bf4fd3d6da5641e4ffee diff --git a/.github/jsonnet/README.md b/.github/jsonnet/README.md index 805e8f6..b719864 100644 --- a/.github/jsonnet/README.md +++ b/.github/jsonnet/README.md @@ -1,2 +1,2 @@ These files come from https://www.github.com/gynzy/lib-jsonnet/ -Do not update here, but extend the libraries upstream. +Do not update here, but extend the libraries upstream. \ No newline at end of file diff --git a/.github/jsonnet/base.jsonnet b/.github/jsonnet/base.jsonnet index 0b85fa8..2b4875e 100644 --- a/.github/jsonnet/base.jsonnet +++ b/.github/jsonnet/base.jsonnet @@ -1,6 +1,3 @@ -local images = import 'images.jsonnet'; -local misc = import 'misc.jsonnet'; - { pipeline(name, jobs, event=['pull_request'], permissions=null, concurrency=null):: { [name + '.yml']: @@ -17,7 +14,7 @@ local misc = import 'misc.jsonnet'; name, timeoutMinutes=30, runsOn=null, - image=images.default_job_image, + image=$.default_job_image, steps=[], ifClause=null, needs=null, @@ -39,7 +36,7 @@ local misc = import 'misc.jsonnet'; { container: { image: image, - } + (if useCredentials then { credentials: { username: '_json_key', password: misc.secret('docker_gcr_io') } } else {}), + } + (if useCredentials then { credentials: { username: '_json_key', password: $.secret('docker_gcr_io') } } else {}), } ) + { @@ -52,7 +49,7 @@ local misc = import 'misc.jsonnet'; (if permissions == null then {} else { permissions: permissions }) + (if concurrency == null then {} else { concurrency: concurrency }) + (if continueOnError == null then {} else { 'continue-on-error': continueOnError }) + - (if env == null then {} else { env: env }), + (if env == null then {} else { env: env }) }, ghExternalJob( @@ -68,7 +65,7 @@ local misc = import 'misc.jsonnet'; } else {}), }, - step(name, run, env=null, workingDirectory=null, ifClause=null, id=null, continueOnError=null, shell=null):: + step(name, run, env=null, workingDirectory=null, ifClause=null, id=null, continueOnError=null):: [ { name: name, @@ -77,8 +74,7 @@ local misc = import 'misc.jsonnet'; + (if env != null then { env: env } else {}) + (if ifClause != null then { 'if': ifClause } else {}) + (if id != null then { id: id } else {}) - + (if continueOnError == null then {} else { 'continue-on-error': continueOnError }) - + (if shell == null then {} else { 'shell': shell }), + + (if continueOnError == null then {} else { 'continue-on-error': continueOnError }), ], action(name, uses, env=null, with=null, id=null, ifClause=null, continueOnError=null):: diff --git a/.github/jsonnet/buckets.jsonnet b/.github/jsonnet/buckets.jsonnet deleted file mode 100644 index b2298d9..0000000 --- a/.github/jsonnet/buckets.jsonnet +++ /dev/null @@ -1,152 +0,0 @@ -{ - // Uploads all files in the source folder to the destination bucket, including compression and TTL headers. - // - // Warnings: - // - remote/destination files not included in the source will be DELETED recursively if pruneRemote is true! - // - the files in the source directory will be modified. Do not attempt to use this directory after running this command. - // - must be run with bash shell. - // - // Parameters: - // sourcePath: The source directory to upload. Can be a local folder of a path in a bucket, depending on sourceBucket. Required. - // sourceBucket: The source bucket. If null, the sourcePath is a local directory. Defaults to null. - // destinationBucket: The destination bucket. Required. - // destinationPath: The destination directory in the bucket. Required. - // - // pruneRemote: If true, all files in the destination bucket that are not in the source will be deleted. Can only be used with destinationPath containing 'pr-'. - // - // compressFileExtentions: A list of file extentions that will be compressed. Set to an empty list to disable compression. - // compressJobs: The number of parallel gzip compression jobs. Use 4 for arc-runner-2 and 16 for arc-runner-16. Defaults to 4. - // - // lowTTLfiles: A list of files, or a single regex, that will be uploaded with a low TTL. Use this for files that are not fingerprinted. - // - // lowTTL: The TTL for lowTTLfiles. Defaults to 60 seconds. - // lowTTLStaleWhileRevalidate: The stale-while-revalidate value for lowTTLfiles. Defaults to 60 seconds. - // lowTTLHeader: The Cache-Control header for lowTTLfiles. This is generated from lowTTL and lowTTLStaleWhileRevalidate. - // - // highTTL: The TTL for all other files. Defaults to 1 week. - // highTTLStaleWhileRevalidate: The stale-while-revalidate value for all other files. Defaults to 1 day. - // highTTLHeader: The Cache-Control header for all other files. This is generated from highTTL and highTTLStaleWhileRevalidate. - // - // additionalHeaders: Additional headers to add to all uploaded files. This should be an array of strings. - uploadFilesToBucketCommand( - sourcePath, - sourceBucket=null, - destinationBucket, - destinationPath, - pruneRemote=false, - compressFileExtentions=['css', 'svg', 'html', 'json', 'js', 'xml', 'txt', 'map'], - compressJobs=4, - lowTTLfiles=[], - lowTTL=60, - lowTTLStaleWhileRevalidate=60, - lowTTLHeader='Cache-Control: public, max-age=' + lowTTL + (if lowTTLStaleWhileRevalidate == 0 then '' else ', stale-while-revalidate=' + lowTTLStaleWhileRevalidate), - highTTL=604800, // 1 week - highTTLStaleWhileRevalidate=86400, // 1 day - highTTLHeader='Cache-Control: public, max-age=' + highTTL + (if highTTLStaleWhileRevalidate == 0 then '' else ', stale-while-revalidate=' + highTTLStaleWhileRevalidate), - additionalHeaders=[], - ):: - // if this function is called with remote pruning, destination must contain pr- - assert !pruneRemote || std.length(std.findSubstr('/pr-', destinationPath)) > 0; - - local hasLowTTLfiles = (std.isArray(lowTTLfiles) && std.length(lowTTLfiles) > 0) || (std.isString(lowTTLfiles) && lowTTLfiles != ''); - local lowTTLfilesRegex = if std.isArray(lowTTLfiles) then '(' + std.strReplace(std.join('|', lowTTLfiles), '.', '\\.') + ')' else lowTTLfiles; - local highTTLfilesRegex = '(?!' + lowTTLfilesRegex + ').*'; - - local hasCompressedFiles = (std.isArray(compressFileExtentions) && std.length(compressFileExtentions) > 0) || (std.isString(compressFileExtentions) && compressFileExtentions != ''); - local compressedFilesRegex = '(' + std.join('|', std.map(function(ext) '((.*(\\.|/))?' + ext + ')', compressFileExtentions)) + ')'; - local uncompressedFilesRegex = '(?!' + compressedFilesRegex + ').*'; - - local compressionHeader = 'Content-Encoding: gzip'; - - - local rsyncCommand = function(name, excludeRegexes, headers) - local excludeRegex = if std.length(excludeRegexes) == 0 then null else '^((' + std.join(')|(', excludeRegexes) + '))$'; - - 'echo "Uploading ' + name + ' files"\n' + - 'gsutil -m ' + std.join(' ', std.map(function(header) '-h "' + header + '" ', headers + additionalHeaders)) + 'rsync -r -c' + - (if excludeRegex == null then '' else ' -x "' + excludeRegex + '"') + - (if pruneRemote then ' -d' else '') + - (if sourceBucket == null then ' ./' else ' gs://' + sourceBucket + '/' + sourcePath + '/') + - ' gs://' + destinationBucket + '/' + destinationPath + '/;\n' + - 'echo "Uploading ' + name + ' files completed"; echo\n' + - '\n'; - - 'set -e -o pipefail;\n' + - (if sourceBucket == null then 'cd ' + sourcePath + ';\n' else '') + - '\n' + - - - if hasCompressedFiles then - ( - if sourceBucket == null then - 'echo "Compressing files in parallel before uploading"\n' + - '{\n' + - " for file in `find . -type f -regextype posix-egrep -regex '" + compressedFilesRegex + "' | sed --expression 's/\\.\\///g'`; do\n" + - ' echo "gzip -9 $file; mv $file.gz $file"\n' + - ' done\n' + - '} | parallel --halt now,fail=1 -j ' + compressJobs + '\n' + - 'echo "Compressing files in parallel completed"\n' + - '\n' - else '' - ) + - - if hasLowTTLfiles then - rsyncCommand( - 'highTTL compressed', - excludeRegexes=[lowTTLfilesRegex, uncompressedFilesRegex], - headers=[highTTLHeader, compressionHeader], - ) + - rsyncCommand( - 'highTTL uncompressed', - excludeRegexes=[lowTTLfilesRegex, compressedFilesRegex], - headers=[highTTLHeader], - ) + - - rsyncCommand( - 'lowTTL compressed', - excludeRegexes=[highTTLfilesRegex, uncompressedFilesRegex], - headers=[lowTTLHeader, compressionHeader], - ) + - rsyncCommand( - 'lowTTL uncompressed', - excludeRegexes=[highTTLfilesRegex, compressedFilesRegex], - headers=[lowTTLHeader], - ) - - - else // no lowTTL files, with compression - rsyncCommand( - 'compressed', - excludeRegexes=[uncompressedFilesRegex], - headers=[highTTLHeader, compressionHeader], - ) + - rsyncCommand( - 'uncompressed', - excludeRegexes=[compressedFilesRegex], - headers=[highTTLHeader], - ) - - - else // no compression - if hasLowTTLfiles then - rsyncCommand( - 'highTTL', - excludeRegexes=[lowTTLfilesRegex], - headers=[highTTLHeader], - ) + - - rsyncCommand( - 'lowTTL', - excludeRegexes=[highTTLfilesRegex], - headers=[lowTTLHeader], - ) - - - else // no lowTTL files, no compression - rsyncCommand( - 'all', - excludeRegexes=[], - headers=[highTTLHeader], - ), - -} diff --git a/.github/jsonnet/cache.jsonnet b/.github/jsonnet/cache.jsonnet deleted file mode 100644 index 9d85bef..0000000 --- a/.github/jsonnet/cache.jsonnet +++ /dev/null @@ -1,128 +0,0 @@ -local base = import 'base.jsonnet'; - -{ - // Fetch a cache from the cache server. - // This is a generic function that can be used to fetch any cache. It is advised to wrap this function - // in a more specific function that fetches a specific cache, setting the cacheName and folders parameters. - // - // To be paired with the uploadCache function. - // - // Parameters: - // cacheName: The name of the cache to fetch. The name of the repository is usually a good option. Required. - // backupCacheName: The name of a backup cache to fetch if the main cache fails. Default is null. - // folders: A list of folders that are in the cache. These will be deleted if the download fails. Can be an empty list if additionalCleanupCommands are used. - // additionalCleanupCommands: A list of additional commands to run if the download fails. Default is an empty list. - // ifClause: An optional if clause to conditionally run this step. Default is null. - // workingDirectory: The working directory for this step. Default is null. - // retry: Whether to retry the download if it fails. Default is true. - // continueWithoutCache: Whether to continue if the cache is not found. Default is true. - fetchCache( - cacheName, - backupCacheName=null, - folders=[], - version='v1', - backupCacheVersion=version, - additionalCleanupCommands=[], - ifClause=null, - workingDirectory=null, - retry=true, - continueWithoutCache=true, - ):: - assert std.length(folders) > 0 || std.length(additionalCleanupCommands) > 0; - - local downloadCommand(cacheName, version, nextSteps, indent = '') = - indent + 'wget -q -O - "https://storage.googleapis.com/files-gynzy-com-test/ci-cache/' + cacheName + '-' + version + '.tar.zst" | unzstd | tar xf -\n' + - indent + 'if [ $? -ne 0 ]; then\n' + - indent + ' echo "Cache download failed, cleanup up partial downloads"\n' + - (if std.length(folders) > 0 then indent + ' rm -rf ' + std.join(' ', folders) + '\n' else '') + - std.join(' ', std.map(function(cmd) indent + ' ' + cmd + '\n', additionalCleanupCommands)) + - indent + ' echo "Cleanup complete"; echo\n\n' + - nextSteps + - indent + 'fi\n'; - - local downloadCommandWithRetry(cacheName, version, nextSteps, indent = '') = - downloadCommand( - cacheName, - version, - if retry then - indent + ' echo "Retrying download..."\n' + - downloadCommand(cacheName, version, nextSteps, indent + ' ') - else - nextSteps, - indent, - ); - - local backupIndent = (if retry then ' ' else ' '); - - local downloadFailedCommand = backupIndent + 'echo "Cache download failed :( ' + (if continueWithoutCache then 'Continuing without cache"' else 'Aborting"; exit 1') + '\n'; - - base.step( - 'download ' + cacheName + ' cache', - run= - 'set +e;\n' + - 'command -v zstd || { apt update && apt install -y zstd; }\n' + - 'echo "Downloading cache"\n' + - downloadCommandWithRetry( - cacheName, - version, - if backupCacheName != null then - backupIndent + 'echo "Downloading backup cache"\n' + - downloadCommandWithRetry(backupCacheName, backupCacheVersion, backupIndent + downloadFailedCommand, indent=backupIndent) - else - downloadFailedCommand, - ), - ifClause=ifClause, - workingDirectory=workingDirectory, - ), - - // Uploads a cache to the cache server. - // This is a generic function that can be used to upload any cache. It is advised to wrap this function - // in a more specific function that uploads a specific cache, setting the cacheName and folders parameters. - // - // To be paired with the fetchCache function. - // - // Parameters: - // cacheName: The name of the cache to upload. The name of the repository is usually a good option. Required. - // folders: A list of folders to include in the cache. Required unless tarCommand is given. - // compressionLevel: The compression level to use for zstd. Default is 10. - // tarCommand: The command to run to create the tar file. Default is 'tar -c ' + std.join(' ', folders). - uploadCache( - cacheName, - folders=null, - version='v1', - compressionLevel=10, - tarCommand='tar -c ' + std.join(' ', folders), - ):: - local cacheBucketPath = function(temp=false) - 'gs://files-gynzy-com-test/ci-cache/' + cacheName + '-' + version + '.tar.zst' + (if temp then '.tmp' else ''); - - base.step( - 'upload-gatsby-cache', - run= - 'set -e\n' + - '\n' + - 'command -v zstd || { apt update && apt install -y zstd; }\n' + - '\n' + - 'echo "Create and upload cache"\n' + - tarCommand + ' | zstdmt -' + compressionLevel + ' | gsutil cp - "' + cacheBucketPath(temp=true) + '"\n' + - 'gsutil mv "' + cacheBucketPath(temp=true) + '" "' + cacheBucketPath(temp=false) + '"\n' + - - 'echo "Upload finished"\n' - ), - - // Removes a cache from the cache server. - // This is a generic function that can be used to remove any cache. It is advised to wrap this function - // in a more specific function that removes a specific cache, setting the cacheName parameter. - // - // Parameters: - // cacheName: The name of the cache to remove. The name of the repository is usually a good option. Required. - // version: The version of the cache to remove. Default is 'v1'. - removeCache(cacheName, version='v1'):: - base.step( - 'remove ' + cacheName + ' cache', - run= - 'set +e;\n' + - 'gsutil rm "gs://files-gynzy-com-test/ci-cache/' + cacheName + '-' + version + '.tar.zst"\n' + - 'echo "Cache removed"\n' - ), -} diff --git a/.github/jsonnet/clusters.jsonnet b/.github/jsonnet/clusters.jsonnet index 10c9b65..3ebe0e7 100644 --- a/.github/jsonnet/clusters.jsonnet +++ b/.github/jsonnet/clusters.jsonnet @@ -1,27 +1,24 @@ -local misc = import 'misc.jsonnet'; - { - test: { - project: 'gynzy-test-project', - name: 'test', - zone: 'europe-west4-b', - secret: misc.secret('GCE_NEW_TEST_JSON'), - jobNodeSelectorType: 'preemptible', - }, + clusters: { + test: { + project: 'gynzy-test-project', + name: 'test', + zone: 'europe-west4-b', + secret: '${{ secrets.GCE_NEW_TEST_JSON }}', + }, - prod: { - project: 'unicorn-985', - name: 'prod-europe-west4', - zone: 'europe-west4', - secret: misc.secret('GCE_JSON'), - jobNodeSelectorType: 'worker', - }, + prod: { + project: 'unicorn-985', + name: 'prod-europe-west4', + zone: 'europe-west4', + secret: '${{ secrets.GCE_JSON }}', + }, - 'gynzy-intern': { - project: 'gynzy-intern', - name: 'gynzy-intern', - zone: 'europe-west4', - secret: misc.secret('CONTINUOUS_DEPLOYMENT_GCE_JSON'), - jobNodeSelectorType: 'worker', + 'gynzy-intern': { + project: 'gynzy-intern', + name: 'gynzy-intern', + zone: 'europe-west4', + secret: '${{ secrets.CONTINUOUS_DEPLOYMENT_GCE_JSON }}', + }, }, } diff --git a/.github/jsonnet/complete-workflows.jsonnet b/.github/jsonnet/complete-workflows.jsonnet index 096278e..d688931 100644 --- a/.github/jsonnet/complete-workflows.jsonnet +++ b/.github/jsonnet/complete-workflows.jsonnet @@ -1,7 +1,3 @@ -local base = import 'base.jsonnet'; -local misc = import 'misc.jsonnet'; -local yarn = import 'yarn.jsonnet'; - { /* @param {string[]} repositories - The repositories to publish to @@ -13,21 +9,21 @@ local yarn = import 'yarn.jsonnet'; workflowJavascriptPackage(repositories=['gynzy'], isPublicFork=true, checkVersionBump=true, testJob=null, branch='main'):: local runsOn = (if isPublicFork then 'ubuntu-latest' else null); - base.pipeline( + $.pipeline( 'misc', - [misc.verifyJsonnet(fetch_upstream=false, runsOn=runsOn)], + [$.verifyJsonnet(fetch_upstream=false, runsOn=runsOn)], ) + - base.pipeline( + $.pipeline( 'publish-prod', [ - yarn.yarnPublishJob(repositories=repositories, runsOn=runsOn), + $.yarnPublishJob(repositories=repositories, runsOn=runsOn), ], - event={ push: { branches: [branch] } }, + event={ push: { branches: [ branch ] } }, ) + - base.pipeline( + $.pipeline( 'pr', [ - yarn.yarnPublishPreviewJob(repositories=repositories, runsOn=runsOn, checkVersionBump=checkVersionBump), + $.yarnPublishPreviewJob(repositories=repositories, runsOn=runsOn, checkVersionBump=checkVersionBump), ] + (if testJob != null then [testJob] diff --git a/.github/jsonnet/databases.jsonnet b/.github/jsonnet/databases.jsonnet index c048cbf..baa7755 100644 --- a/.github/jsonnet/databases.jsonnet +++ b/.github/jsonnet/databases.jsonnet @@ -1,6 +1,3 @@ -local base = import 'base.jsonnet'; -local images = import 'images.jsonnet'; - { database_servers: { test: { @@ -29,6 +26,13 @@ local images = import 'images.jsonnet'; region: 'europe-west4', project: 'unicorn-985', }, + 'eu-w4-metrics-production': { + type: 'mysql', + server: 'eu-w4-metrics-production', + region: 'europe-west4', + project: 'unicorn-985', + lifecycle: 'deprecated', + }, 'gynzy-test': { type: 'mysql', server: 'gynzy-test', @@ -93,9 +97,9 @@ local images = import 'images.jsonnet'; // delete database by setting it to null and calling prune afterwards local pluginOptions = std.prune(mysqlActionOptions { task: 'clone', database: null }); - base.action( + $.action( 'copy-database', - images.mysql_action_image, + $.mysql_action_image, with=pluginOptions ), @@ -106,9 +110,9 @@ local images = import 'images.jsonnet'; // delete database by setting it to null and calling prune afterwards local pluginOptions = std.prune(mysqlActionOptions { task: 'remove', database: null }); - base.action( + $.action( 'delete-database', - images.mysql_action_image, + $.mysql_action_image, with=pluginOptions ), } diff --git a/.github/jsonnet/deployment.jsonnet b/.github/jsonnet/deployment.jsonnet index d6171cb..60de82c 100644 --- a/.github/jsonnet/deployment.jsonnet +++ b/.github/jsonnet/deployment.jsonnet @@ -1,13 +1,8 @@ -local base = import 'base.jsonnet'; -local images = import 'images.jsonnet'; -local misc = import 'misc.jsonnet'; -local notifications = import 'notifications.jsonnet'; - { _assertMergeShaIsLatestCommit(branch, sha='${{ github.sha }}', repository='${{ github.repository }}'):: [ - base.step('install jq curl', 'apk add --no-cache jq curl'), - base.step( + $.step('install jq curl', 'apk add --no-cache jq curl'), + $.step( 'assert merge sha is latest commit', ||| HEAD_SHA=$(curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/${GITHUB_REPOSITORY}/branches/${TARGET_BRANCH} | jq -r .commit.sha); @@ -40,112 +35,78 @@ local notifications = import 'notifications.jsonnet'; * - PR B is merged into the default branch * - now github closes both PRs and without this additional sanity check, both would create a deploy event * - * For more complex deployment scenarios, use the branchMergeDeploymentEventHook instead - * * params: * deployToTest {boolean} - if true, a deployment event is also created for the test environment * prodBranch {string|null} - the branch to deploy to production. defaults to the default branch of the repository. but can be set to a differring release branch * testBranch {string|null} - the branch to deploy to test. defaults to the default branch of the repository. but can be set to a differring test branch * extraDeployTargets {string[]} - deploy targets to create deployment events for. defaults to ['production']. these targets will triger based on the configured prodBranch * runsOn {string|null} - the name of the runner to run this job on. defaults to null, which will later on means the default self hosted runner will be used - * notifyOnTestDeploy {boolean} - if true, a slack message is sent when a test deployment is created */ - masterMergeDeploymentEventHook(deployToTest=false, prodBranch=null, testBranch=null, deployTargets=['production'], runsOn=null, notifyOnTestDeploy=false):: - local branches = [ - { - branch: (if prodBranch != null then prodBranch else '_default_'), - deployments: deployTargets, - notifyOnDeploy: true, - }, - ] + (if deployToTest then [ - { - branch: (if testBranch != null then testBranch else '_default_'), - deployments: ['test'], - notifyOnDeploy: notifyOnTestDeploy, - }, - ] else []); - - self.branchMergeDeploymentEventHook(branches, runsOn=runsOn), - - - /* - * Creates a production deployment event on pr-close if the following conditions are met: - * - the pr is merged - * - the pr is not merged by the virko user - * - the pr is merged into the default branch - * - the merge sha is the latest commit on the default branch. - * this prevents a deployment from beeing created in a specific edge case: - * - PR A is merged into PR B - * - PR B is merged into the default branch - * - now github closes both PRs and without this additional sanity check, both would create a deploy event - * - * params: - * branches {{branch: string, deployments: string[], notifyOnDeploy: boolean}[]} - an array of the branches to create deployment events for. - * Each branch object has the following properties: - * branch {string} - the branch to which the PR has to be merged into. If '_default_' is used, the default branch of the repository is used. - * deployments {string[]} - the environments to deploy to. e.g. ['production', 'test'] - * notifyOnDeploy {boolean} - if true, a slack message is sent when a deployment is created - * runsOn {string|null} - the name of the runner to run this job on. defaults to null, which will later on means the default self hosted runner will be used - */ - branchMergeDeploymentEventHook(branches, runsOn=null):: - base.pipeline( + masterMergeDeploymentEventHook(deployToTest=false, prodBranch=null, testBranch=null, deployTargets=['production'], runsOn=null):: + $.pipeline( 'create-merge-deployment', [ - ( - local branchName = if branch.branch == '_default_' then '${{ github.event.pull_request.base.repo.default_branch }}' else branch.branch; - local branchNameForJob = if branch.branch == '_default_' then 'default-branch' else branch.branch; - local branchNameInExpression = if branch.branch == '_default_' then 'github.event.pull_request.base.repo.default_branch' else "'" + branch.branch + "'"; - - local ifClause = '${{ github.event.pull_request.base.ref == ' + branchNameInExpression + " && steps.assert-merge-sha-is-latest-commit.outputs.CREATE_DEPLOY_EVENT == 'true' }}"; - - base.ghJob( - 'create-merge-deployment-' + branchNameForJob + '-to-' + std.join('-', branch.deployments), - useCredentials=false, - runsOn=runsOn, - permissions={ deployments: 'write', contents: 'read' }, - ifClause="${{ github.actor != 'gynzy-virko' && github.event.pull_request.merged == true}}", - steps=self._assertMergeShaIsLatestCommit(branch=branchName) + - std.map( - function(deploymentTarget) - base.action( - 'publish-deploy-' + deploymentTarget + '-event', - 'chrnorm/deployment-action@v2', - ifClause=ifClause, - with={ - token: misc.secret('VIRKO_GITHUB_TOKEN'), - environment: deploymentTarget, - 'auto-merge': 'false', - ref: '${{ github.event.pull_request.head.sha }}', - description: 'Auto deploy ' + deploymentTarget + ' on PR merge. pr: ${{ github.event.number }} ref: ${{ github.event.pull_request.head.sha }}', - payload: '{ "pr" : ${{ github.event.number }}, "branch": "${{ github.head_ref }}", "base_ref": "${{ github.event.pull_request.base.sha }}", "head_ref": "${{ github.event.pull_request.head.sha }}" }', - } - ), - branch.deployments, - ) + - ( - if branch.notifyOnDeploy then - [ - notifications.sendSlackMessage( - message='Deploy to ' + std.join(' and ', branch.deployments) + ' of started!\nTitle: ${{ github.event.pull_request.title }}\nBranch: ${{ github.head_ref }}', - ifClause=ifClause, - ), - ] - else [] + $.ghJob( + 'create-merge-deployment-prod', + useCredentials=false, + runsOn=runsOn, + permissions={ deployments: 'write', contents: 'read' }, + ifClause="${{ github.actor != 'gynzy-virko' && github.event.pull_request.merged == true}}", + steps=$._assertMergeShaIsLatestCommit(branch=(if prodBranch != null then prodBranch else '${{ github.event.pull_request.base.repo.default_branch }}')) + + std.map( + function(deploymentTarget) + $.action( + 'publish-deploy-' + deploymentTarget + '-event', + 'chrnorm/deployment-action@v2', + ifClause='${{ github.event.pull_request.base.ref == ' + (if prodBranch != null then "'" + prodBranch + "'" else 'github.event.pull_request.base.repo.default_branch') + " && steps.assert-merge-sha-is-latest-commit.outputs.CREATE_DEPLOY_EVENT == 'true' }}", + with={ + token: $.secret('VIRKO_GITHUB_TOKEN'), + environment: deploymentTarget, + 'auto-merge': 'false', + ref: '${{ github.event.pull_request.head.sha }}', + description: 'Auto deploy ' + deploymentTarget + ' on PR merge. pr: ${{ github.event.number }} ref: ${{ github.event.pull_request.head.sha }}', + payload: '{ "pr" : ${{ github.event.number }}, "branch": "${{ github.head_ref }}", "base_ref": "${{ github.event.pull_request.base.sha }}", "head_ref": "${{ github.event.pull_request.head.sha }}" }', + } + ), + deployTargets, + ) + + [ + $.sendSlackMessage( + message='Deploy to prod of pr: ${{ github.event.number }} with title: ${{ github.event.pull_request.title }} branch: ${{ github.head_ref }} started!', + ifClause='${{ github.event.pull_request.base.ref == ' + (if prodBranch != null then "'" + prodBranch + "'" else 'github.event.pull_request.base.repo.default_branch') + " && steps.assert-merge-sha-is-latest-commit.outputs.CREATE_DEPLOY_EVENT == 'true' }}", ), - ) - ) - for branch in branches - ], + ], + ), + ] + + (if deployToTest == true && std.member(deployTargets, 'test') == false then + [ + $.ghJob( + 'create-merge-deployment-test', + useCredentials=false, + permissions={ deployments: 'write', contents: 'read' }, + ifClause="${{ github.actor != 'gynzy-virko' && github.event.pull_request.merged == true}}", + steps=$._assertMergeShaIsLatestCommit(branch=(if testBranch != null then testBranch else '${{ github.event.pull_request.base.repo.default_branch }}')) + + [ + $.action( + 'publish-deploy-test-event', + 'chrnorm/deployment-action@v2', + ifClause='${{ github.event.pull_request.base.ref == ' + (if testBranch != null then "'" + testBranch + "'" else 'github.event.pull_request.base.repo.default_branch') + " && steps.assert-merge-sha-is-latest-commit.outputs.CREATE_DEPLOY_EVENT == 'true' }}", + with={ + token: $.secret('VIRKO_GITHUB_TOKEN'), + environment: 'test', + 'auto-merge': 'false', + ref: '${{ github.event.pull_request.head.sha }}', + description: 'Auto deploy test on PR merge. pr: ${{ github.event.number }} ref: ${{ github.event.pull_request.head.sha }}', + payload: '{ "pr" : ${{ github.event.number }}, "branch": "${{ github.head_ref }}", "base_ref": "${{ github.event.pull_request.base.sha }}", "head_ref": "${{ github.event.pull_request.head.sha }}" }', + } + ), + ], + ), + ] else []), event={ pull_request: { types: ['closed'], }, }, ), - - /* - * Generate a github ifClause for the provided deployment targets: - */ - deploymentTargets(targets):: - '${{ ' + std.join(' || ', std.map(function(target) "github.event.deployment.environment == '" + target + "'", targets)) + ' }}', } diff --git a/.github/jsonnet/docker.jsonnet b/.github/jsonnet/docker.jsonnet index a23ddc1..d33a9c7 100644 --- a/.github/jsonnet/docker.jsonnet +++ b/.github/jsonnet/docker.jsonnet @@ -1,7 +1,3 @@ -local base = import 'base.jsonnet'; -local images = import 'images.jsonnet'; -local misc = import 'misc.jsonnet'; - { buildDocker( imageName, @@ -13,12 +9,12 @@ local misc = import 'misc.jsonnet'; registry='eu.gcr.io', project='unicorn-985', ):: - base.action( - 'build-docker ' + imageName, - images.docker_action_image, + $.action( + 'build-docker', + $.docker_action_image, with={ context: context, - gcloud_service_key: misc.secret('docker_gcr_io_base64'), + gcloud_service_key: $.secret('docker_gcr_io_base64'), image_name: imageName, image_tag: imageTag, project_id: project, diff --git a/.github/jsonnet/helm.jsonnet b/.github/jsonnet/helm.jsonnet index 8e033f8..8b7a871 100644 --- a/.github/jsonnet/helm.jsonnet +++ b/.github/jsonnet/helm.jsonnet @@ -1,15 +1,8 @@ -local base = import 'base.jsonnet'; -local clusters = import 'clusters.jsonnet'; -local databases = import 'databases.jsonnet'; -local images = import 'images.jsonnet'; -local misc = import 'misc.jsonnet'; -local services = import 'services.jsonnet'; - { deployHelm(cluster, release, values, chartPath, delete=false, useHelm3=true, title=null, ifClause=null, ttl=null, namespace='default'):: - base.action( + $.action( (if title == null then if delete then 'delete-helm' else 'deploy-helm' else title), - images.helm_action_image, + $.helm_action_image, with={ clusterProject: cluster.project, clusterLocation: cluster.zone, @@ -34,10 +27,10 @@ local services = import 'services.jsonnet'; helmPath='./helm/' + serviceName, deploymentName=serviceName + '-prod', ifClause=null, - cluster=clusters.prod, + cluster=$.clusters.prod, namespace='default', ):: - self.deployHelm( + $.deployHelm( cluster, deploymentName, { @@ -59,17 +52,17 @@ local services = import 'services.jsonnet'; options={}, helmPath='./helm/' + serviceName, deploymentName=serviceName + '-prod', - image=images.default_job_image, + image=$.default_job_image, useCredentials=false, ):: - base.ghJob( + $.ghJob( 'deploy-prod', ifClause="${{ github.event.deployment.environment == 'production' }}", image=image, useCredentials=useCredentials, steps=[ - misc.checkout(), - self.helmDeployProd(serviceName, options, helmPath, deploymentName), + $.checkout(), + $.helmDeployProd(serviceName, options, helmPath, deploymentName), ], ), @@ -78,10 +71,10 @@ local services = import 'services.jsonnet'; options={}, helmPath='./helm/' + serviceName, deploymentName=serviceName + '-master', - cluster=clusters.test, + cluster=$.clusters.test, namespace='default', ):: - self.deployHelm( + $.deployHelm( cluster, deploymentName, { @@ -102,17 +95,17 @@ local services = import 'services.jsonnet'; options={}, helmPath='./helm/' + serviceName, deploymentName=serviceName + '-master', - image=images.default_job_image, + image=$.default_job_image, useCredentials=false, ):: - base.ghJob( + $.ghJob( 'deploy-test', ifClause="${{ github.event.deployment.environment == 'test' }}", image=image, useCredentials=useCredentials, steps=[ - misc.checkout(), - self.helmDeployTest(serviceName, options, helmPath, deploymentName), + $.checkout(), + $.helmDeployTest(serviceName, options, helmPath, deploymentName), ], ), @@ -121,10 +114,10 @@ local services = import 'services.jsonnet'; options={}, helmPath='./helm/' + serviceName, deploymentName=serviceName + '-pr-${{ github.event.number }}', - cluster=clusters.test, + cluster=$.clusters.test, namespace='default', ):: - self.deployHelm( + $.deployHelm( cluster, deploymentName, { @@ -145,16 +138,16 @@ local services = import 'services.jsonnet'; options={}, helmPath='./helm/' + serviceName, deploymentName=serviceName + '-pr-${{ github.event.number }}', - image=images.default_job_image, + image=$.default_job_image, useCredentials=false, ):: - base.ghJob( + $.ghJob( 'deploy-pr', image=image, useCredentials=useCredentials, steps=[ - misc.checkout(), - self.helmDeployPR(serviceName, options, helmPath, deploymentName), + $.checkout(), + $.helmDeployPR(serviceName, options, helmPath, deploymentName), ], ), @@ -163,10 +156,10 @@ local services = import 'services.jsonnet'; options={}, helmPath='./helm/' + serviceName, deploymentName=serviceName + '-pr-${{ github.event.number }}', - cluster=clusters.test, + cluster=$.clusters.test, namespace='default', ):: - self.deployHelm( + $.deployHelm( cluster, deploymentName, options, @@ -183,16 +176,16 @@ local services = import 'services.jsonnet'; deploymentName=serviceName + '-pr-${{ github.event.number }}', mysqlDeleteOptions={ enabled: false }, ):: - base.ghJob( + $.ghJob( 'helm-delete-pr', - image=images.default_job_image, + image=$.default_job_image, useCredentials=false, steps=[ - misc.checkout(), - self.helmDeletePr(serviceName, options, helmPath, deploymentName), + $.checkout(), + $.helmDeletePr(serviceName, options, helmPath, deploymentName), ] + - (if mysqlDeleteOptions.enabled then [databases.deleteDatabase(mysqlDeleteOptions)] else []), - services=(if mysqlDeleteOptions.enabled then { 'cloudsql-proxy': services.cloudsql_proxy_service(mysqlDeleteOptions.database) } else null), + (if mysqlDeleteOptions.enabled then [$.deleteDatabase(mysqlDeleteOptions)] else []), + services=(if mysqlDeleteOptions.enabled then { 'cloudsql-proxy': $.cloudsql_proxy_service(mysqlDeleteOptions.database) } else null), ), helmDeletePRPipeline( @@ -201,10 +194,10 @@ local services = import 'services.jsonnet'; helmPath='./helm/' + serviceName, deploymentName=serviceName + '-pr-${{ github.event.number }}', ):: - base.pipeline( + $.pipeline( 'close-pr', [ - self.helmDeletePRJob(serviceName, options, helmPath, deploymentName), + $.helmDeletePRJob(serviceName, options, helmPath, deploymentName), ], event={ pull_request: { @@ -220,8 +213,8 @@ local services = import 'services.jsonnet'; deploymentName=serviceName + '-canary', ifClause=null, ):: - self.deployHelm( - clusters.prod, + $.deployHelm( + $.clusters.prod, deploymentName, { identifier: 'prod', @@ -241,17 +234,17 @@ local services = import 'services.jsonnet'; options={}, helmPath='./helm/' + serviceName + '-canary', deploymentName=serviceName + '-canary', - image=images.default_job_image, + image=$.default_job_image, useCredentials=false, ):: - base.ghJob( + $.ghJob( 'deploy-canary', image=image, useCredentials=useCredentials, ifClause="${{ github.event.deployment.environment == 'canary' }}", steps=[ - misc.checkout(), - self.helmDeployCanary(serviceName, options, helmPath, deploymentName), + $.checkout(), + $.helmDeployCanary(serviceName, options, helmPath, deploymentName), ], ), @@ -262,8 +255,8 @@ local services = import 'services.jsonnet'; deploymentName=serviceName + '-canary', ifClause=null, ):: - self.deployHelm( - clusters.prod, + $.deployHelm( + $.clusters.prod, deploymentName, { identifier: 'prod', @@ -284,14 +277,14 @@ local services = import 'services.jsonnet'; helmPath='./helm/' + serviceName + '-canary', deploymentName=serviceName + '-canary', ):: - base.ghJob( + $.ghJob( 'kill-canary', ifClause="${{ github.event.deployment.environment == 'kill-canary' || github.event.deployment.environment == 'production' }}", - image=images.default_job_image, + image=$.default_job_image, useCredentials=false, steps=[ - misc.checkout(), - self.helmKillCanary(serviceName, options, helmPath, deploymentName), + $.checkout(), + $.helmKillCanary(serviceName, options, helmPath, deploymentName), ], ), } diff --git a/.github/jsonnet/images.jsonnet b/.github/jsonnet/images.jsonnet index f7c6492..fbb9b3d 100644 --- a/.github/jsonnet/images.jsonnet +++ b/.github/jsonnet/images.jsonnet @@ -1,20 +1,17 @@ { jsonnet_bin_image: 'eu.gcr.io/unicorn-985/docker-images_jsonnet:v1', helm_action_image: 'docker://europe-docker.pkg.dev/gynzy-test-project/public-images/helm-action:v2', - mysql_action_image: 'docker://europe-docker.pkg.dev/gynzy-test-project/public-images/docker-images_mysql-cloner-action:v1', + mysql_action_image: 'docker://europe-docker.pkg.dev/gynzy-test-project/public-images/mysql-action:v1', docker_action_image: 'docker://europe-docker.pkg.dev/gynzy-test-project/public-images/push-to-gcr-github-action:v1', - default_job_image: 'alpine:3.20.0', + default_job_image: 'alpine:3.18.3', default_mysql57_image: 'eu.gcr.io/unicorn-985/docker-images_mysql57_utf8mb4:v1', default_mysql8_image: 'eu.gcr.io/unicorn-985/docker-images_mysql8_utf8mb4:v1', default_cloudsql_image: 'eu.gcr.io/unicorn-985/docker-images_cloudsql-sidecar:v1', default_redis_image: 'redis:5.0.6', default_unicorns_image: 'node:18.15', default_pubsub_image: 'messagebird/gcloud-pubsub-emulator:latest', - default_backend_nest_image: 'node:20.12.1', + default_backend_nest_image: 'node:18.15', default_mongodb_image: 'eu.gcr.io/unicorn-985/docker-images_mongo6-replicated:v1', - mongo_job_image: 'europe-docker.pkg.dev/gynzy-test-project/public-images/docker-images_mongo-cloner-job:v1', - default_python_image: 'python:3.12.1', - default_pulumi_node_image: 'node:18', - job_poster_image: 'europe-docker.pkg.dev/gynzy-test-project/public-images/docker-images_job-poster:v1', - newrelic_deployment_marker_image: 'newrelic/deployment-marker-action@v2.5.0', + mongo_action_image: 'docker://europe-docker.pkg.dev/gynzy-test-project/public-images/action-mongo-cloner:v1', + default_python_image: 'python:3.11.4', } diff --git a/.github/jsonnet/index.jsonnet b/.github/jsonnet/index.jsonnet index 2a17154..c26c847 100644 --- a/.github/jsonnet/index.jsonnet +++ b/.github/jsonnet/index.jsonnet @@ -1,5 +1,5 @@ (import 'base.jsonnet') + -{ clusters: import 'clusters.jsonnet' } + +(import 'clusters.jsonnet') + (import 'databases.jsonnet') + (import 'docker.jsonnet') + (import 'helm.jsonnet') + @@ -15,6 +15,4 @@ (import 'deployment.jsonnet') + (import 'notifications.jsonnet') + (import 'complete-workflows.jsonnet') + -{ pnpm: import 'pnpm.jsonnet' } + -{ cache: import 'cache.jsonnet' } + -{ buckets: import 'buckets.jsonnet' } +{} diff --git a/.github/jsonnet/misc.jsonnet b/.github/jsonnet/misc.jsonnet index 93e2cf3..5d97734 100644 --- a/.github/jsonnet/misc.jsonnet +++ b/.github/jsonnet/misc.jsonnet @@ -1,57 +1,44 @@ -local base = import 'base.jsonnet'; -local images = import 'images.jsonnet'; - { checkout(ifClause=null, fullClone=false, ref=null):: local with = (if fullClone then { 'fetch-depth': 0 } else {}) + (if ref != null then { ref: ref } else {}); - base.action( + $.action( 'Check out repository code', 'actions/checkout@v3', with=with, ifClause=ifClause - ) + - base.step('git safe directory', "command -v git && git config --global --add safe.directory '*' || true"), + ), lint(service):: - base.step('lint-' + service, - './node_modules/.bin/eslint "./packages/' + service + '/{app,lib,tests,config,addon}/**/*.js" --quiet'), + $.step('lint-' + service, + './node_modules/.bin/eslint "./packages/' + service + '/{app,lib,tests,config,addon}/**/*.js" --quiet'), lintAll():: - base.step('lint', 'yarn lint'), + $.step('lint', 'yarn lint'), verifyGoodFences():: - base.step('verify-good-fences', 'yarn run gf'), + $.step('verify-good-fences', 'yarn run gf'), improvedAudit():: - base.step('audit', 'yarn improved-audit'), - - verifyJsonnetWorkflow():: - base.pipeline( - 'misc', - [ - self.verifyJsonnet(fetch_upstream=false), - ], - event='pull_request', - ), + $.step('audit', 'yarn improved-audit'), verifyJsonnet(fetch_upstream=true, runsOn=null):: - base.ghJob( + $.ghJob( 'verify-jsonnet-gh-actions', runsOn=runsOn, - image=images.jsonnet_bin_image, + image=$.jsonnet_bin_image, steps=[ - self.checkout(ref='${{ github.event.pull_request.head.sha }}'), - base.step('remove-workflows', 'rm -f .github/workflows/*'), + $.checkout(ref='${{ github.event.pull_request.head.sha }}'), + $.step('remove-workflows', 'rm -f .github/workflows/*'), ] + ( - if fetch_upstream then [base.step('fetch latest lib-jsonnet', - ' rm -rf .github/jsonnet/;\n mkdir .github/jsonnet/;\n cd .github;\n curl https://files.gynzy.net/lib-jsonnet/v1/jsonnet-prod.tar.gz | tar xvzf -;\n ')] else [] + if fetch_upstream then [$.step('fetch latest lib-jsonnet', + ' rm -rf .github/jsonnet/;\n mkdir .github/jsonnet/;\n cd .github;\n curl https://files.gynzy.net/lib-jsonnet/v1/jsonnet-prod.tar.gz | tar xvzf -;\n ')] else [] ) + [ - base.step('generate-workflows', 'jsonnet -m .github/workflows/ -S .github.jsonnet;'), - base.step('git workaround', 'git config --global --add safe.directory $PWD'), - base.step('check-jsonnet-diff', 'git diff --exit-code'), - base.step( + $.step('generate-workflows', 'jsonnet -m .github/workflows/ -S .github.jsonnet;'), + $.step('git workaround', 'git config --global --add safe.directory $PWD'), + $.step('check-jsonnet-diff', 'git diff --exit-code'), + $.step( 'possible-causes-for-error', 'echo "Possible causes: \n' + '1. You updated jsonnet files, but did not regenerate the workflows. \n' + @@ -72,16 +59,16 @@ local images = import 'images.jsonnet'; titleUpdateAction='prefix', otherOptions={}, ):: - base.pipeline( + $.pipeline( 'update-pr-description', event={ pull_request: { types: ['opened'] }, }, jobs=[ - base.ghJob( + $.ghJob( 'update-pr-description', steps=[ - base.action( + $.action( 'update-pr-description', 'gynzy/pr-update-action@v2', with={ @@ -111,7 +98,7 @@ local images = import 'images.jsonnet'; '${{ secrets.' + secretName + ' }}', pollUrlForContent(url, expectedContent, name='verify-deploy', attempts='100', interval='2000', ifClause=null):: - base.action( + $.action( name, 'gynzy/wait-for-http-content@v1', with={ @@ -124,16 +111,16 @@ local images = import 'images.jsonnet'; ), cleanupOldBranchesPipelineCron():: - base.pipeline( + $.pipeline( 'purge-old-branches', [ - base.ghJob( + $.ghJob( 'purge-old-branches', useCredentials=false, steps=[ - base.step('setup', 'apk add git bash'), - self.checkout(), - base.action( + $.step('setup', 'apk add git bash'), + $.checkout(), + $.action( 'Run delete-old-branches-action', 'beatlabs/delete-old-branches-action@6e94df089372a619c01ae2c2f666bf474f890911', with={ @@ -157,6 +144,35 @@ local images = import 'images.jsonnet'; }, ), + codiumAIPRAgent():: + $.pipeline( + 'codium-ai', + [ + $.ghJob( + 'pr_agent_job', + useCredentials=false, + ifClause='${{ github.event.pull_request.draft == false }}', + steps=[ + $.action( + 'PR Agent action step', + 'gynzy/pr-agent@712f0ff0c37b71c676398f73c6ea0198eb9cdd03', + continueOnError=true, + env={ + OPENAI_KEY: $.secret('OPENAI_KEY'), + GITHUB_TOKEN: $.secret('GITHUB_TOKEN'), + }, + ), + ] + ), + ], + event={ + pull_request: { + types: ['opened', 'reopened', 'ready_for_review'], + }, + issue_comment: {}, + } + ), + // Test if the changed files match the given glob patterns. // Can test for multiple pattern groups, and sets multiple outputs. // @@ -172,7 +188,7 @@ local images = import 'images.jsonnet'; // Requires the 'pull-requests': 'read' permission // // Example: - // misc.testForChangedFiles({ + // $.testForChangedFiles({ // 'app': ['packages/*/app/**/*', 'package.json'], // 'lib': ['packages/*/lib/**/*'], // }) @@ -187,8 +203,8 @@ local images = import 'images.jsonnet'; // See https://github.com/dorny/paths-filter for more information. testForChangedFiles(changedFiles, headRef=null, baseRef=null):: [ - base.step('git safe directory', 'git config --global --add safe.directory $PWD'), - base.action( + $.step('git safe directory', 'git config --global --add safe.directory $PWD'), + $.action( 'check-for-changes', uses='dorny/paths-filter@v2', id='changes', @@ -203,15 +219,6 @@ local images = import 'images.jsonnet'; ), ], - // Wait for the given jobs to finish. - // Exits successfully if all jobs are successful, otherwise exits with an error. - // - // Parameters: - // name: the name of the github job - // jobs: a list of job names to wait for - // - // Returns: - // a job that waits for the given jobs to finish awaitJob(name, jobs):: local dependingJobs = std.flatMap( function(job) @@ -220,18 +227,18 @@ local images = import 'images.jsonnet'; jobs ); [ - base.ghJob( + $.ghJob( 'await-' + name, ifClause='${{ always() }}', needs=dependingJobs, useCredentials=false, steps=[ - base.step( + $.step( 'success', 'exit 0', ifClause="${{ contains(join(needs.*.result, ','), 'success') }}" ), - base.step( + $.step( 'failure', 'exit 1', ifClause="${{ contains(join(needs.*.result, ','), 'failure') }}" @@ -239,30 +246,4 @@ local images = import 'images.jsonnet'; ], ), ], - - // Post a job to a kubernetes cluster - // - // Parameters: - // name: the name of the github job - // job_name: the name of the job to be posted - // cluster: the cluster to post the job to. This should be an object from the clusters module - // image: the image to use for the job - // environment: a map of environment variables to pass to the job - // command: the command to run in the job (optional) - postJob(name, job_name, cluster, image, environment, command=''):: - base.action( - name, - 'docker://' + images.job_poster_image, - env={ - JOB_NAME: job_name, - IMAGE: image, - COMMAND: command, - ENVIRONMENT: std.join(' ', std.objectFields(environment)), - GCE_JSON: cluster.secret, - GKE_PROJECT: cluster.project, - GKE_ZONE: cluster.zone, - GKE_CLUSTER: cluster.name, - NODESELECTOR_TYPE: cluster.jobNodeSelectorType, - } + environment, - ), } diff --git a/.github/jsonnet/mongo.jsonnet b/.github/jsonnet/mongo.jsonnet index efd6668..6e3a933 100644 --- a/.github/jsonnet/mongo.jsonnet +++ b/.github/jsonnet/mongo.jsonnet @@ -1,203 +1,65 @@ -local base = import 'base.jsonnet'; -local clusters = import 'clusters.jsonnet'; -local images = import 'images.jsonnet'; -local misc = import 'misc.jsonnet'; - -local testProjectId = '5da5889579358e19bf4b16ea'; -local prodProjectId = '5dde7f71a6f239a82fa155f4'; - -// clusterId can be found in the collections-page url: -// https://cloud.mongodb.com/v2/#/metrics/replicaSet//explorer - -local testProjectSettings = { - projectId: testProjectId, - gke_cluster: clusters.test, - CIUsername: 'github-actions', - CIPassword: misc.secret('MONGO_CI_PASSWORD_TEST'), - lifecycle: 'test', - type: 'mongodb', -}; -local prodProjectSettings = { - projectId: prodProjectId, - gke_cluster: clusters.prod, - CIUsername: 'github-actions', - CIPassword: misc.secret('MONGO_CI_PASSWORD_PROD'), - lifecycle: 'prod', - type: 'mongodb', -}; - { - // List of available MongoDB clusters. - mongo_clusters: { - test: testProjectSettings { + mongo_servers: { + test: { + type: 'mongo', name: 'test', connectionString: 'test-pri.kl1s6.gcp.mongodb.net', - clusterId: '5db8186b79358e06a806d134', + gke_cluster: 'test', + gke_project: 'gynzy-test-project', + gke_zone: 'europe-west4-b', + password_secret: 'mongodb-pass-test', + gce_json: $.secret('gce_new_test_json'), + lifecycle: 'test', + projectId: '5da5889579358e19bf4b16ea' // test }, - prod: prodProjectSettings { + prod: { + type: 'mongo', name: 'prod', connectionString: 'prod-pri.silfd.gcp.mongodb.net', - clusterId: '5dde8398cf09a237555dda74', - description: 'This cluster contains data for the scores service. It was the first mongoCluster created, and unfortunately named "prod".', + projectId: '5dde7f71a6f239a82fa155f4', // prod }, - 'board-prod': prodProjectSettings { + 'board-prod': { + type: 'mongo', name: 'board-prod', connectionString: 'board-prod-pri.silfd.mongodb.net', - clusterId: '61b0820f1ddfbc0366c41ddc', + projectId: '5dde7f71a6f239a82fa155f4', // prod }, - 'adaptive-learning-prod': prodProjectSettings { + 'adaptive-learning-prod': { + type: 'mongo', name: 'adaptive-learning-prod', connectionString: 'adaptive-learning-prod-pri.silfd.mongodb.net', - clusterId: '6239cfa3b065b60fe7af89bb', + projectId: '5dde7f71a6f239a82fa155f4', // prod }, - 'accounts-prod': prodProjectSettings { + 'accounts-prod': { + type: 'mongo', name: 'accounts-prod', connectionString: 'accounts-prod-pri.silfd.mongodb.net', - clusterId: '6322ef3b3f5f105b67ecb904', + projectId: '5dde7f71a6f239a82fa155f4', // prod }, - 'interaction-prod': prodProjectSettings { + 'interaction-prod': { + type: 'mongo', name: 'interaction-prod', connectionString: 'interaction-prod-pri.silfd.mongodb.net', - clusterId: '63d7bfaf42209e2ca71398ce', + projectId: '5dde7f71a6f239a82fa155f4', // prod }, }, - // TODO: remove - mongo_servers: self.mongo_clusters, - - // Generate a deeplink to the Atlas UI for a given cluster and database. - // - // If the database is null, the link will point to the cluster overview. - // Otherwise, it will point to the database explorer. - // - // Parameters: - // cluster: The MongoDB cluster. One of the objects from the mongo_servers list. - // database: The name of the database (optional). - // - // Returns: - // string The deeplink to the Atlas UI. - atlasDeeplink(mongoCluster, database=null):: - if database == null || mongoCluster.clusterId == null then - 'https://cloud.mongodb.com/v2/' + mongoCluster.projectId + '#clusters/detail/' + mongoCluster.name - else - 'https://cloud.mongodb.com/v2/' + mongoCluster.projectId + '#/metrics/replicaSet/' + mongoCluster.clusterId + '/explorer/' + database, - // Copy a MongoDB database to a new database. - // It does this by posting a job that runs the mongo-action image with the clone task. - // - // Parameters: - // service: The name of the service. - // mongoCluster: The MongoDB cluster to connect to. One of the objects from the mongo_servers list. - // testDatabase: The name of the source database. - // prDatabase: The name of the PR database. - // - // Returns: - // The job definition. - copyMongoDatabase(service, mongoCluster, testDatabase, prDatabase):: - assert std.length(std.findSubstr('_pr_', prDatabase)) > 0; // target db gets deleted. must contain _pr_ - - misc.postJob( - name='copy-mongo-db', - job_name='mongo-copy-' + service + '-${{ github.run_number }}-${{ github.run_attempt }}', - cluster=mongoCluster.gke_cluster, - image=images.mongo_job_image, - environment={ - TASK: 'clone', - MONGO_SRC: testDatabase, - MONGO_DST: prDatabase, - MONGO_HOST: mongoCluster.connectionString, - MONGO_USER: mongoCluster.CIUsername, - MONGO_PASS: mongoCluster.CIPassword, - }, - ), - - // Delete a MongoDB PR database. - // It does this by posting a job that runs the mongo-action image with the delete task. - // - // Parameters: - // service: The name of the service. - // mongoCluster: The MongoDB cluster to connect to. One of the objects from the mongo_servers list. - // prDatabase: The name of the PR database. - // - // Returns: - // The job definition. - deleteMongoPrDatabase(service, mongoCluster, prDatabase):: - assert std.length(std.findSubstr('_pr_', prDatabase)) > 0; // target db gets deleted. must contain _pr_ - - misc.postJob( - name='delete-mongo-db', - job_name='mongo-delete-' + service + '-${{ github.run_number }}-${{ github.run_attempt }}', - cluster=mongoCluster.gke_cluster, - image=images.mongo_job_image, - environment={ - TASK: 'delete', - MONGO_DST: prDatabase, - MONGO_HOST: mongoCluster.connectionString, - MONGO_USER: mongoCluster.CIUsername, - MONGO_PASS: mongoCluster.CIPassword, - }, - ), + copyMongoDatabase(mongoActionsOptions):: + assert std.length(std.findSubstr('_pr_', mongoActionsOptions.MONGO_DST)) > 0; // target db gets deleted. must contain _pr_ - // Sync the indexes of a MongoDB database with the current codebase. - // - // Parameters: - // service: The name of the service. - // image: The name of the Docker image to use. - // mongoCluster: The MongoDB cluster to connect to. One of the objects from the mongo_servers list. - // database: The name of the database. - // - // Returns: - // The job definition. - mongoSyncIndexes(service, image, mongoCluster, database):: - misc.postJob( - name='sync-mongo-indexes-' + mongoCluster.name + '-' + database, - job_name='mongo-sync-' + service + '-${{ github.run_number }}-${{ github.run_attempt }}', - cluster=mongoCluster.gke_cluster, - image=image, - environment={ - SERVICE: service, - MONGO_SYNC_INDEXES: 'true', - MONGO_DB: database, - MONGO_HOST: mongoCluster.connectionString, - MONGO_USER: mongoCluster.CIUsername, - MONGO_PASS: mongoCluster.CIPassword, - IS_ATLAS_MONGO: 'true', - }, - command='docker/mongo-sync-indexes.sh', + $.action( + 'copy-mongo-db', + $.mongo_action_image, + env=mongoActionsOptions { TASK: 'clone' } ), - // Generate a diff of the indexes of a MongoDB database and the currect codebase. - // The diff is posted as a comment on the pull request. - // - // Parameters: - // service: The name of the service. - // image: The name of the Docker image to use. - // mongoCluster: The MongoDB cluster to connect to. One of the objects from the mongo_servers list. - // database: The name of the database. - // - // Returns: - // The job definition. - mongoDiffIndexes(service, image, mongoCluster, database):: - local mongoDBLink = self.atlasDeeplink(mongoCluster, database); + deleteMongoPrDatabase(mongoActionsOptions):: + assert std.length(std.findSubstr('_pr_', mongoActionsOptions.MONGO_DST)) > 0; // target db gets deleted. must contain _pr_ - misc.postJob( - name='diff-mongo-indexes-' + mongoCluster.name + '-' + database, - job_name='mongo-diff-' + service + '-${{ github.run_number }}-${{ github.run_attempt }}', - cluster=mongoCluster.gke_cluster, - image=image, - environment={ - SERVICE: service, - MONGO_DIFF_INDEXES: 'true', - GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}', - GITHUB_PR: '${{ github.event.number }}', - MONGO_DB: database, - MONGO_HOST: mongoCluster.connectionString, - MONGO_USER: mongoCluster.CIUsername, - MONGO_PASS: mongoCluster.CIPassword, - MONGO_CLUSTER: mongoCluster.name, - MONGO_DEEPLINK: mongoDBLink, - IS_ATLAS_MONGO: 'true', - }, - command='docker/mongo-sync-indexes.sh', + $.action( + 'delete-mongo-db', + $.mongo_action_image, + env=mongoActionsOptions { TASK: 'delete' } ), } diff --git a/.github/jsonnet/newrelic.jsonnet b/.github/jsonnet/newrelic.jsonnet index a0b2e48..d01dccf 100644 --- a/.github/jsonnet/newrelic.jsonnet +++ b/.github/jsonnet/newrelic.jsonnet @@ -1,24 +1,20 @@ -local base = import 'base.jsonnet'; -local images = import 'images.jsonnet'; -local misc = import 'misc.jsonnet'; -local yarn = import 'yarn.jsonnet'; - { postReleaseToNewRelicJob( apps, + ):: - base.ghJob( + $.ghJob( 'post-newrelic-release', - image=images.default_backend_nest_image, + image=$.default_backend_nest_image, useCredentials=false, ifClause="${{ github.event.deployment.environment == 'production' }}", steps=[ - yarn.checkoutAndYarn(ref='${{ github.sha }}'), - base.step( + $.checkoutAndYarn(ref='${{ github.sha }}'), + $.step( 'post-newrelic-release', 'node .github/scripts/newrelic.js', env={ - NEWRELIC_API_KEY: misc.secret('NEWRELIC_API_KEY'), + NEWRELIC_API_KEY: $.secret('NEWRELIC_API_KEY'), NEWRELIC_APPS: std.join( ' ', std.flatMap( function(app) diff --git a/.github/jsonnet/notifications.jsonnet b/.github/jsonnet/notifications.jsonnet index fc1dfa5..6cb0e7d 100644 --- a/.github/jsonnet/notifications.jsonnet +++ b/.github/jsonnet/notifications.jsonnet @@ -1,9 +1,6 @@ -local base = import 'base.jsonnet'; -local images = import 'images.jsonnet'; - { notifiyDeployFailure(channel='#dev-deployments', name='notify-failure', environment='production'):: - base.action( + $.action( name, 'act10ns/slack@v2', with={ @@ -16,7 +13,7 @@ local images = import 'images.jsonnet'; ), sendSlackMessage(channel='#dev-deployments', stepName='sendSlackMessage', message=null, ifClause=null):: - base.action( + $.action( stepName, 'act10ns/slack@v2', with={ @@ -27,18 +24,4 @@ local images = import 'images.jsonnet'; }, ifClause=ifClause, ), - - // This action is used to create a deployment marker in New Relic. - // GUID is the entity guid of the application in New Relic. It can be found by All Entities > (select service) > Metadata > Entity GUID - newrelicCreateDeploymentMarker(stepName='newrelic-deployment', entityGuid):: - base.action( - stepName, - images.newrelic_deployment_marker_image, - with={ - apiKey: $.secret('NEWRELIC_API_KEY'), - guid: entityGuid, - commit: '${{ github.sha }}', - version: '${{ github.sha }}', - }, - ), } diff --git a/.github/jsonnet/pnpm.jsonnet b/.github/jsonnet/pnpm.jsonnet deleted file mode 100644 index cf25f94..0000000 --- a/.github/jsonnet/pnpm.jsonnet +++ /dev/null @@ -1,15 +0,0 @@ -local base = import 'base.jsonnet'; - -{ - install(args=[], with={}):: - base.action( - 'Install application code', - 'pnpm/action-setup@v2', - with={ - version: '^8.14.0', - run_install: ||| - - args: %(args)s - ||| % { args: args }, - } - ), -} diff --git a/.github/jsonnet/pubsub.jsonnet b/.github/jsonnet/pubsub.jsonnet index 9a6b445..702be96 100644 --- a/.github/jsonnet/pubsub.jsonnet +++ b/.github/jsonnet/pubsub.jsonnet @@ -1,19 +1,15 @@ -local base = import 'base.jsonnet'; -local misc = import 'misc.jsonnet'; -local yarn = import 'yarn.jsonnet'; - { deletePrPubsubSubscribersJob(needs=null):: - base.ghJob( + $.ghJob( 'delete-pubsub-pr-subscribers', useCredentials=false, image='google/cloud-sdk:alpine', steps=[ - yarn.configureGoogleAuth(misc.secret('GCE_NEW_TEST_JSON')), - base.step('install jq', 'apk add jq'), - base.step('show auth', 'gcloud auth list'), - base.step('wait for pod termination', 'sleep 60'), - base.step( + $.configureGoogleAuth($.secret('GCE_NEW_TEST_JSON')), + $.step('install jq', 'apk add jq'), + $.step('show auth', 'gcloud auth list'), + $.step('wait for pod termination', 'sleep 60'), + $.step( 'delete subscriptions', "\n gcloud --project gynzy-test-project pubsub subscriptions list --format json | jq -r '.[].name' | grep -- '-pr-${{ github.event.number }}' | xargs -r gcloud --project gynzy-test-project pubsub subscriptions delete" ), ], diff --git a/.github/jsonnet/pulumi.jsonnet b/.github/jsonnet/pulumi.jsonnet index 76c1c05..035061e 100644 --- a/.github/jsonnet/pulumi.jsonnet +++ b/.github/jsonnet/pulumi.jsonnet @@ -1,90 +1,36 @@ -local base = import 'base.jsonnet'; -local images = import 'images.jsonnet'; -local misc = import 'misc.jsonnet'; -local notifications = import 'notifications.jsonnet'; -local yarn = import 'yarn.jsonnet'; - -local pulumiSetupSteps = - base.action( - 'auth', - uses='google-github-actions/auth@v2', - id='auth', - with={ - credentials_json: misc.secret('PULUMI_SERVICE_ACCOUNT'), - } - ) + - base.action('setup-gcloud', uses='google-github-actions/setup-gcloud@v2') + - base.action('pulumi-cli-setup', 'pulumi/actions@v5') + - base.action('jsonnet-setup', 'kobtea/setup-jsonnet-action@v1'); - { - pulumiPreview( - stack, - pulumiDir=null, - stepName='pulumi-preview-' + stack, - environmentVariables={}, - ):: - base.action( - name=stepName, - uses='pulumi/actions@v5', + local pulumiSetupSteps = [ + $.action( + 'auth', + uses='google-github-actions/auth@v1', + id='auth', with={ - command: 'preview', - 'stack-name': stack, - 'work-dir': pulumiDir, - 'comment-on-pr': true, - 'github-token': '${{ secrets.GITHUB_TOKEN }}', - upsert: true, - refresh: true, - }, - env={ - PULUMI_CONFIG_PASSPHRASE: '${{ secrets.PULUMI_CONFIG_PASSPHRASE }}', - } + environmentVariables, + credentials_json: $.secret('PULUMI_SERVICE_ACCOUNT'), + } ), + $.action('setup-gcloud', uses='google-github-actions/setup-gcloud@v0'), + $.action('pulumi-cli-setup', 'pulumi/actions@v4'), + ], - pulumiDeploy( - stack, - pulumiDir=null, - stepName='pulumi-deploy-' + stack, - environmentVariables={}, - ):: - base.action( - name=stepName, - uses='pulumi/actions@v5', - with={ - command: 'up', - 'stack-name': stack, - 'work-dir': pulumiDir, - upsert: true, - refresh: true, - }, - env={ - PULUMI_CONFIG_PASSPHRASE: '${{ secrets.PULUMI_CONFIG_PASSPHRASE }}', - } + environmentVariables, - ), - - pulumiDestroy( + pulumiPreview( stack, pulumiDir=null, - stepName='pulumi-destroy-' + stack, - environmentVariables={}, - ):: - // pulumi destroy is a destructive operation, so we only want to run it on stacks that contain pr- - assert std.length(std.findSubstr('pr-', stack)) > 0; - - base.action( - name=stepName, - uses='pulumi/actions@v5', - with={ - command: 'destroy', - remove: true, - 'stack-name': stack, - 'work-dir': pulumiDir, - refresh: true, - }, - env={ - PULUMI_CONFIG_PASSPHRASE: '${{ secrets.PULUMI_CONFIG_PASSPHRASE }}', - } + environmentVariables, - ), + ): $.action( + 'pulumi-preview-' + stack, + uses='pulumi/actions@v4', + with={ + command: 'preview', + 'stack-name': stack, + 'work-dir': pulumiDir, + 'comment-on-pr': true, + 'github-token': '${{ secrets.GITHUB_TOKEN }}', + upsert: true, + refresh: true, + }, + env={ + PULUMI_CONFIG_PASSPHRASE: '${{ secrets.PULUMI_CONFIG_PASSPHRASE }}', + } + ), pulumiPreviewJob( stack, @@ -92,252 +38,116 @@ local pulumiSetupSteps = yarnDir=null, gitCloneRef='${{ github.event.pull_request.head.sha }}', cacheName=null, - image=images.default_pulumi_node_image, - yarnNpmSource=null, - environmentVariables={}, - additionalSetupSteps=[], - ):: - base.ghJob( - 'pulumi-preview-' + stack, - image=image, - useCredentials=false, - steps=[ - yarn.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir, source=yarnNpmSource), - pulumiSetupSteps, - additionalSetupSteps, - self.pulumiPreview(stack, pulumiDir=pulumiDir, environmentVariables=environmentVariables), - ], - ), + ): $.ghJob( + 'pulumi-preview-' + stack, + image='node:18', + useCredentials=false, + steps=[$.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir)] + + pulumiSetupSteps + + [$.pulumiPreview(stack, pulumiDir=pulumiDir)], + ), pulumiPreviewTestJob( stack='test', pulumiDir=null, yarnDir=null, - yarnNpmSource=null, gitCloneRef='${{ github.event.pull_request.head.sha }}', cacheName=null, - image=images.default_pulumi_node_image, - environmentVariables={}, - additionalSetupSteps=[], - ):: - self.pulumiPreviewJob( - stack, - pulumiDir=pulumiDir, - yarnDir=yarnDir, - yarnNpmSource=yarnNpmSource, - gitCloneRef=gitCloneRef, - cacheName=cacheName, - image=image, - environmentVariables=environmentVariables, - additionalSetupSteps=additionalSetupSteps, - ), + ): $.pulumiPreviewJob(stack, pulumiDir=pulumiDir, yarnDir=yarnDir, gitCloneRef=gitCloneRef, cacheName=cacheName), pulumiPreviewProdJob( stack='prod', pulumiDir=null, yarnDir=null, - yarnNpmSource=null, gitCloneRef='${{ github.event.pull_request.head.sha }}', cacheName=null, - image=images.default_pulumi_node_image, - environmentVariables={}, - additionalSetupSteps=[], - ):: - self.pulumiPreviewJob( - stack, - pulumiDir=pulumiDir, - yarnDir=yarnDir, - yarnNpmSource=yarnNpmSource, - gitCloneRef=gitCloneRef, - cacheName=cacheName, - image=image, - environmentVariables=environmentVariables, - additionalSetupSteps=additionalSetupSteps, - ), + ): $.pulumiPreviewJob(stack, pulumiDir=pulumiDir, yarnDir=yarnDir, gitCloneRef=gitCloneRef, cacheName=cacheName), pulumiPreviewTestAndProdJob( pulumiDir=null, yarnDir=null, - yarnNpmSource=null, gitCloneRef='${{ github.event.pull_request.head.sha }}', cacheName=null, - image=images.default_pulumi_node_image, - productionStack='prod', - testStack='test', - environmentVariables={}, - additionalSetupSteps=[], - ):: - base.ghJob( - 'pulumi-preview', - image=image, - useCredentials=false, - steps=[ - yarn.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir, source=yarnNpmSource), - pulumiSetupSteps, - additionalSetupSteps, - self.pulumiPreview(testStack, pulumiDir=pulumiDir, environmentVariables=environmentVariables), - self.pulumiPreview(productionStack, pulumiDir=pulumiDir, environmentVariables=environmentVariables), - ], - ), + ): $.ghJob( + 'pulumi-preview', + image='node:18', + useCredentials=false, + steps=[$.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir)] + + pulumiSetupSteps + + [ + $.pulumiPreview('test', pulumiDir=pulumiDir), + $.pulumiPreview('prod', pulumiDir=pulumiDir), + ], + ), pulumiDeployJob( stack, pulumiDir=null, yarnDir=null, - yarnNpmSource=null, gitCloneRef='${{ github.sha }}', cacheName=null, ifClause=null, - image=images.default_pulumi_node_image, - jobName='pulumi-deploy-' + stack, - notifyOnFailure=true, - environmentVariables={}, - additionalSetupSteps=[], - ):: - base.ghJob( - name=jobName, - ifClause=ifClause, - image=image, - useCredentials=false, - steps=[ - yarn.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir, source=yarnNpmSource), - pulumiSetupSteps, - additionalSetupSteps, - self.pulumiDeploy(stack, pulumiDir=pulumiDir, stepName=jobName, environmentVariables=environmentVariables), - if notifyOnFailure then notifications.notifiyDeployFailure(environment=stack) else [], - ] - ), + ): $.ghJob( + 'pulumi-deploy-' + stack, + ifClause=ifClause, + image='node:18', + useCredentials=false, + steps=[$.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir)] + + pulumiSetupSteps + + [ + $.action( + 'pulumi-deploy-' + stack, + uses='pulumi/actions@v4', + with={ + command: 'up', + 'stack-name': stack, + 'work-dir': pulumiDir, + upsert: true, + refresh: true, + }, + env={ + PULUMI_CONFIG_PASSPHRASE: '${{ secrets.PULUMI_CONFIG_PASSPHRASE }}', + } + ), + $.notifiyDeployFailure(), + ], + ), pulumiDeployTestJob( stack='test', pulumiDir=null, yarnDir=null, - yarnNpmSource=null, gitCloneRef='${{ github.sha }}', cacheName=null, - image=images.default_pulumi_node_image, ifClause="${{ github.event.deployment.environment == 'test' }}", - environmentVariables={}, - additionalSetupSteps=[], - ):: - self.pulumiDeployJob( - stack, - pulumiDir=pulumiDir, - yarnDir=yarnDir, - yarnNpmSource=yarnNpmSource, - gitCloneRef=gitCloneRef, - cacheName=cacheName, - ifClause=ifClause, - image=image, - environmentVariables=environmentVariables, - additionalSetupSteps=additionalSetupSteps, - ), + ): $.pulumiDeployJob(stack, pulumiDir=pulumiDir, yarnDir=yarnDir, gitCloneRef=gitCloneRef, cacheName=cacheName, ifClause=ifClause), pulumiDeployProdJob( stack='prod', pulumiDir=null, yarnDir=null, - yarnNpmSource=null, gitCloneRef='${{ github.sha }}', cacheName=null, - image=images.default_pulumi_node_image, ifClause="${{ github.event.deployment.environment == 'prod' || github.event.deployment.environment == 'production' }}", - environmentVariables={}, - additionalSetupSteps=[], - ):: - self.pulumiDeployJob( - stack, - pulumiDir=pulumiDir, - yarnDir=yarnDir, - yarnNpmSource=yarnNpmSource, - gitCloneRef=gitCloneRef, - cacheName=cacheName, - ifClause=ifClause, - image=image, - environmentVariables=environmentVariables, - additionalSetupSteps=additionalSetupSteps, - ), - - pulumiDestroyJob( - stack, - pulumiDir=null, - yarnDir=null, - yarnNpmSource=null, - gitCloneRef='${{ github.sha }}', - cacheName=null, - ifClause=null, - image=images.default_pulumi_node_image, - jobName='pulumi-destroy-' + stack, - notifyOnFailure=true, - environmentVariables={}, - additionalSetupSteps=[], - ):: - base.ghJob( - name=jobName, - ifClause=ifClause, - image=image, - useCredentials=false, - steps=[ - yarn.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir, source=yarnNpmSource), - pulumiSetupSteps, - additionalSetupSteps, - self.pulumiDestroy(stack, pulumiDir=pulumiDir, stepName=jobName, environmentVariables=environmentVariables), - if notifyOnFailure then notifications.notifiyDeployFailure(environment=stack) else [], - ], - ), - + ): $.pulumiDeployJob(stack, pulumiDir=pulumiDir, yarnDir=yarnDir, gitCloneRef=gitCloneRef, cacheName=cacheName, ifClause=ifClause), pulumiDefaultPipeline( pulumiDir='.', yarnDir=null, - yarnNpmSource=null, cacheName=null, deployTestWithProd=false, - image=images.default_pulumi_node_image, - testStack='test', - productionStack='prod', - environmentVariables={}, - additionalSetupSteps=[], - ):: - base.pipeline( + ): + $.pipeline( 'pulumi-preview', [ - self.pulumiPreviewTestAndProdJob( - pulumiDir=pulumiDir, - yarnDir=yarnDir, - yarnNpmSource=yarnNpmSource, - cacheName=cacheName, - image=image, - productionStack=productionStack, - testStack=testStack, - environmentVariables=environmentVariables, - additionalSetupSteps=additionalSetupSteps, - ), + $.pulumiPreviewTestAndProdJob(pulumiDir=pulumiDir, yarnDir=yarnDir, cacheName=cacheName), ], ) + - base.pipeline( + $.pipeline( 'pulumi-deploy', [ - self.pulumiDeployTestJob( - pulumiDir=pulumiDir, - yarnDir=yarnDir, - yarnNpmSource=yarnNpmSource, - cacheName=cacheName, - image=image, - environmentVariables=environmentVariables, - additionalSetupSteps=additionalSetupSteps, - ifClause=if deployTestWithProd then "${{ github.event.deployment.environment == 'test' || github.event.deployment.environment == 'prod' || github.event.deployment.environment == 'production' }}" else "${{ github.event.deployment.environment == 'test' }}" - ), - self.pulumiDeployProdJob( - pulumiDir=pulumiDir, - yarnDir=yarnDir, - yarnNpmSource=yarnNpmSource, - cacheName=cacheName, - image=image, - stack=productionStack, - environmentVariables=environmentVariables, - additionalSetupSteps=additionalSetupSteps, - ), + $.pulumiDeployTestJob(pulumiDir=pulumiDir, yarnDir=yarnDir, cacheName=cacheName, ifClause=if deployTestWithProd then "${{ github.event.deployment.environment == 'test' || github.event.deployment.environment == 'prod' || github.event.deployment.environment == 'production' }}" else "${{ github.event.deployment.environment == 'test' }}"), + $.pulumiDeployProdJob(pulumiDir=pulumiDir, yarnDir=yarnDir, cacheName=cacheName), ], event='deployment', ), diff --git a/.github/jsonnet/ruby.jsonnet b/.github/jsonnet/ruby.jsonnet index e0cef8d..bafda85 100644 --- a/.github/jsonnet/ruby.jsonnet +++ b/.github/jsonnet/ruby.jsonnet @@ -1,11 +1,3 @@ -local base = import 'base.jsonnet'; -local database = import 'databases.jsonnet'; -local docker = import 'docker.jsonnet'; -local helm = import 'helm.jsonnet'; -local misc = import 'misc.jsonnet'; -local notifications = import 'notifications.jsonnet'; -local services = import 'services.jsonnet'; - { rubyDeployPRPipeline( serviceName, @@ -25,7 +17,7 @@ local services = import 'services.jsonnet'; database_name_source: serviceName, database_host: 'cloudsql-proxy', database_username: serviceName, - database_password: misc.secret('database_password_test'), + database_password: $.secret('database_password_test'), } + mysqlCloneOptions; local migrateOptionsWithDefaults = { @@ -33,35 +25,34 @@ local services = import 'services.jsonnet'; RAILS_ENV: 'production', RAILS_DB_HOST: 'cloudsql-proxy', RAILS_DB_NAME: serviceName + '_pr_${{ github.event.number }}', - RAILS_DB_PASSWORD: misc.secret('database_password_test'), + RAILS_DB_PASSWORD: $.secret('database_password_test'), RAILS_DB_USER: serviceName, - SECRET_KEY_BASE: misc.secret('rails_secret_test'), } + migrateOptions; - base.pipeline( + $.pipeline( 'deploy-pr', [ - base.ghJob( + $.ghJob( 'deploy-pr', image=rubyImageName, steps=[ - misc.checkout(ref='${{ github.event.pull_request.head.sha }}'), - self.setVerionFile(), + $.checkout(ref='${{ github.event.pull_request.head.sha }}'), + $.setVerionFile(), ] + - (if mysqlCloneOptionsWithDefaults.enabled then [database.copyDatabase(mysqlCloneOptionsWithDefaults)] else []) + - (if migrateOptionsWithDefaults.enabled then self.rubyMigrate(migrateOptionsWithDefaults) else []) + + (if mysqlCloneOptionsWithDefaults.enabled then [$.copyDatabase(mysqlCloneOptionsWithDefaults)] else []) + + (if migrateOptionsWithDefaults.enabled then $.rubyMigrate(migrateOptionsWithDefaults) else []) + [ - docker.buildDocker( + $.buildDocker( dockerImageName, env={ - BUNDLE_GITHUB__COM: misc.secret('BUNDLE_GITHUB__COM'), + BUNDLE_GITHUB__COM: $.secret('BUNDLE_GITHUB__COM'), }, - build_args='BUNDLE_GITHUB__COM=' + misc.secret('BUNDLE_GITHUB__COM'), + build_args='BUNDLE_GITHUB__COM=' + $.secret('BUNDLE_GITHUB__COM'), ), - helm.helmDeployPR(serviceName, helmDeployOptions), + $.helmDeployPR(serviceName, helmDeployOptions), ], services={} + - (if mysqlCloneOptionsWithDefaults.enabled then { 'cloudsql-proxy': services.cloudsql_proxy_service(mysqlCloneOptionsWithDefaults.database) } else {}) + (if mysqlCloneOptionsWithDefaults.enabled then { 'cloudsql-proxy': $.cloudsql_proxy_service(mysqlCloneOptionsWithDefaults.database) } else {}) ), ], event='pull_request', @@ -69,20 +60,18 @@ local services = import 'services.jsonnet'; rubyMigrate(migrateOptions):: local env = { - BUNDLE_GITHUB__COM: misc.secret('BUNDLE_GITHUB__COM'), + BUNDLE_GITHUB__COM: $.secret('BUNDLE_GITHUB__COM'), SSO_PUBLIC_KEY: '', RAILS_ENV: 'production', RAILS_DB_HOST: migrateOptions.RAILS_DB_HOST, RAILS_DB_NAME: migrateOptions.RAILS_DB_NAME, RAILS_DB_PASSWORD: migrateOptions.RAILS_DB_PASSWORD, RAILS_DB_USER: migrateOptions.RAILS_DB_USER, - SECRET_KEY_BASE: migrateOptions.SECRET_KEY_BASE, }; [ - base.step('bundle install', 'bundle install', env={ BUNDLE_GITHUB__COM: misc.secret('BUNDLE_GITHUB__COM') }), - base.step('migrate-db', 'rails db:migrate;', env=env), - base.step('seed-db', 'rails db:seed;', env=env), + $.step('bundle install', 'bundle install', env={ BUNDLE_GITHUB__COM: $.secret('BUNDLE_GITHUB__COM') }), + $.step('migrate-db', 'rails db:migrate;', env=env), ] , @@ -91,24 +80,24 @@ local services = import 'services.jsonnet'; enableDatabase=false, generateCommands=null, extra_env={}, - services={ db: services.mysql57service(database='ci', password='ci', root_password='1234test', username='ci') }, + services={ db: $.mysql57service(database='ci', password='ci', root_password='1234test', username='ci') }, rubyImageName=null, ):: assert rubyImageName != null; - base.ghJob( + $.ghJob( 'apidocs', image=rubyImageName, ifClause="${{ github.event.deployment.environment == 'production' }}", steps=[ - misc.checkout(), - base.step( + $.checkout(), + $.step( 'generate', (if generateCommands != null then generateCommands else ' bundle config --delete without;\n bundle install;\n bundle exec rails db:test:prepare;\n bundle exec rails docs:generate;\n '), env={ RAILS_ENV: 'test', - GOOGLE_PRIVATE_KEY: misc.secret('GOOGLE_PRIVATE_KEY'), - BUNDLE_GITHUB__COM: misc.secret('BUNDLE_GITHUB__COM'), + GOOGLE_PRIVATE_KEY: $.secret('GOOGLE_PRIVATE_KEY'), + BUNDLE_GITHUB__COM: $.secret('BUNDLE_GITHUB__COM'), } + (if enableDatabase then { @@ -118,22 +107,22 @@ local services = import 'services.jsonnet'; RAILS_DB_USER: 'ci', } else {}) + extra_env ), - base.action( + $.action( 'setup auth', - 'google-github-actions/auth@v2', + 'google-github-actions/auth@v1', with={ - credentials_json: misc.secret('GCE_JSON'), + credentials_json: $.secret('GCE_JSON'), }, id='auth', ), - base.action('setup-gcloud', 'google-github-actions/setup-gcloud@v2'), - base.step('deploy-api-docs', 'gsutil -m cp -r doc/api/** gs://apidocs.gynzy.com/' + serviceName + '/'), + $.action('setup-gcloud', 'google-github-actions/setup-gcloud@v0'), + $.step('deploy-api-docs', 'gsutil -m cp -r doc/api/** gs://apidocs.gynzy.com/' + serviceName + '/'), ], services=(if enableDatabase then services else null), ), setVerionFile(version='${{ github.event.pull_request.head.sha }}', file='VERSION'):: - base.step( + $.step( 'set-version', 'echo "' + version + '" > ' + file + ';\n echo "Generated version number:";\n cat ' + file + ';\n ' ), @@ -150,13 +139,13 @@ local services = import 'services.jsonnet'; database_name_target: serviceName + '_pr_${{ github.event.number }}', database_host: 'cloudsql-proxy', database_username: serviceName, - database_password: misc.secret('database_password_test'), + database_password: $.secret('database_password_test'), } + mysqlDeleteOptions; - base.pipeline( + $.pipeline( 'close-pr', [ - helm.helmDeletePRJob(serviceName, options, helmPath, deploymentName, mysqlDeleteOptionsWithDefaults), + $.helmDeletePRJob(serviceName, options, helmPath, deploymentName, mysqlDeleteOptionsWithDefaults), ], event={ pull_request: { @@ -180,22 +169,21 @@ local services = import 'services.jsonnet'; RAILS_ENV: 'production', RAILS_DB_HOST: 'cloudsql-proxy', RAILS_DB_NAME: serviceName, - RAILS_DB_PASSWORD: misc.secret('database_password_test'), + RAILS_DB_PASSWORD: $.secret('database_password_test'), RAILS_DB_USER: serviceName, - SECRET_KEY_BASE: misc.secret('rails_secret_test'), } + migrateOptions; - base.ghJob( + $.ghJob( 'deploy-test', ifClause="${{ github.event.deployment.environment == 'test' }}", image=image, useCredentials=useCredentials, steps= - [misc.checkout()] + - (if migrateOptionsWithDefaults.enabled then self.rubyMigrate(migrateOptionsWithDefaults) else []) + - [helm.helmDeployTest(serviceName, options, helmPath, deploymentName)], + [$.checkout()] + + (if migrateOptionsWithDefaults.enabled then $.rubyMigrate(migrateOptionsWithDefaults) else []) + + [$.helmDeployTest(serviceName, options, helmPath, deploymentName)], services={} + - (if migrateOptionsWithDefaults.enabled then { 'cloudsql-proxy': services.cloudsql_proxy_service(migrateOptionsWithDefaults.database) } else {}) + (if migrateOptionsWithDefaults.enabled then { 'cloudsql-proxy': $.cloudsql_proxy_service(migrateOptionsWithDefaults.database) } else {}) ), rubyDeployProdJob( @@ -213,20 +201,21 @@ local services = import 'services.jsonnet'; RAILS_ENV: 'production', RAILS_DB_HOST: 'cloudsql-proxy', RAILS_DB_NAME: serviceName, - RAILS_DB_PASSWORD: misc.secret('database_password_production'), + RAILS_DB_PASSWORD: $.secret('database_password_production'), RAILS_DB_USER: serviceName, - SECRET_KEY_BASE: misc.secret('rails_secret_production'), } + migrateOptions; - base.ghJob( + $.ghJob( 'deploy-prod', ifClause="${{ github.event.deployment.environment == 'production' }}", image=image, useCredentials=useCredentials, - steps=[misc.checkout()] + - (if migrateOptionsWithDefaults.enabled then self.rubyMigrate(migrateOptionsWithDefaults) else []) + - [helm.helmDeployProd(serviceName, options, helmPath, deploymentName)] + [notifications.notifiyDeployFailure()], + steps=[$.checkout()] + + (if migrateOptionsWithDefaults.enabled then $.rubyMigrate(migrateOptionsWithDefaults) else []) + + [$.helmDeployProd(serviceName, options, helmPath, deploymentName)] + [$.notifiyDeployFailure()], services={} + - (if migrateOptionsWithDefaults.enabled then { 'cloudsql-proxy': services.cloudsql_proxy_service(migrateOptionsWithDefaults.database) } else {}) + (if migrateOptionsWithDefaults.enabled then { 'cloudsql-proxy': $.cloudsql_proxy_service(migrateOptionsWithDefaults.database) } else {}) ), + + } diff --git a/.github/jsonnet/services.jsonnet b/.github/jsonnet/services.jsonnet index f1fd3ae..35fbb63 100644 --- a/.github/jsonnet/services.jsonnet +++ b/.github/jsonnet/services.jsonnet @@ -1,13 +1,10 @@ -local images = import 'images.jsonnet'; -local misc = import 'misc.jsonnet'; - { mysql57service(database=null, password=null, root_password=null, username=null, port='3306'):: { - image: images.default_mysql57_image, + image: $.default_mysql57_image, credentials: { username: '_json_key', - password: misc.secret('docker_gcr_io'), + password: $.secret('docker_gcr_io'), }, env: { MYSQL_DATABASE: database, @@ -22,10 +19,10 @@ local misc = import 'misc.jsonnet'; mysql8service(database=null, password=null, root_password=null, username=null, port='3306'):: { - image: images.default_mysql8_image, + image: $.default_mysql8_image, credentials: { username: '_json_key', - password: misc.secret('docker_gcr_io'), + password: $.secret('docker_gcr_io'), }, env: { MYSQL_DATABASE: database, @@ -40,27 +37,27 @@ local misc = import 'misc.jsonnet'; cloudsql_proxy_service(database):: { - image: images.default_cloudsql_image, + image: $.default_cloudsql_image, credentials: { username: '_json_key', - password: misc.secret('docker_gcr_io'), + password: $.secret('docker_gcr_io'), }, env: { GOOGLE_PROJECT: database.project, CLOUDSQL_ZONE: database.region, CLOUDSQL_INSTANCE: database.server, - SERVICE_JSON: misc.secret('GCE_JSON'), + SERVICE_JSON: $.secret('GCE_JSON'), }, ports: ['3306:3306'], }, redis_service():: { - image: images.default_redis_image, + image: $.default_redis_image, ports: ['6379:6379'], }, pubsub_service():: { - image: images.default_pubsub_image, + image: $.default_pubsub_image, ports: ['8681:8681'], }, @@ -71,11 +68,11 @@ local misc = import 'misc.jsonnet'; password='therootpass', ):: { [name]: { - image: images.default_mongodb_image, + image: $.default_mongodb_image, ports: ['27017:27017'], credentials: { username: '_json_key', - password: misc.secret('docker_gcr_io'), + password: $.secret('docker_gcr_io'), }, env: { MONGO_INITDB_ROOT_USERNAME: username, diff --git a/.github/jsonnet/yarn.jsonnet b/.github/jsonnet/yarn.jsonnet index a688544..752cca5 100644 --- a/.github/jsonnet/yarn.jsonnet +++ b/.github/jsonnet/yarn.jsonnet @@ -1,21 +1,16 @@ -local base = import 'base.jsonnet'; -local cache = import 'cache.jsonnet'; -local images = import 'images.jsonnet'; -local misc = import 'misc.jsonnet'; - { yarn(ifClause=null, prod=false, workingDirectory=null):: - base.step( + $.step( 'yarn' + (if prod then '-prod' else ''), run='yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline' + (if prod then ' --prod' else '') + ' || yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline' + (if prod then ' --prod' else ''), ifClause=ifClause, workingDirectory=workingDirectory ), - setNpmToken(ifClause=null, workingDirectory=null):: self.setGynzyNpmToken(ifClause=ifClause, workingDirectory=workingDirectory), + setNpmToken(ifClause=null, workingDirectory=null):: $.setGynzyNpmToken(ifClause=ifClause, workingDirectory=workingDirectory), setGynzyNpmToken(ifClause=null, workingDirectory=null):: - base.step( + $.step( 'set gynzy npm_token', run= ||| @@ -26,14 +21,14 @@ local misc = import 'misc.jsonnet'; EOF |||, env={ - NPM_TOKEN: misc.secret('npm_token'), + NPM_TOKEN: $.secret('npm_token'), }, ifClause=ifClause, workingDirectory=workingDirectory, ), setGithubNpmToken(ifClause=null, workingDirectory=null):: - base.step( + $.step( 'set github npm_token', run= ||| @@ -44,53 +39,83 @@ local misc = import 'misc.jsonnet'; EOF |||, env={ - NODE_AUTH_TOKEN: misc.secret('GITHUB_TOKEN'), + NODE_AUTH_TOKEN: $.secret('GITHUB_TOKEN'), }, ifClause=ifClause, workingDirectory=workingDirectory, ), - checkoutAndYarn(cacheName=null, ifClause=null, fullClone=false, ref=null, prod=false, workingDirectory=null, source='gynzy'):: - misc.checkout(ifClause=ifClause, fullClone=fullClone, ref=ref) + - (if source == 'gynzy' then self.setGynzyNpmToken(ifClause=ifClause, workingDirectory=workingDirectory) else []) + - (if source == 'github' then self.setGithubNpmToken(ifClause=ifClause, workingDirectory=workingDirectory) else []) + - (if cacheName == null then [] else self.fetchYarnCache(cacheName, ifClause=ifClause, workingDirectory=workingDirectory)) + - self.yarn(ifClause=ifClause, prod=prod, workingDirectory=workingDirectory), - - fetchYarnCache(cacheName, ifClause=null, workingDirectory=null):: - cache.fetchCache( - cacheName=cacheName, - folders=['.yarncache'], - additionalCleanupCommands=["find . -type d -name 'node_modules' | xargs rm -rf"], - ifClause=ifClause, - workingDirectory=workingDirectory - ), + checkoutAndYarn(cacheName=null, ifClause=null, fullClone=false, ref=null, prod=false, workingDirectory=null):: + $.checkout(ifClause=ifClause, fullClone=fullClone, ref=ref) + + $.setGynzyNpmToken(ifClause=ifClause, workingDirectory=workingDirectory) + + (if cacheName == null then [] else $.fetchYarnCache(cacheName, ifClause=ifClause, workingDirectory=workingDirectory)) + + $.yarn(ifClause=ifClause, prod=prod, workingDirectory=workingDirectory), + + fetchYarnCache(cacheName, ifClause=null, workingDirectory=null):: $.step( + 'download yarn cache', + run= + ||| + set +e; + echo "Downloading yarn cache && node_modules" + wget -q -O - "https://storage.googleapis.com/files-gynzy-com-test/yarn-cache/${CACHE_NAME}.tar.gz" | tar xfz - + if [ $? -ne 0 ]; then + # download failed. cleanup node_modules because it can contain a yarn integrity file but not have all the data as specified + echo "Cache download failed, cleanup up caches and run yarn without cache" + find . -type d -name 'node_modules' | xargs rm -rf + rm -rf .yarncache + echo "Cleanup complete" + else + echo "Finished downloading yarn cache && node_modules" + fi + |||, + ifClause=ifClause, + workingDirectory=workingDirectory, + env={ + CACHE_NAME: cacheName, + }, + ), updateYarnCachePipeline(cacheName, appsDir='packages', image=null, useCredentials=null):: - base.pipeline( + $.pipeline( 'update-yarn-cache', [ - base.ghJob( + $.ghJob( 'update-yarn-cache', image=image, useCredentials=useCredentials, ifClause="${{ github.event.deployment.environment == 'production' || github.event.deployment.environment == 'prod' }}", steps=[ - misc.checkout() + - self.setGynzyNpmToken() + - self.yarn(), - base.action( + $.checkout() + + $.setGynzyNpmToken() + + $.yarn(), + $.action( 'setup auth', - 'google-github-actions/auth@v2', + 'google-github-actions/auth@v1', with={ - credentials_json: misc.secret('SERVICE_JSON'), + credentials_json: $.secret('SERVICE_JSON'), }, id='auth', ), - base.action('setup-gcloud', 'google-github-actions/setup-gcloud@v2'), - cache.uploadCache( - cacheName=cacheName, - tarCommand='ls "' + appsDir + '/*/node_modules" -1 -d 2>/dev/null | xargs tar -c .yarncache node_modules', + $.action('setup-gcloud', 'google-github-actions/setup-gcloud@v0'), + $.step( + 'upload-yarn-cache', + ||| + set -e + + echo "Creating cache archive" + # v1 + ls "${APPS_DIR}/*/node_modules" -1 -d 2>/dev/null | xargs tar -czf "${CACHE_NAME}.tar.gz" .yarncache node_modules + + echo "Upload cache" + gsutil cp "${CACHE_NAME}.tar.gz" "gs://files-gynzy-com-test/yarn-cache/${CACHE_NAME}.tar.gz.tmp" + gsutil mv "gs://files-gynzy-com-test/yarn-cache/${CACHE_NAME}.tar.gz.tmp" "gs://files-gynzy-com-test/yarn-cache/${CACHE_NAME}.tar.gz" + + echo "Upload finished" + |||, + env={ + CACHE_NAME: cacheName, + APPS_DIR: appsDir, + } ), ], ), @@ -98,7 +123,7 @@ local misc = import 'misc.jsonnet'; event='deployment', ), - configureGoogleAuth(secret):: base.step( + configureGoogleAuth(secret):: $.step( 'activate google service account', run= ||| @@ -111,7 +136,7 @@ local misc = import 'misc.jsonnet'; ), yarnPublish(isPr=true, ifClause=null):: - base.step( + $.step( 'publish', ||| bash -c 'set -xeo pipefail; @@ -150,8 +175,8 @@ local misc = import 'misc.jsonnet'; yarnPublishToRepositories(isPr, repositories, ifClause=null):: (std.flatMap(function(repository) - if repository == 'gynzy' then [self.setGynzyNpmToken(ifClause=ifClause), self.yarnPublish(isPr=isPr, ifClause=ifClause)] - else if repository == 'github' then [self.setGithubNpmToken(ifClause=ifClause), self.yarnPublish(isPr=isPr, ifClause=ifClause)] + if repository == 'gynzy' then [$.setGynzyNpmToken(ifClause=ifClause), $.yarnPublish(isPr=isPr, ifClause=ifClause)] + else if repository == 'github' then [$.setGithubNpmToken(ifClause=ifClause), $.yarnPublish(isPr=isPr, ifClause=ifClause)] else error 'Unknown repository type given.', repositories)), @@ -160,30 +185,30 @@ local misc = import 'misc.jsonnet'; image='node:18', useCredentials=false, gitCloneRef='${{ github.event.pull_request.head.sha }}', - buildSteps=[base.step('build', 'yarn build')], + buildSteps=[$.step('build', 'yarn build')], checkVersionBump=true, repositories=['gynzy'], onChangedFiles=false, changedFilesHeadRef=null, changedFilesBaseRef=null, runsOn=null, - ):: + ): local ifClause = (if onChangedFiles != false then "steps.changes.outputs.package == 'true'" else null); - base.ghJob( + $.ghJob( 'yarn-publish-preview', runsOn=runsOn, image='node:18', useCredentials=false, steps= - [self.checkoutAndYarn(ref=gitCloneRef, fullClone=false)] + - (if onChangedFiles != false then misc.testForChangedFiles({ package: onChangedFiles }, headRef=changedFilesHeadRef, baseRef=changedFilesBaseRef) else []) + + [$.checkoutAndYarn(ref=gitCloneRef, fullClone=false)] + + (if onChangedFiles != false then $.testForChangedFiles({ package: onChangedFiles }, headRef=changedFilesHeadRef, baseRef=changedFilesBaseRef) else []) + (if checkVersionBump then [ - base.action('check-version-bump', uses='del-systems/check-if-version-bumped@v1', with={ + $.action('check-version-bump', uses='del-systems/check-if-version-bumped@v1', with={ token: '${{ github.token }}', }, ifClause=ifClause), ] else []) + (if onChangedFiles != false then std.map(function(step) std.map(function(s) s { 'if': ifClause }, step), buildSteps) else buildSteps) + - self.yarnPublishToRepositories(isPr=true, repositories=repositories, ifClause=ifClause), + $.yarnPublishToRepositories(isPr=true, repositories=repositories, ifClause=ifClause), permissions={ packages: 'write', contents: 'read', 'pull-requests': 'read' }, ), @@ -191,25 +216,25 @@ local misc = import 'misc.jsonnet'; image='node:18', useCredentials=false, gitCloneRef='${{ github.sha }}', - buildSteps=[base.step('build', 'yarn build')], + buildSteps=[$.step('build', 'yarn build')], repositories=['gynzy'], onChangedFiles=false, changedFilesHeadRef=null, changedFilesBaseRef=null, ifClause=null, runsOn=null, - ):: + ): local stepIfClause = (if onChangedFiles != false then "steps.changes.outputs.package == 'true'" else null); - base.ghJob( + $.ghJob( 'yarn-publish', image='node:18', runsOn=runsOn, useCredentials=false, steps= - [misc.checkoutAndYarn(ref=gitCloneRef, fullClone=false)] + - (if onChangedFiles != false then misc.testForChangedFiles({ package: onChangedFiles }, headRef=changedFilesHeadRef, baseRef=changedFilesBaseRef) else []) + + [$.checkoutAndYarn(ref=gitCloneRef, fullClone=false)] + + (if onChangedFiles != false then $.testForChangedFiles({ package: onChangedFiles }, headRef=changedFilesHeadRef, baseRef=changedFilesBaseRef) else []) + (if onChangedFiles != false then std.map(function(step) std.map(function(s) s { 'if': stepIfClause }, step), buildSteps) else buildSteps) + - self.yarnPublishToRepositories(isPr=false, repositories=repositories, ifClause=stepIfClause), + $.yarnPublishToRepositories(isPr=false, repositories=repositories, ifClause=stepIfClause), permissions={ packages: 'write', contents: 'read', 'pull-requests': 'read' }, ifClause=ifClause, ), diff --git a/.github/workflows/misc.yml b/.github/workflows/misc.yml new file mode 100644 index 0000000..174a45d --- /dev/null +++ b/.github/workflows/misc.yml @@ -0,0 +1,28 @@ +"jobs": + "verify-jsonnet-gh-actions": + "container": + "credentials": + "password": "${{ secrets.docker_gcr_io }}" + "username": "_json_key" + "image": "eu.gcr.io/unicorn-985/docker-images_jsonnet:v1" + "runs-on": "ubuntu-latest" + "steps": + - "name": "Check out repository code" + "uses": "actions/checkout@v3" + "with": + "ref": "${{ github.event.pull_request.head.sha }}" + - "name": "remove-workflows" + "run": "rm -f .github/workflows/*" + - "name": "generate-workflows" + "run": "jsonnet -m .github/workflows/ -S .github.jsonnet;" + - "name": "git workaround" + "run": "git config --global --add safe.directory $PWD" + - "name": "check-jsonnet-diff" + "run": "git diff --exit-code" + - "if": "failure()" + "name": "possible-causes-for-error" + "run": "echo \"Possible causes: \n1. You updated jsonnet files, but did not regenerate the workflows. \nTo fix, run 'yarn github:generate' locally and commit the changes. If this helps, check if your pre-commit hooks work.\n2. You used the wrong jsonnet binary. In this case, the newlines at the end of the files differ.\nTo fix, install the go binary. On mac, run 'brew uninstall jsonnet && brew install jsonnet-go'\"" + "timeout-minutes": 30 +"name": "misc" +"on": +- "pull_request" \ No newline at end of file diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 0000000..5774dd6 --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,94 @@ +"jobs": + "test": + "runs-on": "ubuntu-latest" + "steps": + - "name": "Check out repository code" + "uses": "actions/checkout@v3" + "with": + "ref": "${{ github.event.pull_request.head.ref }}" + - "name": "setup node" + "uses": "actions/setup-node@v3" + "with": + "node-version": 18 + - "name": "Start MongoDB" + "uses": "supercharge/mongodb-github-action@v1" + "with": + "mongodb-replica-set": "rs0" + "mongodb-version": 5 + - "name": "yarn" + "run": "yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline || yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline" + - "name": "test" + "run": "yarn test" + "timeout-minutes": 30 + "yarn-publish-preview": + "container": + "image": "node:18" + "permissions": + "contents": "read" + "packages": "write" + "pull-requests": "read" + "runs-on": "ubuntu-latest" + "steps": + - "name": "Check out repository code" + "uses": "actions/checkout@v3" + "with": + "ref": "${{ github.event.pull_request.head.sha }}" + - "env": + "NPM_TOKEN": "${{ secrets.npm_token }}" + "name": "set gynzy npm_token" + "run": | + cat < .npmrc + registry=https://npm.gynzy.net/ + always-auth="true" + "//npm.gynzy.net/:_authToken"="${NPM_TOKEN}" + EOF + - "name": "yarn" + "run": "yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline || yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline" + - "name": "build" + "run": "yarn build" + - "env": + "NPM_TOKEN": "${{ secrets.npm_token }}" + "name": "set gynzy npm_token" + "run": | + cat < .npmrc + registry=https://npm.gynzy.net/ + always-auth="true" + "//npm.gynzy.net/:_authToken"="${NPM_TOKEN}" + EOF + - "env": + "PR_NUMBER": "${{ github.event.number }}" + "name": "publish" + "run": | + bash -c 'set -xeo pipefail; + + cp package.json package.json.bak; + + VERSION=$(yarn version --non-interactive 2>/dev/null | grep "Current version" | grep -o -P '[0-9a-zA-Z_.-]+$' ); + if [[ ! -z "${PR_NUMBER}" ]]; then + echo "Setting tag/version for pr build."; + TAG=pr-$PR_NUMBER; + PUBLISHVERSION="$VERSION-pr$PR_NUMBER.$GITHUB_RUN_NUMBER"; + elif [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then + if [[ "${GITHUB_REF_NAME}" != "${VERSION}" ]]; then + echo "Tag version does not match package version. They should match. Exiting"; + exit 1; + fi + echo "Setting tag/version for release/tag build."; + PUBLISHVERSION=$VERSION; + TAG="latest"; + elif [[ "${GITHUB_REF_TYPE}" == "branch" && ( "${GITHUB_REF_NAME}" == "main" || "${GITHUB_REF_NAME}" == "master" ) ]] || [[ "${GITHUB_EVENT_NAME}" == "deployment" ]]; then + echo "Setting tag/version for release/tag build."; + PUBLISHVERSION=$VERSION; + TAG="latest"; + else + exit 1 + fi + + yarn publish --non-interactive --no-git-tag-version --tag "$TAG" --new-version "$PUBLISHVERSION"; + + mv package.json.bak package.json; + '; + "timeout-minutes": 30 +"name": "pr" +"on": +- "pull_request" \ No newline at end of file diff --git a/.github/workflows/publish-prod.yml b/.github/workflows/publish-prod.yml new file mode 100644 index 0000000..d31f5bb --- /dev/null +++ b/.github/workflows/publish-prod.yml @@ -0,0 +1,74 @@ +"jobs": + "yarn-publish": + "container": + "image": "node:18" + "permissions": + "contents": "read" + "packages": "write" + "pull-requests": "read" + "runs-on": "ubuntu-latest" + "steps": + - "name": "Check out repository code" + "uses": "actions/checkout@v3" + "with": + "ref": "${{ github.sha }}" + - "env": + "NPM_TOKEN": "${{ secrets.npm_token }}" + "name": "set gynzy npm_token" + "run": | + cat < .npmrc + registry=https://npm.gynzy.net/ + always-auth="true" + "//npm.gynzy.net/:_authToken"="${NPM_TOKEN}" + EOF + - "name": "yarn" + "run": "yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline || yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline" + - "name": "build" + "run": "yarn build" + - "env": + "NPM_TOKEN": "${{ secrets.npm_token }}" + "name": "set gynzy npm_token" + "run": | + cat < .npmrc + registry=https://npm.gynzy.net/ + always-auth="true" + "//npm.gynzy.net/:_authToken"="${NPM_TOKEN}" + EOF + - "env": {} + "name": "publish" + "run": | + bash -c 'set -xeo pipefail; + + cp package.json package.json.bak; + + VERSION=$(yarn version --non-interactive 2>/dev/null | grep "Current version" | grep -o -P '[0-9a-zA-Z_.-]+$' ); + if [[ ! -z "${PR_NUMBER}" ]]; then + echo "Setting tag/version for pr build."; + TAG=pr-$PR_NUMBER; + PUBLISHVERSION="$VERSION-pr$PR_NUMBER.$GITHUB_RUN_NUMBER"; + elif [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then + if [[ "${GITHUB_REF_NAME}" != "${VERSION}" ]]; then + echo "Tag version does not match package version. They should match. Exiting"; + exit 1; + fi + echo "Setting tag/version for release/tag build."; + PUBLISHVERSION=$VERSION; + TAG="latest"; + elif [[ "${GITHUB_REF_TYPE}" == "branch" && ( "${GITHUB_REF_NAME}" == "main" || "${GITHUB_REF_NAME}" == "master" ) ]] || [[ "${GITHUB_EVENT_NAME}" == "deployment" ]]; then + echo "Setting tag/version for release/tag build."; + PUBLISHVERSION=$VERSION; + TAG="latest"; + else + exit 1 + fi + + yarn publish --non-interactive --no-git-tag-version --tag "$TAG" --new-version "$PUBLISHVERSION"; + + mv package.json.bak package.json; + '; + "timeout-minutes": 30 +"name": "publish-prod" +"on": + "push": + "branches": + - "main" \ No newline at end of file