From 3fe89192f632fd198e943261602a22a56fce999c Mon Sep 17 00:00:00 2001 From: javarome Date: Mon, 19 Aug 2024 01:58:28 +0200 Subject: [PATCH] docs --- build.ts | 6 +- org/eu/fr/cnes/geipan/GeipanCaseSummary.ts | 2 +- .../cnes/geipan/GeipanCaseSummaryRR0Mapper.ts | 5 +- org/eu/fr/cnes/geipan/GeipanHttpDatasource.ts | 2 +- ...n.36a0bf81ff285eb6cad31df608651a2f.en.html | 1528 ----------------- .../crypto/ufo/enquete/dossier/CaseService.ts | 8 +- search/SearchCommand.ts | 4 +- ...egistry.ts => PersistentSourceRegistry.ts} | 7 +- source/SourceFactory.ts | 34 +- source/SourceRegistry.ts | 4 +- time/datasource/nuforc/NuforcCaseSummary.ts | 4 + .../datasource/nuforc/NuforcHttpDatasource.ts | 6 +- time/datasource/nuforc/NuforcRR0Mapper.ts | 18 +- time/datasource/ufo-search/UfoSearchCase.ts | 4 + .../ufo-search/UfoSearchCaseRR0Mapper.ts | 7 +- .../ufo-search/UfoSearchHttpDatasource.ts | 12 +- time/datasource/urecat/UrecatDatasource.ts | 9 +- time/datasource/urecat/UrecatRR0Mapper.ts | 4 + 18 files changed, 88 insertions(+), 1576 deletions(-) rename source/{PersisentSourceRegistry.ts => PersistentSourceRegistry.ts} (77%) diff --git a/build.ts b/build.ts index c08ca6e08b..b53929e9bf 100644 --- a/build.ts +++ b/build.ts @@ -72,7 +72,7 @@ import { ChronologyReplacerActions } from "./time/datasource/ChronologyReplacerA import { SourceReplacer } from "./source/SourceReplacer" import { NoteReplacer } from "./note/NoteReplacer" import { NoteFileCounter } from "./note/NoteFileCounter" -import { PersisentSourceRegistry } from "./source/PersisentSourceRegistry" +import { PersistentSourceRegistry } from "./source/PersistentSourceRegistry" import { SourceIndexStep } from "./source/SourceIndexStep" import { SourceFileCounter } from "./source/SourceFileCounter" import { TimeElementFactory } from "./time/TimeElementFactory" @@ -236,7 +236,7 @@ timeService.getFiles().then(async (timeFiles) => { const sourceRegistryFileName = "source/index.json" const baseUrl = "https://rr0.org" const http = new HttpSource() - const sourceFactory = new PersisentSourceRegistry(dataService, http, baseUrl, sourceRegistryFileName) + const sourceFactory = new PersistentSourceRegistry(dataService, http, baseUrl, sourceRegistryFileName, timeFormat) const noteCounter = new NoteFileCounter() const noteRenderer = new NoteRenderer(noteCounter) const caseRenderer = new CaseSummaryRenderer(noteRenderer, sourceFactory, sourceRenderer) @@ -264,7 +264,7 @@ timeService.getFiles().then(async (timeFiles) => { new SsiSetVarReplaceCommand("title", (_match: string, ...args: any[]) => `${args[0]}`), new SsiSetVarReplaceCommand("url", (_match: string, ...args: any[]) => ``), - new SsiLastModifiedReplaceCommand(context.time.options), + new SsiLastModifiedReplaceCommand(timeFormat), new SsiTitleReplaceCommand([timeDefaultHandler]), new DescriptionReplaceCommand("UFO data for french-reading people", "abstract"), new AuthorReplaceCommand(timeService) diff --git a/org/eu/fr/cnes/geipan/GeipanCaseSummary.ts b/org/eu/fr/cnes/geipan/GeipanCaseSummary.ts index f29edade82..3adac64e67 100644 --- a/org/eu/fr/cnes/geipan/GeipanCaseSummary.ts +++ b/org/eu/fr/cnes/geipan/GeipanCaseSummary.ts @@ -13,7 +13,7 @@ export type GeipanZoneCode = FranceDepartementCode | FranceRegionCode | CountryC */ export interface GeipanCaseSummary { id: string - url: URL + url: string dateTime: TimeContext sightingType?: GeipanSightingType city: string diff --git a/org/eu/fr/cnes/geipan/GeipanCaseSummaryRR0Mapper.ts b/org/eu/fr/cnes/geipan/GeipanCaseSummaryRR0Mapper.ts index 7bfd4400b6..8ef51e6b56 100644 --- a/org/eu/fr/cnes/geipan/GeipanCaseSummaryRR0Mapper.ts +++ b/org/eu/fr/cnes/geipan/GeipanCaseSummaryRR0Mapper.ts @@ -43,11 +43,14 @@ export class GeipanCaseSummaryRR0Mapper implements CaseMapper - - - diff --git a/science/crypto/ufo/enquete/dossier/CaseService.ts b/science/crypto/ufo/enquete/dossier/CaseService.ts index a825c47d8a..9dafb63c05 100644 --- a/science/crypto/ufo/enquete/dossier/CaseService.ts +++ b/science/crypto/ufo/enquete/dossier/CaseService.ts @@ -32,13 +32,7 @@ export class CaseService { const caseContext = context.clone() if (time) { caseContext.time = time - const options = caseContext.time.options - options.year = "numeric" - options.month = undefined - options.day = undefined - options.hour = undefined - options.weekday = undefined - options.minute = undefined + const options: Intl.DateTimeFormatOptions = {year: "numeric"} const {result, replacement} = this.timeElementFactory.renderer.renderContent(caseContext, undefined, {url: true}, options) result.append(replacement) diff --git a/search/SearchCommand.ts b/search/SearchCommand.ts index 0a26b8bb3b..85d5e4a8ea 100644 --- a/search/SearchCommand.ts +++ b/search/SearchCommand.ts @@ -67,8 +67,8 @@ export class SearchCommand implements ReplaceCommand { context.warn(`Title "${title}" with URL ${url} is already indexed with URL ${titleIndexed.url}`) } const indexContext = context.clone() - indexContext.time.options.weekday = undefined - const time = this.timeTextBuilder.build(indexContext).toLowerCase() + const time = this.timeTextBuilder.build(indexContext, true, + {year: "numeric", month: "short", day: "numeric"}).toLowerCase() indexedPages.push({title, url, time}) } if (this.config.indexWords) { diff --git a/source/PersisentSourceRegistry.ts b/source/PersistentSourceRegistry.ts similarity index 77% rename from source/PersisentSourceRegistry.ts rename to source/PersistentSourceRegistry.ts index b383a01deb..cab24ac2a4 100644 --- a/source/PersisentSourceRegistry.ts +++ b/source/PersistentSourceRegistry.ts @@ -7,10 +7,11 @@ import { FileContents } from "ssg-api" /** * Create Source objects and register them. */ -export class PersisentSourceRegistry extends SourceRegistry { +export class PersistentSourceRegistry extends SourceRegistry { - constructor(dataService: DataService, http: HttpSource, baseUrl: string, protected fileName: string) { - super(dataService, http, baseUrl) + constructor(dataService: DataService, http: HttpSource, baseUrl: string, protected fileName: string, + options: Intl.DateTimeFormatOptions) { + super(dataService, http, baseUrl, options) const registryFileContents = FileContents.read(fileName, "utf-8").contents this.registry = JSON.parse(registryFileContents) } diff --git a/source/SourceFactory.ts b/source/SourceFactory.ts index 1be30166fa..34a081bda3 100644 --- a/source/SourceFactory.ts +++ b/source/SourceFactory.ts @@ -12,7 +12,8 @@ import { HttpSource } from "../time/datasource/HttpSource" */ export class SourceFactory { - constructor(protected dataService: DataService, protected http: HttpSource, protected baseUrl: string) { + constructor(protected dataService: DataService, protected http: HttpSource, protected baseUrl: string, + protected options: Intl.DateTimeFormatOptions) { } /** @@ -54,15 +55,16 @@ export class SourceFactory { source = await this.fromPage(href, hash) break case ".json": - const sources = await this.dataService.getFromDir(path.dirname(href), sourceTypes, [path.basename(href)]) + const sources = await this.dataService.getFromDir(path.dirname(href), sourceTypes, + [path.basename(href)]) source = sources?.[0] break default: { - const sources = await this.dataService.getFromDir(ext ? path.dirname(href) : href, sourceTypes, + const sources = await this.dataService.getFromDir(ext ? path.dirname(href) : href, sourceTypes, ["index.json", "people.json"]) source = sources?.[0] if (!source) { - source = this.fromPage(path.join(href, "index.html"), hash) + source = await this.fromPage(path.join(href, "index.html"), hash) } } } @@ -97,9 +99,15 @@ export class SourceFactory { title = href } publisher = resOut.headers.get("host") - const time = lastModif ? TimeContext.fromDate(new Date(lastModif), context.time.options) : context.time + const time = lastModif ? TimeContext.fromDate(new Date(lastModif), this.options) : context.time const publication: Publication = {publisher, time} - return {title, url: href, publication} + return { + title, + url: href, + events: [], + previousSourceRefs: [], + publication + } } async fromPage(href: string, hash = ""): Promise { @@ -118,11 +126,15 @@ export class SourceFactory { } else { title = doc.querySelector("title").textContent } + const publisher = doc.querySelector("meta[name='copyright']")?.getAttribute("content") + const authors = Array.from(doc.querySelectorAll("meta[name='author']")).map(meta => meta.getAttribute("content")) return { - title: title, - authors: Array.from(doc.querySelectorAll("meta[name='author']")).map(meta => meta.getAttribute("content")), - publication: {publisher: doc.querySelector("meta[name='copyright']")?.getAttribute("content")}, - url - } as Source + title, + url: url.href, + events: [], + previousSourceRefs: [], + authors, + publication: {publisher, time: undefined} + } } } diff --git a/source/SourceRegistry.ts b/source/SourceRegistry.ts index eae28cb8bc..19f7fa21f1 100644 --- a/source/SourceRegistry.ts +++ b/source/SourceRegistry.ts @@ -11,8 +11,8 @@ export class SourceRegistry extends SourceFactory { protected registry = {} - constructor(dataService: DataService, http: HttpSource, baseUrl: string) { - super(dataService, http, baseUrl) + constructor(dataService: DataService, http: HttpSource, baseUrl: string, options: Intl.DateTimeFormatOptions) { + super(dataService, http, baseUrl, options) } /** diff --git a/time/datasource/nuforc/NuforcCaseSummary.ts b/time/datasource/nuforc/NuforcCaseSummary.ts index 74a26d3a8a..790a713329 100644 --- a/time/datasource/nuforc/NuforcCaseSummary.ts +++ b/time/datasource/nuforc/NuforcCaseSummary.ts @@ -1,8 +1,12 @@ import { NuforcState } from "./NuforcState" import { NuforcCountry } from "./NuforcCountry" import { NuforcShape } from "./NuforcShape" +import { TimeContext } from "../../TimeContext" export interface NuforcCaseSummary { + readonly id: string + readonly time: TimeContext + readonly url: string readonly city: string readonly state: NuforcState readonly country: NuforcCountry diff --git a/time/datasource/nuforc/NuforcHttpDatasource.ts b/time/datasource/nuforc/NuforcHttpDatasource.ts index ae1f2aade2..e6e82b7ad4 100644 --- a/time/datasource/nuforc/NuforcHttpDatasource.ts +++ b/time/datasource/nuforc/NuforcHttpDatasource.ts @@ -37,7 +37,7 @@ export class NuforcHttpDatasource extends NuforcDatasource { protected async readCases(context: RR0SsgContext): Promise { const searchUrl = this.queryUrl(context) - const doc = await this.http.get(searchUrl.href, {headers: {accept: "text/html;charset=utf-8"}}) + const doc = await this.http.get(searchUrl, {headers: {accept: "text/html;charset=utf-8"}}) const rowEls = doc.querySelectorAll("#table_1 tbody tr") return Array.from(rowEls).map(row => this.getNativeCase(context, row)) } @@ -90,7 +90,7 @@ export class NuforcHttpDatasource extends NuforcDatasource { const columns = row.querySelectorAll("td") const url = this.getLink(columns[0]) const caseNumber = new URLSearchParams(url.search).get("id") - const dateTime = this.getTime(columns[1], context) + const time = this.getTime(columns[1], context) const city = columns[2].textContent const state = this.getState(columns[3]) const country = this.getCountry(columns[4]) @@ -99,6 +99,6 @@ export class NuforcHttpDatasource extends NuforcDatasource { const reportDate = this.dateFromField(columns[7]) const postDate = this.dateFromField(columns[8]) const image = this.getImage(columns[9]) - return {id: caseNumber, url, city, state, country, dateTime, shape, summary, reportDate, postDate, image} + return {id: caseNumber, url: url.href, city, state, country, time, shape, summary, reportDate, postDate, image} } } diff --git a/time/datasource/nuforc/NuforcRR0Mapper.ts b/time/datasource/nuforc/NuforcRR0Mapper.ts index 7319de364b..287ae29d49 100644 --- a/time/datasource/nuforc/NuforcRR0Mapper.ts +++ b/time/datasource/nuforc/NuforcRR0Mapper.ts @@ -24,6 +24,8 @@ import { southKorea } from "../../../org/kr/SouthKorea" import { uk } from "../../../org/uk/Uk" import { usa } from "../../../org/us/Usa" import { Source } from "../../../source/Source" +import { algeria } from "../../../org/dz/Algeria" +import { panama } from "../../../org/pa/Panama" export class NuforcRR0Mapper implements CaseMapper { @@ -64,26 +66,30 @@ export class NuforcRR0Mapper implements CaseMapper heading.textContent.indexOf("Date") >= 0) const itemContext = context.clone() - const dateTime = itemContext.time.reset() + const time = itemContext.time.reset() let url: URL let caseNumber if (dateLabel) { const dateLink = dateLabel.nextElementSibling as HTMLAnchorElement url = new URL(dateLink.href, this.baseUrl) caseNumber = url.hash.substring(1) - this.setDate(dateTime, dateLink.textContent) + this.setDate(time, dateLink.textContent) } const timeLabel = fieldsHeadings.find(heading => heading.textContent.indexOf("Time") >= 0) if (timeLabel) { - this.setTime(dateTime, timeLabel) + this.setTime(time, timeLabel) } const locationLabel = fieldsHeadings.find(heading => heading.textContent.indexOf("Location") >= 0) let location: string | undefined @@ -114,7 +114,11 @@ export class UfoSearchHttpDatasource extends UfoSearchDatasource { let attributes let extraData return { - id: caseNumber, dateTime, location, desc: type, + id: caseNumber, + time, + url: url.href, + location, + desc: type, key_vals: {url}, attributes, ref: "", diff --git a/time/datasource/urecat/UrecatDatasource.ts b/time/datasource/urecat/UrecatDatasource.ts index 6d798c5a7b..5d1f9f0e00 100644 --- a/time/datasource/urecat/UrecatDatasource.ts +++ b/time/datasource/urecat/UrecatDatasource.ts @@ -14,12 +14,13 @@ export abstract class UrecatDatasource implements Datasource { } async fetch(context: RR0SsgContext): Promise { - const day = context.time.getDayOfMonth() - const month = context.time.getMonth() - const year = context.time.getYear() + const time = context.time + const day = time.getDayOfMonth() + const month = time.getMonth() + const year = time.getYear() const summaries = await this.getSummaries(context) return summaries.filter(c => { - const sightingTime = c.dateTime + const sightingTime = c.time return (!year || year === sightingTime.getYear()) && (!month || month === sightingTime.getMonth()) && (!day || day === sightingTime.getDayOfMonth()) }) } diff --git a/time/datasource/urecat/UrecatRR0Mapper.ts b/time/datasource/urecat/UrecatRR0Mapper.ts index ef67d6a60e..857dd43682 100644 --- a/time/datasource/urecat/UrecatRR0Mapper.ts +++ b/time/datasource/urecat/UrecatRR0Mapper.ts @@ -23,6 +23,7 @@ export class UrecatRR0Mapper implements CaseMapper