From 57f317a24054023670fbac35d6a1bd266df350c8 Mon Sep 17 00:00:00 2001 From: Nathaniel Paulus Date: Wed, 12 Feb 2025 19:30:05 -0500 Subject: [PATCH] Improve handling of language codes --- src/SIL.XForge.Scripture/ClientApp/biome.json | 38 +++++++++++++++++++ .../draft-sources.component.html | 8 ++-- .../draft-sources.component.spec.ts | 10 +++++ .../draft-sources/draft-sources.component.ts | 34 ++++++++++------- .../draft-sources/draft-sources.stories.ts | 18 --------- .../translate/draft-generation/draft-utils.ts | 23 +++++++++++ .../Controllers/ParatextController.cs | 2 +- 7 files changed, 96 insertions(+), 37 deletions(-) create mode 100644 src/SIL.XForge.Scripture/ClientApp/biome.json create mode 100644 src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-utils.ts diff --git a/src/SIL.XForge.Scripture/ClientApp/biome.json b/src/SIL.XForge.Scripture/ClientApp/biome.json new file mode 100644 index 0000000000..3d2f6a413a --- /dev/null +++ b/src/SIL.XForge.Scripture/ClientApp/biome.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://biomejs.dev/schemas/1.9.4/schema.json", + "vcs": { + "enabled": false, + "clientKind": "git", + "useIgnoreFile": false + }, + "files": { + "ignoreUnknown": false, + "ignore": [] + }, + "formatter": { + "enabled": true, + "indentStyle": "space", + "indentWidth": 2 + }, + "organizeImports": { + "enabled": true + }, + "linter": { + "enabled": true, + "rules": { + "recommended": true, + "style": { + "useImportType": "off", + "noInferrableTypes": "off" + } + } + }, + "javascript": { + "formatter": { + "quoteStyle": "single", + "trailingCommas": "none", + "lineWidth": 120, + "arrowParentheses": "asNeeded" + } + } +} diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-sources/draft-sources.component.html b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-sources/draft-sources.component.html index 3a36503419..578d44ccc1 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-sources/draft-sources.component.html +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-sources/draft-sources.component.html @@ -121,7 +121,7 @@

Translated project {{ parentheses(targetLanguageDisplayName) }}

{{ project.name }} - {{ t("language_code") }} {{ project.writingSystem?.tag }} + {{ t("language_code") }} {{ project.writingSystem.tag }}
} @@ -156,7 +156,7 @@

Source {{ parentheses(sourceLanguageDisplayName) }}

- @if (sourceSideLanguageCodes.length > 1) { + @if (multipleSourceSideLanguages) {

All source and reference projects should be in the same language

@@ -187,7 +187,7 @@

Source and target languages are both {{ i18n.getLanguageDisplayName(targetLa } - @if (sourceSideLanguageCodes.length === 1 && !showSourceAndTargetLanguagesIdenticalWarning) { + @if (!multipleSourceSideLanguages && !showSourceAndTargetLanguagesIdenticalWarning) {

{{ t("incorrect_language_codes_reduce_quality") }}

Please make sure all the language codes are correct before saving.

@@ -214,7 +214,7 @@

{{ t("incorrect_language_codes_reduce_quality") }}

@if (getControlState("projectSettings") === ElementState.Submitting) { -
Saving
+
Saving
} @else {
checkmark All changes saved
} diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-sources/draft-sources.component.spec.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-sources/draft-sources.component.spec.ts index 46e01214dd..7c94260841 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-sources/draft-sources.component.spec.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-sources/draft-sources.component.spec.ts @@ -62,6 +62,16 @@ describe('DraftSourcesComponent', () => { expect(env.component.resources).toBeDefined(); })); + it('should not save when language codes are not confirmed', fakeAsync(() => { + const env = new TestEnvironment(); + env.component.languageCodesConfirmed = false; + + env.component.save(); + tick(); + + verify(mockedSFProjectService.onlineUpdateSettings(anything(), anything())).never(); + })); + it('updates control state during save', fakeAsync(() => { const env = new TestEnvironment(); when(mockedSFProjectService.onlineUpdateSettings(anything(), anything())).thenResolve(); diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-sources/draft-sources.component.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-sources/draft-sources.component.ts index 83f9c22961..53057da01e 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-sources/draft-sources.component.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-sources/draft-sources.component.ts @@ -6,7 +6,7 @@ import { MatCardModule } from '@angular/material/card'; import { MatCheckboxChange, MatCheckboxModule } from '@angular/material/checkbox'; import { MatRippleModule } from '@angular/material/core'; import { MatIconModule } from '@angular/material/icon'; -import { MatProgressSpinner } from '@angular/material/progress-spinner'; +import { MatProgressSpinnerModule } from '@angular/material/progress-spinner'; import { Router } from '@angular/router'; import { TranslocoModule } from '@ngneat/transloco'; import { SFProjectProfile } from 'realtime-server/lib/esm/scriptureforge/models/sf-project'; @@ -27,6 +27,7 @@ import { ParatextService, SelectableProject, SelectableProjectWithLanguageCode } import { SFProjectService } from '../../../core/sf-project.service'; import { NoticeComponent } from '../../../shared/notice/notice.component'; import { isSFProjectSyncing } from '../../../sync/sync.component'; +import { countNonEquivalentLanguageCodes } from '../draft-utils'; function translateSourceToSelectableProjectWithLanguageTag( project: TranslateSource @@ -113,7 +114,7 @@ export function projectToDraftSources(project: SFProjectProfile): DraftSourcesAs TranslocoModule, NoticeComponent, MatCheckboxModule, - MatProgressSpinner + MatProgressSpinnerModule ], templateUrl: './draft-sources.component.html', styleUrl: './draft-sources.component.scss' @@ -190,17 +191,25 @@ export class DraftSourcesComponent extends DataLoadingComponent implements OnIni return value ? `(${value})` : ''; } + get multipleSourceSideLanguages(): boolean { + return countNonEquivalentLanguageCodes(this.sourceSideLanguageCodes) > 1; + } + get sourceSideLanguageCodes(): string[] { return Array.from( - // FIXME Handle language codes that may be equivalent by not identical strings new Set([...this.draftingSources, ...this.trainingSources].filter(s => s != null).map(s => s.languageTag)) ); } get showSourceAndTargetLanguagesIdenticalWarning(): boolean { + // Show the warning when there's only one language on the source side, but that one language is equivalent to the + // target language. const sourceCodes = this.sourceSideLanguageCodes; - // FIXME Handle language codes that may be equivalent by not identical strings - return sourceCodes.length === 1 && sourceCodes[0] === this.trainingTargets[0]!.writingSystem.tag; + return ( + sourceCodes.length > 0 && + countNonEquivalentLanguageCodes(sourceCodes) === 1 && + countNonEquivalentLanguageCodes([sourceCodes[0], this.targetLanguageTag]) < 2 + ); } get targetLanguageTag(): string { @@ -226,18 +235,15 @@ export class DraftSourcesComponent extends DataLoadingComponent implements OnIni const { trainingSources, trainingTargets, draftingSources } = projectToDraftSources(projectDoc.data); if (trainingSources.length > 2) throw new Error('More than 2 training sources is not supported'); - const mappedTrainingSources = trainingSources - // FIXME No actual nullish elements, but TS doesn't know because of how we set the array length - .filter(s => s != null) - .map(translateSourceToSelectableProjectWithLanguageTag) as SelectableProjectWithLanguageCode[] & - ({ length: 0 } | { length: 1 } | { length: 2 }); + const mappedTrainingSources = trainingSources.map( + translateSourceToSelectableProjectWithLanguageTag + ) as SelectableProjectWithLanguageCode[] & ({ length: 0 } | { length: 1 } | { length: 2 }); this.trainingSources = mappedTrainingSources; this.trainingTargets = trainingTargets; - const mappedDraftingSources: SelectableProjectWithLanguageCode[] = draftingSources - // FIXME No actual nullish elements, but TS doesn't know because of how we set the array length - .filter(s => s != null) - .map(translateSourceToSelectableProjectWithLanguageTag); + const mappedDraftingSources: SelectableProjectWithLanguageCode[] = draftingSources.map( + translateSourceToSelectableProjectWithLanguageTag + ); this.draftingSources = [mappedDraftingSources[0]]; if (this.draftingSources.length < 1) this.draftingSources.push(undefined); diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-sources/draft-sources.stories.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-sources/draft-sources.stories.ts index e2d0eb2e05..707d2fc987 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-sources/draft-sources.stories.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-sources/draft-sources.stories.ts @@ -85,24 +85,6 @@ export default { ] }) ], - // render: args => { - // // setUpMocks(args as DraftSourcesComponentStoryState); - // return { - // // moduleMetadata: { - // // providers: [ - // // { provide: ActivatedProjectService, useValue: instance(mockActivatedProjectService) }, - // // { provide: DestroyRef, useValue: instance(mockDestroyRef) }, - // // { provide: ParatextService, useValue: instance(mockParatextService) }, - // // { provide: DialogService, useValue: instance(mockDialogService) }, - // // { provide: SFProjectService, useValue: instance(mockProjectService) }, - // // { provide: SFUserProjectsService, useValue: instance(mockUserProjectsService) }, - // // { provide: Router, useValue: instance(mockRouter) }, - // // { provide: FeatureFlagService, useValue: instance(mockFeatureFlags) } - // // ] - // // }, - // template: `` - // }; - // }, args: defaultArgs, parameters: { controls: { diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-utils.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-utils.ts new file mode 100644 index 0000000000..905bea72f5 --- /dev/null +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-utils.ts @@ -0,0 +1,23 @@ +export function englishNameFromCode(code: string): string { + const locale = new Intl.Locale(code); + return new Intl.DisplayNames(['en'], { type: 'language' }).of(locale.language) ?? ''; +} + +export function areLanguageCodesEquivalent(code1: string, code2: string): boolean { + return code1 === code2 || englishNameFromCode(code1) === englishNameFromCode(code2); +} + +/** + * Given a list of language codes, counts the number of codes that are "unique" in the sense that they have different + * English names. + * + * This is a *very rough* way of determining whether languages are "equivalent" for the purposes of training a model. + * + * As an example, zh and cmn both map to "Chinese" in English, so they would be considered equivalent. + * + * TODO Create a more robust way of determining whether languages are equivalent, that isn't browser-dependent. + */ +export function countNonEquivalentLanguageCodes(languageCodes: string[]): number { + const uniqueTags = new Set(languageCodes); + return new Set(Array.from(uniqueTags).map(englishNameFromCode)).size; +} diff --git a/src/SIL.XForge.Scripture/Controllers/ParatextController.cs b/src/SIL.XForge.Scripture/Controllers/ParatextController.cs index c3e5611438..910cb9a0e0 100644 --- a/src/SIL.XForge.Scripture/Controllers/ParatextController.cs +++ b/src/SIL.XForge.Scripture/Controllers/ParatextController.cs @@ -209,7 +209,7 @@ int chapter /// /// /// The resources were successfully retrieved. A dictionary is returned where the Paratext Id is the key, and the - /// values are an array with the short name followed by the name. + /// values are an array containing: [shortName, name, languageTag]. /// /// The user does not have permission to access Paratext. /// The user's Paratext tokens have expired, and the user must log in again.