Skip to content

Commit

Permalink
#28: Compacting IRIs in storage (WIP)
Browse files Browse the repository at this point in the history
  • Loading branch information
gsvarovsky committed Nov 19, 2020
1 parent d400c5b commit 2fc73d8
Show file tree
Hide file tree
Showing 14 changed files with 183 additions and 46 deletions.
3 changes: 2 additions & 1 deletion src/engine/MeldEncoding.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ import { compact } from 'jsonld';
import { flatten } from './util';
import { Context, ExpandedTermDef } from '../jrql-support';
import { Iri } from 'jsonld/jsonld-spec';
import { Triple, tripleKey, rdfToJson, jsonToRdf, TripleMap } from './quads';
import { Triple, tripleKey, TripleMap } from './quads';
import { rdfToJson, jsonToRdf } from "./jsonld";

export class DomainContext implements Context {
'@base': Iri;
Expand Down
11 changes: 4 additions & 7 deletions src/engine/dataset/JrqlGraph.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ import { from, of, EMPTY, Observable, throwError } from 'rxjs';
import { flatten, fromArrayPromise } from '../util';
import { QuadSolution, VarValues } from './QuadSolution';
import { array, shortId } from '../../util';
import { jsonToRdf, rdfToJson, TriplePos } from '../quads';
import { TriplePos } from '../quads';
import { activeCtx, expandTerm, jsonToRdf, rdfToJson } from "../jsonld";

/**
* A graph wrapper that provides low-level json-rql handling for queries. The
Expand Down Expand Up @@ -222,12 +223,8 @@ export async function toSubject(quads: Quad[], context: Context): Promise<object
return compact(await rdfToJson(quads), context || {}) as unknown as Subject;
}

export async function resolve(iri: Iri, context?: Context): Promise<NamedNode> {
return namedNode(context ? (await compact({
'@id': iri,
'http://json-rql.org/predicate': 1,
'@context': context
}, {}) as any)['@id'] : iri);
async function resolve(iri: Iri, context?: Context): Promise<NamedNode> {
return namedNode(context ? expandTerm(iri, await activeCtx(context)) : iri);
}

function asMatchTerms(quad: Quad):
Expand Down
3 changes: 2 additions & 1 deletion src/engine/dataset/SuSetDataset.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ import { LocalLock } from '../local';
import { SUSET_CONTEXT, qsName, toPrefixedId } from './SuSetGraph';
import { SuSetJournalGraph, SuSetJournalEntry } from './SuSetJournal';
import { MeldConfig, Read } from '../..';
import { QuadMap, TripleMap, Triple, rdfToJson } from '../quads';
import { QuadMap, TripleMap, Triple } from '../quads';
import { rdfToJson } from "../jsonld";

interface HashTid extends Subject {
'@id': Iri; // hash:<hashed triple id>
Expand Down
33 changes: 25 additions & 8 deletions src/engine/dataset/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ import { QuadSet } from '../quads';
import { Filter } from '../indices';
import dataFactory = require('@rdfjs/data-model');
import { TermName } from 'quadstore/dist/lib/types';
import { Context } from 'jsonld/jsonld-spec';
import { activeCtx, compactIri, expandTerm } from '../jsonld';
import { ActiveContext } from 'jsonld/lib/context';

/**
* Atomically-applied patch to a quad-store.
Expand Down Expand Up @@ -116,22 +119,36 @@ export interface Graph {

export class QuadStoreDataset implements Dataset {
readonly location: string;
private readonly store: Quadstore;
private /* readonly */ store: Quadstore;
private readonly activeCtx?: Promise<ActiveContext>;
private readonly lock = new LockManager;
private isClosed: boolean = false;

constructor(backend: AbstractLevelDOWN) {
this.store = new Quadstore({
backend, dataFactory, indexes: [
[TermName.GRAPH, TermName.SUBJECT, TermName.PREDICATE, TermName.OBJECT],
[TermName.GRAPH, TermName.OBJECT, TermName.SUBJECT, TermName.PREDICATE],
[TermName.GRAPH, TermName.PREDICATE, TermName.OBJECT, TermName.SUBJECT]
] });
constructor(private readonly backend: AbstractLevelDOWN, context?: Context) {
// Internal of level-js and leveldown
this.location = (<any>backend).location ?? uuid();
if (context != null)
this.activeCtx = activeCtx(context);
}

async initialise(): Promise<QuadStoreDataset> {
const activeCtx = await this.activeCtx;
this.store = new Quadstore({
backend : this.backend,
dataFactory,
indexes: [
[TermName.GRAPH, TermName.SUBJECT, TermName.PREDICATE, TermName.OBJECT],
[TermName.GRAPH, TermName.OBJECT, TermName.SUBJECT, TermName.PREDICATE],
[TermName.GRAPH, TermName.PREDICATE, TermName.OBJECT, TermName.SUBJECT]
],
prefixes: activeCtx == null ? undefined : {
expandTerm: term => {
console.log('expanding', term)
return expandTerm(term, activeCtx);
},
compactIri: iri => compactIri(iri, activeCtx)
}
});
await this.store.open();
return this;
}
Expand Down
31 changes: 31 additions & 0 deletions src/engine/jsonld.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
import { Quad } from 'rdf-js';
import { fromRDF, toRDF, Options, processContext } from 'jsonld';
import { cloneQuad } from './quads';
import { Context, Iri } from 'jsonld/jsonld-spec';
import { getInitialContext, expandIri, ActiveContext } from 'jsonld/lib/context';
import { compactIri as _compactIri } from 'jsonld/lib/compact';
export { Options } from 'jsonld';
export { ActiveContext } from 'jsonld/lib/context';

export function rdfToJson(quads: Quad[]): Promise<any> {
// Using native types to avoid unexpected value objects
return fromRDF(quads, { useNativeTypes: true });
}

export async function jsonToRdf(json: any): Promise<Quad[]> {
const quads = await toRDF(json) as Quad[];
// jsonld produces quad members without equals
return quads.map(cloneQuad);
}

export function expandTerm(value: string, ctx: ActiveContext, options?: Options.Expand): Iri {
return expandIri(ctx, value, { base: true }, options ?? {});
}

export function compactIri(iri: Iri, ctx: ActiveContext, options?: Options.CompactIri): string {
return _compactIri({ activeCtx: ctx, iri, ...options });
}

export async function activeCtx(ctx: Context, options?: Options.DocLoader): Promise<ActiveContext> {
return await processContext(getInitialContext({}), ctx, options ?? {});
}
11 changes: 0 additions & 11 deletions src/engine/quads.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import { Quad, Term, Literal } from 'rdf-js';
import { namedNode, defaultGraph, variable, blankNode, literal, quad as newQuad } from '@rdfjs/data-model';
import { fromRDF, toRDF } from 'jsonld';
import { IndexMap, IndexSet } from "./indices";

export type Triple = Omit<Quad, 'graph'>;
Expand Down Expand Up @@ -55,16 +54,6 @@ function quadIndexKey(quad: Quad): string {
return [quad.graph.value].concat(tripleKey(quad)).join('^');
}

export function rdfToJson(quads: Quad[]): Promise<any> {
// Using native types to avoid unexpected value objects
return fromRDF(quads, { useNativeTypes: true });
}

export function jsonToRdf(json: any): Promise<Quad[]> {
// jsonld produces quad members without equals
return toRDF(json).then((quads: Quad[]) => quads.map(cloneQuad));
}

export function cloneQuad(quad: Quad): Quad {
return newQuad(
cloneTerm(quad.subject),
Expand Down
2 changes: 1 addition & 1 deletion src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ export async function clone(
constraint?: MeldConstraint): Promise<MeldClone> {

const context = new DomainContext(config['@domain'], config['@context']);
const dataset = await new QuadStoreDataset(backend).initialise();
const dataset = await new QuadStoreDataset(backend, context).initialise();

if (typeof remotes == 'function')
remotes = new remotes(config);
Expand Down
10 changes: 10 additions & 0 deletions src/types/jsonld-compact.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
declare module 'jsonld/lib/compact' {
import { Iri, Url } from 'jsonld/jsonld-spec';
import { ActiveContext } from 'jsonld/lib/context';
import { Options } from 'jsonld';

function compactIri(opts: {
activeCtx: ActiveContext,
iri: Iri
} & Options.CompactIri): string;
}
14 changes: 14 additions & 0 deletions src/types/jsonld-context.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
declare module 'jsonld/lib/context' {
import { Context, Iri } from 'jsonld/jsonld-spec';
import { Options } from 'jsonld';

// Using the 'protected' field to prevent type mistakes
type ActiveContext = Context & { protected: {} };

function getInitialContext(options: { processingMode?: boolean }): ActiveContext;
function expandIri(
activeCtx: Context,
value: string,
relativeTo: { vocab?: boolean, base?: boolean },
options: Options.Common): Iri;
}
19 changes: 19 additions & 0 deletions src/types/jsonld.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import { Context, Url } from 'jsonld/jsonld-spec';
import { ActiveContext } from 'jsonld/lib/context';

declare module 'jsonld' {
namespace Options {
export interface CompactIri {
value?: any, // TODO
relativeTo?: { vocab: boolean; },
reverse?: boolean,
base?: Url
}
}

function processContext(
activeCtx: ActiveContext,
localCtx: Context,
options: Options.DocLoader):
Promise<ActiveContext>;
}
24 changes: 12 additions & 12 deletions test/DatasetEngine.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -226,15 +226,15 @@ describe('Dataset engine', () => {
});

describe('as post-genesis clone', () => {
let ldb: AbstractLevelDOWN;
let backend: AbstractLevelDOWN;
let config: MeldConfig;
let remoteTime: TreeClock;

beforeEach(async () => {
ldb = new MemDown();
backend = new MemDown();
config = testConfig();
// Start a temporary genesis clone to initialise the store
let clone = new DatasetEngine({ dataset: await memStore(ldb), remotes: mockRemotes(), config });
let clone = new DatasetEngine({ dataset: await memStore({ backend }), remotes: mockRemotes(), config });
await clone.initialise();
remoteTime = await clone.newClock(); // Forks the clock so no longer genesis
await clone.close();
Expand All @@ -245,7 +245,7 @@ describe('Dataset engine', () => {
// Re-start on the same data, with a rev-up that never completes
const remotes = mockRemotes(NEVER, [true]);
remotes.revupFrom = async () => ({ lastTime: remoteTime, updates: NEVER });
const clone = new DatasetEngine({ dataset: await memStore(ldb), remotes, config: testConfig() });
const clone = new DatasetEngine({ dataset: await memStore({ backend }), remotes, config: testConfig() });

// Check that we are never not outdated
const everNotOutdated = clone.status.becomes({ outdated: false });
Expand All @@ -260,7 +260,7 @@ describe('Dataset engine', () => {
// Re-start on the same data, with a rev-up that completes with no updates
const remotes = mockRemotes(NEVER, [true]);
remotes.revupFrom = async () => ({ lastTime: remoteTime, updates: EMPTY });
const clone = new DatasetEngine({ dataset: await memStore(ldb), remotes, config: testConfig() });
const clone = new DatasetEngine({ dataset: await memStore({ backend }), remotes, config: testConfig() });

// Check that we do transition through an outdated state
const wasOutdated = clone.status.becomes({ outdated: true });
Expand All @@ -274,7 +274,7 @@ describe('Dataset engine', () => {

test('is not outdated if immediately siloed', async () => {
const remotes = mockRemotes(NEVER, [null, false]);
const clone = new DatasetEngine({ dataset: await memStore(ldb), remotes, config: testConfig() });
const clone = new DatasetEngine({ dataset: await memStore({ backend }), remotes, config: testConfig() });

await clone.initialise();

Expand All @@ -288,15 +288,15 @@ describe('Dataset engine', () => {
.mockReturnValueOnce(Promise.resolve({ lastTime: remoteTime, updates: throwError('boom') }))
.mockReturnValueOnce(Promise.resolve({ lastTime: remoteTime, updates: EMPTY }));
remotes.revupFrom = revupFrom;
const clone = new DatasetEngine({ dataset: await memStore(ldb), remotes, config: testConfig() });
const clone = new DatasetEngine({ dataset: await memStore({ backend }), remotes, config: testConfig() });
await clone.initialise();
await expect(clone.status.becomes({ outdated: false })).resolves.toBeDefined();
expect(revupFrom.mock.calls.length).toBe(2);
});

test('maintains fifo during rev-up', async () => {
// We need local siloed update
let clone = new DatasetEngine({ dataset: await memStore(ldb), remotes: mockRemotes(), config });
let clone = new DatasetEngine({ dataset: await memStore({ backend }), remotes: mockRemotes(), config });
await clone.initialise();
await clone.write({
'@id': 'http://test.m-ld.org/fred',
Expand All @@ -308,7 +308,7 @@ describe('Dataset engine', () => {
const revUps = new Source<DeltaMessage>();
remotes.revupFrom = async () => ({ lastTime: remoteTime.ticked(), updates: revUps });
// The clone will initialise into a revving-up state, waiting for a revUp
clone = new DatasetEngine({ dataset: await memStore(ldb), remotes, config: testConfig() });
clone = new DatasetEngine({ dataset: await memStore({ backend }), remotes, config: testConfig() });
const observedTicks = clone.updates.pipe(map(next => next.time.ticks), take(2), toArray()).toPromise();
await clone.initialise();
// Do a new update during rev-up, this will immediately produce an update
Expand All @@ -329,7 +329,7 @@ describe('Dataset engine', () => {
const remoteUpdates = new Source<DeltaMessage>();
const remotes = mockRemotes(remoteUpdates, [true]);
remotes.revupFrom = async () => ({ lastTime: remoteTime, updates: EMPTY });
const clone = new DatasetEngine({ dataset: await memStore(ldb), remotes, config: testConfig() });
const clone = new DatasetEngine({ dataset: await memStore({ backend }), remotes, config: testConfig() });
await clone.initialise();
await clone.status.becomes({ outdated: false });

Expand All @@ -346,7 +346,7 @@ describe('Dataset engine', () => {
const remoteUpdates = new Source<DeltaMessage>();
const remotes = mockRemotes(remoteUpdates, [true]);
remotes.revupFrom = async () => ({ lastTime: remoteTime, updates: EMPTY });
const clone = new DatasetEngine({ dataset: await memStore(ldb), remotes, config: testConfig() });
const clone = new DatasetEngine({ dataset: await memStore({ backend }), remotes, config: testConfig() });
await clone.initialise();
await clone.status.becomes({ outdated: false });

Expand All @@ -362,7 +362,7 @@ describe('Dataset engine', () => {
await clone.close(); // Will complete the updates
const arrived = await updates;
expect(arrived.length).toBe(1);
expect(arrived[0]).toMatchObject({ '@insert': [{ "@id": "http://test.m-ld.org/wilma" }]})
expect(arrived[0]).toMatchObject({ '@insert': [{ "@id": "http://test.m-ld.org/wilma" }] })
});
});
});
9 changes: 7 additions & 2 deletions test/MeldState.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,14 @@ describe('Meld State API', () => {
let api: ApiStateMachine;

beforeEach(async () => {
let clone = new DatasetEngine({ dataset: await memStore(), remotes: mockRemotes(), config: testConfig() });
const context = new DomainContext('test.m-ld.org');
let clone = new DatasetEngine({
dataset: await memStore({ context }),
remotes: mockRemotes(),
config: testConfig()
});
await clone.initialise();
api = new ApiStateMachine(new DomainContext('test.m-ld.org'), clone);
api = new ApiStateMachine(context, clone);
});

test('retrieves a JSON-LD subject', async () => {
Expand Down
50 changes: 50 additions & 0 deletions test/jsonld.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import { expandTerm, compactIri, ActiveContext, activeCtx } from '../src/engine/jsonld';

describe('JSON-LD', () => {
const context = {
'@base': 'http://example.org/',
'@vocab': 'http://example.org/#',
'mld': 'http://m-ld.org/'
};
let ctx: ActiveContext;

beforeAll(async () => {
ctx = await activeCtx(context);
});

describe('expand term', () => {
test('expand prefix', () => {
expect(expandTerm('mld:hello', ctx)).toBe('http://m-ld.org/hello');
});

test('ignores undefined prefix', () => {
expect(expandTerm('foo:hello', ctx)).toBe('foo:hello');
});

test('leaves an IRI', () => {
expect(expandTerm('http://hello.com/', ctx)).toBe('http://hello.com/');
});

test('expands against base', () => {
expect(expandTerm('hello', ctx)).toBe('http://example.org/hello');
});
});

describe('compact IRI', () => {
test('compacts with prefix', () => {
expect(compactIri('http://m-ld.org/hello', ctx)).toBe('mld:hello');
});

test('ignores prefix', () => {
expect(compactIri('foo:hello', ctx)).toBe('foo:hello');
});

test('leaves an unmatched IRI', () => {
expect(compactIri('http://hello.com/', ctx)).toBe('http://hello.com/');
});

test('compacts using base', () => {
expect(compactIri('http://example.org/hello', ctx)).toBe('hello');
});
});
});
Loading

0 comments on commit 2fc73d8

Please sign in to comment.