diff --git a/deno.json b/deno.json index 3bda8f9..71d2bab 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "name": "@valkyr/db", - "version": "3.0.1", + "version": "3.0.2", "exports": { ".": "./src/mod.ts" }, diff --git a/src/databases/indexeddb/storage.ts b/src/databases/indexeddb/storage.ts index 51cbd27..a4c5bd0 100644 --- a/src/databases/indexeddb/storage.ts +++ b/src/databases/indexeddb/storage.ts @@ -1,13 +1,12 @@ import type { IDBPDatabase } from "idb"; -import { Query, update } from "mingo"; +import { update } from "mingo"; import type { Criteria } from "mingo/types"; import type { Modifier } from "mingo/updater"; -import type { IndexSpec } from "../../index/manager.ts"; +import { IndexManager, type IndexSpec } from "../../index/manager.ts"; import { type DBLogger, InsertLog, QueryLog, RemoveLog, UpdateLog } from "../../logger.ts"; import { addOptions, type QueryOptions, Storage, type UpdateResult } from "../../storage.ts"; -import type { AnyDocument } from "../../types.ts"; -import { IndexedDBCache } from "./cache.ts"; +import type { AnyDocument, StringKeyOf } from "../../types.ts"; const OBJECT_PROTOTYPE = Object.getPrototypeOf({}); const OBJECT_TAG = "[object Object]"; @@ -16,9 +15,9 @@ export class IndexedDBStorage extends readonly pkey: string; readonly log: DBLogger; - readonly #cache = new IndexedDBCache(); + readonly #index: IndexManager; - readonly #promise: Promise; + readonly #promise: Promise; #db?: IDBPDatabase; @@ -30,7 +29,16 @@ export class IndexedDBStorage extends } this.pkey = index.field; this.log = log ?? function log() {}; - this.#promise = promise; + this.#index = new IndexManager(indexes); + this.#promise = this.#preload(promise); + } + + async #preload(promise: Promise): Promise { + this.#db = await promise; + const records = await this.#db.getAll(this.name); + for (const record of records) { + await this.#index.insert(record); + } } get db(): IDBPDatabase { @@ -40,67 +48,15 @@ export class IndexedDBStorage extends return this.#db; } + get documents(): TSchema[] { + return this.#index.primary.documents; + } + async resolve(): Promise { - if (this.#db === undefined) { - this.#db = await this.#promise; - } + await this.#promise; return this; } - /* - |-------------------------------------------------------------------------------- - | Indexes - |-------------------------------------------------------------------------------- - */ - - #isPrimaryIndex(key: string): boolean { - for (const { field, kind } of this.indexes) { - if (key === field && kind === "primary") { - return true; - } - } - return false; - } - - #isUniqueIndex(key: string): boolean { - for (const { field, kind } of this.indexes) { - if (key === field && kind === "unique") { - return true; - } - } - return false; - } - - #isSharedIndex(key: string): boolean { - for (const { field, kind } of this.indexes) { - if (key === field && kind === "shared") { - return true; - } - } - return false; - } - - #getOptimalIndex(keys: string[]): string { - let best: string | undefined; - - for (const key of keys) { - if (this.#isPrimaryIndex(key)) { - return key; // cannot beat primary - } - - if (this.#isUniqueIndex(key)) { - best ??= key; - continue; - } - - if (best === undefined && this.#isSharedIndex(key)) { - best = key; - } - } - - return best ?? keys[0]; - } - /* |-------------------------------------------------------------------------------- | Insert @@ -124,7 +80,9 @@ export class IndexedDBStorage extends await tx.done; this.broadcast("insert", documents); - this.#cache.flush(); + for (const document of documents) { + this.#index.insert(document); + } this.log(logger.result()); } @@ -135,153 +93,25 @@ export class IndexedDBStorage extends |-------------------------------------------------------------------------------- */ - async getByIndex(index: string, value: string): Promise { - return this.db.getAllFromIndex(this.name, index, value); + async getByIndex(field: StringKeyOf, value: string): Promise { + return this.#index.getByIndex(field, value); } async find(condition: Criteria = {}, options?: QueryOptions): Promise { const logger = new QueryLog(this.name, { condition, options }); - const hashCode = this.#cache.hash(condition, options); - const cached = this.#cache.get(hashCode); - if (cached !== undefined) { - this.log(logger.result({ cached: true })); - return cached; - } - - const indexes = this.#resolveIndexes(condition); - - let cursor = new Query(condition).find(await this.#getAll({ ...options }, indexes)); + const cursor = this.#index.getByCondition(condition); if (options !== undefined) { - cursor = addOptions(cursor, options); + addOptions(cursor, options); } - const documents = cursor.all(); - this.#cache.set(hashCode, documents); + const documents = await cursor.all(); this.log(logger.result()); return documents; } - /** - * TODO: Prototype! Needs to cover more mongodb query cases and investigation around - * nested indexing in indexeddb. - */ - #resolveIndexes(filter: any): { [key: string]: any } | undefined { - const indexNames = this.db.transaction(this.name, "readonly").store.indexNames; - const index: { [key: string]: any } = {}; - for (const key in filter) { - if (indexNames.contains(key) === true) { - let val: any; - if (isObject(filter[key]) === true) { - if ((filter as any)[key].$in !== undefined) { - val = (filter as any)[key].$in; - } - } else { - val = filter[key]; - } - if (val !== undefined) { - index[key] = val; - } - } - } - if (Object.keys(index).length > 0) { - return index; - } - } - - async #getAll( - { offset, range, limit }: QueryOptions, - indexes?: { [key: string]: IDBKeyRange | undefined }, - ): Promise { - const tx = this.db.transaction(this.name, "readonly"); - - const store = tx.objectStore(this.name); - - // ### Indexed - // Fetch all records by optimal index - - if (indexes) { - const indexName = this.#getOptimalIndex(Object.keys(indexes)); - const index = store.index(indexName); - - const key = indexes[indexName]; - - const results: TSchema[] = []; - - // Handle $in - - if (Array.isArray(key)) { - for (const value of key) { - const records = await index.getAll(value); - results.push(...records); - } - - // Deduplicate (required for $in) - - const unique = new Map(); - for (const doc of results) { - unique.set(this.pkey, doc); // adjust PK if needed - } - - await tx.done; - return [...unique.values()]; - } - - // Single-key lookup - - const records = await index.getAll(key); - - await tx.done; - return records; - } - - // ### Range - // Fetch records in a given range. - - if (range) { - return store.getAll(IDBKeyRange.bound(range.from, range.to), limit); - } - - // ### Offset - // Offset-based query (cursor-based) - - if (offset) { - return this.#getAllByOffset(offset.value, offset.direction, limit); - } - - // ### Default - // Fetch all records - - return store.getAll(undefined, limit); - } - - async #getAllByOffset(value: string, direction: 1 | -1, limit?: number) { - if (direction === 1) { - return this.db.getAll(this.name, IDBKeyRange.lowerBound(value), limit); - } - return this.#getAllByDescOffset(value, limit); - } - - async #getAllByDescOffset(value: string, limit?: number) { - if (limit === undefined) { - return this.db.getAll(this.name, IDBKeyRange.upperBound(value)); - } - const result = []; - let cursor = await this.db - .transaction(this.name, "readonly") - .store.openCursor(IDBKeyRange.upperBound(value), "prev"); - for (let i = 0; i < limit; i++) { - if (cursor === null) { - break; - } - result.push(cursor.value); - cursor = await cursor.continue(); - } - return result.reverse(); - } - /* |-------------------------------------------------------------------------------- | Update @@ -320,7 +150,9 @@ export class IndexedDBStorage extends await tx.done; this.broadcast("update", documents); - this.#cache.flush(); + for (const document of documents) { + this.#index.update(document); + } this.log(logger.result()); @@ -343,7 +175,9 @@ export class IndexedDBStorage extends await tx.done; this.broadcast("remove", documents); - this.#cache.flush(); + for (const document of documents) { + this.#index.remove(document); + } this.log(logger.result({ count: documents.length })); @@ -371,6 +205,7 @@ export class IndexedDBStorage extends async flush(): Promise { await this.db.clear(this.name); + this.#index.flush(); } } diff --git a/src/databases/memory/storage.ts b/src/databases/memory/storage.ts index 2f03b7a..6f5af79 100644 --- a/src/databases/memory/storage.ts +++ b/src/databases/memory/storage.ts @@ -35,7 +35,7 @@ export class MemoryStorage extends St } async find(condition: Criteria = {}, options?: QueryOptions): Promise { - const cursor = new Query(condition).find(this.documents); + const cursor = this.index.getByCondition(condition); if (options !== undefined) { return addOptions(cursor, options).all(); } diff --git a/src/index/manager.ts b/src/index/manager.ts index ee8ba00..10dac80 100644 --- a/src/index/manager.ts +++ b/src/index/manager.ts @@ -1,10 +1,14 @@ +import { Query } from "mingo"; +import type { Cursor } from "mingo/cursor"; import type { Criteria } from "mingo/types"; -import type { AnyDocument, StringKeyOf } from "../types.ts"; +import type { AnyDocument, QueryCriteria, StringKeyOf } from "../types.ts"; import { PrimaryIndex, type PrimaryKey } from "./primary.ts"; import { SharedIndex } from "./shared.ts"; import { UniqueIndex } from "./unique.ts"; +const OBJECT_PROTOTYPE = Object.getPrototypeOf({}); +const OBJECT_TAG = "[object Object]"; const EMPTY_SET: ReadonlySet = Object.freeze(new Set()); export class IndexManager { @@ -13,7 +17,9 @@ export class IndexManager { readonly unique: Map, UniqueIndex> = new Map, UniqueIndex>(); readonly shared: Map, SharedIndex> = new Map, SharedIndex>(); - constructor(readonly specs: IndexSpec[]) { + readonly specs: IndexSpec[]; + + constructor(specs: IndexSpec[]) { const primary = specs.find((spec) => spec.kind === "primary"); if (primary === undefined) { throw new Error("Primary index is required"); @@ -31,6 +37,55 @@ export class IndexManager { } } } + this.specs = specs; + } + + #isPrimaryIndex(key: string): boolean { + for (const { field, kind } of this.specs) { + if (key === field && kind === "primary") { + return true; + } + } + return false; + } + + #isUniqueIndex(key: string): boolean { + for (const { field, kind } of this.specs) { + if (key === field && kind === "unique") { + return true; + } + } + return false; + } + + #isSharedIndex(key: string): boolean { + for (const { field, kind } of this.specs) { + if (key === field && kind === "shared") { + return true; + } + } + return false; + } + + #getOptimalIndex(keys: string[]): string { + let best: string | undefined; + + for (const key of keys) { + if (this.#isPrimaryIndex(key)) { + return key; // cannot beat primary + } + + if (this.#isUniqueIndex(key)) { + best ??= key; + continue; + } + + if (best === undefined && this.#isSharedIndex(key)) { + best = key; + } + } + + return best ?? keys[0]; } /** @@ -48,12 +103,32 @@ export class IndexManager { try { for (const [field, index] of this.unique) { - index.insert(document[field], pk); - insertedUniques.push([field, document[field]]); + const value = document[field] as any; + if (value !== undefined) { + if (Array.isArray(value)) { + for (const innerValue of value) { + index.insert(innerValue, pk); + insertedUniques.push([field, innerValue]); + } + } else { + index.insert(value, pk); + insertedUniques.push([field, value]); + } + } } for (const [field, index] of this.shared) { - index.insert(document[field], pk); - insertedShared.push([field, document[field]]); + const value = document[field] as any; + if (value !== undefined) { + if (Array.isArray(value)) { + for (const innerValue of value) { + index.insert(innerValue, pk); + insertedShared.push([field, innerValue]); + } + } else { + index.insert(value, pk); + insertedShared.push([field, value]); + } + } } this.primary.insert(pk, document); } catch (err) { @@ -67,67 +142,30 @@ export class IndexManager { } } - getByCondition(condition: Criteria): TSchema[] { - const indexedKeys = Array.from( - new Set([this.primary.key as StringKeyOf, ...this.unique.keys(), ...this.shared.keys()]), - ); + getByCondition(condition: Criteria = {}): Cursor { + const indexes = resolveIndexesFromCondition(condition, this.specs); - const candidatePKs: PrimaryKey[] = []; - - // ### Primary Keys - // Collect primary keys for indexed equality conditions - - const pkSets: ReadonlySet[] = []; - - for (const key of indexedKeys) { - const value = (condition as any)[key]; - if (value !== undefined) { - // Use index if available - const pks = this.getPrimaryKeysByIndex(key, value); - pkSets.push(pks); - } + if (indexes === undefined) { + return new Query(condition, {}).find(this.primary.documents); } - // ### Intersect - // Intersect all sets to find candidates + const index = this.#getOptimalIndex(Object.keys(indexes)); + const value = indexes[index]; - if (pkSets.length > 0) { - const sortedSets = pkSets.sort((a, b) => a.size - b.size); - const intersection = new Set(sortedSets[0]); - for (let i = 1; i < sortedSets.length; i++) { - for (const pk of intersection) { - if (!sortedSets[i].has(pk)) { - intersection.delete(pk); - } - } + if (Array.isArray(value)) { + const results: TSchema[] = []; + for (const innerValue of value) { + const records = this.getByIndex(index as any, innerValue); + results.push(...records); } - candidatePKs.push(...intersection); - } else { - candidatePKs.push(...this.primary.keys()); // no indexed fields → scan all primary keys + const unique = new Map(); + for (const doc of results) { + unique.set(doc[this.primary.key], doc); + } + return new Query(condition, {}).find(Array.from(unique.values())); } - // ### Filter - // Filter candidates by remaining condition - - const results: TSchema[] = []; - for (const pk of candidatePKs) { - const doc = this.primary.get(pk); - if (doc === undefined) { - continue; - } - let match = true; - for (const [field, expected] of Object.entries(condition)) { - if ((doc as any)[field] !== expected) { - match = false; - break; - } - } - if (match) { - results.push(doc); - } - } - - return results; + return new Query(condition, {}).find(this.getByIndex(index as any, value)); } /** @@ -305,6 +343,55 @@ export class IndexManager { } } +/* + |-------------------------------------------------------------------------------- + | Utils + |-------------------------------------------------------------------------------- + */ + +function resolveIndexesFromCondition( + condition: QueryCriteria, + indexes: IndexSpec[], +): Record, any> | undefined { + const indexNames = indexes.map(({ field }) => field); + + const index: any = {}; + + for (const key in condition) { + if (indexNames.includes(key as any) === true) { + let val: any; + if (isObject(condition[key]) === true) { + if ((condition as any)[key].$in !== undefined) { + val = (condition as any)[key].$in; + } + } else { + val = condition[key]; + } + if (val !== undefined) { + index[key] = val; + } + } + } + + if (Object.keys(index).length > 0) { + return index; + } +} + +function isObject(v: any): v is object { + if (!v) { + return false; + } + const proto = Object.getPrototypeOf(v); + return (proto === OBJECT_PROTOTYPE || proto === null) && OBJECT_TAG === Object.prototype.toString.call(v); +} + +/* + |-------------------------------------------------------------------------------- + | Types + |-------------------------------------------------------------------------------- + */ + export type IndexSpec = { field: StringKeyOf; kind: IndexKind; diff --git a/src/index/primary.ts b/src/index/primary.ts index c91bcf9..1859f6c 100644 --- a/src/index/primary.ts +++ b/src/index/primary.ts @@ -3,9 +3,13 @@ import type { AnyDocument } from "../types.ts"; export type PrimaryKey = string; export class PrimaryIndex { + readonly key: string; + readonly #index = new Map(); - constructor(readonly key: string) {} + constructor(key: string) { + this.key = key; + } get documents(): TSchema[] { return Array.from(this.#index.values()); @@ -21,7 +25,8 @@ export class PrimaryIndex { insert(pk: PrimaryKey, document: TSchema): void { if (this.#index.has(pk)) { - throw new Error(`Duplicate primary key: ${pk}`); + console.warn(`Duplicate primary key: ${pk}`); + return; } this.#index.set(pk, document); } diff --git a/tests/collection.test.ts b/tests/collection.test.ts index 5d0d943..cb6b6be 100644 --- a/tests/collection.test.ts +++ b/tests/collection.test.ts @@ -29,7 +29,17 @@ describe("Collection", { sanitizeOps: false, sanitizeResources: false }, () => { name: string; storage: MemoryStorage; schema: UserSchema; - indexes: [{ field: "id"; kind: "primary" }]; + indexes: [ + { field: "id"; kind: "primary" }, + { + field: "emails"; + kind: "unique"; + }, + { + field: "name"; + kind: "shared"; + }, + ]; }>; beforeEach(() => { @@ -40,25 +50,29 @@ describe("Collection", { sanitizeOps: false, sanitizeResources: false }, () => { field: "id", kind: "primary", }, + { + field: "emails", + kind: "unique", + }, + { + field: "name", + kind: "shared", + }, ]), - schema: { - id: z.string(), - name: z.string().optional(), - fullName: z.string().optional(), - emails: z.array(z.email()), - friends: z.array( - z.object({ - id: z.string(), - type: z.union([z.literal("family"), z.literal("close")]), - }), - ), - age: z.number(), - }, + schema, indexes: [ { field: "id", kind: "primary", }, + { + field: "emails", + kind: "unique", + }, + { + field: "name", + kind: "shared", + }, ], }); }); @@ -239,6 +253,18 @@ describe("Collection", { sanitizeOps: false, sanitizeResources: false }, () => { expect(docs.every((d) => d.age >= 30)).toBe(true); }); + it("should find documents by indexed condition", async () => { + const docs = await collection.findMany({ name: "Bob" }); + expect(docs).toHaveLength(1); + expect(docs[0].name).toBe("Bob"); + }); + + it("should find documents by nested indexed condition", async () => { + const docs = await collection.findMany({ emails: { $in: ["charlie@test.com"] } }); + expect(docs).toHaveLength(1); + expect(docs[0].emails[0]).toBe("charlie@test.com"); + }); + it("should support limit option", async () => { const docs = await collection.findMany({}, { limit: 2 }); expect(docs).toHaveLength(2); diff --git a/tests/index-manager.test.ts b/tests/index-manager.test.ts index c8b1a13..e2b3044 100644 --- a/tests/index-manager.test.ts +++ b/tests/index-manager.test.ts @@ -159,7 +159,7 @@ describe("IndexManager", { sanitizeOps: false, sanitizeResources: false }, () => const user = { id: "u1", email: "a@example.com", group: "staff", name: "Alice", active: true }; manager.insert(user); - const results = manager.getByCondition({ id: "u1" }); + const results = manager.getByCondition({ id: "u1" }).all(); expect(results).toHaveLength(1); expect(results[0]).toEqual(user); }); @@ -170,7 +170,7 @@ describe("IndexManager", { sanitizeOps: false, sanitizeResources: false }, () => const user = { id: "u1", email: "a@example.com", group: "staff", name: "Alice", active: true }; manager.insert(user); - const results = manager.getByCondition({ email: "a@example.com" }); + const results = manager.getByCondition({ email: "a@example.com" }).all(); expect(results).toHaveLength(1); expect(results[0]).toEqual(user); }); @@ -186,12 +186,12 @@ describe("IndexManager", { sanitizeOps: false, sanitizeResources: false }, () => manager.insert(user2); manager.insert(user3); - const staff = manager.getByCondition({ group: "staff" }); + const staff = manager.getByCondition({ group: "staff" }).all(); expect(staff).toHaveLength(2); expect(staff).toContainEqual(user1); expect(staff).toContainEqual(user2); - const admin = manager.getByCondition({ group: "admin" }); + const admin = manager.getByCondition({ group: "admin" }).all(); expect(admin).toHaveLength(1); expect(admin[0]).toEqual(user3); }); @@ -201,11 +201,14 @@ describe("IndexManager", { sanitizeOps: false, sanitizeResources: false }, () => const user1 = { id: "u1", email: "a@example.com", group: "staff", name: "Alice", active: true }; const user2 = { id: "u2", email: "b@example.com", group: "staff", name: "Bob", active: false }; + manager.insert(user1); manager.insert(user2); // Lookup by shared + non-indexed field - const results = manager.getByCondition({ group: "staff", active: true }); + + const results = manager.getByCondition({ group: "staff", active: true }).all(); + expect(results).toHaveLength(1); expect(results[0]).toEqual(user1); }); @@ -216,10 +219,10 @@ describe("IndexManager", { sanitizeOps: false, sanitizeResources: false }, () => const user = { id: "u1", email: "a@example.com", group: "staff", name: "Alice", active: true }; manager.insert(user); - const results = manager.getByCondition({ group: "admin" }); + const results = manager.getByCondition({ group: "admin" }).all(); expect(results).toEqual([]); - const results2 = manager.getByCondition({ email: "nonexistent@example.com" }); + const results2 = manager.getByCondition({ email: "nonexistent@example.com" }).all(); expect(results2).toEqual([]); }); });