feat: release 3.0.2

deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@valkyr/db",
-  "version": "3.0.1",
+  "version": "3.0.2",
   "exports": {
     ".": "./src/mod.ts"
   },

src/databases/indexeddb/storage.ts

@@ -1,13 +1,12 @@
 import type { IDBPDatabase } from "idb";
-import { Query, update } from "mingo";
+import { update } from "mingo";
 import type { Criteria } from "mingo/types";
 import type { Modifier } from "mingo/updater";
 
-import type { IndexSpec } from "../../index/manager.ts";
+import { IndexManager, type IndexSpec } from "../../index/manager.ts";
 import { type DBLogger, InsertLog, QueryLog, RemoveLog, UpdateLog } from "../../logger.ts";
 import { addOptions, type QueryOptions, Storage, type UpdateResult } from "../../storage.ts";
-import type { AnyDocument } from "../../types.ts";
+import type { AnyDocument, StringKeyOf } from "../../types.ts";
-import { IndexedDBCache } from "./cache.ts";
 
 const OBJECT_PROTOTYPE = Object.getPrototypeOf({});
 const OBJECT_TAG = "[object Object]";
@@ -16,9 +15,9 @@ export class IndexedDBStorage<TSchema extends AnyDocument = AnyDocument> extends
   readonly pkey: string;
   readonly log: DBLogger;
 
-  readonly #cache = new IndexedDBCache<TSchema>();
+  readonly #index: IndexManager<TSchema>;
 
-  readonly #promise: Promise<IDBPDatabase>;
+  readonly #promise: Promise<void>;
 
   #db?: IDBPDatabase;
 
@@ -30,7 +29,16 @@ export class IndexedDBStorage<TSchema extends AnyDocument = AnyDocument> extends
     }
     this.pkey = index.field;
     this.log = log ?? function log() {};
-    this.#promise = promise;
+    this.#index = new IndexManager(indexes);
+    this.#promise = this.#preload(promise);
+  }
+
+  async #preload(promise: Promise<IDBPDatabase>): Promise<void> {
+    this.#db = await promise;
+    const records = await this.#db.getAll(this.name);
+    for (const record of records) {
+      await this.#index.insert(record);
+    }
   }
 
   get db(): IDBPDatabase {
@@ -40,67 +48,15 @@ export class IndexedDBStorage<TSchema extends AnyDocument = AnyDocument> extends
     return this.#db;
   }
 
+  get documents(): TSchema[] {
+    return this.#index.primary.documents;
+  }
+
   async resolve(): Promise<this> {
-    if (this.#db === undefined) {
+    await this.#promise;
-      this.#db = await this.#promise;
-    }
     return this;
   }
 
-  /*
-   |--------------------------------------------------------------------------------
-   | Indexes
-   |--------------------------------------------------------------------------------
-   */
-
-  #isPrimaryIndex(key: string): boolean {
-    for (const { field, kind } of this.indexes) {
-      if (key === field && kind === "primary") {
-        return true;
-      }
-    }
-    return false;
-  }
-
-  #isUniqueIndex(key: string): boolean {
-    for (const { field, kind } of this.indexes) {
-      if (key === field && kind === "unique") {
-        return true;
-      }
-    }
-    return false;
-  }
-
-  #isSharedIndex(key: string): boolean {
-    for (const { field, kind } of this.indexes) {
-      if (key === field && kind === "shared") {
-        return true;
-      }
-    }
-    return false;
-  }
-
-  #getOptimalIndex(keys: string[]): string {
-    let best: string | undefined;
-
-    for (const key of keys) {
-      if (this.#isPrimaryIndex(key)) {
-        return key; // cannot beat primary
-      }
-
-      if (this.#isUniqueIndex(key)) {
-        best ??= key;
-        continue;
-      }
-
-      if (best === undefined && this.#isSharedIndex(key)) {
-        best = key;
-      }
-    }
-
-    return best ?? keys[0];
-  }
-
   /*
    |--------------------------------------------------------------------------------
    | Insert
@@ -124,7 +80,9 @@ export class IndexedDBStorage<TSchema extends AnyDocument = AnyDocument> extends
     await tx.done;
 
     this.broadcast("insert", documents);
-    this.#cache.flush();
+    for (const document of documents) {
+      this.#index.insert(document);
+    }
 
     this.log(logger.result());
   }
@@ -135,153 +93,25 @@ export class IndexedDBStorage<TSchema extends AnyDocument = AnyDocument> extends
    |--------------------------------------------------------------------------------
    */
 
-  async getByIndex(index: string, value: string): Promise<TSchema[]> {
+  async getByIndex(field: StringKeyOf<TSchema>, value: string): Promise<TSchema[]> {
-    return this.db.getAllFromIndex(this.name, index, value);
+    return this.#index.getByIndex(field, value);
   }
 
   async find(condition: Criteria<TSchema> = {}, options?: QueryOptions): Promise<TSchema[]> {
     const logger = new QueryLog(this.name, { condition, options });
 
-    const hashCode = this.#cache.hash(condition, options);
+    const cursor = this.#index.getByCondition(condition);
-    const cached = this.#cache.get(hashCode);
-    if (cached !== undefined) {
-      this.log(logger.result({ cached: true }));
-      return cached;
-    }
-
-    const indexes = this.#resolveIndexes(condition);
-
-    let cursor = new Query(condition).find<TSchema>(await this.#getAll({ ...options }, indexes));
     if (options !== undefined) {
-      cursor = addOptions(cursor, options);
+      addOptions(cursor, options);
     }
 
-    const documents = cursor.all();
+    const documents = await cursor.all();
-    this.#cache.set(hashCode, documents);
 
     this.log(logger.result());
 
     return documents;
   }
 
-  /**
-   * TODO: Prototype! Needs to cover more mongodb query cases and investigation around
-   * nested indexing in indexeddb.
-   */
-  #resolveIndexes(filter: any): { [key: string]: any } | undefined {
-    const indexNames = this.db.transaction(this.name, "readonly").store.indexNames;
-    const index: { [key: string]: any } = {};
-    for (const key in filter) {
-      if (indexNames.contains(key) === true) {
-        let val: any;
-        if (isObject(filter[key]) === true) {
-          if ((filter as any)[key].$in !== undefined) {
-            val = (filter as any)[key].$in;
-          }
-        } else {
-          val = filter[key];
-        }
-        if (val !== undefined) {
-          index[key] = val;
-        }
-      }
-    }
-    if (Object.keys(index).length > 0) {
-      return index;
-    }
-  }
-
-  async #getAll(
-    { offset, range, limit }: QueryOptions,
-    indexes?: { [key: string]: IDBKeyRange | undefined },
-  ): Promise<TSchema[]> {
-    const tx = this.db.transaction(this.name, "readonly");
-
-    const store = tx.objectStore(this.name);
-
-    // ### Indexed
-    // Fetch all records by optimal index
-
-    if (indexes) {
-      const indexName = this.#getOptimalIndex(Object.keys(indexes));
-      const index = store.index(indexName);
-
-      const key = indexes[indexName];
-
-      const results: TSchema[] = [];
-
-      // Handle $in
-
-      if (Array.isArray(key)) {
-        for (const value of key) {
-          const records = await index.getAll(value);
-          results.push(...records);
-        }
-
-        // Deduplicate (required for $in)
-
-        const unique = new Map<any, TSchema>();
-        for (const doc of results) {
-          unique.set(this.pkey, doc); // adjust PK if needed
-        }
-
-        await tx.done;
-        return [...unique.values()];
-      }
-
-      // Single-key lookup
-
-      const records = await index.getAll(key);
-
-      await tx.done;
-      return records;
-    }
-
-    // ### Range
-    // Fetch records in a given range.
-
-    if (range) {
-      return store.getAll(IDBKeyRange.bound(range.from, range.to), limit);
-    }
-
-    // ### Offset
-    // Offset-based query (cursor-based)
-
-    if (offset) {
-      return this.#getAllByOffset(offset.value, offset.direction, limit);
-    }
-
-    // ### Default
-    // Fetch all records
-
-    return store.getAll(undefined, limit);
-  }
-
-  async #getAllByOffset(value: string, direction: 1 | -1, limit?: number) {
-    if (direction === 1) {
-      return this.db.getAll(this.name, IDBKeyRange.lowerBound(value), limit);
-    }
-    return this.#getAllByDescOffset(value, limit);
-  }
-
-  async #getAllByDescOffset(value: string, limit?: number) {
-    if (limit === undefined) {
-      return this.db.getAll(this.name, IDBKeyRange.upperBound(value));
-    }
-    const result = [];
-    let cursor = await this.db
-      .transaction(this.name, "readonly")
-      .store.openCursor(IDBKeyRange.upperBound(value), "prev");
-    for (let i = 0; i < limit; i++) {
-      if (cursor === null) {
-        break;
-      }
-      result.push(cursor.value);
-      cursor = await cursor.continue();
-    }
-    return result.reverse();
-  }
-
   /*
    |--------------------------------------------------------------------------------
    | Update
@@ -320,7 +150,9 @@ export class IndexedDBStorage<TSchema extends AnyDocument = AnyDocument> extends
     await tx.done;
 
     this.broadcast("update", documents);
-    this.#cache.flush();
+    for (const document of documents) {
+      this.#index.update(document);
+    }
 
     this.log(logger.result());
 
@@ -343,7 +175,9 @@ export class IndexedDBStorage<TSchema extends AnyDocument = AnyDocument> extends
     await tx.done;
 
     this.broadcast("remove", documents);
-    this.#cache.flush();
+    for (const document of documents) {
+      this.#index.remove(document);
+    }
 
     this.log(logger.result({ count: documents.length }));
 
@@ -371,6 +205,7 @@ export class IndexedDBStorage<TSchema extends AnyDocument = AnyDocument> extends
 
   async flush(): Promise<void> {
     await this.db.clear(this.name);
+    this.#index.flush();
   }
 }
 

src/databases/memory/storage.ts

@@ -35,7 +35,7 @@ export class MemoryStorage<TSchema extends AnyDocument = AnyDocument> extends St
   }
 
   async find(condition: Criteria<TSchema> = {}, options?: QueryOptions): Promise<TSchema[]> {
-    const cursor = new Query(condition).find<TSchema>(this.documents);
+    const cursor = this.index.getByCondition(condition);
     if (options !== undefined) {
       return addOptions(cursor, options).all();
     }

src/index/manager.ts

@@ -1,10 +1,14 @@
+import { Query } from "mingo";
+import type { Cursor } from "mingo/cursor";
 import type { Criteria } from "mingo/types";
 
-import type { AnyDocument, StringKeyOf } from "../types.ts";
+import type { AnyDocument, QueryCriteria, StringKeyOf } from "../types.ts";
 import { PrimaryIndex, type PrimaryKey } from "./primary.ts";
 import { SharedIndex } from "./shared.ts";
 import { UniqueIndex } from "./unique.ts";
 
+const OBJECT_PROTOTYPE = Object.getPrototypeOf({});
+const OBJECT_TAG = "[object Object]";
 const EMPTY_SET: ReadonlySet<PrimaryKey> = Object.freeze(new Set<PrimaryKey>());
 
 export class IndexManager<TSchema extends AnyDocument> {
@@ -13,7 +17,9 @@ export class IndexManager<TSchema extends AnyDocument> {
   readonly unique: Map<StringKeyOf<TSchema>, UniqueIndex> = new Map<StringKeyOf<TSchema>, UniqueIndex>();
   readonly shared: Map<StringKeyOf<TSchema>, SharedIndex> = new Map<StringKeyOf<TSchema>, SharedIndex>();
 
-  constructor(readonly specs: IndexSpec<TSchema>[]) {
+  readonly specs: IndexSpec<TSchema>[];
+
+  constructor(specs: IndexSpec<TSchema>[]) {
     const primary = specs.find((spec) => spec.kind === "primary");
     if (primary === undefined) {
       throw new Error("Primary index is required");
@@ -31,6 +37,55 @@ export class IndexManager<TSchema extends AnyDocument> {
         }
       }
     }
+    this.specs = specs;
+  }
+
+  #isPrimaryIndex(key: string): boolean {
+    for (const { field, kind } of this.specs) {
+      if (key === field && kind === "primary") {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  #isUniqueIndex(key: string): boolean {
+    for (const { field, kind } of this.specs) {
+      if (key === field && kind === "unique") {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  #isSharedIndex(key: string): boolean {
+    for (const { field, kind } of this.specs) {
+      if (key === field && kind === "shared") {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  #getOptimalIndex(keys: string[]): string {
+    let best: string | undefined;
+
+    for (const key of keys) {
+      if (this.#isPrimaryIndex(key)) {
+        return key; // cannot beat primary
+      }
+
+      if (this.#isUniqueIndex(key)) {
+        best ??= key;
+        continue;
+      }
+
+      if (best === undefined && this.#isSharedIndex(key)) {
+        best = key;
+      }
+    }
+
+    return best ?? keys[0];
   }
 
   /**
@@ -48,12 +103,32 @@ export class IndexManager<TSchema extends AnyDocument> {
 
     try {
       for (const [field, index] of this.unique) {
-        index.insert(document[field], pk);
+        const value = document[field] as any;
-        insertedUniques.push([field, document[field]]);
+        if (value !== undefined) {
+          if (Array.isArray(value)) {
+            for (const innerValue of value) {
+              index.insert(innerValue, pk);
+              insertedUniques.push([field, innerValue]);
+            }
+          } else {
+            index.insert(value, pk);
+            insertedUniques.push([field, value]);
+          }
+        }
       }
       for (const [field, index] of this.shared) {
-        index.insert(document[field], pk);
+        const value = document[field] as any;
-        insertedShared.push([field, document[field]]);
+        if (value !== undefined) {
+          if (Array.isArray(value)) {
+            for (const innerValue of value) {
+              index.insert(innerValue, pk);
+              insertedShared.push([field, innerValue]);
+            }
+          } else {
+            index.insert(value, pk);
+            insertedShared.push([field, value]);
+          }
+        }
       }
       this.primary.insert(pk, document);
     } catch (err) {
@@ -67,67 +142,30 @@ export class IndexManager<TSchema extends AnyDocument> {
     }
   }
 
-  getByCondition(condition: Criteria<TSchema>): TSchema[] {
+  getByCondition(condition: Criteria<TSchema> = {}): Cursor<TSchema> {
-    const indexedKeys = Array.from(
+    const indexes = resolveIndexesFromCondition(condition, this.specs);
-      new Set([this.primary.key as StringKeyOf<TSchema>, ...this.unique.keys(), ...this.shared.keys()]),
-    );
 
-    const candidatePKs: PrimaryKey[] = [];
+    if (indexes === undefined) {
-
+      return new Query(condition, {}).find<TSchema>(this.primary.documents);
-    // ### Primary Keys
-    // Collect primary keys for indexed equality conditions
-
-    const pkSets: ReadonlySet<PrimaryKey>[] = [];
-
-    for (const key of indexedKeys) {
-      const value = (condition as any)[key];
-      if (value !== undefined) {
-        // Use index if available
-        const pks = this.getPrimaryKeysByIndex(key, value);
-        pkSets.push(pks);
-      }
     }
 
-    // ### Intersect
+    const index = this.#getOptimalIndex(Object.keys(indexes));
-    // Intersect all sets to find candidates
+    const value = indexes[index];
-
-    if (pkSets.length > 0) {
-      const sortedSets = pkSets.sort((a, b) => a.size - b.size);
-      const intersection = new Set(sortedSets[0]);
-      for (let i = 1; i < sortedSets.length; i++) {
-        for (const pk of intersection) {
-          if (!sortedSets[i].has(pk)) {
-            intersection.delete(pk);
-          }
-        }
-      }
-      candidatePKs.push(...intersection);
-    } else {
-      candidatePKs.push(...this.primary.keys()); // no indexed fields → scan all primary keys
-    }
-
-    // ### Filter
-    // Filter candidates by remaining condition
 
+    if (Array.isArray(value)) {
       const results: TSchema[] = [];
-    for (const pk of candidatePKs) {
+      for (const innerValue of value) {
-      const doc = this.primary.get(pk);
+        const records = this.getByIndex(index as any, innerValue);
-      if (doc === undefined) {
+        results.push(...records);
-        continue;
       }
-      let match = true;
+      const unique = new Map<any, TSchema>();
-      for (const [field, expected] of Object.entries(condition)) {
+      for (const doc of results) {
-        if ((doc as any)[field] !== expected) {
+        unique.set(doc[this.primary.key], doc);
-          match = false;
-          break;
-        }
-      }
-      if (match) {
-        results.push(doc);
       }
+      return new Query(condition, {}).find<TSchema>(Array.from(unique.values()));
     }
 
-    return results;
+    return new Query(condition, {}).find<TSchema>(this.getByIndex(index as any, value));
   }
 
   /**
@@ -305,6 +343,55 @@ export class IndexManager<TSchema extends AnyDocument> {
   }
 }
 
+/*
+ |--------------------------------------------------------------------------------
+ | Utils
+ |--------------------------------------------------------------------------------
+ */
+
+function resolveIndexesFromCondition<TSchema extends AnyDocument>(
+  condition: QueryCriteria<TSchema>,
+  indexes: IndexSpec<TSchema>[],
+): Record<StringKeyOf<TSchema>, any> | undefined {
+  const indexNames = indexes.map(({ field }) => field);
+
+  const index: any = {};
+
+  for (const key in condition) {
+    if (indexNames.includes(key as any) === true) {
+      let val: any;
+      if (isObject(condition[key]) === true) {
+        if ((condition as any)[key].$in !== undefined) {
+          val = (condition as any)[key].$in;
+        }
+      } else {
+        val = condition[key];
+      }
+      if (val !== undefined) {
+        index[key] = val;
+      }
+    }
+  }
+
+  if (Object.keys(index).length > 0) {
+    return index;
+  }
+}
+
+function isObject(v: any): v is object {
+  if (!v) {
+    return false;
+  }
+  const proto = Object.getPrototypeOf(v);
+  return (proto === OBJECT_PROTOTYPE || proto === null) && OBJECT_TAG === Object.prototype.toString.call(v);
+}
+
+/*
+ |--------------------------------------------------------------------------------
+ | Types
+ |--------------------------------------------------------------------------------
+ */
+
 export type IndexSpec<TSchema extends AnyDocument> = {
   field: StringKeyOf<TSchema>;
   kind: IndexKind;

src/index/primary.ts

@@ -3,9 +3,13 @@ import type { AnyDocument } from "../types.ts";
 export type PrimaryKey = string;
 
 export class PrimaryIndex<TSchema extends AnyDocument> {
+  readonly key: string;
+
   readonly #index = new Map<PrimaryKey, TSchema>();
 
-  constructor(readonly key: string) {}
+  constructor(key: string) {
+    this.key = key;
+  }
 
   get documents(): TSchema[] {
     return Array.from(this.#index.values());
@@ -21,7 +25,8 @@ export class PrimaryIndex<TSchema extends AnyDocument> {
 
   insert(pk: PrimaryKey, document: TSchema): void {
     if (this.#index.has(pk)) {
-      throw new Error(`Duplicate primary key: ${pk}`);
+      console.warn(`Duplicate primary key: ${pk}`);
+      return;
     }
     this.#index.set(pk, document);
   }

tests/collection.test.ts

@@ -29,7 +29,17 @@ describe("Collection", { sanitizeOps: false, sanitizeResources: false }, () => {
     name: string;
     storage: MemoryStorage;
     schema: UserSchema;
-    indexes: [{ field: "id"; kind: "primary" }];
+    indexes: [
+      { field: "id"; kind: "primary" },
+      {
+        field: "emails";
+        kind: "unique";
+      },
+      {
+        field: "name";
+        kind: "shared";
+      },
+    ];
   }>;
 
   beforeEach(() => {
@@ -40,25 +50,29 @@ describe("Collection", { sanitizeOps: false, sanitizeResources: false }, () => {
           field: "id",
           kind: "primary",
         },
-      ]),
+        {
-      schema: {
+          field: "emails",
-        id: z.string(),
+          kind: "unique",
-        name: z.string().optional(),
-        fullName: z.string().optional(),
-        emails: z.array(z.email()),
-        friends: z.array(
-          z.object({
-            id: z.string(),
-            type: z.union([z.literal("family"), z.literal("close")]),
-          }),
-        ),
-        age: z.number(),
         },
+        {
+          field: "name",
+          kind: "shared",
+        },
+      ]),
+      schema,
       indexes: [
         {
           field: "id",
           kind: "primary",
         },
+        {
+          field: "emails",
+          kind: "unique",
+        },
+        {
+          field: "name",
+          kind: "shared",
+        },
       ],
     });
   });
@@ -239,6 +253,18 @@ describe("Collection", { sanitizeOps: false, sanitizeResources: false }, () => {
         expect(docs.every((d) => d.age >= 30)).toBe(true);
       });
 
+      it("should find documents by indexed condition", async () => {
+        const docs = await collection.findMany({ name: "Bob" });
+        expect(docs).toHaveLength(1);
+        expect(docs[0].name).toBe("Bob");
+      });
+
+      it("should find documents by nested indexed condition", async () => {
+        const docs = await collection.findMany({ emails: { $in: ["charlie@test.com"] } });
+        expect(docs).toHaveLength(1);
+        expect(docs[0].emails[0]).toBe("charlie@test.com");
+      });
+
       it("should support limit option", async () => {
         const docs = await collection.findMany({}, { limit: 2 });
         expect(docs).toHaveLength(2);

tests/index-manager.test.ts

@@ -159,7 +159,7 @@ describe("IndexManager", { sanitizeOps: false, sanitizeResources: false }, () =>
       const user = { id: "u1", email: "a@example.com", group: "staff", name: "Alice", active: true };
       manager.insert(user);
 
-      const results = manager.getByCondition({ id: "u1" });
+      const results = manager.getByCondition({ id: "u1" }).all();
       expect(results).toHaveLength(1);
       expect(results[0]).toEqual(user);
     });
@@ -170,7 +170,7 @@ describe("IndexManager", { sanitizeOps: false, sanitizeResources: false }, () =>
       const user = { id: "u1", email: "a@example.com", group: "staff", name: "Alice", active: true };
       manager.insert(user);
 
-      const results = manager.getByCondition({ email: "a@example.com" });
+      const results = manager.getByCondition({ email: "a@example.com" }).all();
       expect(results).toHaveLength(1);
       expect(results[0]).toEqual(user);
     });
@@ -186,12 +186,12 @@ describe("IndexManager", { sanitizeOps: false, sanitizeResources: false }, () =>
       manager.insert(user2);
       manager.insert(user3);
 
-      const staff = manager.getByCondition({ group: "staff" });
+      const staff = manager.getByCondition({ group: "staff" }).all();
       expect(staff).toHaveLength(2);
       expect(staff).toContainEqual(user1);
       expect(staff).toContainEqual(user2);
 
-      const admin = manager.getByCondition({ group: "admin" });
+      const admin = manager.getByCondition({ group: "admin" }).all();
       expect(admin).toHaveLength(1);
       expect(admin[0]).toEqual(user3);
     });
@@ -201,11 +201,14 @@ describe("IndexManager", { sanitizeOps: false, sanitizeResources: false }, () =>
 
       const user1 = { id: "u1", email: "a@example.com", group: "staff", name: "Alice", active: true };
       const user2 = { id: "u2", email: "b@example.com", group: "staff", name: "Bob", active: false };
+
       manager.insert(user1);
       manager.insert(user2);
 
       // Lookup by shared + non-indexed field
-      const results = manager.getByCondition({ group: "staff", active: true });
+
+      const results = manager.getByCondition({ group: "staff", active: true }).all();
+
       expect(results).toHaveLength(1);
       expect(results[0]).toEqual(user1);
     });
@@ -216,10 +219,10 @@ describe("IndexManager", { sanitizeOps: false, sanitizeResources: false }, () =>
       const user = { id: "u1", email: "a@example.com", group: "staff", name: "Alice", active: true };
       manager.insert(user);
 
-      const results = manager.getByCondition({ group: "admin" });
+      const results = manager.getByCondition({ group: "admin" }).all();
       expect(results).toEqual([]);
 
-      const results2 = manager.getByCondition({ email: "nonexistent@example.com" });
+      const results2 = manager.getByCondition({ email: "nonexistent@example.com" }).all();
       expect(results2).toEqual([]);
     });
   });