kodemon/db
1
0
Fork 0
Code Issues 1 Actions Wiki Activity

feat: release 3.0.2

Browse Source
This commit is contained in:
kodemon
2026-01-07 02:34:18 +01:00
parent 72355de920
commit 53b43f6253
7 changed files with 241 additions and 285 deletions
Download Patch File Download Diff File Expand all files Collapse all files

235
src/databases/indexeddb/storage.ts
View File

@@ -1,13 +1,12 @@
import type { IDBPDatabase } from "idb";
import { Query, update } from "mingo";
import { update } from "mingo";
import type { Criteria } from "mingo/types";
import type { Modifier } from "mingo/updater";
import type { IndexSpec } from "../../index/manager.ts";
import { IndexManager, type IndexSpec } from "../../index/manager.ts";
import { type DBLogger, InsertLog, QueryLog, RemoveLog, UpdateLog } from "../../logger.ts";
import { addOptions, type QueryOptions, Storage, type UpdateResult } from "../../storage.ts";
import type { AnyDocument } from "../../types.ts";
import { IndexedDBCache } from "./cache.ts";
import type { AnyDocument, StringKeyOf } from "../../types.ts";
const OBJECT_PROTOTYPE = Object.getPrototypeOf({});
const OBJECT_TAG = "[object Object]";
@@ -16,9 +15,9 @@ export class IndexedDBStorage<TSchema extends AnyDocument = AnyDocument> extends
readonly pkey: string;
readonly log: DBLogger;
readonly #cache = new IndexedDBCache<TSchema>();
readonly #index: IndexManager<TSchema>;
readonly #promise: Promise<IDBPDatabase>;
readonly #promise: Promise<void>;
#db?: IDBPDatabase;
@@ -30,7 +29,16 @@ export class IndexedDBStorage<TSchema extends AnyDocument = AnyDocument> extends
}
this.pkey = index.field;
this.log = log ?? function log() {};
this.#promise = promise;
this.#index = new IndexManager(indexes);
this.#promise = this.#preload(promise);
}
async #preload(promise: Promise<IDBPDatabase>): Promise<void> {
this.#db = await promise;
const records = await this.#db.getAll(this.name);
for (const record of records) {
await this.#index.insert(record);
}
}
get db(): IDBPDatabase {
@@ -40,67 +48,15 @@ export class IndexedDBStorage<TSchema extends AnyDocument = AnyDocument> extends
return this.#db;
}
get documents(): TSchema[] {
return this.#index.primary.documents;
}
async resolve(): Promise<this> {
if (this.#db === undefined) {
this.#db = await this.#promise;
}
await this.#promise;
return this;
}
/*
|--------------------------------------------------------------------------------
| Indexes
|--------------------------------------------------------------------------------
*/
#isPrimaryIndex(key: string): boolean {
for (const { field, kind } of this.indexes) {
if (key === field && kind === "primary") {
return true;
}
}
return false;
}
#isUniqueIndex(key: string): boolean {
for (const { field, kind } of this.indexes) {
if (key === field && kind === "unique") {
return true;
}
}
return false;
}
#isSharedIndex(key: string): boolean {
for (const { field, kind } of this.indexes) {
if (key === field && kind === "shared") {
return true;
}
}
return false;
}
#getOptimalIndex(keys: string[]): string {
let best: string | undefined;
for (const key of keys) {
if (this.#isPrimaryIndex(key)) {
return key; // cannot beat primary
}
if (this.#isUniqueIndex(key)) {
best ??= key;
continue;
}
if (best === undefined && this.#isSharedIndex(key)) {
best = key;
}
}
return best ?? keys[0];
}
/*
|--------------------------------------------------------------------------------
| Insert
@@ -124,7 +80,9 @@ export class IndexedDBStorage<TSchema extends AnyDocument = AnyDocument> extends
await tx.done;
this.broadcast("insert", documents);
this.#cache.flush();
for (const document of documents) {
this.#index.insert(document);
}
this.log(logger.result());
}
@@ -135,153 +93,25 @@ export class IndexedDBStorage<TSchema extends AnyDocument = AnyDocument> extends
|--------------------------------------------------------------------------------
*/
async getByIndex(index: string, value: string): Promise<TSchema[]> {
return this.db.getAllFromIndex(this.name, index, value);
async getByIndex(field: StringKeyOf<TSchema>, value: string): Promise<TSchema[]> {
return this.#index.getByIndex(field, value);
}
async find(condition: Criteria<TSchema> = {}, options?: QueryOptions): Promise<TSchema[]> {
const logger = new QueryLog(this.name, { condition, options });
const hashCode = this.#cache.hash(condition, options);
const cached = this.#cache.get(hashCode);
if (cached !== undefined) {
this.log(logger.result({ cached: true }));
return cached;
}
const indexes = this.#resolveIndexes(condition);
let cursor = new Query(condition).find<TSchema>(await this.#getAll({ ...options }, indexes));
const cursor = this.#index.getByCondition(condition);
if (options !== undefined) {
cursor = addOptions(cursor, options);
addOptions(cursor, options);
}
const documents = cursor.all();
this.#cache.set(hashCode, documents);
const documents = await cursor.all();
this.log(logger.result());
return documents;
}
/**
* TODO: Prototype! Needs to cover more mongodb query cases and investigation around
* nested indexing in indexeddb.
*/
#resolveIndexes(filter: any): { [key: string]: any } | undefined {
const indexNames = this.db.transaction(this.name, "readonly").store.indexNames;
const index: { [key: string]: any } = {};
for (const key in filter) {
if (indexNames.contains(key) === true) {
let val: any;
if (isObject(filter[key]) === true) {
if ((filter as any)[key].$in !== undefined) {
val = (filter as any)[key].$in;
}
} else {
val = filter[key];
}
if (val !== undefined) {
index[key] = val;
}
}
}
if (Object.keys(index).length > 0) {
return index;
}
}
async #getAll(
{ offset, range, limit }: QueryOptions,
indexes?: { [key: string]: IDBKeyRange | undefined },
): Promise<TSchema[]> {
const tx = this.db.transaction(this.name, "readonly");
const store = tx.objectStore(this.name);
// ### Indexed
// Fetch all records by optimal index
if (indexes) {
const indexName = this.#getOptimalIndex(Object.keys(indexes));
const index = store.index(indexName);
const key = indexes[indexName];
const results: TSchema[] = [];
// Handle $in
if (Array.isArray(key)) {
for (const value of key) {
const records = await index.getAll(value);
results.push(...records);
}
// Deduplicate (required for $in)
const unique = new Map<any, TSchema>();
for (const doc of results) {
unique.set(this.pkey, doc); // adjust PK if needed
}
await tx.done;
return [...unique.values()];
}
// Single-key lookup
const records = await index.getAll(key);
await tx.done;
return records;
}
// ### Range
// Fetch records in a given range.
if (range) {
return store.getAll(IDBKeyRange.bound(range.from, range.to), limit);
}
// ### Offset
// Offset-based query (cursor-based)
if (offset) {
return this.#getAllByOffset(offset.value, offset.direction, limit);
}
// ### Default
// Fetch all records
return store.getAll(undefined, limit);
}
async #getAllByOffset(value: string, direction: 1 | -1, limit?: number) {
if (direction === 1) {
return this.db.getAll(this.name, IDBKeyRange.lowerBound(value), limit);
}
return this.#getAllByDescOffset(value, limit);
}
async #getAllByDescOffset(value: string, limit?: number) {
if (limit === undefined) {
return this.db.getAll(this.name, IDBKeyRange.upperBound(value));
}
const result = [];
let cursor = await this.db
.transaction(this.name, "readonly")
.store.openCursor(IDBKeyRange.upperBound(value), "prev");
for (let i = 0; i < limit; i++) {
if (cursor === null) {
break;
}
result.push(cursor.value);
cursor = await cursor.continue();
}
return result.reverse();
}
/*
|--------------------------------------------------------------------------------
| Update
@@ -320,7 +150,9 @@ export class IndexedDBStorage<TSchema extends AnyDocument = AnyDocument> extends
await tx.done;
this.broadcast("update", documents);
this.#cache.flush();
for (const document of documents) {
this.#index.update(document);
}
this.log(logger.result());
@@ -343,7 +175,9 @@ export class IndexedDBStorage<TSchema extends AnyDocument = AnyDocument> extends
await tx.done;
this.broadcast("remove", documents);
this.#cache.flush();
for (const document of documents) {
this.#index.remove(document);
}
this.log(logger.result({ count: documents.length }));
@@ -371,6 +205,7 @@ export class IndexedDBStorage<TSchema extends AnyDocument = AnyDocument> extends
async flush(): Promise<void> {
await this.db.clear(this.name);
this.#index.flush();
}
}

2
src/databases/memory/storage.ts
View File

@@ -35,7 +35,7 @@ export class MemoryStorage<TSchema extends AnyDocument = AnyDocument> extends St
}
async find(condition: Criteria<TSchema> = {}, options?: QueryOptions): Promise<TSchema[]> {
const cursor = new Query(condition).find<TSchema>(this.documents);
const cursor = this.index.getByCondition(condition);
if (options !== undefined) {
return addOptions(cursor, options).all();
}

207
src/index/manager.ts
View File

@@ -1,10 +1,14 @@
import { Query } from "mingo";
import type { Cursor } from "mingo/cursor";
import type { Criteria } from "mingo/types";
import type { AnyDocument, StringKeyOf } from "../types.ts";
import type { AnyDocument, QueryCriteria, StringKeyOf } from "../types.ts";
import { PrimaryIndex, type PrimaryKey } from "./primary.ts";
import { SharedIndex } from "./shared.ts";
import { UniqueIndex } from "./unique.ts";
const OBJECT_PROTOTYPE = Object.getPrototypeOf({});
const OBJECT_TAG = "[object Object]";
const EMPTY_SET: ReadonlySet<PrimaryKey> = Object.freeze(new Set<PrimaryKey>());
export class IndexManager<TSchema extends AnyDocument> {
@@ -13,7 +17,9 @@ export class IndexManager<TSchema extends AnyDocument> {
readonly unique: Map<StringKeyOf<TSchema>, UniqueIndex> = new Map<StringKeyOf<TSchema>, UniqueIndex>();
readonly shared: Map<StringKeyOf<TSchema>, SharedIndex> = new Map<StringKeyOf<TSchema>, SharedIndex>();
constructor(readonly specs: IndexSpec<TSchema>[]) {
readonly specs: IndexSpec<TSchema>[];
constructor(specs: IndexSpec<TSchema>[]) {
const primary = specs.find((spec) => spec.kind === "primary");
if (primary === undefined) {
throw new Error("Primary index is required");
@@ -31,6 +37,55 @@ export class IndexManager<TSchema extends AnyDocument> {
}
}
}
this.specs = specs;
}
#isPrimaryIndex(key: string): boolean {
for (const { field, kind } of this.specs) {
if (key === field && kind === "primary") {
return true;
}
}
return false;
}
#isUniqueIndex(key: string): boolean {
for (const { field, kind } of this.specs) {
if (key === field && kind === "unique") {
return true;
}
}
return false;
}
#isSharedIndex(key: string): boolean {
for (const { field, kind } of this.specs) {
if (key === field && kind === "shared") {
return true;
}
}
return false;
}
#getOptimalIndex(keys: string[]): string {
let best: string | undefined;
for (const key of keys) {
if (this.#isPrimaryIndex(key)) {
return key; // cannot beat primary
}
if (this.#isUniqueIndex(key)) {
best ??= key;
continue;
}
if (best === undefined && this.#isSharedIndex(key)) {
best = key;
}
}
return best ?? keys[0];
}
/**
@@ -48,12 +103,32 @@ export class IndexManager<TSchema extends AnyDocument> {
try {
for (const [field, index] of this.unique) {
index.insert(document[field], pk);
insertedUniques.push([field, document[field]]);
const value = document[field] as any;
if (value !== undefined) {
if (Array.isArray(value)) {
for (const innerValue of value) {
index.insert(innerValue, pk);
insertedUniques.push([field, innerValue]);
}
} else {
index.insert(value, pk);
insertedUniques.push([field, value]);
}
}
}
for (const [field, index] of this.shared) {
index.insert(document[field], pk);
insertedShared.push([field, document[field]]);
const value = document[field] as any;
if (value !== undefined) {
if (Array.isArray(value)) {
for (const innerValue of value) {
index.insert(innerValue, pk);
insertedShared.push([field, innerValue]);
}
} else {
index.insert(value, pk);
insertedShared.push([field, value]);
}
}
}
this.primary.insert(pk, document);
} catch (err) {
@@ -67,67 +142,30 @@ export class IndexManager<TSchema extends AnyDocument> {
}
}
getByCondition(condition: Criteria<TSchema>): TSchema[] {
const indexedKeys = Array.from(
new Set([this.primary.key as StringKeyOf<TSchema>, ...this.unique.keys(), ...this.shared.keys()]),
);
getByCondition(condition: Criteria<TSchema> = {}): Cursor<TSchema> {
const indexes = resolveIndexesFromCondition(condition, this.specs);
const candidatePKs: PrimaryKey[] = [];
// ### Primary Keys
// Collect primary keys for indexed equality conditions
const pkSets: ReadonlySet<PrimaryKey>[] = [];
for (const key of indexedKeys) {
const value = (condition as any)[key];
if (value !== undefined) {
// Use index if available
const pks = this.getPrimaryKeysByIndex(key, value);
pkSets.push(pks);
}
if (indexes === undefined) {
return new Query(condition, {}).find<TSchema>(this.primary.documents);
}
// ### Intersect
// Intersect all sets to find candidates
const index = this.#getOptimalIndex(Object.keys(indexes));
const value = indexes[index];
if (pkSets.length > 0) {
const sortedSets = pkSets.sort((a, b) => a.size - b.size);
const intersection = new Set(sortedSets[0]);
for (let i = 1; i < sortedSets.length; i++) {
for (const pk of intersection) {
if (!sortedSets[i].has(pk)) {
intersection.delete(pk);
}
}
if (Array.isArray(value)) {
const results: TSchema[] = [];
for (const innerValue of value) {
const records = this.getByIndex(index as any, innerValue);
results.push(...records);
}
candidatePKs.push(...intersection);
} else {
candidatePKs.push(...this.primary.keys()); // no indexed fields → scan all primary keys
const unique = new Map<any, TSchema>();
for (const doc of results) {
unique.set(doc[this.primary.key], doc);
}
return new Query(condition, {}).find<TSchema>(Array.from(unique.values()));
}
// ### Filter
// Filter candidates by remaining condition
const results: TSchema[] = [];
for (const pk of candidatePKs) {
const doc = this.primary.get(pk);
if (doc === undefined) {
continue;
}
let match = true;
for (const [field, expected] of Object.entries(condition)) {
if ((doc as any)[field] !== expected) {
match = false;
break;
}
}
if (match) {
results.push(doc);
}
}
return results;
return new Query(condition, {}).find<TSchema>(this.getByIndex(index as any, value));
}
/**
@@ -305,6 +343,55 @@ export class IndexManager<TSchema extends AnyDocument> {
}
}
/*
|--------------------------------------------------------------------------------
| Utils
|--------------------------------------------------------------------------------
*/
function resolveIndexesFromCondition<TSchema extends AnyDocument>(
condition: QueryCriteria<TSchema>,
indexes: IndexSpec<TSchema>[],
): Record<StringKeyOf<TSchema>, any> | undefined {
const indexNames = indexes.map(({ field }) => field);
const index: any = {};
for (const key in condition) {
if (indexNames.includes(key as any) === true) {
let val: any;
if (isObject(condition[key]) === true) {
if ((condition as any)[key].$in !== undefined) {
val = (condition as any)[key].$in;
}
} else {
val = condition[key];
}
if (val !== undefined) {
index[key] = val;
}
}
}
if (Object.keys(index).length > 0) {
return index;
}
}
function isObject(v: any): v is object {
if (!v) {
return false;
}
const proto = Object.getPrototypeOf(v);
return (proto === OBJECT_PROTOTYPE || proto === null) && OBJECT_TAG === Object.prototype.toString.call(v);
}
/*
|--------------------------------------------------------------------------------
| Types
|--------------------------------------------------------------------------------
*/
export type IndexSpec<TSchema extends AnyDocument> = {
field: StringKeyOf<TSchema>;
kind: IndexKind;

9
src/index/primary.ts
View File

@@ -3,9 +3,13 @@ import type { AnyDocument } from "../types.ts";
export type PrimaryKey = string;
export class PrimaryIndex<TSchema extends AnyDocument> {
readonly key: string;
readonly #index = new Map<PrimaryKey, TSchema>();
constructor(readonly key: string) {}
constructor(key: string) {
this.key = key;
}
get documents(): TSchema[] {
return Array.from(this.#index.values());
@@ -21,7 +25,8 @@ export class PrimaryIndex<TSchema extends AnyDocument> {
insert(pk: PrimaryKey, document: TSchema): void {
if (this.#index.has(pk)) {
throw new Error(`Duplicate primary key: ${pk}`);
console.warn(`Duplicate primary key: ${pk}`);
return;
}
this.#index.set(pk, document);
}