Allow parsing dirty PHP objects (#10)

* Rename DoctrineArray to PHPArray

* Rename PHPArray to PHPObject

* Allow parsing dirty PHP objects

components/List/ListItem/ListItem.tsx

@@ -1,5 +1,5 @@
 import { Trans } from '@lingui/macro';
-import { DoctrineArray } from '../../../db/DoctrineArray';
+import { PHPObject } from '../../../db/PHPObject';
 import { Item } from '../../../types/game/Item';
 import { UserList } from '../../../types/universalis/user';
 import GameItemIcon from '../../GameItemIcon/GameItemIcon';

components/Market/MarketNav/MarketNav.tsx

@@ -1,7 +1,7 @@
 import { t, Trans } from '@lingui/macro';
 import { useState, useRef } from 'react';
 import { sprintf } from 'sprintf-js';
-import { DoctrineArray } from '../../../db/DoctrineArray';
+import { PHPObject } from '../../../db/PHPObject';
 import { UserList, UserListCustomType } from '../../../types/universalis/user';
 import LoggedIn from '../../LoggedIn/LoggedIn';
 import { useModalCover } from '../../UniversalisLayout/components/ModalCover/ModalCover';
@@ -49,7 +49,7 @@ export default function MarketNav({ hasSession, lists, dispatch, itemId }: Marke
       return;
     }
 
-    const items = new DoctrineArray();
+    const items = new PHPObject();
     items.push(...(faves ?? []));
     if (favourite) {
       items.splice(items.indexOf(itemId), 1);
@@ -92,7 +92,7 @@ export default function MarketNav({ hasSession, lists, dispatch, itemId }: Marke
 
     const list = lists.find((list) => list.id === listId)!;
     const addingItem = !list.items.includes(itemId);
-    const items = new DoctrineArray();
+    const items = new PHPObject();
     items.push(...list.items);
     if (addingItem) {
       items.unshift(itemId);

db/DalamudAdapter.ts

@@ -5,7 +5,7 @@ import { v4 as uuidv4 } from 'uuid';
 import * as db from './user';
 import * as listsDb from './user-list';
 import { unix } from './util';
-import { DoctrineArray } from './DoctrineArray';
+import { PHPObject } from './PHPObject';
 
 export default function DalamudAdapter(): Adapter {
   return {
@@ -45,10 +45,10 @@ export default function DalamudAdapter(): Adapter {
         await db.createUser(mogUser, conn);
 
         // Create the user's custom lists
-        const recentlyViewed = listsDb.RecentlyViewedList(uuidv4(), id, new DoctrineArray());
+        const recentlyViewed = listsDb.RecentlyViewedList(uuidv4(), id, new PHPObject());
         await listsDb.createUserList(recentlyViewed, conn);
 
-        const faves = listsDb.FavouritesList(uuidv4(), id, new DoctrineArray());
+        const faves = listsDb.FavouritesList(uuidv4(), id, new PHPObject());
         await listsDb.createUserList(faves, conn);
 
         return {

db/PHPObject.test.ts

@@ -0,0 +1,71 @@
+import { PHPObject } from './PHPObject';
+
+test('PHPArray can serialize an empty array', () => {
+  const php = new PHPObject();
+  const serialized = php.serialize();
+  expect(serialized).toEqual('a:0:{}');
+});
+
+test('PHPArray can serialize an array with one number', () => {
+  const php = new PHPObject();
+  php.push(33027);
+  const serialized = php.serialize();
+  expect(serialized).toEqual('a:1:{i:0;i:33027;}');
+});
+
+test('PHPArray can serialize an array with two numbers', () => {
+  const php = new PHPObject();
+  php.push(...[33027, 5333]);
+  const serialized = php.serialize();
+  expect(serialized).toEqual('a:2:{i:0;i:33027;i:1;i:5333;}');
+});
+
+test('PHPArray can deserialize an empty array', () => {
+  const php = PHPObject.deserialize('a:0:{}');
+  expect(php).toHaveLength(0);
+});
+
+test('PHPArray can deserialize an array with one number', () => {
+  const php = PHPObject.deserialize('a:1:{i:0;i:33027;}');
+  expect(php).toHaveLength(1);
+  expect(php[0]).toStrictEqual(33027);
+});
+
+test('PHPArray can deserialize an array with two numbers', () => {
+  const php = PHPObject.deserialize('a:2:{i:0;i:33027;i:1;i:5333;}');
+  expect(php).toHaveLength(2);
+  expect(php[0]).toStrictEqual(33027);
+  expect(php[1]).toStrictEqual(5333);
+});
+
+test('PHPArray can serialize and deserialize an array with many numbers', () => {
+  const data = new Array(1000).fill(0).map(() => Math.floor(Math.random() * 1000));
+  const php = new PHPObject();
+  php.push(...data);
+  const serialized = php.serialize();
+  const result = PHPObject.deserialize(serialized);
+
+  expect(result).toHaveLength(data.length);
+  for (let i = 0; i < data.length; i++) {
+    expect(result[i]).toStrictEqual(data[i]);
+  }
+});
+
+test('PHPArray fails to deserialize arrays that have had elements removed without being reindexed', () => {
+  expect(async () =>
+    PHPObject.deserialize(
+      'a:15:{i:0;i:36904;i:1;i:36906;i:2;i:36005;i:3;i:36003;i:4;i:33684;i:5;i:33674;i:6;i:33145;i:7;i:32846;i:8;i:30861;i:10;i:30862;i:11;i:30865;i:12;i:28125;i:13;i:28917;i:14;i:23023;i:15;i:16564;}'
+    )
+  ).rejects.toThrow();
+});
+
+test('PHPArray can optionally deserialize arrays that have had elements removed without being reindexed', () => {
+  // It is impossible to know which element has been removed; instead, we just assume it was the last element.
+  const php = PHPObject.deserialize(
+    'a:15:{i:0;i:36904;i:1;i:36906;i:2;i:36005;i:3;i:36003;i:4;i:33684;i:5;i:33674;i:6;i:33145;i:7;i:32846;i:8;i:30861;i:10;i:30862;i:11;i:30865;i:12;i:28125;i:13;i:28917;i:14;i:23023;i:15;i:16564;}',
+    { allowDirtyArrays: true }
+  );
+
+  // The result should not have lost any data.
+  expect(php).toHaveLength(16);
+});

db/DoctrineArray.ts → db/PHPObject.ts

@@ -12,13 +12,33 @@ enum DoctrineArrayParseState {
   ExpectingArrayInt,
 }
 
+export interface PHPDeserializationOptions {
+  /**
+   * Whether or not to fail when attempting to deserialize an array that
+   * has not been reindexed after modification. Defaults to `false`.
+   *
+   * References:
+   * https://www.php.net/manual/en/language.types.array.php#language.types.array.useful-funcs
+   */
+  allowDirtyArrays?: boolean;
+}
+
+type GetOption<
+  T extends PHPDeserializationOptions = PHPDeserializationOptions,
+  K extends keyof T = keyof T
+> = (k: K) => NonNullable<T[K]>;
+
+const defaultOptions: Required<PHPDeserializationOptions> = {
+  allowDirtyArrays: false,
+};
+
 class ArrayWithEnd<T> extends Array<T> {
   public end() {
     return this[this.length - 1];
   }
 }
 
-export class DoctrineArray extends ArrayWithEnd<any> {
+export class PHPObject extends ArrayWithEnd<any> {
   public constructor(..._errorProne: never) {
     super();
   }
@@ -36,7 +56,11 @@ export class DoctrineArray extends ArrayWithEnd<any> {
     return fragments.join('');
   }
 
-  public static deserialize(data: string) {
+  public static deserialize(data: string, options?: PHPDeserializationOptions) {
+    const getOption: GetOption = (k) => {
+      return (options && options[k]) ?? defaultOptions[k];
+    };
+
     // Preconditions
     if (data.length === 0) {
       throw new Error('Input string was empty.');
@@ -66,7 +90,7 @@ export class DoctrineArray extends ArrayWithEnd<any> {
           }
 
           if (nextType === 'a') {
-            containers.push(new DoctrineArray());
+            containers.push(new PHPObject());
             states.push(
               DoctrineArrayParseState.ExpectingArrayEnd,
               DoctrineArrayParseState.ExpectingArrayStart,
@@ -118,7 +142,7 @@ export class DoctrineArray extends ArrayWithEnd<any> {
           if (data[ptr] !== '}') {
             throw new Error(`Expected array end; got ${data[ptr]}.`);
           }
-          if (containers.end().length !== lengths.end()) {
+          if (!getOption('allowDirtyArrays') && containers.end().length !== lengths.end()) {
             throw new Error(`Expected ${lengths.end()} elements; got ${containers.end().length}.`);
           }
           break;
@@ -171,7 +195,7 @@ export class DoctrineArray extends ArrayWithEnd<any> {
       ptr++;
     }
 
-    return containers[0] as DoctrineArray;
+    return containers[0] as PHPObject;
   }
 }
 

db/user-list.ts

@@ -1,12 +1,12 @@
 import { UserList, UserListCustomType } from '../types/universalis/user';
 import mariadb from 'mariadb';
-import { DoctrineArray } from './DoctrineArray';
+import { PHPObject } from './PHPObject';
 import { unix } from './util';
 
 export const USER_LIST_MAX_ITEMS = 100;
 export const USER_LIST_MAX = 12;
 
-export const RecentlyViewedList = (id: string, userId: string, items: DoctrineArray): UserList => {
+export const RecentlyViewedList = (id: string, userId: string, items: PHPObject): UserList => {
   return {
     id,
     userId,
@@ -19,7 +19,7 @@ export const RecentlyViewedList = (id: string, userId: string, items: DoctrineAr
   };
 };
 
-export const FavouritesList = (id: string, userId: string, items: DoctrineArray): UserList => {
+export const FavouritesList = (id: string, userId: string, items: PHPObject): UserList => {
   return {
     id,
     userId,
@@ -114,7 +114,7 @@ export function updateUserListName(
 export function updateUserListItems(
   userId: string,
   listId: string,
-  items: DoctrineArray,
+  items: PHPObject,
   conn: mariadb.Connection
 ) {
   return conn.execute(
@@ -144,6 +144,6 @@ function rowToUserList(row: Record<string, any>): UserList {
     name: row['name'],
     custom: row['custom'],
     customType: row['custom_type'],
-    items: DoctrineArray.deserialize(row['items']),
+    items: PHPObject.deserialize(row['items'], { allowDirtyArrays: true }),
   };
 }

pages/api/web/lists/[listId].ts

@@ -1,7 +1,7 @@
 import { NextApiRequest, NextApiResponse } from 'next';
 import { getServerSession } from 'next-auth';
 import { acquireConn, releaseConn } from '../../../../db/connect';
-import { DoctrineArray } from '../../../../db/DoctrineArray';
+import { PHPObject } from '../../../../db/PHPObject';
 import * as db from '../../../../db/user-list';
 import { authOptions } from '../../auth/[...nextauth]';
 
@@ -33,7 +33,7 @@ async function put(req: NextApiRequest, res: NextApiResponse) {
     return res.status(441).json({ message: 'List is at maximum capacity.' });
   }
 
-  const itemsProcessed = Array.isArray(items) ? new DoctrineArray() : null;
+  const itemsProcessed = Array.isArray(items) ? new PHPObject() : null;
   itemsProcessed?.push(...items);
 
   const conn = await acquireConn();

pages/api/web/lists/faves.ts

@@ -1,7 +1,7 @@
 import { NextApiRequest, NextApiResponse } from 'next';
 import { getServerSession } from 'next-auth';
 import { acquireConn, releaseConn } from '../../../../db/connect';
-import { DoctrineArray } from '../../../../db/DoctrineArray';
+import { PHPObject } from '../../../../db/PHPObject';
 import * as db from '../../../../db/user-list';
 import { unix } from '../../../../db/util';
 import { UserListCustomType } from '../../../../types/universalis/user';
@@ -31,7 +31,7 @@ async function put(req: NextApiRequest, res: NextApiResponse) {
       .json({ message: 'List is at maximum capacity; please remove some items first.' });
   }
 
-  const items = new DoctrineArray();
+  const items = new PHPObject();
   items.push(...itemsBody);
 
   const conn = await acquireConn();

pages/api/web/lists/index.ts

@@ -1,7 +1,7 @@
 import { NextApiRequest, NextApiResponse } from 'next';
 import { getServerSession } from 'next-auth';
 import { acquireConn, releaseConn } from '../../../../db/connect';
-import { DoctrineArray } from '../../../../db/DoctrineArray';
+import { PHPObject } from '../../../../db/PHPObject';
 import { unix } from '../../../../db/util';
 import { UserList, UserListCustomType } from '../../../../types/universalis/user';
 import { authOptions } from '../../auth/[...nextauth]';
@@ -37,7 +37,7 @@ async function post(req: NextApiRequest, res: NextApiResponse) {
   }
   const name = nameBody;
 
-  const items = new DoctrineArray();
+  const items = new PHPObject();
   const itemsBody: unknown = req.body.items;
   if (
     !Array.isArray(itemsBody) ||

pages/market/[itemId].tsx

@@ -17,7 +17,7 @@ import { DataCenter } from '../../types/game/DataCenter';
 import { UserList, UserListCustomType } from '../../types/universalis/user';
 import { authOptions } from '../api/auth/[...nextauth]';
 import { v4 as uuidv4 } from 'uuid';
-import { DoctrineArray } from '../../db/DoctrineArray';
+import { PHPObject } from '../../db/PHPObject';
 import { ListsDispatchAction } from '../../components/Market/MarketNav/MarketNav';
 import MarketDataCenter from '../../components/Market/MarketDataCenter/MarketDataCenter';
 import MarketWorld from '../../components/Market/MarketWorld/MarketWorld';
@@ -144,7 +144,7 @@ async function addToRecentlyViewed(userId: string, itemId: number, conn: Connect
   const recents = await getUserListCustom(userId, UserListCustomType.RecentlyViewed, conn);
 
   if (recents) {
-    const items = new DoctrineArray();
+    const items = new PHPObject();
     items.push(...recents.items.filter((item) => item !== itemId));
     items.unshift(itemId);
     while (items.length > USER_LIST_MAX_ITEMS) {
@@ -153,7 +153,7 @@ async function addToRecentlyViewed(userId: string, itemId: number, conn: Connect
 
     await updateUserListItems(userId, recents.id, items, conn);
   } else {
-    const items = new DoctrineArray();
+    const items = new PHPObject();
     items.push(itemId);
     await createUserList(RecentlyViewedList(uuidv4(), userId, items), conn);
   }

types/universalis/user.ts

@@ -1,4 +1,4 @@
-import { DoctrineArray } from '../../db/DoctrineArray';
+import { PHPObject } from '../../db/PHPObject';
 
 export interface User {
   id: string;
@@ -41,7 +41,7 @@ export interface UserList {
   name: string;
   custom: boolean;
   customType: UserListCustomType | null;
-  items: DoctrineArray;
+  items: PHPObject;
 }
 
 export interface UserCharacter {