Fix None encoding and decoding for SurrealDB v2.2.x and later (surrealdb#184)

Author: mumoshu

src/surrealdb/cbor2/_decoder.py

@@ -604,6 +604,14 @@ def decode_bigfloat(self) -> Decimal:
 
         return self.set_shareable(Decimal(sig) * (2 ** Decimal(exp)))
 
+    def decode_none(self) -> None:
+        # Semantic tag 6
+        value = self._decode()
+        if not isinstance(value, type(None)):
+            raise CBORDecodeValueError("invalid None value " + str(value))
+
+        return self.set_shareable(None)
+
     def decode_stringref(self) -> str | bytes:
         # Semantic tag 25
         if self._stringref_namespace is None:
@@ -786,7 +794,7 @@ def decode_float64(self) -> float:
     3: CBORDecoder.decode_negative_bignum,
     4: CBORDecoder.decode_fraction,
     5: CBORDecoder.decode_bigfloat,
-    6: lambda self: None,
+    6: CBORDecoder.decode_none,
     25: CBORDecoder.decode_stringref,
     28: CBORDecoder.decode_shareable,
     29: CBORDecoder.decode_sharedref,

src/surrealdb/cbor2/_encoder.py

@@ -658,8 +658,14 @@ def encode_boolean(self, value: bool) -> None:
 
     def encode_none(self, value: None) -> None:
         # tag(6, None)
-        self._fp_write(b"\xd9\x00\x06\xf6")
-        # self._fp_write(b"\xf6")
+
+        # Note that although just \xf6 (major type 6, 22) is already null in CBOR,
+        # which this cbor2 implementation decodes it as Python None,
+        # SurrealDB's variant of CBOR wraps it with \xc6 (major type 6, tag 6)
+        # which results in \xc6\xf6.
+        # We try to be compliant as much as SurrealDB's variant of CBOR
+        # and therefore encode it as \xc6\xf6.
+        self._fp_write(b"\xc6\xf6")
 
     def encode_undefined(self, value: UndefinedType) -> None:
         self._fp_write(b"\xf7")

tests/unit_tests/data_types/test_none.py

@@ -6,6 +6,7 @@
     will have to look into schema objects for more complete serialization.
 """
 from unittest import main, IsolatedAsyncioTestCase
+from os import environ
 
 from surrealdb.connections.async_ws import AsyncWsSurrealConnection
 from surrealdb.data.types.record_id import RecordID
@@ -30,7 +31,7 @@ async def asyncSetUp(self):
         await self.connection.use(namespace=self.namespace, database=self.database_name)
 
         # Cleanup
-        await self.connection.query("DELETE person;")
+        await self.connection.query("REMOVE TABLE person;")
 
     async def test_none(self):
         schema = """
@@ -63,5 +64,45 @@ async def test_none(self):
         await self.connection.query("REMOVE TABLE person;")
         await self.connection.close()
 
+    async def test_none_with_query(self):
+        is_sdb21 = environ.get("SURREALDB_VERSION", "v2.1.4").startswith("v2.1.")
+        schema = """
+            DEFINE TABLE person SCHEMAFULL TYPE NORMAL;
+            DEFINE FIELD name ON person TYPE string;
+            DEFINE FIELD nums ON person TYPE array<option<int>>;
+        """
+        await self.connection.query(schema)
+        outcome = await self.connection.query(
+            "UPSERT person MERGE {id: $id, name: $name, nums: $nums}",
+            {"id": [1,2], "name": "John", "nums": [None]}
+        )
+        record_check = RecordID(table_name="person", identifier=[1, 2])
+        self.assertEqual(1, len(outcome))
+        self.assertEqual(record_check, outcome[0]["id"])
+        self.assertEqual("John", outcome[0]["name"])
+        if is_sdb21:
+            self.assertEqual([], outcome[0].get("nums"))
+        else:
+            self.assertEqual([None], outcome[0].get("nums"))
+
+        outcome = await self.connection.query(
+            "UPSERT person MERGE {id: $id, name: $name, nums: $nums}",
+            {"id": [3,4], "name": "Dave", "nums": [None]}
+        )
+        record_check = RecordID(table_name="person", identifier=[3, 4])
+        self.assertEqual(1, len(outcome))
+        self.assertEqual(record_check, outcome[0]["id"])
+        self.assertEqual("Dave", outcome[0]["name"])
+        if is_sdb21:
+            self.assertEqual([], outcome[0].get("nums"))
+        else:
+            self.assertEqual([None], outcome[0].get("nums"))
+
+        outcome = await self.connection.query("SELECT * FROM person")
+        self.assertEqual(2, len(outcome))
+
+        await self.connection.query("REMOVE TABLE person;")
+        await self.connection.close()
+
 if __name__ == "__main__":
     main()