From 98957b6f0129d699cd14f2b3e21caea410ad8b7e Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Sun, 9 May 2021 22:14:50 +0200
Subject: [PATCH 01/47] Expose sqlite3_serialize and sqlite3_deserialize

---
 Modules/_sqlite/clinic/connection.c.h | 101 +++++++++++++++++++++++++-
 Modules/_sqlite/connection.c          |  99 +++++++++++++++++++++++++
 2 files changed, 199 insertions(+), 1 deletion(-)

diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h
index f231ecc2ae78be..d344ff8b894ad0 100644
--- a/Modules/_sqlite/clinic/connection.c.h
+++ b/Modules/_sqlite/clinic/connection.c.h
@@ -648,6 +648,105 @@ pysqlite_connection_create_collation(pysqlite_Connection *self, PyObject *const
     return return_value;
 }
 
+PyDoc_STRVAR(serialize__doc__,
+"serialize($self, /, *, schema=\'main\')\n"
+"--\n"
+"\n");
+
+#define SERIALIZE_METHODDEF    \
+    {"serialize", (PyCFunction)(void(*)(void))serialize, METH_FASTCALL|METH_KEYWORDS, serialize__doc__},
+
+static PyObject *
+serialize_impl(pysqlite_Connection *self, const char *schema);
+
+static PyObject *
+serialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+    PyObject *return_value = NULL;
+    static const char * const _keywords[] = {"schema", NULL};
+    static _PyArg_Parser _parser = {NULL, _keywords, "serialize", 0};
+    PyObject *argsbuf[1];
+    Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
+    const char *schema = "main";
+
+    args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    if (!noptargs) {
+        goto skip_optional_kwonly;
+    }
+    if (!PyUnicode_Check(args[0])) {
+        _PyArg_BadArgument("serialize", "argument 'schema'", "str", args[0]);
+        goto exit;
+    }
+    Py_ssize_t schema_length;
+    schema = PyUnicode_AsUTF8AndSize(args[0], &schema_length);
+    if (schema == NULL) {
+        goto exit;
+    }
+    if (strlen(schema) != (size_t)schema_length) {
+        PyErr_SetString(PyExc_ValueError, "embedded null character");
+        goto exit;
+    }
+skip_optional_kwonly:
+    return_value = serialize_impl(self, schema);
+
+exit:
+    return return_value;
+}
+
+PyDoc_STRVAR(deserialize__doc__,
+"deserialize($self, data, /, *, schema=\'main\')\n"
+"--\n"
+"\n");
+
+#define DESERIALIZE_METHODDEF    \
+    {"deserialize", (PyCFunction)(void(*)(void))deserialize, METH_FASTCALL|METH_KEYWORDS, deserialize__doc__},
+
+static PyObject *
+deserialize_impl(pysqlite_Connection *self, PyObject *data,
+                 const char *schema);
+
+static PyObject *
+deserialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+    PyObject *return_value = NULL;
+    static const char * const _keywords[] = {"", "schema", NULL};
+    static _PyArg_Parser _parser = {NULL, _keywords, "deserialize", 0};
+    PyObject *argsbuf[2];
+    Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1;
+    PyObject *data;
+    const char *schema = "main";
+
+    args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    data = args[0];
+    if (!noptargs) {
+        goto skip_optional_kwonly;
+    }
+    if (!PyUnicode_Check(args[1])) {
+        _PyArg_BadArgument("deserialize", "argument 'schema'", "str", args[1]);
+        goto exit;
+    }
+    Py_ssize_t schema_length;
+    schema = PyUnicode_AsUTF8AndSize(args[1], &schema_length);
+    if (schema == NULL) {
+        goto exit;
+    }
+    if (strlen(schema) != (size_t)schema_length) {
+        PyErr_SetString(PyExc_ValueError, "embedded null character");
+        goto exit;
+    }
+skip_optional_kwonly:
+    return_value = deserialize_impl(self, data, schema);
+
+exit:
+    return return_value;
+}
+
 PyDoc_STRVAR(pysqlite_connection_enter__doc__,
 "__enter__($self, /)\n"
 "--\n"
@@ -710,4 +809,4 @@ pysqlite_connection_exit(pysqlite_Connection *self, PyObject *const *args, Py_ss
 #ifndef PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF
     #define PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF
 #endif /* !defined(PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF) */
-/*[clinic end generated code: output=c1bf09db3bcd0105 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=f0f43cd2277f99fd input=a9049054013a1b77]*/
diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 9199c347caab0d..5bb65a9bd3685e 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1789,6 +1789,103 @@ pysqlite_connection_create_collation_impl(pysqlite_Connection *self,
     return Py_NewRef(Py_None);
 }
 
+/*[clinic input]
+_sqlite3.Connection.serialize as serialize
+
+    *
+    schema: str = "main"
+
+[clinic start generated code]*/
+
+static PyObject *
+serialize_impl(pysqlite_Connection *self, const char *schema)
+/*[clinic end generated code: output=b246381f2b3f1d84 input=f6782d98caa63008]*/
+{
+    if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) {
+        return NULL;
+    }
+
+    sqlite3_int64 size;
+    const unsigned int flags = 0;
+    const char *data = (const char *)sqlite3_serialize(self->db, schema, &size,
+                                                       flags);
+
+    PyObject *ret = PyTuple_New(2);
+    if (ret == NULL) {
+        return NULL;
+    }
+    PyTuple_SET_ITEM(ret, 0, PyBytes_FromStringAndSize(data, (Py_ssize_t)size));
+    PyTuple_SET_ITEM(ret, 1, PyLong_FromUnsignedLong(size));
+    return ret;
+}
+
+static void
+reset_all_statements(sqlite3 *db)
+{
+    assert(db != NULL);
+    sqlite3_stmt *stmt = NULL;
+    while ((stmt = sqlite3_next_stmt(db, stmt))) {
+        if (sqlite3_stmt_busy(stmt)) {
+            (void)sqlite3_reset(stmt);
+        }
+    }
+}
+
+/*[clinic input]
+_sqlite3.Connection.deserialize as deserialize
+
+
+    data: object
+    /
+    *
+    schema: str = "main"
+
+[clinic start generated code]*/
+
+static PyObject *
+deserialize_impl(pysqlite_Connection *self, PyObject *data,
+                 const char *schema)
+/*[clinic end generated code: output=264709775101cd18 input=fc2cc53bd3736b89]*/
+{
+    if (PyObject_CheckBuffer(data) < 0) {
+        PyErr_SetString(PyExc_ValueError,
+                        "could not convert 'data' to buffer");
+        return NULL;
+    }
+
+    Py_buffer view;
+    if (PyObject_GetBuffer(data, &view, PyBUF_SIMPLE) < 0) {
+        return NULL;
+    }
+
+    if (view.len > 9223372036854775807) {
+        PyErr_SetString(PyExc_OverflowError, "'data' is too large");
+        PyBuffer_Release(&view);
+        return NULL;
+    }
+
+    if (self->db) {
+        reset_all_statements(self->db);
+    }
+
+    Py_ssize_t size = view.len;
+    unsigned char *buf = view.buf;
+    const unsigned int flags = 0;
+    int rc;
+    Py_BEGIN_ALLOW_THREADS
+    rc = sqlite3_deserialize(self->db, schema, buf, size, size, flags);
+    Py_END_ALLOW_THREADS
+
+    PyBuffer_Release(&view);
+    if (rc != SQLITE_OK) {
+        _pysqlite_seterror(self->db);
+        return NULL;
+    }
+
+    Py_RETURN_TRUE;
+}
+
+
 /*[clinic input]
 _sqlite3.Connection.__enter__ as pysqlite_connection_enter
 
@@ -1874,6 +1971,8 @@ static PyMethodDef connection_methods[] = {
     PYSQLITE_CONNECTION_SET_AUTHORIZER_METHODDEF
     PYSQLITE_CONNECTION_SET_PROGRESS_HANDLER_METHODDEF
     PYSQLITE_CONNECTION_SET_TRACE_CALLBACK_METHODDEF
+    SERIALIZE_METHODDEF
+    DESERIALIZE_METHODDEF
     {NULL, NULL}
 };
 

From 79ecb6c52860434c6fbd6c0c44be8c68cd0cfe12 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Mon, 14 Jun 2021 23:48:16 +0200
Subject: [PATCH 02/47] Wrap in ifdef for now

---
 Modules/_sqlite/clinic/connection.c.h | 18 +++++++++++++++++-
 Modules/_sqlite/connection.c          |  7 +++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h
index d344ff8b894ad0..379ae1ac79fd50 100644
--- a/Modules/_sqlite/clinic/connection.c.h
+++ b/Modules/_sqlite/clinic/connection.c.h
@@ -648,6 +648,8 @@ pysqlite_connection_create_collation(pysqlite_Connection *self, PyObject *const
     return return_value;
 }
 
+#if defined(HAVE_SERIALIZE_API)
+
 PyDoc_STRVAR(serialize__doc__,
 "serialize($self, /, *, schema=\'main\')\n"
 "--\n"
@@ -696,6 +698,10 @@ serialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, Py
     return return_value;
 }
 
+#endif /* defined(HAVE_SERIALIZE_API) */
+
+#if defined(HAVE_SERIALIZE_API)
+
 PyDoc_STRVAR(deserialize__doc__,
 "deserialize($self, data, /, *, schema=\'main\')\n"
 "--\n"
@@ -747,6 +753,8 @@ deserialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs,
     return return_value;
 }
 
+#endif /* defined(HAVE_SERIALIZE_API) */
+
 PyDoc_STRVAR(pysqlite_connection_enter__doc__,
 "__enter__($self, /)\n"
 "--\n"
@@ -809,4 +817,12 @@ pysqlite_connection_exit(pysqlite_Connection *self, PyObject *const *args, Py_ss
 #ifndef PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF
     #define PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF
 #endif /* !defined(PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF) */
-/*[clinic end generated code: output=f0f43cd2277f99fd input=a9049054013a1b77]*/
+
+#ifndef SERIALIZE_METHODDEF
+    #define SERIALIZE_METHODDEF
+#endif /* !defined(SERIALIZE_METHODDEF) */
+
+#ifndef DESERIALIZE_METHODDEF
+    #define DESERIALIZE_METHODDEF
+#endif /* !defined(DESERIALIZE_METHODDEF) */
+/*[clinic end generated code: output=f88e471ed19d47fa input=a9049054013a1b77]*/
diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 5bb65a9bd3685e..e081afdd1962eb 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -36,6 +36,11 @@
 #define HAVE_TRACE_V2
 #endif
 
+// Opt. feature since 3.32.0, always included since 3.36.0
+#if SQLITE_VERSION_NUMBER >= 3036000
+#define HAVE_SERIALIZE_API
+#endif
+
 #include "clinic/connection.c.h"
 /*[clinic input]
 module _sqlite3
@@ -1789,6 +1794,7 @@ pysqlite_connection_create_collation_impl(pysqlite_Connection *self,
     return Py_NewRef(Py_None);
 }
 
+#ifdef HAVE_SERIALIZE_API
 /*[clinic input]
 _sqlite3.Connection.serialize as serialize
 
@@ -1884,6 +1890,7 @@ deserialize_impl(pysqlite_Connection *self, PyObject *data,
 
     Py_RETURN_TRUE;
 }
+#endif
 
 
 /*[clinic input]

From 74ab8a760ce9f6812d0c344b125bcc120a0d1c0a Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Wed, 16 Jun 2021 00:20:21 +0200
Subject: [PATCH 03/47] Add basic tests

---
 Lib/sqlite3/test/dbapi.py | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py
index 4bda6aa393e3ff..79f8a23dcedbb1 100644
--- a/Lib/sqlite3/test/dbapi.py
+++ b/Lib/sqlite3/test/dbapi.py
@@ -202,6 +202,22 @@ def test_in_transaction_ro(self):
         with self.assertRaises(AttributeError):
             self.cx.in_transaction = True
 
+
+class SerializeTests(unittest.TestCase):
+    def test_serialize_deserialize(self):
+        with sqlite.connect(":memory:") as cx:
+            cx.execute("create table t(t)")
+            data, size = cx.serialize()
+        cx.close()
+
+        self.assertEqual(len(data), size)
+
+        cx = sqlite.connect(":memory:")
+        cx.deserialize(data)
+        cx.execute("select t from t")
+        cx.close()
+
+
 class OpenTests(unittest.TestCase):
     _sql = "create table test(id integer)"
 
@@ -649,6 +665,23 @@ def run(con, errors):
         if len(errors) > 0:
             self.fail("\n".join(errors))
 
+    def test_con_serialize(self):
+        def run(con, err):
+            try:
+                con.serialize()
+                err.append("did not raise ProgrammingError")
+            except sqlite.ProgrammingError:
+                pass
+            except:
+                err.append("raised wrong exception")
+
+        err = []
+        t = threading.Thread(target=run, kwargs={"con": self.con, "err": err})
+        t.start()
+        t.join()
+        if len(err) > 0:
+            self.fail("\n".join(err))
+
     def test_cur_implicit_begin(self):
         def run(cur, errors):
             try:
@@ -986,6 +1019,7 @@ def suite():
         CursorTests,
         ExtensionTests,
         ModuleTests,
+        SerializeTests,
         SqliteOnConflictTests,
         ThreadTests,
     ]

From 125a58c1abaece3f1b18f8257bafacd41f352eb7 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Wed, 16 Jun 2021 13:25:25 +0200
Subject: [PATCH 04/47] Let AC convert data to buffer

---
 Modules/_sqlite/clinic/connection.c.h | 31 ++++++++++++++++++++++-----
 Modules/_sqlite/connection.c          | 26 ++++++----------------
 2 files changed, 32 insertions(+), 25 deletions(-)

diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h
index 3fdb43760a24f7..028d3a21e2782e 100644
--- a/Modules/_sqlite/clinic/connection.c.h
+++ b/Modules/_sqlite/clinic/connection.c.h
@@ -711,7 +711,7 @@ PyDoc_STRVAR(deserialize__doc__,
     {"deserialize", (PyCFunction)(void(*)(void))deserialize, METH_FASTCALL|METH_KEYWORDS, deserialize__doc__},
 
 static PyObject *
-deserialize_impl(pysqlite_Connection *self, PyObject *data,
+deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
                  const char *schema);
 
 static PyObject *
@@ -722,14 +722,30 @@ deserialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs,
     static _PyArg_Parser _parser = {NULL, _keywords, "deserialize", 0};
     PyObject *argsbuf[2];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1;
-    PyObject *data;
+    Py_buffer data = {NULL, NULL};
     const char *schema = "main";
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf);
     if (!args) {
         goto exit;
     }
-    data = args[0];
+    if (PyUnicode_Check(args[0])) {
+        Py_ssize_t len;
+        const char *ptr = PyUnicode_AsUTF8AndSize(args[0], &len);
+        if (ptr == NULL) {
+            goto exit;
+        }
+        PyBuffer_FillInfo(&data, args[0], (void *)ptr, len, 1, 0);
+    }
+    else { /* any bytes-like object */
+        if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) {
+            goto exit;
+        }
+        if (!PyBuffer_IsContiguous(&data, 'C')) {
+            _PyArg_BadArgument("deserialize", "argument 1", "contiguous buffer", args[0]);
+            goto exit;
+        }
+    }
     if (!noptargs) {
         goto skip_optional_kwonly;
     }
@@ -747,9 +763,14 @@ deserialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs,
         goto exit;
     }
 skip_optional_kwonly:
-    return_value = deserialize_impl(self, data, schema);
+    return_value = deserialize_impl(self, &data, schema);
 
 exit:
+    /* Cleanup for data */
+    if (data.obj) {
+       PyBuffer_Release(&data);
+    }
+
     return return_value;
 }
 
@@ -825,4 +846,4 @@ pysqlite_connection_exit(pysqlite_Connection *self, PyObject *const *args, Py_ss
 #ifndef DESERIALIZE_METHODDEF
     #define DESERIALIZE_METHODDEF
 #endif /* !defined(DESERIALIZE_METHODDEF) */
-/*[clinic end generated code: output=19c5e77935335c4d input=a9049054013a1b77]*/
+/*[clinic end generated code: output=b84e5a71dd4a0e61 input=a9049054013a1b77]*/
diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 0b9402c1ff70c7..895d2f721c7d0e 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1845,7 +1845,7 @@ reset_all_statements(sqlite3 *db)
 _sqlite3.Connection.deserialize as deserialize
 
 
-    data: object
+    data: Py_buffer(accept={buffer, str})
     /
     *
     schema: str = "main"
@@ -1853,24 +1853,12 @@ _sqlite3.Connection.deserialize as deserialize
 [clinic start generated code]*/
 
 static PyObject *
-deserialize_impl(pysqlite_Connection *self, PyObject *data,
+deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
                  const char *schema)
-/*[clinic end generated code: output=264709775101cd18 input=fc2cc53bd3736b89]*/
+/*[clinic end generated code: output=96b8470aaebf1b25 input=c67fca5dac036eec]*/
 {
-    if (PyObject_CheckBuffer(data) < 0) {
-        PyErr_SetString(PyExc_ValueError,
-                        "could not convert 'data' to buffer");
-        return NULL;
-    }
-
-    Py_buffer view;
-    if (PyObject_GetBuffer(data, &view, PyBUF_SIMPLE) < 0) {
-        return NULL;
-    }
-
-    if (view.len > 9223372036854775807) {
+    if (data->len > 9223372036854775807) {
         PyErr_SetString(PyExc_OverflowError, "'data' is too large");
-        PyBuffer_Release(&view);
         return NULL;
     }
 
@@ -1878,15 +1866,13 @@ deserialize_impl(pysqlite_Connection *self, PyObject *data,
         reset_all_statements(self->db);
     }
 
-    Py_ssize_t size = view.len;
-    unsigned char *buf = view.buf;
+    Py_ssize_t size = data->len;
     const unsigned int flags = 0;
     int rc;
     Py_BEGIN_ALLOW_THREADS
-    rc = sqlite3_deserialize(self->db, schema, buf, size, size, flags);
+    rc = sqlite3_deserialize(self->db, schema, data->buf, size, size, flags);
     Py_END_ALLOW_THREADS
 
-    PyBuffer_Release(&view);
     if (rc != SQLITE_OK) {
         _pysqlite_seterror(self->db);
         return NULL;

From 42f8cd536697e3b9edd3b11e0155116a5d3aafb0 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Wed, 16 Jun 2021 13:39:37 +0200
Subject: [PATCH 05/47] Test deserialize type errors

---
 Lib/sqlite3/test/dbapi.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py
index 79f8a23dcedbb1..e0f89072f4d8dd 100644
--- a/Lib/sqlite3/test/dbapi.py
+++ b/Lib/sqlite3/test/dbapi.py
@@ -217,6 +217,12 @@ def test_serialize_deserialize(self):
         cx.execute("select t from t")
         cx.close()
 
+    def test_deserialize_wrong_args(self):
+        cx = sqlite.connect(":memory:")
+        self.assertRaises(TypeError, cx.deserialize, [])
+        self.assertRaises(TypeError, cx.deserialize, None)
+        self.assertRaises(TypeError, cx.deserialize, 1)
+
 
 class OpenTests(unittest.TestCase):
     _sql = "create table test(id integer)"

From 07d3b5cc492fc15b8da016745607b3851e33476c Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Wed, 16 Jun 2021 18:41:30 +0200
Subject: [PATCH 06/47] Skip tests if serialize API is missing

---
 Lib/sqlite3/test/dbapi.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py
index e0f89072f4d8dd..eae4012af3cfbf 100644
--- a/Lib/sqlite3/test/dbapi.py
+++ b/Lib/sqlite3/test/dbapi.py
@@ -203,6 +203,8 @@ def test_in_transaction_ro(self):
             self.cx.in_transaction = True
 
 
+@unittest.skipIf(hasattr(sqlite.Connection, "serialize") == False,
+                 "Serialize API missing")
 class SerializeTests(unittest.TestCase):
     def test_serialize_deserialize(self):
         with sqlite.connect(":memory:") as cx:
@@ -671,6 +673,8 @@ def run(con, errors):
         if len(errors) > 0:
             self.fail("\n".join(errors))
 
+    @unittest.skipIf(hasattr(sqlite.Connection, "serialize") == False,
+                     "Serialize API missing")
     def test_con_serialize(self):
         def run(con, err):
             try:

From 6c5147dd1c5d4a9dd552c4e19d213cd8c50efc79 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Wed, 16 Jun 2021 23:14:50 +0200
Subject: [PATCH 07/47] Only reset pending statements if needed

---
 Modules/_sqlite/connection.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 895d2f721c7d0e..1bc42c01ce8ded 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1862,10 +1862,6 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
         return NULL;
     }
 
-    if (self->db) {
-        reset_all_statements(self->db);
-    }
-
     Py_ssize_t size = data->len;
     const unsigned int flags = 0;
     int rc;
@@ -1878,6 +1874,10 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
         return NULL;
     }
 
+    if (self->db) {
+        reset_all_statements(self->db);
+    }
+
     Py_RETURN_TRUE;
 }
 #endif

From d24fc6102132087a87e078b1d404b85c317d3d54 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Wed, 16 Jun 2021 23:32:22 +0200
Subject: [PATCH 08/47] Test deserialize with corrupt data

---
 Lib/sqlite3/test/dbapi.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py
index eae4012af3cfbf..9278927ed9ae57 100644
--- a/Lib/sqlite3/test/dbapi.py
+++ b/Lib/sqlite3/test/dbapi.py
@@ -225,6 +225,14 @@ def test_deserialize_wrong_args(self):
         self.assertRaises(TypeError, cx.deserialize, None)
         self.assertRaises(TypeError, cx.deserialize, 1)
 
+    def test_deserialize_corrupt_database(self):
+        cx = sqlite.connect(":memory:")
+        with self.assertRaises(sqlite.DatabaseError):
+            cx.deserialize(b"\0\1\3")
+            # SQLite does not generate an error until you try to query the
+            # deserialized database, so we query the ever present schema table.
+            cx.execute("select * from sqlite_schema")
+
 
 class OpenTests(unittest.TestCase):
     _sql = "create table test(id integer)"

From 7db9dfb854b5bc2ad59f5114a6699965ae98bb7a Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Wed, 16 Jun 2021 23:13:07 +0200
Subject: [PATCH 09/47] Cast sqlite3_deserialize() arguments

---
 Modules/_sqlite/connection.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 1bc42c01ce8ded..a1c03440e1b035 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1862,11 +1862,12 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
         return NULL;
     }
 
-    Py_ssize_t size = data->len;
+    sqlite3_int64 size = (sqlite3_int64)data->len;
+    unsigned char *buf = (unsigned char *)data->buf;
     const unsigned int flags = 0;
     int rc;
     Py_BEGIN_ALLOW_THREADS
-    rc = sqlite3_deserialize(self->db, schema, data->buf, size, size, flags);
+    rc = sqlite3_deserialize(self->db, schema, buf, size, size, flags);
     Py_END_ALLOW_THREADS
 
     if (rc != SQLITE_OK) {

From 527ee92187219718859c0878a8b2438017a4a570 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Wed, 16 Jun 2021 23:20:19 +0200
Subject: [PATCH 10/47] Refactor tests

---
 Lib/sqlite3/test/dbapi.py | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py
index 9278927ed9ae57..5ebce17eaf2d2a 100644
--- a/Lib/sqlite3/test/dbapi.py
+++ b/Lib/sqlite3/test/dbapi.py
@@ -206,32 +206,32 @@ def test_in_transaction_ro(self):
 @unittest.skipIf(hasattr(sqlite.Connection, "serialize") == False,
                  "Serialize API missing")
 class SerializeTests(unittest.TestCase):
-    def test_serialize_deserialize(self):
-        with sqlite.connect(":memory:") as cx:
-            cx.execute("create table t(t)")
-            data, size = cx.serialize()
-        cx.close()
+    def setUp(self):
+        self.cx = sqlite.connect(":memory:")
 
+    def test_serialize_deserialize(self):
+        with self.cx:
+            self.cx.execute("create table t(t)")
+            data, size = self.cx.serialize()
         self.assertEqual(len(data), size)
 
-        cx = sqlite.connect(":memory:")
-        cx.deserialize(data)
-        cx.execute("select t from t")
-        cx.close()
+        # Remove test table, then load the saved database
+        with self.cx:
+            self.cx.execute("drop table t")
+        self.cx.deserialize(data)
+        self.cx.execute("select t from t")
 
     def test_deserialize_wrong_args(self):
-        cx = sqlite.connect(":memory:")
-        self.assertRaises(TypeError, cx.deserialize, [])
-        self.assertRaises(TypeError, cx.deserialize, None)
-        self.assertRaises(TypeError, cx.deserialize, 1)
+        self.assertRaises(TypeError, self.cx.deserialize, [])
+        self.assertRaises(TypeError, self.cx.deserialize, None)
+        self.assertRaises(TypeError, self.cx.deserialize, 1)
 
     def test_deserialize_corrupt_database(self):
-        cx = sqlite.connect(":memory:")
         with self.assertRaises(sqlite.DatabaseError):
-            cx.deserialize(b"\0\1\3")
+            self.cx.deserialize(b"\0\1\3")
             # SQLite does not generate an error until you try to query the
             # deserialized database, so we query the ever present schema table.
-            cx.execute("select * from sqlite_schema")
+            self.cx.execute("select * from sqlite_schema")
 
 
 class OpenTests(unittest.TestCase):

From 6348eaa7f0b7a3e92224a9416bfd16fbcdad5edc Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Wed, 16 Jun 2021 23:26:10 +0200
Subject: [PATCH 11/47] Check thread and connection when deserializing

---
 Modules/_sqlite/connection.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index a1c03440e1b035..b88bbd7dfd1699 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1857,6 +1857,10 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
                  const char *schema)
 /*[clinic end generated code: output=96b8470aaebf1b25 input=c67fca5dac036eec]*/
 {
+    if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) {
+        return NULL;
+    }
+
     if (data->len > 9223372036854775807) {
         PyErr_SetString(PyExc_OverflowError, "'data' is too large");
         return NULL;

From 6e4344f4be0ce3610da6b330b79a29067be1a489 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Wed, 16 Jun 2021 23:39:32 +0200
Subject: [PATCH 12/47] Simplify serialize return value

Don't return size; use len() on the returned bytes object instead.
---
 Lib/sqlite3/test/dbapi.py    | 4 ++--
 Modules/_sqlite/connection.c | 9 +--------
 2 files changed, 3 insertions(+), 10 deletions(-)

diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py
index 5ebce17eaf2d2a..c469184606b7d0 100644
--- a/Lib/sqlite3/test/dbapi.py
+++ b/Lib/sqlite3/test/dbapi.py
@@ -212,8 +212,8 @@ def setUp(self):
     def test_serialize_deserialize(self):
         with self.cx:
             self.cx.execute("create table t(t)")
-            data, size = self.cx.serialize()
-        self.assertEqual(len(data), size)
+            data = self.cx.serialize()
+        self.assertEqual(len(data), 8192)
 
         # Remove test table, then load the saved database
         with self.cx:
diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index b88bbd7dfd1699..d854eee2f2c412 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1819,14 +1819,7 @@ serialize_impl(pysqlite_Connection *self, const char *schema)
     const unsigned int flags = 0;
     const char *data = (const char *)sqlite3_serialize(self->db, schema, &size,
                                                        flags);
-
-    PyObject *ret = PyTuple_New(2);
-    if (ret == NULL) {
-        return NULL;
-    }
-    PyTuple_SET_ITEM(ret, 0, PyBytes_FromStringAndSize(data, (Py_ssize_t)size));
-    PyTuple_SET_ITEM(ret, 1, PyLong_FromUnsignedLong(size));
-    return ret;
+    return PyBytes_FromStringAndSize(data, (Py_ssize_t)size);
 }
 
 static void

From bde320b920c1bd3404a8003402ad2323a65129fe Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Wed, 16 Jun 2021 23:52:15 +0200
Subject: [PATCH 13/47] Reset cursors and statements before deserialize

Add basic test for this
---
 Lib/sqlite3/test/dbapi.py    | 10 ++++++++++
 Modules/_sqlite/connection.c | 21 +++------------------
 2 files changed, 13 insertions(+), 18 deletions(-)

diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py
index c469184606b7d0..f1e96f3ff4b279 100644
--- a/Lib/sqlite3/test/dbapi.py
+++ b/Lib/sqlite3/test/dbapi.py
@@ -233,6 +233,16 @@ def test_deserialize_corrupt_database(self):
             # deserialized database, so we query the ever present schema table.
             self.cx.execute("select * from sqlite_schema")
 
+    def test_fetch_across_deserialize(self):
+        with self.cx:
+            self.cx.execute("create table t(t)")
+        data = self.cx.serialize()
+        cu = self.cx.execute("select * from sqlite_schema")
+        self.cx.deserialize(data)
+        schema = [('table', 't', 't', 2, 'CREATE TABLE t(t)')]
+        with self.assertRaises(sqlite.InterfaceError):
+            cu.fetchall()
+
 
 class OpenTests(unittest.TestCase):
     _sql = "create table test(id integer)"
diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index d854eee2f2c412..b8586d2d44141b 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1822,22 +1822,9 @@ serialize_impl(pysqlite_Connection *self, const char *schema)
     return PyBytes_FromStringAndSize(data, (Py_ssize_t)size);
 }
 
-static void
-reset_all_statements(sqlite3 *db)
-{
-    assert(db != NULL);
-    sqlite3_stmt *stmt = NULL;
-    while ((stmt = sqlite3_next_stmt(db, stmt))) {
-        if (sqlite3_stmt_busy(stmt)) {
-            (void)sqlite3_reset(stmt);
-        }
-    }
-}
-
 /*[clinic input]
 _sqlite3.Connection.deserialize as deserialize
 
-
     data: Py_buffer(accept={buffer, str})
     /
     *
@@ -1848,7 +1835,7 @@ _sqlite3.Connection.deserialize as deserialize
 static PyObject *
 deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
                  const char *schema)
-/*[clinic end generated code: output=96b8470aaebf1b25 input=c67fca5dac036eec]*/
+/*[clinic end generated code: output=96b8470aaebf1b25 input=ed3de6f0e6fe4aa2]*/
 {
     if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) {
         return NULL;
@@ -1859,6 +1846,8 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
         return NULL;
     }
 
+    pysqlite_do_all_statements(self, ACTION_RESET, 1);
+
     sqlite3_int64 size = (sqlite3_int64)data->len;
     unsigned char *buf = (unsigned char *)data->buf;
     const unsigned int flags = 0;
@@ -1872,10 +1861,6 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
         return NULL;
     }
 
-    if (self->db) {
-        reset_all_statements(self->db);
-    }
-
     Py_RETURN_TRUE;
 }
 #endif

From 3d8c6426da430cdb7d1e348c06f1217be0a6e420 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Thu, 17 Jun 2021 00:03:10 +0200
Subject: [PATCH 14/47] Add NEWS stub

---
 .../next/Library/2021-06-17-00-02-58.bpo-41930.JS6fsd.rst      | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 Misc/NEWS.d/next/Library/2021-06-17-00-02-58.bpo-41930.JS6fsd.rst

diff --git a/Misc/NEWS.d/next/Library/2021-06-17-00-02-58.bpo-41930.JS6fsd.rst b/Misc/NEWS.d/next/Library/2021-06-17-00-02-58.bpo-41930.JS6fsd.rst
new file mode 100644
index 00000000000000..ce494e7225e22c
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-06-17-00-02-58.bpo-41930.JS6fsd.rst
@@ -0,0 +1,3 @@
+Add :meth:`~sqlite3.Connection.serialize` and
+:meth:`~sqlite3.Connection.deserialize` support to :mod:`sqlite3`. Patch by
+Erlend E. Aasland.

From fac7cdcf5ceb39d10de135f2ce0f146b9dacae93 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Fri, 30 Jul 2021 10:01:00 +0200
Subject: [PATCH 15/47] Free SQLite buffer after Py conversion

---
 Modules/_sqlite/connection.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 1ad8b2e7edc4f8..746cbd87ab0d31 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1796,7 +1796,14 @@ serialize_impl(pysqlite_Connection *self, const char *schema)
     const unsigned int flags = 0;
     const char *data = (const char *)sqlite3_serialize(self->db, schema, &size,
                                                        flags);
-    return PyBytes_FromStringAndSize(data, (Py_ssize_t)size);
+    if (data == NULL) {
+        PyErr_Format(self->OperationalError, "unable to serialize '%s'",
+                     schema);
+        return NULL;
+    }
+    PyObject *res = PyBytes_FromStringAndSize(data, (Py_ssize_t)size);
+    sqlite3_free((void *)data);
+    return res;
 }
 
 /*[clinic input]

From 9df648fca6aeb31bd9a89ed77968e431ecf15ba8 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Fri, 30 Jul 2021 10:10:33 +0200
Subject: [PATCH 16/47] Transfer ownership of the deserialized data to SQLite

---
 Modules/_sqlite/connection.c | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 746cbd87ab0d31..20f2a7a28c36c7 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1832,16 +1832,27 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
 
     pysqlite_do_all_statements(self, ACTION_RESET, 1);
 
+    /* Transfer ownership of the buffer to SQLite:
+     * - Move buffer from Py to SQLite
+     * - Tell SQLite to free buffer memory
+     * - Tell SQLite that it is permitted to grow the resulting database
+     */
     sqlite3_int64 size = (sqlite3_int64)data->len;
-    unsigned char *buf = (unsigned char *)data->buf;
-    const unsigned int flags = 0;
+    unsigned char *buf = sqlite3_malloc(size);
+    if (buf == NULL) {
+        return PyErr_NoMemory();
+    }
+    (void)memcpy(buf, data->buf, data->len);
+
+    const unsigned int flags = SQLITE_DESERIALIZE_FREEONCLOSE |
+                               SQLITE_DESERIALIZE_RESIZEABLE;
     int rc;
     Py_BEGIN_ALLOW_THREADS
     rc = sqlite3_deserialize(self->db, schema, buf, size, size, flags);
     Py_END_ALLOW_THREADS
 
     if (rc != SQLITE_OK) {
-        _pysqlite_seterror(self->state, self->db);
+        (void)_pysqlite_seterror(self->state, self->db);
         return NULL;
     }
 

From 02d0e45ada683a4f1066f374693bf3b4078595ca Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Fri, 30 Jul 2021 10:18:42 +0200
Subject: [PATCH 17/47] Add docstrings

---
 Modules/_sqlite/clinic/connection.c.h | 28 ++++++++++++++++++++++++---
 Modules/_sqlite/connection.c          | 21 ++++++++++++++++++--
 2 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h
index 28339b0602fc7a..54e089ba1a5a3c 100644
--- a/Modules/_sqlite/clinic/connection.c.h
+++ b/Modules/_sqlite/clinic/connection.c.h
@@ -760,7 +760,16 @@ pysqlite_connection_create_collation(pysqlite_Connection *self, PyObject *const
 PyDoc_STRVAR(serialize__doc__,
 "serialize($self, /, *, schema=\'main\')\n"
 "--\n"
-"\n");
+"\n"
+"Serialize a database into a byte string. Non-standard.\n"
+"\n"
+"  schema\n"
+"    Which database to serialize.\n"
+"\n"
+"For an ordinary on-disk database file, the serialization is just a copy of the\n"
+"disk file. For an in-memory database or a \"temp\" database, the serialization is\n"
+"the same sequence of bytes which would be written to disk if that database\n"
+"where backed up to disk.");
 
 #define SERIALIZE_METHODDEF    \
     {"serialize", (PyCFunction)(void(*)(void))serialize, METH_FASTCALL|METH_KEYWORDS, serialize__doc__},
@@ -812,7 +821,20 @@ serialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, Py
 PyDoc_STRVAR(deserialize__doc__,
 "deserialize($self, data, /, *, schema=\'main\')\n"
 "--\n"
-"\n");
+"\n"
+"Load a serialized database. Non-standard.\n"
+"\n"
+"  data\n"
+"    The serialized database content\n"
+"  schema\n"
+"    Which database to reopen with the deserialization.\n"
+"\n"
+"The deserialize interface causes the database connection to disconnect from the\n"
+"target database, and then reopen it as an in-memory database based on the given\n"
+"serialized data.\n"
+"\n"
+"The deserialize interface will fail with SQLITE_BUSY if the database is\n"
+"currently in a read transaction or is involved in a backup operation.");
 
 #define DESERIALIZE_METHODDEF    \
     {"deserialize", (PyCFunction)(void(*)(void))deserialize, METH_FASTCALL|METH_KEYWORDS, deserialize__doc__},
@@ -953,4 +975,4 @@ pysqlite_connection_exit(pysqlite_Connection *self, PyObject *const *args, Py_ss
 #ifndef DESERIALIZE_METHODDEF
     #define DESERIALIZE_METHODDEF
 #endif /* !defined(DESERIALIZE_METHODDEF) */
-/*[clinic end generated code: output=c79744de72ec493b input=a9049054013a1b77]*/
+/*[clinic end generated code: output=246bf62d1e58e291 input=a9049054013a1b77]*/
diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 20f2a7a28c36c7..cc7d47cc258e82 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1781,12 +1781,19 @@ _sqlite3.Connection.serialize as serialize
 
     *
     schema: str = "main"
+        Which database to serialize.
 
+Serialize a database into a byte string. Non-standard.
+
+For an ordinary on-disk database file, the serialization is just a copy of the
+disk file. For an in-memory database or a "temp" database, the serialization is
+the same sequence of bytes which would be written to disk if that database
+where backed up to disk.
 [clinic start generated code]*/
 
 static PyObject *
 serialize_impl(pysqlite_Connection *self, const char *schema)
-/*[clinic end generated code: output=b246381f2b3f1d84 input=f6782d98caa63008]*/
+/*[clinic end generated code: output=b246381f2b3f1d84 input=618605a185793d6b]*/
 {
     if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) {
         return NULL;
@@ -1810,16 +1817,26 @@ serialize_impl(pysqlite_Connection *self, const char *schema)
 _sqlite3.Connection.deserialize as deserialize
 
     data: Py_buffer(accept={buffer, str})
+        The serialized database content
     /
     *
     schema: str = "main"
+        Which database to reopen with the deserialization.
+
+Load a serialized database. Non-standard.
+
+The deserialize interface causes the database connection to disconnect from the
+target database, and then reopen it as an in-memory database based on the given
+serialized data.
 
+The deserialize interface will fail with SQLITE_BUSY if the database is
+currently in a read transaction or is involved in a backup operation.
 [clinic start generated code]*/
 
 static PyObject *
 deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
                  const char *schema)
-/*[clinic end generated code: output=96b8470aaebf1b25 input=ed3de6f0e6fe4aa2]*/
+/*[clinic end generated code: output=96b8470aaebf1b25 input=dfe957b21dc48572]*/
 {
     if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) {
         return NULL;

From bf5ab3d5d44a0d6dea756d2c884f5084c01dd2ea Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Sat, 31 Jul 2021 22:55:01 +0200
Subject: [PATCH 18/47] Add What's New entry

---
 Doc/whatsnew/3.11.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst
index 88b6f8fa7314e0..93349f921c7b31 100644
--- a/Doc/whatsnew/3.11.rst
+++ b/Doc/whatsnew/3.11.rst
@@ -218,6 +218,11 @@ sqlite3
   now raise :exc:`UnicodeEncodeError` instead of :exc:`sqlite3.ProgrammingError`.
   (Contributed by Erlend E. Aasland in :issue:`44688`.)
 
+* Add :meth:`~sqlite3.Connection.serialize` and
+  :meth:`~sqlite3.Connection.deserialize` to :class:`sqlite3.Connection` for
+  serializing and deserializing databases.
+  (Contributed by Erlend E. Aasland in :issue:`41930`.)
+
 
 Removed
 =======

From 5a7f974e66d7927ab16e36faeb610b445b3c39a0 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Mon, 2 Aug 2021 10:46:03 +0200
Subject: [PATCH 19/47] Add Docs

---
 Doc/library/sqlite3.rst | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst
index 6399bed7ed52c6..ae3ce2b6fda37a 100644
--- a/Doc/library/sqlite3.rst
+++ b/Doc/library/sqlite3.rst
@@ -616,6 +616,31 @@ Connection Objects
       .. versionadded:: 3.7
 
 
+   .. method:: serialize(*, schema="main")
+
+      This method serializes a database into a :class:`bytes` sequence.  For an
+      ordinary on-disk database file, the serialization is just a copy of the
+      disk file.  For an in-memory database or a "temp" database, the
+      serialization is the same sequence of bytes which would be written to
+      disk if that database where backed up to disk.
+
+      *schema* is the database to be serialized, and defaults to the main
+      database.
+
+      .. versionadded:: 3.11
+
+
+   .. method:: deserialize(data, /, *, schema="main")
+
+      This method causes the database connection to disconnect from database
+      *schema*, and reopen *schema* as an in-memory database based on the
+      serialization contained in *data*.  Deserialization will fail with
+      ``SQLITE_BUSY`` if the database is currently in a read transaction or is
+      involved in a backup operation.
+
+      .. versionadded:: 3.11
+
+
 .. _sqlite3-cursor-objects:
 
 Cursor Objects

From 5c7cb032fda2cbf408ee8ec3e536ab36b4eb7f81 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Mon, 2 Aug 2021 10:47:25 +0200
Subject: [PATCH 20/47] Improve naming: schema => name

---
 Doc/library/sqlite3.rst               |  8 ++---
 Modules/_sqlite/clinic/connection.c.h | 46 +++++++++++++--------------
 Modules/_sqlite/connection.c          | 18 +++++------
 3 files changed, 36 insertions(+), 36 deletions(-)

diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst
index ae3ce2b6fda37a..b0b82fcbe4803e 100644
--- a/Doc/library/sqlite3.rst
+++ b/Doc/library/sqlite3.rst
@@ -616,7 +616,7 @@ Connection Objects
       .. versionadded:: 3.7
 
 
-   .. method:: serialize(*, schema="main")
+   .. method:: serialize(*, name="main")
 
       This method serializes a database into a :class:`bytes` sequence.  For an
       ordinary on-disk database file, the serialization is just a copy of the
@@ -624,16 +624,16 @@ Connection Objects
       serialization is the same sequence of bytes which would be written to
       disk if that database where backed up to disk.
 
-      *schema* is the database to be serialized, and defaults to the main
+      *name* is the database to be serialized, and defaults to the main
       database.
 
       .. versionadded:: 3.11
 
 
-   .. method:: deserialize(data, /, *, schema="main")
+   .. method:: deserialize(data, /, *, name="main")
 
       This method causes the database connection to disconnect from database
-      *schema*, and reopen *schema* as an in-memory database based on the
+      *name*, and reopen *name* as an in-memory database based on the
       serialization contained in *data*.  Deserialization will fail with
       ``SQLITE_BUSY`` if the database is currently in a read transaction or is
       involved in a backup operation.
diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h
index 54e089ba1a5a3c..b0d91ba96922f3 100644
--- a/Modules/_sqlite/clinic/connection.c.h
+++ b/Modules/_sqlite/clinic/connection.c.h
@@ -758,12 +758,12 @@ pysqlite_connection_create_collation(pysqlite_Connection *self, PyObject *const
 #if defined(HAVE_SERIALIZE_API)
 
 PyDoc_STRVAR(serialize__doc__,
-"serialize($self, /, *, schema=\'main\')\n"
+"serialize($self, /, *, name=\'main\')\n"
 "--\n"
 "\n"
 "Serialize a database into a byte string. Non-standard.\n"
 "\n"
-"  schema\n"
+"  name\n"
 "    Which database to serialize.\n"
 "\n"
 "For an ordinary on-disk database file, the serialization is just a copy of the\n"
@@ -775,17 +775,17 @@ PyDoc_STRVAR(serialize__doc__,
     {"serialize", (PyCFunction)(void(*)(void))serialize, METH_FASTCALL|METH_KEYWORDS, serialize__doc__},
 
 static PyObject *
-serialize_impl(pysqlite_Connection *self, const char *schema);
+serialize_impl(pysqlite_Connection *self, const char *name);
 
 static PyObject *
 serialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"schema", NULL};
+    static const char * const _keywords[] = {"name", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "serialize", 0};
     PyObject *argsbuf[1];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
-    const char *schema = "main";
+    const char *name = "main";
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf);
     if (!args) {
@@ -795,20 +795,20 @@ serialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, Py
         goto skip_optional_kwonly;
     }
     if (!PyUnicode_Check(args[0])) {
-        _PyArg_BadArgument("serialize", "argument 'schema'", "str", args[0]);
+        _PyArg_BadArgument("serialize", "argument 'name'", "str", args[0]);
         goto exit;
     }
-    Py_ssize_t schema_length;
-    schema = PyUnicode_AsUTF8AndSize(args[0], &schema_length);
-    if (schema == NULL) {
+    Py_ssize_t name_length;
+    name = PyUnicode_AsUTF8AndSize(args[0], &name_length);
+    if (name == NULL) {
         goto exit;
     }
-    if (strlen(schema) != (size_t)schema_length) {
+    if (strlen(name) != (size_t)name_length) {
         PyErr_SetString(PyExc_ValueError, "embedded null character");
         goto exit;
     }
 skip_optional_kwonly:
-    return_value = serialize_impl(self, schema);
+    return_value = serialize_impl(self, name);
 
 exit:
     return return_value;
@@ -819,14 +819,14 @@ serialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, Py
 #if defined(HAVE_SERIALIZE_API)
 
 PyDoc_STRVAR(deserialize__doc__,
-"deserialize($self, data, /, *, schema=\'main\')\n"
+"deserialize($self, data, /, *, name=\'main\')\n"
 "--\n"
 "\n"
 "Load a serialized database. Non-standard.\n"
 "\n"
 "  data\n"
 "    The serialized database content\n"
-"  schema\n"
+"  name\n"
 "    Which database to reopen with the deserialization.\n"
 "\n"
 "The deserialize interface causes the database connection to disconnect from the\n"
@@ -841,18 +841,18 @@ PyDoc_STRVAR(deserialize__doc__,
 
 static PyObject *
 deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
-                 const char *schema);
+                 const char *name);
 
 static PyObject *
 deserialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"", "schema", NULL};
+    static const char * const _keywords[] = {"", "name", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "deserialize", 0};
     PyObject *argsbuf[2];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1;
     Py_buffer data = {NULL, NULL};
-    const char *schema = "main";
+    const char *name = "main";
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf);
     if (!args) {
@@ -879,20 +879,20 @@ deserialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs,
         goto skip_optional_kwonly;
     }
     if (!PyUnicode_Check(args[1])) {
-        _PyArg_BadArgument("deserialize", "argument 'schema'", "str", args[1]);
+        _PyArg_BadArgument("deserialize", "argument 'name'", "str", args[1]);
         goto exit;
     }
-    Py_ssize_t schema_length;
-    schema = PyUnicode_AsUTF8AndSize(args[1], &schema_length);
-    if (schema == NULL) {
+    Py_ssize_t name_length;
+    name = PyUnicode_AsUTF8AndSize(args[1], &name_length);
+    if (name == NULL) {
         goto exit;
     }
-    if (strlen(schema) != (size_t)schema_length) {
+    if (strlen(name) != (size_t)name_length) {
         PyErr_SetString(PyExc_ValueError, "embedded null character");
         goto exit;
     }
 skip_optional_kwonly:
-    return_value = deserialize_impl(self, &data, schema);
+    return_value = deserialize_impl(self, &data, name);
 
 exit:
     /* Cleanup for data */
@@ -975,4 +975,4 @@ pysqlite_connection_exit(pysqlite_Connection *self, PyObject *const *args, Py_ss
 #ifndef DESERIALIZE_METHODDEF
     #define DESERIALIZE_METHODDEF
 #endif /* !defined(DESERIALIZE_METHODDEF) */
-/*[clinic end generated code: output=246bf62d1e58e291 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=e0cd7297d88c73eb input=a9049054013a1b77]*/
diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 99fcc7da56a4a1..fb72a001f36d27 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1796,7 +1796,7 @@ pysqlite_connection_create_collation_impl(pysqlite_Connection *self,
 _sqlite3.Connection.serialize as serialize
 
     *
-    schema: str = "main"
+    name: str = "main"
         Which database to serialize.
 
 Serialize a database into a byte string. Non-standard.
@@ -1808,8 +1808,8 @@ where backed up to disk.
 [clinic start generated code]*/
 
 static PyObject *
-serialize_impl(pysqlite_Connection *self, const char *schema)
-/*[clinic end generated code: output=b246381f2b3f1d84 input=618605a185793d6b]*/
+serialize_impl(pysqlite_Connection *self, const char *name)
+/*[clinic end generated code: output=97342b0e55239dd3 input=4b6efe5a4d524470]*/
 {
     if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) {
         return NULL;
@@ -1817,11 +1817,11 @@ serialize_impl(pysqlite_Connection *self, const char *schema)
 
     sqlite3_int64 size;
     const unsigned int flags = 0;
-    const char *data = (const char *)sqlite3_serialize(self->db, schema, &size,
+    const char *data = (const char *)sqlite3_serialize(self->db, name, &size,
                                                        flags);
     if (data == NULL) {
         PyErr_Format(self->OperationalError, "unable to serialize '%s'",
-                     schema);
+                     name);
         return NULL;
     }
     PyObject *res = PyBytes_FromStringAndSize(data, (Py_ssize_t)size);
@@ -1836,7 +1836,7 @@ _sqlite3.Connection.deserialize as deserialize
         The serialized database content
     /
     *
-    schema: str = "main"
+    name: str = "main"
         Which database to reopen with the deserialization.
 
 Load a serialized database. Non-standard.
@@ -1851,8 +1851,8 @@ currently in a read transaction or is involved in a backup operation.
 
 static PyObject *
 deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
-                 const char *schema)
-/*[clinic end generated code: output=96b8470aaebf1b25 input=dfe957b21dc48572]*/
+                 const char *name)
+/*[clinic end generated code: output=e394c798b98bad89 input=648327de6cdc107e]*/
 {
     if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) {
         return NULL;
@@ -1881,7 +1881,7 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
                                SQLITE_DESERIALIZE_RESIZEABLE;
     int rc;
     Py_BEGIN_ALLOW_THREADS
-    rc = sqlite3_deserialize(self->db, schema, buf, size, size, flags);
+    rc = sqlite3_deserialize(self->db, name, buf, size, size, flags);
     Py_END_ALLOW_THREADS
 
     if (rc != SQLITE_OK) {

From 729778c4a3bd512dcee8d0a7f1409cd2585e54f7 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Thu, 12 Aug 2021 15:09:46 +0200
Subject: [PATCH 21/47] Add more tests

---
 Lib/sqlite3/test/dbapi.py | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py
index 39658037c355ed..4dff9b924131a4 100644
--- a/Lib/sqlite3/test/dbapi.py
+++ b/Lib/sqlite3/test/dbapi.py
@@ -263,28 +263,45 @@ def test_uninit_operations(self):
                                        func)
 
 
-@unittest.skipIf(hasattr(sqlite.Connection, "serialize") == False,
-                 "Serialize API missing")
+@unittest.skipUnless(hasattr(sqlite.Connection, "serialize"),
+                     "requires serialize API")
 class SerializeTests(unittest.TestCase):
     def setUp(self):
         self.cx = sqlite.connect(":memory:")
 
+    def tearDown(self):
+        self.cx.close()
+
     def test_serialize_deserialize(self):
         with self.cx:
             self.cx.execute("create table t(t)")
             data = self.cx.serialize()
         self.assertEqual(len(data), 8192)
 
-        # Remove test table, then load the saved database
+        # Remove test table, verify that it was removed
         with self.cx:
             self.cx.execute("drop table t")
+        try:
+            self.cx.execute("select t from t")
+        except sqlite.OperationalError:
+            pass
+        else:
+            self.fail()
+
+        # Load the serialized database, verify that the table is back again
         self.cx.deserialize(data)
         self.cx.execute("select t from t")
 
     def test_deserialize_wrong_args(self):
-        self.assertRaises(TypeError, self.cx.deserialize, [])
-        self.assertRaises(TypeError, self.cx.deserialize, None)
-        self.assertRaises(TypeError, self.cx.deserialize, 1)
+        dataset = [
+            (BufferError, memoryview(b"blob")[::2]),
+            (TypeError, []),
+            (TypeError, 1),
+            (TypeError, None),
+        ]
+        for exc, arg in dataset:
+            with self.subTest(exc=exc, arg=arg):
+                self.assertRaises(exc, self.cx.deserialize, arg)
 
     def test_deserialize_corrupt_database(self):
         with self.assertRaises(sqlite.DatabaseError):

From 5214fadb460709a0f7ae570ae6399bf285e92b82 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Thu, 12 Aug 2021 22:19:36 +0200
Subject: [PATCH 22/47] Allow threads while serializing

---
 Modules/_sqlite/connection.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 9d899861df816c..e46c6d0328e049 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1821,8 +1821,12 @@ serialize_impl(pysqlite_Connection *self, const char *name)
 
     sqlite3_int64 size;
     const unsigned int flags = 0;
-    const char *data = (const char *)sqlite3_serialize(self->db, name, &size,
-                                                       flags);
+    const char *data;
+
+    Py_BEGIN_ALLOW_THREADS
+    data = (const char *)sqlite3_serialize(self->db, name, &size, flags);
+    Py_END_ALLOW_THREADS
+
     if (data == NULL) {
         PyErr_Format(self->OperationalError, "unable to serialize '%s'",
                      name);

From 5aa2ec54f232e1053f9ec2f94cad78d1c9576eb5 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Fri, 13 Aug 2021 09:21:16 +0200
Subject: [PATCH 23/47] skip test if SQLite version < 3.36.0

---
 Lib/sqlite3/test/dbapi.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py
index 4dff9b924131a4..d1340062ae627e 100644
--- a/Lib/sqlite3/test/dbapi.py
+++ b/Lib/sqlite3/test/dbapi.py
@@ -263,8 +263,8 @@ def test_uninit_operations(self):
                                        func)
 
 
-@unittest.skipUnless(hasattr(sqlite.Connection, "serialize"),
-                     "requires serialize API")
+@unittest.skipIf(sqlite.sqlite_version_info < (3, 36, 0),
+                 "Requires SQLite 3.36.0 or higher")
 class SerializeTests(unittest.TestCase):
     def setUp(self):
         self.cx = sqlite.connect(":memory:")

From a95eb8a2b98d2f41926f65a81dd622dd6e681305 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Sun, 12 Sep 2021 18:14:22 +0200
Subject: [PATCH 24/47] Fix merge

---
 Modules/_sqlite/connection.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 6a5aa7ed7cc192..d18ffb617919b6 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1833,7 +1833,7 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
         return NULL;
     }
 
-    pysqlite_do_all_statements(self, ACTION_RESET, 1);
+    pysqlite_do_all_statements(self);
 
     /* Transfer ownership of the buffer to SQLite:
      * - Move buffer from Py to SQLite

From 315d993217b925c8dfadf21b6dbc148e1d8c7f54 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Wed, 17 Nov 2021 21:30:49 +0100
Subject: [PATCH 25/47] Improve tests

---
 Lib/test/test_sqlite3/test_dbapi.py | 74 ++++++++++++++---------------
 1 file changed, 35 insertions(+), 39 deletions(-)

diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py
index bbfd536b02447e..a05079f66de25d 100644
--- a/Lib/test/test_sqlite3/test_dbapi.py
+++ b/Lib/test/test_sqlite3/test_dbapi.py
@@ -606,59 +606,55 @@ def test_uninit_operations(self):
 @unittest.skipIf(sqlite.sqlite_version_info < (3, 36, 0),
                  "Requires SQLite 3.36.0 or higher")
 class SerializeTests(unittest.TestCase):
-    def setUp(self):
-        self.cx = sqlite.connect(":memory:")
-
-    def tearDown(self):
-        self.cx.close()
-
     def test_serialize_deserialize(self):
-        with self.cx:
-            self.cx.execute("create table t(t)")
-            data = self.cx.serialize()
-        self.assertEqual(len(data), 8192)
-
-        # Remove test table, verify that it was removed
-        with self.cx:
-            self.cx.execute("drop table t")
-        try:
-            self.cx.execute("select t from t")
-        except sqlite.OperationalError:
-            pass
-        else:
-            self.fail()
+        with memory_database() as cx:
+            with cx:
+                cx.execute("create table t(t)")
+            data = cx.serialize()
+            self.assertEqual(len(data), 8192)
+
+            # Remove test table, verify that it was removed.
+            with cx:
+                cx.execute("drop table t")
+            regex = "no such table"
+            with self.assertRaisesRegex(sqlite.OperationalError, regex):
+                cx.execute("select t from t")
 
-        # Load the serialized database, verify that the table is back again
-        self.cx.deserialize(data)
-        self.cx.execute("select t from t")
+            # Deserialize and verify that test table is restored.
+            cx.deserialize(data)
+            cx.execute("select t from t")
 
     def test_deserialize_wrong_args(self):
-        dataset = [
+        dataset = (
             (BufferError, memoryview(b"blob")[::2]),
             (TypeError, []),
             (TypeError, 1),
             (TypeError, None),
-        ]
+        )
         for exc, arg in dataset:
             with self.subTest(exc=exc, arg=arg):
-                self.assertRaises(exc, self.cx.deserialize, arg)
+                with memory_database() as cx:
+                    self.assertRaises(exc, cx.deserialize, arg)
 
     def test_deserialize_corrupt_database(self):
-        with self.assertRaises(sqlite.DatabaseError):
-            self.cx.deserialize(b"\0\1\3")
-            # SQLite does not generate an error until you try to query the
-            # deserialized database, so we query the ever present schema table.
-            self.cx.execute("select * from sqlite_schema")
+        with memory_database() as cx:
+            regex = "file is not a database"
+            with self.assertRaisesRegex(sqlite.DatabaseError, regex):
+                cx.deserialize(b"\0\1\3")
+                # SQLite does not generate an error until you try to query the
+                # deserialized database.
+                cx.execute("create table fail(f)")
 
     def test_fetch_across_deserialize(self):
-        with self.cx:
-            self.cx.execute("create table t(t)")
-        data = self.cx.serialize()
-        cu = self.cx.execute("select * from sqlite_schema")
-        self.cx.deserialize(data)
-        schema = [('table', 't', 't', 2, 'CREATE TABLE t(t)')]
-        with self.assertRaises(sqlite.InterfaceError):
-            cu.fetchall()
+        with memory_database() as cx:
+            with cx:
+                cx.execute("create table t(t)")
+            data = cx.serialize()
+            cu = cx.execute("select t from t")
+            cx.deserialize(data)
+            regex = "Cursor.*can no longer be fetched from"
+            with self.assertRaisesRegex(sqlite.InterfaceError, regex):
+                cu.fetchall()
 
 
 class OpenTests(unittest.TestCase):

From ce20059750ed05e2e047dc97b6215860359794e1 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Sat, 20 Nov 2021 13:08:50 +0100
Subject: [PATCH 26/47] Use autoconf to detect serialize API

---
 Modules/_sqlite/connection.c |  4 ++--
 configure                    | 42 ++++++++++++++++++++++++++++++++++++
 configure.ac                 |  3 +++
 pyconfig.h.in                |  3 +++
 4 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 99e3142b0beb4a..bd1f24f5eb0b98 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1840,7 +1840,7 @@ pysqlite_connection_create_collation_impl(pysqlite_Connection *self,
     Py_RETURN_NONE;
 }
 
-#ifdef HAVE_SERIALIZE_API
+#ifdef PY_SQLITE_HAVE_SERIALIZE
 /*[clinic input]
 _sqlite3.Connection.serialize as serialize
 
@@ -1944,7 +1944,7 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
 
     Py_RETURN_TRUE;
 }
-#endif
+#endif  // PY_SQLITE_HAVE_SERIALIZE
 
 
 /*[clinic input]
diff --git a/configure b/configure
index 9340cb072199c3..95237bbdd5eafc 100755
--- a/configure
+++ b/configure
@@ -11162,6 +11162,48 @@ _ACEOF
 
   LIBS="-lsqlite3 $LIBS"
 
+fi
+
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_serialize in -lsqlite3" >&5
+$as_echo_n "checking for sqlite3_serialize in -lsqlite3... " >&6; }
+if ${ac_cv_lib_sqlite3_sqlite3_serialize+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lsqlite3  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char sqlite3_serialize ();
+int
+main ()
+{
+return sqlite3_serialize ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_lib_sqlite3_sqlite3_serialize=yes
+else
+  ac_cv_lib_sqlite3_sqlite3_serialize=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_serialize" >&5
+$as_echo "$ac_cv_lib_sqlite3_sqlite3_serialize" >&6; }
+if test "x$ac_cv_lib_sqlite3_sqlite3_serialize" = xyes; then :
+
+$as_echo "#define PY_SQLITE_HAVE_SERIALIZE 1" >>confdefs.h
+
 fi
 
 
diff --git a/configure.ac b/configure.ac
index 924659713e4747..a3b6c1fa86927d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3197,6 +3197,9 @@ AC_CHECK_HEADER([sqlite3.h], [
     ], [have_supported_sqlite3=yes], [have_supported_sqlite3=no])
   ], [have_sqlite3=no])
   AC_CHECK_LIB([sqlite3], [sqlite3_load_extension])
+  AC_CHECK_LIB([sqlite3], [sqlite3_serialize],
+               [AC_DEFINE([PY_SQLITE_HAVE_SERIALIZE], [1],
+                          ["Define if SQLite was compiled with the serialize API"])])
 ])
 
 AS_VAR_COPY([CFLAGS], [save_CFLAGS])
diff --git a/pyconfig.h.in b/pyconfig.h.in
index 0cc593fdfc569e..68e38fb40bab8c 100644
--- a/pyconfig.h.in
+++ b/pyconfig.h.in
@@ -1452,6 +1452,9 @@
 /* Define to 1 to build the sqlite module with loadable extensions support. */
 #undef PY_SQLITE_ENABLE_LOAD_EXTENSION
 
+/* "Define if SQLite was compiled with the serialize API" */
+#undef PY_SQLITE_HAVE_SERIALIZE
+
 /* Default cipher suites list for ssl module. 1: Python's preferred selection,
    2: leave OpenSSL defaults untouched, 0: custom string */
 #undef PY_SSL_DEFAULT_CIPHERS

From 516c59f0c0842f1d55eef840ca9b5a9179b3624f Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Sat, 20 Nov 2021 13:20:26 +0100
Subject: [PATCH 27/47] Regen clinic

---
 Modules/_sqlite/clinic/connection.c.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h
index a9d35329881ed3..f9150c0e0e6b70 100644
--- a/Modules/_sqlite/clinic/connection.c.h
+++ b/Modules/_sqlite/clinic/connection.c.h
@@ -697,7 +697,7 @@ pysqlite_connection_create_collation(pysqlite_Connection *self, PyTypeObject *cl
     return return_value;
 }
 
-#if defined(HAVE_SERIALIZE_API)
+#if defined(PY_SQLITE_HAVE_SERIALIZE)
 
 PyDoc_STRVAR(serialize__doc__,
 "serialize($self, /, *, name=\'main\')\n"
@@ -756,9 +756,9 @@ serialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, Py
     return return_value;
 }
 
-#endif /* defined(HAVE_SERIALIZE_API) */
+#endif /* defined(PY_SQLITE_HAVE_SERIALIZE) */
 
-#if defined(HAVE_SERIALIZE_API)
+#if defined(PY_SQLITE_HAVE_SERIALIZE)
 
 PyDoc_STRVAR(deserialize__doc__,
 "deserialize($self, data, /, *, name=\'main\')\n"
@@ -845,7 +845,7 @@ deserialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs,
     return return_value;
 }
 
-#endif /* defined(HAVE_SERIALIZE_API) */
+#endif /* defined(PY_SQLITE_HAVE_SERIALIZE) */
 
 PyDoc_STRVAR(pysqlite_connection_enter__doc__,
 "__enter__($self, /)\n"
@@ -994,4 +994,4 @@ getlimit(pysqlite_Connection *self, PyObject *arg)
 #ifndef DESERIALIZE_METHODDEF
     #define DESERIALIZE_METHODDEF
 #endif /* !defined(DESERIALIZE_METHODDEF) */
-/*[clinic end generated code: output=f89f071412ba75f9 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=aefa6b5036964d5a input=a9049054013a1b77]*/

From 3b59ec02794b1dc7ac4a172622ab6231c402674b Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Sat, 20 Nov 2021 14:41:20 +0100
Subject: [PATCH 28/47] Fix test deps

---
 Lib/test/test_sqlite3/test_dbapi.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py
index a05079f66de25d..411f953a79f9a7 100644
--- a/Lib/test/test_sqlite3/test_dbapi.py
+++ b/Lib/test/test_sqlite3/test_dbapi.py
@@ -603,8 +603,7 @@ def test_uninit_operations(self):
                                        func)
 
 
-@unittest.skipIf(sqlite.sqlite_version_info < (3, 36, 0),
-                 "Requires SQLite 3.36.0 or higher")
+@unittest.skipUnless(hasattr(sqlite3.Connection, "serialize"))
 class SerializeTests(unittest.TestCase):
     def test_serialize_deserialize(self):
         with memory_database() as cx:

From eec900083c187757ccb28146caf1b2ec82b8cf27 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Sat, 20 Nov 2021 14:45:31 +0100
Subject: [PATCH 29/47] Build sqlite3 with serialize on Windows

---
 PCbuild/_sqlite3.vcxproj | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/PCbuild/_sqlite3.vcxproj b/PCbuild/_sqlite3.vcxproj
index e268c473f4c985..96bf40f5bc2eb8 100644
--- a/PCbuild/_sqlite3.vcxproj
+++ b/PCbuild/_sqlite3.vcxproj
@@ -94,6 +94,7 @@
   <ItemDefinitionGroup>
     <ClCompile>
       <AdditionalIncludeDirectories>$(sqlite3Dir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>PY_SQLITE_HAVE_SERIALIZE</PreprocessorDefinitions>
     </ClCompile>
   </ItemDefinitionGroup>
   <ItemGroup>
@@ -132,4 +133,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
\ No newline at end of file
+</Project>

From a902bd93dff3a3d0d335d4baa9d55f31b006cbe0 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Sat, 20 Nov 2021 17:27:55 +0100
Subject: [PATCH 30/47] Typo

---
 Lib/test/test_sqlite3/test_dbapi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py
index 411f953a79f9a7..ba6de5f5ca579c 100644
--- a/Lib/test/test_sqlite3/test_dbapi.py
+++ b/Lib/test/test_sqlite3/test_dbapi.py
@@ -603,7 +603,7 @@ def test_uninit_operations(self):
                                        func)
 
 
-@unittest.skipUnless(hasattr(sqlite3.Connection, "serialize"))
+@unittest.skipUnless(hasattr(sqlite.Connection, "serialize"))
 class SerializeTests(unittest.TestCase):
     def test_serialize_deserialize(self):
         with memory_database() as cx:

From 3b316c8dd4432dcddc381007e3be67f1c3ca0da3 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Sat, 20 Nov 2021 17:59:14 +0100
Subject: [PATCH 31/47] Fix skipIf decorator

---
 Lib/test/test_sqlite3/test_dbapi.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py
index ba6de5f5ca579c..c6faa79e777057 100644
--- a/Lib/test/test_sqlite3/test_dbapi.py
+++ b/Lib/test/test_sqlite3/test_dbapi.py
@@ -603,7 +603,8 @@ def test_uninit_operations(self):
                                        func)
 
 
-@unittest.skipUnless(hasattr(sqlite.Connection, "serialize"))
+@unittest.skipUnless(hasattr(sqlite.Connection, "serialize"),
+                     "Needs SQLite serialize API")
 class SerializeTests(unittest.TestCase):
     def test_serialize_deserialize(self):
         with memory_database() as cx:

From 3babef8fb04aa10be0dac7f63bb052d31dd9ed35 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Mon, 29 Nov 2021 18:07:26 +0100
Subject: [PATCH 32/47] Note that these API's are only available if the
 underlying SQLite lib supports them

---
 Doc/library/sqlite3.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst
index a1d319b9358a08..a85d4cc84f1902 100644
--- a/Doc/library/sqlite3.rst
+++ b/Doc/library/sqlite3.rst
@@ -745,6 +745,11 @@ Connection Objects
       *name* is the database to be serialized, and defaults to the main
       database.
 
+      .. note::
+
+         This method is only available if the underlying SQLite library has the
+         serialize API.
+
       .. versionadded:: 3.11
 
 
@@ -756,6 +761,11 @@ Connection Objects
       ``SQLITE_BUSY`` if the database is currently in a read transaction or is
       involved in a backup operation.
 
+      .. note::
+
+         This method is only available if the underlying SQLite library has the
+         serialize API.
+
       .. versionadded:: 3.11
 
 

From 9dffda67c50a77dc47ad52f70a66d48e068e8369 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Tue, 18 Jan 2022 22:49:48 +0100
Subject: [PATCH 33/47] Fix merge: pysqlite_do_all_statements is an ex-function

---
 Lib/test/test_sqlite3/test_dbapi.py | 11 -----------
 Modules/_sqlite/connection.c        |  2 --
 2 files changed, 13 deletions(-)

diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py
index c6faa79e777057..0e067a20b1243e 100644
--- a/Lib/test/test_sqlite3/test_dbapi.py
+++ b/Lib/test/test_sqlite3/test_dbapi.py
@@ -645,17 +645,6 @@ def test_deserialize_corrupt_database(self):
                 # deserialized database.
                 cx.execute("create table fail(f)")
 
-    def test_fetch_across_deserialize(self):
-        with memory_database() as cx:
-            with cx:
-                cx.execute("create table t(t)")
-            data = cx.serialize()
-            cu = cx.execute("select t from t")
-            cx.deserialize(data)
-            regex = "Cursor.*can no longer be fetched from"
-            with self.assertRaisesRegex(sqlite.InterfaceError, regex):
-                cu.fetchall()
-
 
 class OpenTests(unittest.TestCase):
     _sql = "create table test(id integer)"
diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 8847713078950f..fdddebe545426b 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1892,8 +1892,6 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
         return NULL;
     }
 
-    pysqlite_do_all_statements(self);
-
     /* Transfer ownership of the buffer to SQLite:
      * - Move buffer from Py to SQLite
      * - Tell SQLite to free buffer memory

From ac2f552c3aa573599f5fa4511c8e5dcaea62ce92 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Thu, 20 Jan 2022 13:27:43 +0100
Subject: [PATCH 34/47] Use sqlite3_malloc64 and add bigmemtest

---
 Lib/test/test_sqlite3/test_dbapi.py |  9 +++++++++
 Modules/_sqlite/connection.c        | 20 +++++++++++++-------
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py
index 0e067a20b1243e..a1533a496f2bf8 100644
--- a/Lib/test/test_sqlite3/test_dbapi.py
+++ b/Lib/test/test_sqlite3/test_dbapi.py
@@ -29,6 +29,7 @@
 
 from test.support import (
     SHORT_TIMEOUT,
+    bigmemtest,
     check_disallow_instantiation,
     threading_helper,
 )
@@ -645,6 +646,13 @@ def test_deserialize_corrupt_database(self):
                 # deserialized database.
                 cx.execute("create table fail(f)")
 
+    @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform')
+    @bigmemtest(size=2**63, memuse=3, dry_run=False)
+    def test_deserialize_too_much_data_64bit(self):
+        with memory_database() as cx:
+            with self.assertRaisesRegex(OverflowError, "'data' is too large"):
+                cx.deserialize(b"b" * size)
+
 
 class OpenTests(unittest.TestCase):
     _sql = "create table test(id integer)"
@@ -1074,6 +1082,7 @@ def test_check_connection_thread(self):
         ]
         if hasattr(sqlite.Connection, "serialize"):
             fns.append(lambda: self.con.serialize())
+            fns.append(lambda: self.con.deserialize(b""))
 
         for fn in fns:
             with self.subTest(fn=fn):
diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index fdddebe545426b..444848b922f6d6 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1853,7 +1853,7 @@ serialize_impl(pysqlite_Connection *self, const char *name)
                      name);
         return NULL;
     }
-    PyObject *res = PyBytes_FromStringAndSize(data, (Py_ssize_t)size);
+    PyObject *res = PyBytes_FromStringAndSize(data, size);
     sqlite3_free((void *)data);
     return res;
 }
@@ -1887,18 +1887,24 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
         return NULL;
     }
 
-    if (data->len > 9223372036854775807) {
-        PyErr_SetString(PyExc_OverflowError, "'data' is too large");
-        return NULL;
-    }
-
     /* Transfer ownership of the buffer to SQLite:
      * - Move buffer from Py to SQLite
      * - Tell SQLite to free buffer memory
      * - Tell SQLite that it is permitted to grow the resulting database
+     *
+     * Make sure we don't overflow sqlite3_deserialize(); it accepts a signed
+     * 64-bit int as its data size argument, but sqlite3_malloc64 accepts an
+     * unsigned 64-bit int as its size argument
+     *
+     * We can safely use sqlite3_malloc64 here, since it was introduced before
+     * the serialize APIs.
      */
+    if (data->len > 9223372036854775807) {
+        PyErr_SetString(PyExc_OverflowError, "'data' is too large");
+        return NULL;
+    }
     sqlite3_int64 size = (sqlite3_int64)data->len;
-    unsigned char *buf = sqlite3_malloc(size);
+    unsigned char *buf = sqlite3_malloc64(size);
     if (buf == NULL) {
         return PyErr_NoMemory();
     }

From 7cdc5bafbd4375feaa08129e1f6f9c5405fd7310 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Thu, 20 Jan 2022 13:36:32 +0100
Subject: [PATCH 35/47] Remove spurious newline in VS docs

---
 PCbuild/_sqlite3.vcxproj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/PCbuild/_sqlite3.vcxproj b/PCbuild/_sqlite3.vcxproj
index 96bf40f5bc2eb8..b4c65386dc51d8 100644
--- a/PCbuild/_sqlite3.vcxproj
+++ b/PCbuild/_sqlite3.vcxproj
@@ -133,4 +133,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>
\ No newline at end of file

From 45c3e3eb844b001dfe3df1c88ab0d8409cd65276 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Sat, 26 Feb 2022 23:07:27 +0100
Subject: [PATCH 36/47] Nit: add missing punctuation

---
 Modules/_sqlite/connection.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 51643470333d57..f7c7f62d994ec2 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1863,7 +1863,7 @@ serialize_impl(pysqlite_Connection *self, const char *name)
 _sqlite3.Connection.deserialize as deserialize
 
     data: Py_buffer(accept={buffer, str})
-        The serialized database content
+        The serialized database content.
     /
     *
     name: str = "main"
@@ -1895,7 +1895,7 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
      *
      * Make sure we don't overflow sqlite3_deserialize(); it accepts a signed
      * 64-bit int as its data size argument, but sqlite3_malloc64 accepts an
-     * unsigned 64-bit int as its size argument
+     * unsigned 64-bit int as its size argument.
      *
      * We can safely use sqlite3_malloc64 here, since it was introduced before
      * the serialize APIs.

From fb127a8c6b32a95a9172ad667568036b2a818a22 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Sat, 26 Feb 2022 23:09:40 +0100
Subject: [PATCH 37/47] Allow threads while memcpy'ing

---
 Modules/_sqlite/connection.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index f7c7f62d994ec2..01fd1a174b3a68 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1904,17 +1904,18 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
         PyErr_SetString(PyExc_OverflowError, "'data' is too large");
         return NULL;
     }
+
     sqlite3_int64 size = (sqlite3_int64)data->len;
     unsigned char *buf = sqlite3_malloc64(size);
     if (buf == NULL) {
         return PyErr_NoMemory();
     }
-    (void)memcpy(buf, data->buf, data->len);
 
     const unsigned int flags = SQLITE_DESERIALIZE_FREEONCLOSE |
                                SQLITE_DESERIALIZE_RESIZEABLE;
     int rc;
     Py_BEGIN_ALLOW_THREADS
+    (void)memcpy(buf, data->buf, data->len);
     rc = sqlite3_deserialize(self->db, name, buf, size, size, flags);
     Py_END_ALLOW_THREADS
 

From c76252888989b6225cd026c9d596097590e2a236 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Sat, 26 Feb 2022 23:10:02 +0100
Subject: [PATCH 38/47] Remove extra newline

---
 Modules/_sqlite/connection.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 01fd1a174b3a68..9080b9bc438792 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1923,7 +1923,6 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
         (void)_pysqlite_seterror(self->state, self->db);
         return NULL;
     }
-
     Py_RETURN_TRUE;
 }
 #endif  // PY_SQLITE_HAVE_SERIALIZE

From 609a47820da58a28a3ce2beed68c20b8eb31ae36 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Sat, 26 Feb 2022 23:34:10 +0100
Subject: [PATCH 39/47] Regen clinic

---
 Modules/_sqlite/clinic/connection.c.h | 4 ++--
 Modules/_sqlite/connection.c          | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h
index f9150c0e0e6b70..aca1bec802b0de 100644
--- a/Modules/_sqlite/clinic/connection.c.h
+++ b/Modules/_sqlite/clinic/connection.c.h
@@ -767,7 +767,7 @@ PyDoc_STRVAR(deserialize__doc__,
 "Load a serialized database. Non-standard.\n"
 "\n"
 "  data\n"
-"    The serialized database content\n"
+"    The serialized database content.\n"
 "  name\n"
 "    Which database to reopen with the deserialization.\n"
 "\n"
@@ -994,4 +994,4 @@ getlimit(pysqlite_Connection *self, PyObject *arg)
 #ifndef DESERIALIZE_METHODDEF
     #define DESERIALIZE_METHODDEF
 #endif /* !defined(DESERIALIZE_METHODDEF) */
-/*[clinic end generated code: output=aefa6b5036964d5a input=a9049054013a1b77]*/
+/*[clinic end generated code: output=fc68ba7a5d8a9173 input=a9049054013a1b77]*/
diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 9080b9bc438792..b6aec5ef8d6623 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1882,7 +1882,7 @@ currently in a read transaction or is involved in a backup operation.
 static PyObject *
 deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
                  const char *name)
-/*[clinic end generated code: output=e394c798b98bad89 input=648327de6cdc107e]*/
+/*[clinic end generated code: output=e394c798b98bad89 input=7cf51653d0c40e22]*/
 {
     if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) {
         return NULL;

From f81ca8d35b1e6306b1a23af8165289a3bd73f4cb Mon Sep 17 00:00:00 2001
From: Erlend Egeberg Aasland <erlend.aasland@innova.no>
Date: Sun, 27 Feb 2022 08:40:18 +0100
Subject: [PATCH 40/47] Update Doc/library/sqlite3.rst

Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>
---
 Doc/library/sqlite3.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst
index cb81f4df45ed84..ce97a2e403b0a8 100644
--- a/Doc/library/sqlite3.rst
+++ b/Doc/library/sqlite3.rst
@@ -750,7 +750,7 @@ Connection Objects
 
    .. method:: serialize(*, name="main")
 
-      This method serializes a database into a :class:`bytes` sequence.  For an
+      This method serializes a database into a :class:`bytes` object.  For an
       ordinary on-disk database file, the serialization is just a copy of the
       disk file.  For an in-memory database or a "temp" database, the
       serialization is the same sequence of bytes which would be written to

From 013b4def943336bb1c832d7953be811024f2f51a Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Sun, 27 Feb 2022 08:52:05 +0100
Subject: [PATCH 41/47] Address code review

---
 Doc/library/sqlite3.rst               | 2 +-
 Modules/_sqlite/clinic/connection.c.h | 4 ++--
 Modules/_sqlite/connection.c          | 6 +++---
 configure.ac                          | 2 +-
 pyconfig.h.in                         | 2 +-
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst
index ce97a2e403b0a8..b648fc69af3ee8 100644
--- a/Doc/library/sqlite3.rst
+++ b/Doc/library/sqlite3.rst
@@ -754,7 +754,7 @@ Connection Objects
       ordinary on-disk database file, the serialization is just a copy of the
       disk file.  For an in-memory database or a "temp" database, the
       serialization is the same sequence of bytes which would be written to
-      disk if that database where backed up to disk.
+      disk if that database were backed up to disk.
 
       *name* is the database to be serialized, and defaults to the main
       database.
diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h
index aca1bec802b0de..48a0752d6d3438 100644
--- a/Modules/_sqlite/clinic/connection.c.h
+++ b/Modules/_sqlite/clinic/connection.c.h
@@ -711,7 +711,7 @@ PyDoc_STRVAR(serialize__doc__,
 "For an ordinary on-disk database file, the serialization is just a copy of the\n"
 "disk file. For an in-memory database or a \"temp\" database, the serialization is\n"
 "the same sequence of bytes which would be written to disk if that database\n"
-"where backed up to disk.");
+"were backed up to disk.");
 
 #define SERIALIZE_METHODDEF    \
     {"serialize", (PyCFunction)(void(*)(void))serialize, METH_FASTCALL|METH_KEYWORDS, serialize__doc__},
@@ -994,4 +994,4 @@ getlimit(pysqlite_Connection *self, PyObject *arg)
 #ifndef DESERIALIZE_METHODDEF
     #define DESERIALIZE_METHODDEF
 #endif /* !defined(DESERIALIZE_METHODDEF) */
-/*[clinic end generated code: output=fc68ba7a5d8a9173 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=879ab806f1216663 input=a9049054013a1b77]*/
diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index b6aec5ef8d6623..d727cebd3613b5 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1830,12 +1830,12 @@ Serialize a database into a byte string. Non-standard.
 For an ordinary on-disk database file, the serialization is just a copy of the
 disk file. For an in-memory database or a "temp" database, the serialization is
 the same sequence of bytes which would be written to disk if that database
-where backed up to disk.
+were backed up to disk.
 [clinic start generated code]*/
 
 static PyObject *
 serialize_impl(pysqlite_Connection *self, const char *name)
-/*[clinic end generated code: output=97342b0e55239dd3 input=4b6efe5a4d524470]*/
+/*[clinic end generated code: output=97342b0e55239dd3 input=bb5513432e17fef8]*/
 {
     if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) {
         return NULL;
@@ -1900,7 +1900,7 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
      * We can safely use sqlite3_malloc64 here, since it was introduced before
      * the serialize APIs.
      */
-    if (data->len > 9223372036854775807) {
+    if (data->len > ((1 << 63) - 1)) {
         PyErr_SetString(PyExc_OverflowError, "'data' is too large");
         return NULL;
     }
diff --git a/configure.ac b/configure.ac
index 89912d144647a6..7f6236597cd924 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3484,7 +3484,7 @@ dnl hence CPPFLAGS instead of CFLAGS.
     AC_CHECK_LIB([sqlite3], [sqlite3_serialize], [
       AC_DEFINE(
         [PY_SQLITE_HAVE_SERIALIZE], [1],
-        ["Define if SQLite was compiled with the serialize API"])
+        [Define if SQLite was compiled with the serialize API])
     ])
   ])
 )
diff --git a/pyconfig.h.in b/pyconfig.h.in
index 52f8fcad4173ac..4d0a123fe4f091 100644
--- a/pyconfig.h.in
+++ b/pyconfig.h.in
@@ -1494,7 +1494,7 @@
 /* Define to 1 to build the sqlite module with loadable extensions support. */
 #undef PY_SQLITE_ENABLE_LOAD_EXTENSION
 
-/* "Define if SQLite was compiled with the serialize API" */
+/* Define if SQLite was compiled with the serialize API */
 #undef PY_SQLITE_HAVE_SERIALIZE
 
 /* Default cipher suites list for ssl module. 1: Python's preferred selection,

From f0f3c0aeb9bd7a33bc8218e6536f2edc1ff88d44 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Sun, 27 Feb 2022 10:16:50 +0100
Subject: [PATCH 42/47] Revert int max change

---
 Modules/_sqlite/connection.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index d727cebd3613b5..f9441da7c1690e 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1900,7 +1900,7 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
      * We can safely use sqlite3_malloc64 here, since it was introduced before
      * the serialize APIs.
      */
-    if (data->len > ((1 << 63) - 1)) {
+    if (data->len > 9223372036854775807) {  // (1 << 63) - 1
         PyErr_SetString(PyExc_OverflowError, "'data' is too large");
         return NULL;
     }

From 6d016f08c1e8b757f2d4f4e621d71680d165dbcc Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Sun, 27 Feb 2022 10:55:47 +0100
Subject: [PATCH 43/47] Remove redundant info from comment

---
 Modules/_sqlite/connection.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index f9441da7c1690e..94709bb0f6840b 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1894,8 +1894,7 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
      * - Tell SQLite that it is permitted to grow the resulting database
      *
      * Make sure we don't overflow sqlite3_deserialize(); it accepts a signed
-     * 64-bit int as its data size argument, but sqlite3_malloc64 accepts an
-     * unsigned 64-bit int as its size argument.
+     * 64-bit int as its data size argument.
      *
      * We can safely use sqlite3_malloc64 here, since it was introduced before
      * the serialize APIs.

From cea72193fa2a5f8babbbc680ad3fece9b17ce979 Mon Sep 17 00:00:00 2001
From: Erlend Egeberg Aasland <erlend.aasland@innova.no>
Date: Mon, 28 Feb 2022 07:16:48 +0100
Subject: [PATCH 44/47] Update Doc/library/sqlite3.rst

Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>
---
 Doc/library/sqlite3.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst
index b648fc69af3ee8..51818cba162bab 100644
--- a/Doc/library/sqlite3.rst
+++ b/Doc/library/sqlite3.rst
@@ -778,7 +778,7 @@ Connection Objects
       .. note::
 
          This method is only available if the underlying SQLite library has the
-         serialize API.
+         deserialize API.
 
       .. versionadded:: 3.11
 

From 7ddb4b72dee1db6b938c7ecd961c75823da5fd51 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Mon, 28 Feb 2022 09:21:27 +0100
Subject: [PATCH 45/47] Address review

- adjust docs
- deserialize returns None
- remove 'non-standard' from docstring
---
 Doc/library/sqlite3.rst               |  9 ++++++---
 Modules/_sqlite/clinic/connection.c.h |  6 +++---
 Modules/_sqlite/connection.c          | 10 +++++-----
 3 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst
index 51818cba162bab..af08df16e7b394 100644
--- a/Doc/library/sqlite3.rst
+++ b/Doc/library/sqlite3.rst
@@ -771,9 +771,12 @@ Connection Objects
 
       This method causes the database connection to disconnect from database
       *name*, and reopen *name* as an in-memory database based on the
-      serialization contained in *data*.  Deserialization will fail with
-      ``SQLITE_BUSY`` if the database is currently in a read transaction or is
-      involved in a backup operation.
+      serialization contained in *data*.  Deserialization will raise
+      :exc:`OperationalError` if the database connection is currently involved
+      in a read transaction or a backup operation.  :exc:`DataError` will be
+      raised if ``len(data)`` is larger than ``2**63 - 1``, and
+      :exc:`DatabaseError` will be raised if *data* does not contain a valid
+      SQLite database.
 
       .. note::
 
diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h
index 48a0752d6d3438..7c8624c617b495 100644
--- a/Modules/_sqlite/clinic/connection.c.h
+++ b/Modules/_sqlite/clinic/connection.c.h
@@ -703,7 +703,7 @@ PyDoc_STRVAR(serialize__doc__,
 "serialize($self, /, *, name=\'main\')\n"
 "--\n"
 "\n"
-"Serialize a database into a byte string. Non-standard.\n"
+"Serialize a database into a byte string.\n"
 "\n"
 "  name\n"
 "    Which database to serialize.\n"
@@ -764,7 +764,7 @@ PyDoc_STRVAR(deserialize__doc__,
 "deserialize($self, data, /, *, name=\'main\')\n"
 "--\n"
 "\n"
-"Load a serialized database. Non-standard.\n"
+"Load a serialized database.\n"
 "\n"
 "  data\n"
 "    The serialized database content.\n"
@@ -994,4 +994,4 @@ getlimit(pysqlite_Connection *self, PyObject *arg)
 #ifndef DESERIALIZE_METHODDEF
     #define DESERIALIZE_METHODDEF
 #endif /* !defined(DESERIALIZE_METHODDEF) */
-/*[clinic end generated code: output=879ab806f1216663 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=961f983241c1b845 input=a9049054013a1b77]*/
diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 94709bb0f6840b..321e14ded48081 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1825,7 +1825,7 @@ _sqlite3.Connection.serialize as serialize
     name: str = "main"
         Which database to serialize.
 
-Serialize a database into a byte string. Non-standard.
+Serialize a database into a byte string.
 
 For an ordinary on-disk database file, the serialization is just a copy of the
 disk file. For an in-memory database or a "temp" database, the serialization is
@@ -1835,7 +1835,7 @@ were backed up to disk.
 
 static PyObject *
 serialize_impl(pysqlite_Connection *self, const char *name)
-/*[clinic end generated code: output=97342b0e55239dd3 input=bb5513432e17fef8]*/
+/*[clinic end generated code: output=97342b0e55239dd3 input=d2eb5194a65abe2b]*/
 {
     if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) {
         return NULL;
@@ -1869,7 +1869,7 @@ _sqlite3.Connection.deserialize as deserialize
     name: str = "main"
         Which database to reopen with the deserialization.
 
-Load a serialized database. Non-standard.
+Load a serialized database.
 
 The deserialize interface causes the database connection to disconnect from the
 target database, and then reopen it as an in-memory database based on the given
@@ -1882,7 +1882,7 @@ currently in a read transaction or is involved in a backup operation.
 static PyObject *
 deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
                  const char *name)
-/*[clinic end generated code: output=e394c798b98bad89 input=7cf51653d0c40e22]*/
+/*[clinic end generated code: output=e394c798b98bad89 input=1be4ca1faacf28f2]*/
 {
     if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) {
         return NULL;
@@ -1922,7 +1922,7 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data,
         (void)_pysqlite_seterror(self->state, self->db);
         return NULL;
     }
-    Py_RETURN_TRUE;
+    Py_RETURN_NONE;
 }
 #endif  // PY_SQLITE_HAVE_SERIALIZE
 

From 7edccfed418eca5cdf88517cf9f3e91c2a051bc8 Mon Sep 17 00:00:00 2001
From: "Erlend E. Aasland" <erlend.aasland@innova.no>
Date: Mon, 28 Feb 2022 10:43:05 +0100
Subject: [PATCH 46/47] Address review: try to avoid memory allocations when
 serializing

---
 Modules/_sqlite/connection.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c
index 321e14ded48081..2d910eb8fce604 100644
--- a/Modules/_sqlite/connection.c
+++ b/Modules/_sqlite/connection.c
@@ -1841,12 +1841,19 @@ serialize_impl(pysqlite_Connection *self, const char *name)
         return NULL;
     }
 
+    /* If SQLite has a contiguous memory representation of the database, we can
+     * avoid memory allocations, so we try with the no-copy flag first.
+     */
     sqlite3_int64 size;
-    const unsigned int flags = 0;
+    unsigned int flags = SQLITE_SERIALIZE_NOCOPY;
     const char *data;
 
     Py_BEGIN_ALLOW_THREADS
     data = (const char *)sqlite3_serialize(self->db, name, &size, flags);
+    if (data == NULL) {
+        flags &= ~SQLITE_SERIALIZE_NOCOPY;
+        data = (const char *)sqlite3_serialize(self->db, name, &size, flags);
+    }
     Py_END_ALLOW_THREADS
 
     if (data == NULL) {
@@ -1855,7 +1862,9 @@ serialize_impl(pysqlite_Connection *self, const char *name)
         return NULL;
     }
     PyObject *res = PyBytes_FromStringAndSize(data, size);
-    sqlite3_free((void *)data);
+    if (!(flags & SQLITE_SERIALIZE_NOCOPY)) {
+        sqlite3_free((void *)data);
+    }
     return res;
 }
 

From 1db4093b1f4f9296fec13083d6ec1f8aad53868b Mon Sep 17 00:00:00 2001
From: Erlend Egeberg Aasland <erlend.aasland@innova.no>
Date: Sat, 5 Mar 2022 17:09:56 +0100
Subject: [PATCH 47/47] Update PCbuild/_sqlite3.vcxproj

Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com>
---
 PCbuild/_sqlite3.vcxproj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/PCbuild/_sqlite3.vcxproj b/PCbuild/_sqlite3.vcxproj
index b4c65386dc51d8..9cff43f73e5bec 100644
--- a/PCbuild/_sqlite3.vcxproj
+++ b/PCbuild/_sqlite3.vcxproj
@@ -94,7 +94,7 @@
   <ItemDefinitionGroup>
     <ClCompile>
       <AdditionalIncludeDirectories>$(sqlite3Dir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>PY_SQLITE_HAVE_SERIALIZE</PreprocessorDefinitions>
+      <PreprocessorDefinitions>PY_SQLITE_HAVE_SERIALIZE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
     </ClCompile>
   </ItemDefinitionGroup>
   <ItemGroup>