Implement Rust-based Database Access

AnkiDroid/build.gradle

@@ -241,6 +241,16 @@ dependencies {
     implementation 'androidx.constraintlayout:constraintlayout:2.0.4'
     implementation 'androidx.webkit:webkit:1.4.0'
 
+    // == Rust conversion (from Anki-Android-Backend on GitHub) ==
+    String backendVersion = "0.1.0" // We want both testing and implementation on the same version
+    // build with ./gradlew rsdroid:assembleRelease
+    // In my experience, using `files()` currently requires a reindex operation, which is slow.
+    // implementation files("C:\\GitHub\\Rust-Test\\rsdroid\\build\\outputs\\aar\\rsdroid-release.aar")
+    implementation "io.github.david-allison-1:anki-android-backend:$backendVersion"
+    // build with ./gradlew rsdroid-testing:assembleRelease
+    // RobolectricTest.java: replace RustBackendLoader.init(); with RustBackendLoader.loadRsdroid(path);
+    // A path for a testing library is typically under rsdroid-testing/assets
+    testImplementation "io.github.david-allison-1:anki-android-backend-testing:$backendVersion"
 
     // May need a resolution strategy for support libs to our versions
     implementation'ch.acra:acra-http:5.5.1'

AnkiDroid/src/androidTest/java/com/ichi2/anki/tests/RustTest.java

@@ -0,0 +1,48 @@
+/*
+ *  Copyright (c) 2020 David Allison <[email protected]>
+ *
+ *  This program is free software; you can redistribute it and/or modify it under
+ *  the terms of the GNU General Public License as published by the Free Software
+ *  Foundation; either version 3 of the License, or (at your option) any later
+ *  version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT ANY
+ *  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+ *  PARTICULAR PURPOSE. See the GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along with
+ *  this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package com.ichi2.anki.tests;
+
+import com.ichi2.libanki.Collection;
+import com.ichi2.libanki.Storage;
+
+import net.ankiweb.rsdroid.BackendException;
+
+import org.hamcrest.MatcherAssert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.is;
+
+public class RustTest extends InstrumentedTest {
+
+    /** Ensure that the database can't be locked */
+    @Rule
+    public Timeout timeout = new Timeout(3, TimeUnit.SECONDS);
+
+    @Test
+    public void collectionIsVersion11AfterOpen() throws BackendException, IOException {
+        // This test will be decommissioned, but before we get an upgrade strategy, we need to ensure we're not upgrading the database.
+        String path = Shared.getTestFilePath(getTestContext(), "initial_version_2_12_1.anki2");
+        Collection collection = Storage.Collection(getTestContext(), path);
+        int ver = collection.getDb().queryScalar("select ver from col");
+        MatcherAssert.assertThat(ver, is(11));
+    }
+}

AnkiDroid/src/main/java/com/ichi2/anki/Info.java

@@ -189,11 +189,20 @@ public String copyDebugInfo() {
             Timber.e(e, "Sched name not found");
         }
 
+        Boolean dbV2Enabled = null;
+        try {
+            dbV2Enabled = getCol().isUsingRustBackend();
+        } catch (Throwable e) {
+            Timber.w(e, "Unable to detect Rust Backend");
+        }
+
+
         String debugInfo = "AnkiDroid Version = " + VersionUtils.getPkgVersionName() + "\n\n" +
                 "Android Version = " + Build.VERSION.RELEASE + "\n\n" +
                 "ACRA UUID = " + Installation.id(this) + "\n\n" +
                 "Scheduler = " + schedName + "\n\n" +
-                "Crash Reports Enabled = " + isSendingCrashReports() + "\n";
+                "Crash Reports Enabled = " + isSendingCrashReports() + "\n\n" +
+                "DatabaseV2 Enabled = " + dbV2Enabled + "\n";
 
         android.content.ClipboardManager clipboardManager = (android.content.ClipboardManager)getSystemService(Context.CLIPBOARD_SERVICE);
         if (clipboardManager != null) {

AnkiDroid/src/main/java/com/ichi2/libanki/Collection.java

@@ -34,6 +34,7 @@
 import com.ichi2.anki.exception.ConfirmModSchemaException;
 import com.ichi2.async.CancelListener;
 import com.ichi2.async.CollectionTask;
+import com.ichi2.libanki.backend.DroidBackend;
 import com.ichi2.async.ProgressSender;
 import com.ichi2.async.TaskManager;
 import com.ichi2.libanki.exception.NoSuchDeckException;
@@ -128,6 +129,7 @@ public class Collection {
     private LinkedBlockingDeque<Undoable> mUndo;
 
     private final String mPath;
+    private final DroidBackend mDroidBackend;
     private boolean mDebugLog;
     private PrintWriter mLogHnd;
 
@@ -187,12 +189,13 @@ public String getString(Resources res) {
     private static final int UNDO_SIZE_MAX = 20;
 
     @VisibleForTesting
-    public Collection(Context context, DB db, String path, boolean server, boolean log, @NonNull Time time) {
+    public Collection(Context context, DB db, String path, boolean server, boolean log, @NonNull Time time, @NonNull DroidBackend droidBackend) {
         mContext = context;
         mDebugLog = log;
         mDb = db;
         mPath = path;
         mTime = time;
+        mDroidBackend = droidBackend;
         _openLog();
         log(path, VersionUtils.getPkgVersionName());
         mServer = server;
@@ -469,7 +472,7 @@ public synchronized void close(boolean save) {
     public void reopen() {
         Timber.i("Reopening Database");
         if (mDb == null) {
-            mDb = new DB(mPath);
+            mDb = mDroidBackend.openCollectionDatabase(mPath);
             mMedia.connect();
             _openLog();
         }
@@ -2163,6 +2166,10 @@ public AbstractSched createScheduler(int reportLimit) {
         return mSched;
     }
 
+    public boolean isUsingRustBackend() {
+        return mDroidBackend.isUsingRustBackend();
+    }
+
     /** Allows a mock db to be inserted for testing */
     @VisibleForTesting
     public void setDb(DB database) {

AnkiDroid/src/main/java/com/ichi2/libanki/DB.java

@@ -32,6 +32,9 @@
 import com.ichi2.anki.dialogs.DatabaseErrorDialog;
 import com.ichi2.utils.DatabaseChangeDecorator;
 
+import net.ankiweb.rsdroid.BackendFactory;
+import net.ankiweb.rsdroid.database.RustSQLiteOpenHelperFactory;
+
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
@@ -62,16 +65,21 @@ public class DB {
     private final SupportSQLiteDatabase mDatabase;
     private boolean mMod = false;
 
+    public DB(String ankiFilename) {
+        this(ankiFilename, null);
+    }
+
     /**
      * Open a connection to the SQLite collection database.
      */
-    public DB(String ankiFilename) {
+    public DB(String ankiFilename, @Nullable BackendFactory backendFactory) {
 
         SupportSQLiteOpenHelper.Configuration configuration = SupportSQLiteOpenHelper.Configuration.builder(AnkiDroidApp.getInstance())
                 .name(ankiFilename)
                 .callback(getDBCallback())
                 .build();
-        SupportSQLiteOpenHelper helper = getSqliteOpenHelperFactory().create(configuration);
+        SupportSQLiteOpenHelper helper = getSqliteOpenHelperFactory(backendFactory).create(configuration);
+        // Note: This line creates the database and schema when executed using a Rust backend
         mDatabase = new DatabaseChangeDecorator(helper.getWritableDatabase());
         mDatabase.disableWriteAheadLogging();
         mDatabase.query("PRAGMA synchronous = 2", null);
@@ -90,7 +98,11 @@ public static void setSqliteOpenHelperFactory(@Nullable SupportSQLiteOpenHelper.
     }
 
 
-    private SupportSQLiteOpenHelper.Factory getSqliteOpenHelperFactory() {
+    private SupportSQLiteOpenHelper.Factory getSqliteOpenHelperFactory(@Nullable BackendFactory backendFactory) {
+        if (backendFactory != null) {
+            return new RustSQLiteOpenHelperFactory(backendFactory);
+        }
+
         if (sqliteOpenHelperFactory == null) {
             return new RequerySQLiteOpenHelperFactory();
         }

AnkiDroid/src/main/java/com/ichi2/libanki/Storage.java

@@ -19,8 +19,11 @@
 import android.content.ContentValues;
 import android.content.Context;
 
+import com.ichi2.anki.UIUtils;
 import com.ichi2.anki.exception.ConfirmModSchemaException;
 
+import com.ichi2.libanki.backend.DroidBackend;
+import com.ichi2.libanki.backend.DroidBackendFactory;
 import com.ichi2.libanki.exception.UnknownDatabaseVersionException;
 import com.ichi2.libanki.utils.SystemTime;
 import com.ichi2.libanki.utils.Time;
@@ -42,6 +45,10 @@
         "PMD.SwitchStmtsShouldHaveDefault","PMD.EmptyIfStmt","PMD.SimplifyBooleanReturns","PMD.CollapsibleIfStatements"})
 public class Storage {
 
+    private static boolean sUseBackend = true;
+    private static boolean sUseInMemory = false;
+
+
     /* Open a new or existing collection. Path must be unicode */
     public static Collection Collection(Context context, String path) {
         return Collection(context, path, false, false);
@@ -65,19 +72,20 @@ public static Collection Collection(Context context, String path, boolean server
         assert path.endsWith(".anki2");
         File dbFile = new File(path);
         boolean create = !dbFile.exists();
-        // connect
-        DB db = new DB(path);
+        DroidBackend backend = DroidBackendFactory.getInstance(useBackend());
+        DB db = backend.openCollectionDatabase(sUseInMemory ? ":memory:" : path);
+
         try {
             // initialize
             int ver;
             if (create) {
-                ver = _createDB(db, time);
+                ver = _createDB(db, time, backend);
             } else {
                 ver = _upgradeSchema(db, time);
             }
             db.execute("PRAGMA temp_store = memory");
             // add db to col and do any remaining upgrades
-            Collection col = new Collection(context, db, path, server, log, time);
+            Collection col = new Collection(context, db, path, server, log, time, backend);
             if (ver < Consts.SCHEMA_VERSION) {
                 _upgrade(col, ver);
             } else if (ver > Consts.SCHEMA_VERSION) {
@@ -97,6 +105,14 @@ public static Collection Collection(Context context, String path, boolean server
         }
     }
 
+    /**
+     * Whether the collection should try to be opened with a Rust-based DB Backend
+     * Falls back to Java if init fails.
+     * */
+    protected static boolean useBackend() {
+        return sUseBackend;
+    }
+
 
     private static int _upgradeSchema(DB db, @NonNull Time time) {
         int ver = db.queryScalar("SELECT ver FROM col");
@@ -275,12 +291,19 @@ private static void _upgradeClozeModel(Collection col, Model m) throws ConfirmMo
     }
 
 
-    private static int _createDB(DB db, @NonNull Time time) {
-        db.execute("PRAGMA page_size = 4096");
-        db.execute("PRAGMA legacy_file_format = 0");
-        db.execute("VACUUM");
-        _addSchema(db, time);
-        _updateIndices(db);
+    private static int _createDB(DB db, @NonNull Time time, DroidBackend backend) {
+        if (backend.databaseCreationCreatesSchema()) {
+            _setColVars(db, time);
+            // This line is required for testing - otherwise Rust will override a mocked time.
+            db.execute("update col set crt = ?", UIUtils.getDayStart(time) / 1000);
+        } else {
+            db.execute("PRAGMA page_size = 4096");
+            db.execute("PRAGMA legacy_file_format = 0");
+            db.execute("VACUUM");
+            _addSchema(db, time);
+            _updateIndices(db);
+        }
+
         db.execute("ANALYZE");
         return Consts.SCHEMA_VERSION;
     }
@@ -366,4 +389,13 @@ public static void addIndices(DB db) {
         _updateIndices(db);
     }
 
+
+    public static void setUseBackend(boolean useBackend) {
+        sUseBackend = useBackend;
+    }
+
+
+    public static void setUseInMemory(boolean useInMemoryDatabase) {
+        sUseInMemory = useInMemoryDatabase;
+    }
 }

AnkiDroid/src/main/java/com/ichi2/libanki/backend/DroidBackend.java

@@ -0,0 +1,32 @@
+/*
+ *  Copyright (c) 2020 David Allison <[email protected]>
+ *
+ *  This program is free software; you can redistribute it and/or modify it under
+ *  the terms of the GNU General Public License as published by the Free Software
+ *  Foundation; either version 3 of the License, or (at your option) any later
+ *  version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT ANY
+ *  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+ *  PARTICULAR PURPOSE. See the GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along with
+ *  this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package com.ichi2.libanki.backend;
+
+import com.ichi2.libanki.DB;
+
+/**
+ * Interface to the rust backend listing all currently supported functionality.
+ */
+public interface DroidBackend {
+    DB openCollectionDatabase(String path);
+    void closeCollection();
+
+    /** Whether a call to {@link DroidBackend#openCollectionDatabase(String)} will generate a schema and indices for the database */
+    boolean databaseCreationCreatesSchema();
+
+    boolean isUsingRustBackend();
+}

AnkiDroid/src/main/java/com/ichi2/libanki/backend/DroidBackendFactory.java

@@ -0,0 +1,60 @@
+/*
+ *  Copyright (c) 2020 David Allison <[email protected]>
+ *
+ *  This program is free software; you can redistribute it and/or modify it under
+ *  the terms of the GNU General Public License as published by the Free Software
+ *  Foundation; either version 3 of the License, or (at your option) any later
+ *  version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT ANY
+ *  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+ *  PARTICULAR PURPOSE. See the GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along with
+ *  this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package com.ichi2.libanki.backend;
+
+import com.ichi2.anki.AnkiDroidApp;
+
+import net.ankiweb.rsdroid.BackendFactory;
+import net.ankiweb.rsdroid.RustBackendFailedException;
+
+import androidx.annotation.Nullable;
+import timber.log.Timber;
+
+/** Responsible for selection of either the Rust or Java-based backend */
+public class DroidBackendFactory {
+
+    /** Intentionally private - use {@link DroidBackendFactory#getInstance(boolean)}} */
+    private DroidBackendFactory() {
+
+    }
+
+    /**
+     * Obtains an instance of a {@link DroidBackend}.
+     * Each call to this method will generate a separate instance which can handle a new Anki collection
+     */
+    public static DroidBackend getInstance(boolean useBackend) {
+
+        BackendFactory backendFactory = null;
+        if (useBackend) {
+            try {
+                backendFactory = BackendFactory.createInstance();
+            } catch (RustBackendFailedException e) {
+                Timber.w(e, "Rust backend failed to load - falling back to Java");
+                AnkiDroidApp.sendExceptionReport(e, "DroidBackendFactory::getInstance");
+            }
+        }
+        return getInstance(backendFactory);
+    }
+
+    private static DroidBackend getInstance(@Nullable BackendFactory backendFactory) {
+        if (backendFactory == null) {
+            return new JavaDroidBackend();
+        } else {
+            return new RustDroidBackend(backendFactory);
+        }
+    }
+}