From 0692fd59131f62e1c87e800a39af2f2dde3f5341 Mon Sep 17 00:00:00 2001 From: David Allison <62114487+david-allison-1@users.noreply.github.com> Date: Sun, 23 Aug 2020 16:45:16 +0100 Subject: [PATCH 1/4] Use Rust backend for database queries All tests pass --- AnkiDroid/build.gradle | 11 ++ .../assets/initial_version_2_12_1.anki2 | Bin 0 -> 69632 bytes .../java/com/ichi2/anki/tests/RustTest.java | 48 ++++++ .../java/com/ichi2/libanki/Collection.java | 11 +- .../src/main/java/com/ichi2/libanki/DB.java | 18 ++- .../main/java/com/ichi2/libanki/Storage.java | 66 ++++++++- .../java/com/ichi2/anki/RobolectricTest.java | 13 ++ .../java/com/ichi2/libanki/StorageTest.java | 140 ++++++++++++++++++ 8 files changed, 294 insertions(+), 13 deletions(-) create mode 100644 AnkiDroid/src/androidTest/assets/initial_version_2_12_1.anki2 create mode 100644 AnkiDroid/src/androidTest/java/com/ichi2/anki/tests/RustTest.java create mode 100644 AnkiDroid/src/test/java/com/ichi2/libanki/StorageTest.java diff --git a/AnkiDroid/build.gradle b/AnkiDroid/build.gradle index e013bc70ef8b..08a71c51aad1 100644 --- a/AnkiDroid/build.gradle +++ b/AnkiDroid/build.gradle @@ -241,6 +241,17 @@ dependencies { implementation 'androidx.constraintlayout:constraintlayout:2.0.4' implementation 'androidx.webkit:webkit:1.4.0' + // == Rust conversion (from Anki-Android-Backend on GitHub) == + // use a variable as we want both testing and implementation on the same version + String backendVersion = "0.0.5" + // build with ./gradlew rsdroid:assembleRelease + // In my experience, using `files()` currently requires a reindex operation, which is slow. + // implementation files("C:\\GitHub\\Rust-Test\\rsdroid\\build\\outputs\\aar\\rsdroid-release.aar") + implementation "io.github.david-allison-1:anki-android-backend:$backendVersion" + // build with ./gradlew rsdroid-testing:assembleRelease + // add a one-time call to Runtime.getRuntime().load(path); to test locally + // TODO: Add method in RustBackendLoader to load native library on disk + testImplementation "io.github.david-allison-1:anki-android-backend-testing:$backendVersion" // May need a resolution strategy for support libs to our versions implementation'ch.acra:acra-http:5.5.1' diff --git a/AnkiDroid/src/androidTest/assets/initial_version_2_12_1.anki2 b/AnkiDroid/src/androidTest/assets/initial_version_2_12_1.anki2 new file mode 100644 index 0000000000000000000000000000000000000000..01c878e457952e4222ef9ef33df076efa71efaa6 GIT binary patch literal 69632 zcmeI3TW=f36~{?ik`+6SoCXGwhX9K$f{Fp&<_i55M}MQDnLk$x z*S3FunX2A6|L?-eZ!Z7k(w~-{rN1qGx^!{z4~xf(wS_+~{Ayu`D)9jU5ct*z{OFCP z(!1{#TatAJj~)ve1ok4yWG#DrW&5+8+q*l)?(I7dc8qMzs_cr2V;@)4Gv$vjFO_~! zDl*5j`LT1n=lPOHdsY;?`f6pQ=Kd#lcfK%k^@i^mTE2mDy=a-gBxMQz{DceH&e z*7W;A&c3uv+c(v)F>W~s9hZdzA$*k7YlzJR#wl7N%aEm7OThq9NhB7WTurgx9sO>(3>8T?x;V3}nTIl6@9n zl@ug%EHV$-m}RUENKe?=Xv`GUp@>K_8;DcFtb3MP(99vL zyVacepgYw($)JK&{_5j%7fNrvRs8xW9ddnP`C=iL2vk66Fcqpvta3xG73R9nL};9b zm1jmVvT{70)^JEue)jVeU0V*a^-pVfW_E~NoM()6mhbhlZRB%?eB@ab`C6M>2U-17 zwNt9|wcnVl<)0LnN^5IHO9zs9c1Vk%J(o+Sl41J(d@jV(wZ^LOEhf0JyYt0vS#9F= zPX2|y&XbY_{q?PFV9*c3Bm_Ui)u!v_RF00ck)1V8`;K;XMV z;I|Kp-zzLEE*6)Um%sYxkEO5P`FiEey$Ai@73D`4|It$4e_yLe6ZnS*03Z=ojdB>2L>C*Q->f4thLrTwTv2sj5hk4C(BX zNw33i`>`h{1*H1N>TOthc%!k^s5Lg%uWvMNsCJI!L!acR&BzbsE*;4+o2Gi_{GPDQ zwD1nI4vy4`EVJ2TBBBPEZSVTqaX4;}%)JwbgY+We=>D_zXnqc%! z?ZZ&D)^6Lr6}#M%79E3$TD5v0I|&P=V+lTNd+l}WpAsKI9?D-?paf{33H=p;C3tYODAaeb#*|GMRFi`!red)(&o6N_woC3 zKWVoE7OLZ8o*X9CQF~q9cf4e5Gd)3^Qg{4FYJa$ApVU(=y^5SgUoq+3XS2CR(CIa5 z!rG$nt3xAcv?D4puc^~DM$-1^ysszMddzjiplOgbOxzroDilqlUJZ^_k$OpajR~jk zH4Td_=AkO3^SR6JO&=#gXdp-K_& zgZ+?R#lOY8=!l0Os*)s82J44(j_b2FrIm|EEL$_<#TifB*=900@8p2!H?xfB*=900^960@(kb;o!nG5C8!X009sH z0T2KI5C8!X009sf5y1W*sQ>~X00JNY0w4eaAOHd&00JNY0%xB9_Wx%;#_$jXKmY_l z00ck)1V8`;KmY_l00b^71UUbXL;wL0009sH0T2KI5C8!X009sHfiq12`~Nc?UN{E= zAOHd&00JNY0w4eaAOHd&00No-_Wwu(5C8!X009sH0T2KI5C8!X009s<`vkE6Kl?F; zhadm~AOHd&00JNY0w4eaAOHd&fc-ym00ck)1V8`;KmY_l00ck)1V8`;&OQO`|IdDm z;UNfs00@8p2!H?xfB*=900@8p2w?w@8~_0j009sH0T2KI5C8!X009sHfwNEGGP!te z<*$X6f6^B|AOHd&00JNY0w4eaAOHd&00JNY0QdSJ?1)M&@@=+FmZES8aYpR)2LU2 zV^t*iv0P)q>3dDX;-2K8Dkb%P*fhF=SqJLAOTYclk3D;BRCToPNKRElvtoM8B`eH3 zEOIQm?FpNVX(i+Gdm$M^<$h>uTO;XM2LrR&VbI&hTwTvo>ymta#jWm3c{`T^cIiS+3}jWAov# z?X|rR_Cv$5Z!s@A;^Bv?BuSLP`XSpP5?(As#q5rBlB(`Vm%5w+(M?Iz>O*BG>nk(q zN~x)BG`1SG#^(C& zs3O^J+rAaMln$0)QPir{1KCMfNQ4Df?zN+UTNI-tU;z)6akthTud?BIx;Bn@K)$j* zZ^g2=)kz#Lh$Z)|soI3OQ8aM775a6V+}G6sJr>D<;0bpFIY^u1QgE*?_mg%zpmg?Z zvTc}DN9}ca-|>>M&GZBzOx^J#6|i=jd-h2^)w+HTUB!$e&up#{hC}ivr7LykqN4v@ z=elMI{|Pq|-JOrP)8CiWD}7<#ob=F`-^}#EZ6#0sP4~fF>gH7Z-|m7t$1-GNy)UN^ zjvN|WKIwv*OjG%l?}yp)54x)BC5blkW%j(yt*JiNKp%UKKK5Ojz>HNtkdE&$VT4=_ z2HZB(v|66gt6qtj?PS8dZQI6YsZZlRTmOHVH%}<&S-n06K!es~NcnyS&~P}Ouao5c zTsb}YJd^&(QsE`d`kRfJvwo#TuBB^$TBZKfDxguv2=&U2P-15u*nvS7EH7l{)8XV zlO&Md(yU-MH|SRHC?-3)yS&F@p%#u)>tz}pM}DMRzY{@nO!;SX!3Js_U8~fTdyXg5 zyX%!red|ctKHOuL^h4U=ZB(gkwP^@x6Jj=H7}Lg(mVBHl$v@g-+*Pd9POuw?7%GZK + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation; either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +package com.ichi2.anki.tests; + +import com.ichi2.libanki.Collection; +import com.ichi2.libanki.Storage; + +import net.ankiweb.rsdroid.BackendException; + +import org.hamcrest.MatcherAssert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.is; + +public class RustTest extends InstrumentedTest { + + /** Ensure that the database can't be locked */ + @Rule + public Timeout timeout = new Timeout(3, TimeUnit.SECONDS); + + @Test + public void collectionIsVersion11AfterOpen() throws BackendException, IOException { + // This test will be decommissioned, but before we get an upgrade strategy, we need to ensure we're not upgrading the database. + String path = Shared.getTestFilePath(getTestContext(), "initial_version_2_12_1.anki2"); + Collection collection = Storage.Collection(getTestContext(), path); + int ver = collection.getDb().queryScalar("select ver from col"); + MatcherAssert.assertThat(ver, is(11)); + } +} diff --git a/AnkiDroid/src/main/java/com/ichi2/libanki/Collection.java b/AnkiDroid/src/main/java/com/ichi2/libanki/Collection.java index 5252b4119214..c886e94c7af0 100644 --- a/AnkiDroid/src/main/java/com/ichi2/libanki/Collection.java +++ b/AnkiDroid/src/main/java/com/ichi2/libanki/Collection.java @@ -84,6 +84,11 @@ import androidx.sqlite.db.SupportSQLiteStatement; import timber.log.Timber; + +import net.ankiweb.rsdroid.BackendException; +import net.ankiweb.rsdroid.BackendFactory; +import net.ankiweb.rsdroid.BackendUtils; + import static com.ichi2.async.CancelListener.isCancelled; import static com.ichi2.libanki.Collection.DismissType.REVIEW; import static com.ichi2.libanki.Consts.DECK_DYN; @@ -128,6 +133,7 @@ public class Collection { private LinkedBlockingDeque mUndo; private final String mPath; + private BackendFactory mBackendFactory; private boolean mDebugLog; private PrintWriter mLogHnd; @@ -187,12 +193,13 @@ public String getString(Resources res) { private static final int UNDO_SIZE_MAX = 20; @VisibleForTesting - public Collection(Context context, DB db, String path, boolean server, boolean log, @NonNull Time time) { + public Collection(Context context, DB db, String path, boolean server, boolean log, @NonNull Time time, @Nullable BackendFactory backendFactory) { mContext = context; mDebugLog = log; mDb = db; mPath = path; mTime = time; + mBackendFactory = backendFactory; _openLog(); log(path, VersionUtils.getPkgVersionName()); mServer = server; @@ -469,7 +476,7 @@ public synchronized void close(boolean save) { public void reopen() { Timber.i("Reopening Database"); if (mDb == null) { - mDb = new DB(mPath); + mDb = new DB(mPath, mBackendFactory); mMedia.connect(); _openLog(); } diff --git a/AnkiDroid/src/main/java/com/ichi2/libanki/DB.java b/AnkiDroid/src/main/java/com/ichi2/libanki/DB.java index 5c5b977267c0..1a1297005474 100644 --- a/AnkiDroid/src/main/java/com/ichi2/libanki/DB.java +++ b/AnkiDroid/src/main/java/com/ichi2/libanki/DB.java @@ -32,6 +32,9 @@ import com.ichi2.anki.dialogs.DatabaseErrorDialog; import com.ichi2.utils.DatabaseChangeDecorator; +import net.ankiweb.rsdroid.BackendFactory; +import net.ankiweb.rsdroid.database.RustSQLiteOpenHelperFactory; + import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; @@ -62,16 +65,21 @@ public class DB { private final SupportSQLiteDatabase mDatabase; private boolean mMod = false; + public DB(String ankiFilename) { + this(ankiFilename, null); + } + /** * Open a connection to the SQLite collection database. */ - public DB(String ankiFilename) { + public DB(String ankiFilename, @Nullable BackendFactory backendFactory) { SupportSQLiteOpenHelper.Configuration configuration = SupportSQLiteOpenHelper.Configuration.builder(AnkiDroidApp.getInstance()) .name(ankiFilename) .callback(getDBCallback()) .build(); - SupportSQLiteOpenHelper helper = getSqliteOpenHelperFactory().create(configuration); + SupportSQLiteOpenHelper helper = getSqliteOpenHelperFactory(backendFactory).create(configuration); + // Note: This line creates the database and schema when executed using a Rust backend mDatabase = new DatabaseChangeDecorator(helper.getWritableDatabase()); mDatabase.disableWriteAheadLogging(); mDatabase.query("PRAGMA synchronous = 2", null); @@ -90,7 +98,11 @@ public static void setSqliteOpenHelperFactory(@Nullable SupportSQLiteOpenHelper. } - private SupportSQLiteOpenHelper.Factory getSqliteOpenHelperFactory() { + private SupportSQLiteOpenHelper.Factory getSqliteOpenHelperFactory(@Nullable BackendFactory backendFactory) { + if (backendFactory != null) { + return new RustSQLiteOpenHelperFactory(backendFactory); + } + if (sqliteOpenHelperFactory == null) { return new RequerySQLiteOpenHelperFactory(); } diff --git a/AnkiDroid/src/main/java/com/ichi2/libanki/Storage.java b/AnkiDroid/src/main/java/com/ichi2/libanki/Storage.java index 1205e66e5c57..d9dfcf045244 100644 --- a/AnkiDroid/src/main/java/com/ichi2/libanki/Storage.java +++ b/AnkiDroid/src/main/java/com/ichi2/libanki/Storage.java @@ -19,8 +19,9 @@ import android.content.ContentValues; import android.content.Context; +import com.ichi2.anki.AnkiDroidApp; +import com.ichi2.anki.UIUtils; import com.ichi2.anki.exception.ConfirmModSchemaException; - import com.ichi2.libanki.exception.UnknownDatabaseVersionException; import com.ichi2.libanki.utils.SystemTime; import com.ichi2.libanki.utils.Time; @@ -28,6 +29,10 @@ import com.ichi2.utils.JSONException; import com.ichi2.utils.JSONObject; +import net.ankiweb.rsdroid.BackendFactory; +import net.ankiweb.rsdroid.BackendV1; +import net.ankiweb.rsdroid.RustBackendFailedException; + import java.io.File; import java.util.ArrayList; import java.util.Arrays; @@ -42,6 +47,9 @@ "PMD.SwitchStmtsShouldHaveDefault","PMD.EmptyIfStmt","PMD.SimplifyBooleanReturns","PMD.CollapsibleIfStatements"}) public class Storage { + private static boolean sUseBackend = true; + + /* Open a new or existing collection. Path must be unicode */ public static Collection Collection(Context context, String path) { return Collection(context, path, false, false); @@ -65,8 +73,31 @@ public static Collection Collection(Context context, String path, boolean server assert path.endsWith(".anki2"); File dbFile = new File(path); boolean create = !dbFile.exists(); + BackendFactory backendFactory = null; // connect - DB db = new DB(path); + BackendV1 instance; + + // This isn't ideal - as opening the collection performs some creation operations, but we need it before new DB() + // TODO: Delete the DB if creating and operations fail - as the col data won't be correct. + // but: not if the database already existed + DB db = null; + try { + backendFactory = BackendFactory.createInstance(); + // Note: This will partially create the database + Timber.i("backend: open collection %s", path); + db = new DB(path, backendFactory); + } catch (RustBackendFailedException e) { + Timber.e("Loading Rust Backend failed - falling back to Java"); + AnkiDroidApp.sendExceptionReport(e, "Storage::Collection"); + } + + if (db == null) { + backendFactory = null; + Timber.i("backend: skipping open collection"); + db = new DB(path); + } + + try { // initialize int ver; @@ -77,7 +108,7 @@ public static Collection Collection(Context context, String path, boolean server } db.execute("PRAGMA temp_store = memory"); // add db to col and do any remaining upgrades - Collection col = new Collection(context, db, path, server, log, time); + Collection col = new Collection(context, db, path, server, log, time, backendFactory); if (ver < Consts.SCHEMA_VERSION) { _upgrade(col, ver); } else if (ver > Consts.SCHEMA_VERSION) { @@ -97,6 +128,14 @@ public static Collection Collection(Context context, String path, boolean server } } + /** + * Whether the collection should try to be opened with a Rust-based DB Backend + * Falls back to Java if init fails. + * */ + protected static boolean useBackend() { + return sUseBackend; + } + private static int _upgradeSchema(DB db, @NonNull Time time) { int ver = db.queryScalar("SELECT ver FROM col"); @@ -276,11 +315,18 @@ private static void _upgradeClozeModel(Collection col, Model m) throws ConfirmMo private static int _createDB(DB db, @NonNull Time time) { - db.execute("PRAGMA page_size = 4096"); - db.execute("PRAGMA legacy_file_format = 0"); - db.execute("VACUUM"); - _addSchema(db, time); - _updateIndices(db); + if (useBackend()) { + _setColVars(db, time); + // This line is required for testing - otherwise Rust will override a mocked time. + db.execute("update col set crt = ?", UIUtils.getDayStart(time) / 1000); + } else { + db.execute("PRAGMA page_size = 4096"); + db.execute("PRAGMA legacy_file_format = 0"); + db.execute("VACUUM"); + _addSchema(db, time); + _updateIndices(db); + } + db.execute("ANALYZE"); return Consts.SCHEMA_VERSION; } @@ -366,4 +412,8 @@ public static void addIndices(DB db) { _updateIndices(db); } + + public static void setUseBackend(boolean useBackend) { + sUseBackend = useBackend; + } } diff --git a/AnkiDroid/src/test/java/com/ichi2/anki/RobolectricTest.java b/AnkiDroid/src/test/java/com/ichi2/anki/RobolectricTest.java index 89b893993e28..ed040ce9aa64 100644 --- a/AnkiDroid/src/test/java/com/ichi2/anki/RobolectricTest.java +++ b/AnkiDroid/src/test/java/com/ichi2/anki/RobolectricTest.java @@ -36,6 +36,7 @@ import com.ichi2.libanki.Models; import com.ichi2.libanki.Note; +import com.ichi2.libanki.Storage; import com.ichi2.libanki.sched.AbstractSched; import com.ichi2.libanki.sched.Sched; import com.ichi2.libanki.sched.SchedV2; @@ -43,6 +44,8 @@ import com.ichi2.utils.BooleanGetter; import com.ichi2.utils.JSONException; +import net.ankiweb.rsdroid.testing.RustBackendLoader; + import org.hamcrest.Matcher; import org.junit.After; import org.junit.Assert; @@ -86,11 +89,16 @@ protected boolean useInMemoryDatabase() { @Before public void setUp() { + + RustBackendLoader.init(); + // If you want to see the Android logging (from Timber), you need to set it up here ShadowLog.stream = System.out; // Robolectric can't handle our default sqlite implementation of requery, it needs the framework DB.setSqliteOpenHelperFactory(getHelperFactory()); + // But, don't use the helper unless useLegacyHelper is true + Storage.setUseBackend(!useLegacyHelper()); //Reset static variable for custom tabs failure. CustomTabActivityHelper.resetFailed(); @@ -103,6 +111,11 @@ public void setUp() { } + protected boolean useLegacyHelper() { + return false; + } + + @NonNull protected SupportSQLiteOpenHelper.Factory getHelperFactory() { if (useInMemoryDatabase()) { diff --git a/AnkiDroid/src/test/java/com/ichi2/libanki/StorageTest.java b/AnkiDroid/src/test/java/com/ichi2/libanki/StorageTest.java new file mode 100644 index 000000000000..bed2a5abea39 --- /dev/null +++ b/AnkiDroid/src/test/java/com/ichi2/libanki/StorageTest.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2020 David Allison + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation; either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +package com.ichi2.libanki; + +import android.database.Cursor; + +import com.ichi2.anki.CollectionHelper; +import com.ichi2.anki.RobolectricTest; +import com.ichi2.utils.JSONObject; + +import org.json.JSONException; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import androidx.core.util.Pair; +import androidx.test.ext.junit.runners.AndroidJUnit4; + +import static org.hamcrest.Matchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** Regression test for Rust */ +@RunWith(AndroidJUnit4.class) +public class StorageTest extends RobolectricTest { + + @Override + protected boolean useLegacyHelper() { + return true; + } + + + @Override + public void setUp() { + Storage.setUseBackend(false); + super.setUp(); + } + + + @Test + public void compareNewDatabases() throws JSONException { + + List expected = getResults(); + + // If you don't tear down the database you'll get unexpected IllegalStateExceptions related to connections + CollectionHelper.getInstance().closeCollection(false, "compareNewDatabases"); + + // After every test make sure the CollectionHelper is no longer overridden (done for null testing) + disableNullCollection(); + + Storage.setUseBackend(true); + + List actual = getResults(); + + + for (int i = 0; i < expected.size(); i++) { + if (i == 1 || i == 2 || i == 3) { + continue; + } + if (i == 8) { + JSONObject actualJson = new JSONObject(actual.get(i).toString()); + JSONObject expectedJson = new JSONObject(expected.get(i).toString()); + + actualJson.remove("curModel"); + expectedJson.remove("curModel"); + + assertThat(actualJson.toString(), is(expectedJson.toString())); + continue; + } + + if (i == 9) { + JSONObject actualJson = new JSONObject(actual.get(i).toString()); + JSONObject expectedJson = new JSONObject(expected.get(i).toString()); + + renameKeys(actualJson); + renameKeys(expectedJson); + + for (String k : actualJson) { + actualJson.getJSONObject(k).remove("id"); + expectedJson.getJSONObject(k).remove("id"); + } + + assertThat(actualJson.toString(4), is(expectedJson.toString(4))); + continue; + } + + assertThat(Integer.toString(i), actual.get(i), is(expected.get(i))); + } + } + + + protected void renameKeys(JSONObject actualJson) { + List> keys = new ArrayList<>(); + Iterator keyIt = actualJson.keys(); + while (keyIt.hasNext()) { + String name = keyIt.next(); + keys.add(new Pair<>(name, actualJson.getJSONObject(name).getString("name"))); + } + + keys.sort((x,y) -> x.second.compareTo(y.second)); + + for (int i = 0; i < keys.size(); i++) { + String keyName = keys.get(i).first; + actualJson.put(Integer.toString(i+i), actualJson.get(keyName)); + actualJson.remove(keyName); + } + + } + + + protected List getResults() { + List results = new ArrayList<>(); + try (Cursor c = getCol().getDb().query("select * from col")) { + c.moveToFirst(); + + + for (int i = 0; i < c.getColumnCount(); i++) { + results.add(c.getString(i)); + } + } + return results; + } + +} From 49f846ce76ac544dd6ab3662991d6387c416735f Mon Sep 17 00:00:00 2001 From: David Allison <62114487+david-allison-1@users.noreply.github.com> Date: Mon, 25 Jan 2021 01:48:48 +0000 Subject: [PATCH 2/4] Do not fail on Rust Backend failure This is temporary - we will later require the Rust Backend But for now, we have a java fallback which will work. Mostly aimed at handling class: RustBackendFailedException --- AnkiDroid/build.gradle | 2 +- .../java/com/ichi2/libanki/Collection.java | 14 ++--- .../main/java/com/ichi2/libanki/Storage.java | 48 +++++---------- .../ichi2/libanki/backend/DroidBackend.java | 30 ++++++++++ .../libanki/backend/DroidBackendFactory.java | 60 +++++++++++++++++++ .../libanki/backend/JavaDroidBackend.java | 47 +++++++++++++++ .../libanki/backend/RustDroidBackend.java | 53 ++++++++++++++++ .../java/com/ichi2/anki/RobolectricTest.java | 8 +++ 8 files changed, 219 insertions(+), 43 deletions(-) create mode 100644 AnkiDroid/src/main/java/com/ichi2/libanki/backend/DroidBackend.java create mode 100644 AnkiDroid/src/main/java/com/ichi2/libanki/backend/DroidBackendFactory.java create mode 100644 AnkiDroid/src/main/java/com/ichi2/libanki/backend/JavaDroidBackend.java create mode 100644 AnkiDroid/src/main/java/com/ichi2/libanki/backend/RustDroidBackend.java diff --git a/AnkiDroid/build.gradle b/AnkiDroid/build.gradle index 08a71c51aad1..712a4f005090 100644 --- a/AnkiDroid/build.gradle +++ b/AnkiDroid/build.gradle @@ -243,7 +243,7 @@ dependencies { // == Rust conversion (from Anki-Android-Backend on GitHub) == // use a variable as we want both testing and implementation on the same version - String backendVersion = "0.0.5" + String backendVersion = "0.0.7" // build with ./gradlew rsdroid:assembleRelease // In my experience, using `files()` currently requires a reindex operation, which is slow. // implementation files("C:\\GitHub\\Rust-Test\\rsdroid\\build\\outputs\\aar\\rsdroid-release.aar") diff --git a/AnkiDroid/src/main/java/com/ichi2/libanki/Collection.java b/AnkiDroid/src/main/java/com/ichi2/libanki/Collection.java index c886e94c7af0..86d21ac55b16 100644 --- a/AnkiDroid/src/main/java/com/ichi2/libanki/Collection.java +++ b/AnkiDroid/src/main/java/com/ichi2/libanki/Collection.java @@ -34,6 +34,7 @@ import com.ichi2.anki.exception.ConfirmModSchemaException; import com.ichi2.async.CancelListener; import com.ichi2.async.CollectionTask; +import com.ichi2.libanki.backend.DroidBackend; import com.ichi2.async.ProgressSender; import com.ichi2.async.TaskManager; import com.ichi2.libanki.exception.NoSuchDeckException; @@ -84,11 +85,6 @@ import androidx.sqlite.db.SupportSQLiteStatement; import timber.log.Timber; - -import net.ankiweb.rsdroid.BackendException; -import net.ankiweb.rsdroid.BackendFactory; -import net.ankiweb.rsdroid.BackendUtils; - import static com.ichi2.async.CancelListener.isCancelled; import static com.ichi2.libanki.Collection.DismissType.REVIEW; import static com.ichi2.libanki.Consts.DECK_DYN; @@ -133,7 +129,7 @@ public class Collection { private LinkedBlockingDeque mUndo; private final String mPath; - private BackendFactory mBackendFactory; + private final DroidBackend mDroidBackend; private boolean mDebugLog; private PrintWriter mLogHnd; @@ -193,13 +189,13 @@ public String getString(Resources res) { private static final int UNDO_SIZE_MAX = 20; @VisibleForTesting - public Collection(Context context, DB db, String path, boolean server, boolean log, @NonNull Time time, @Nullable BackendFactory backendFactory) { + public Collection(Context context, DB db, String path, boolean server, boolean log, @NonNull Time time, @NonNull DroidBackend droidBackend) { mContext = context; mDebugLog = log; mDb = db; mPath = path; mTime = time; - mBackendFactory = backendFactory; + mDroidBackend = droidBackend; _openLog(); log(path, VersionUtils.getPkgVersionName()); mServer = server; @@ -476,7 +472,7 @@ public synchronized void close(boolean save) { public void reopen() { Timber.i("Reopening Database"); if (mDb == null) { - mDb = new DB(mPath, mBackendFactory); + mDb = mDroidBackend.openCollectionDatabase(mPath); mMedia.connect(); _openLog(); } diff --git a/AnkiDroid/src/main/java/com/ichi2/libanki/Storage.java b/AnkiDroid/src/main/java/com/ichi2/libanki/Storage.java index d9dfcf045244..18e87a77a695 100644 --- a/AnkiDroid/src/main/java/com/ichi2/libanki/Storage.java +++ b/AnkiDroid/src/main/java/com/ichi2/libanki/Storage.java @@ -19,9 +19,11 @@ import android.content.ContentValues; import android.content.Context; -import com.ichi2.anki.AnkiDroidApp; import com.ichi2.anki.UIUtils; import com.ichi2.anki.exception.ConfirmModSchemaException; + +import com.ichi2.libanki.backend.DroidBackend; +import com.ichi2.libanki.backend.DroidBackendFactory; import com.ichi2.libanki.exception.UnknownDatabaseVersionException; import com.ichi2.libanki.utils.SystemTime; import com.ichi2.libanki.utils.Time; @@ -29,10 +31,6 @@ import com.ichi2.utils.JSONException; import com.ichi2.utils.JSONObject; -import net.ankiweb.rsdroid.BackendFactory; -import net.ankiweb.rsdroid.BackendV1; -import net.ankiweb.rsdroid.RustBackendFailedException; - import java.io.File; import java.util.ArrayList; import java.util.Arrays; @@ -48,6 +46,7 @@ public class Storage { private static boolean sUseBackend = true; + private static boolean sUseInMemory = false; /* Open a new or existing collection. Path must be unicode */ @@ -73,42 +72,20 @@ public static Collection Collection(Context context, String path, boolean server assert path.endsWith(".anki2"); File dbFile = new File(path); boolean create = !dbFile.exists(); - BackendFactory backendFactory = null; - // connect - BackendV1 instance; - - // This isn't ideal - as opening the collection performs some creation operations, but we need it before new DB() - // TODO: Delete the DB if creating and operations fail - as the col data won't be correct. - // but: not if the database already existed - DB db = null; - try { - backendFactory = BackendFactory.createInstance(); - // Note: This will partially create the database - Timber.i("backend: open collection %s", path); - db = new DB(path, backendFactory); - } catch (RustBackendFailedException e) { - Timber.e("Loading Rust Backend failed - falling back to Java"); - AnkiDroidApp.sendExceptionReport(e, "Storage::Collection"); - } - - if (db == null) { - backendFactory = null; - Timber.i("backend: skipping open collection"); - db = new DB(path); - } - + DroidBackend backend = DroidBackendFactory.getInstance(useBackend()); + DB db = backend.openCollectionDatabase(sUseInMemory ? ":memory:" : path); try { // initialize int ver; if (create) { - ver = _createDB(db, time); + ver = _createDB(db, time, backend); } else { ver = _upgradeSchema(db, time); } db.execute("PRAGMA temp_store = memory"); // add db to col and do any remaining upgrades - Collection col = new Collection(context, db, path, server, log, time, backendFactory); + Collection col = new Collection(context, db, path, server, log, time, backend); if (ver < Consts.SCHEMA_VERSION) { _upgrade(col, ver); } else if (ver > Consts.SCHEMA_VERSION) { @@ -314,8 +291,8 @@ private static void _upgradeClozeModel(Collection col, Model m) throws ConfirmMo } - private static int _createDB(DB db, @NonNull Time time) { - if (useBackend()) { + private static int _createDB(DB db, @NonNull Time time, DroidBackend backend) { + if (backend.databaseCreationCreatesSchema()) { _setColVars(db, time); // This line is required for testing - otherwise Rust will override a mocked time. db.execute("update col set crt = ?", UIUtils.getDayStart(time) / 1000); @@ -416,4 +393,9 @@ public static void addIndices(DB db) { public static void setUseBackend(boolean useBackend) { sUseBackend = useBackend; } + + + public static void setUseInMemory(boolean useInMemoryDatabase) { + sUseInMemory = useInMemoryDatabase; + } } diff --git a/AnkiDroid/src/main/java/com/ichi2/libanki/backend/DroidBackend.java b/AnkiDroid/src/main/java/com/ichi2/libanki/backend/DroidBackend.java new file mode 100644 index 000000000000..bad75ad4c9a9 --- /dev/null +++ b/AnkiDroid/src/main/java/com/ichi2/libanki/backend/DroidBackend.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2020 David Allison + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation; either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +package com.ichi2.libanki.backend; + +import com.ichi2.libanki.DB; + +/** + * Interface to the rust backend listing all currently supported functionality. + */ +public interface DroidBackend { + DB openCollectionDatabase(String path); + void closeCollection(); + + /** Whether a call to {@link DroidBackend#openCollectionDatabase(String)} will generate a schema and indices for the database */ + boolean databaseCreationCreatesSchema(); +} diff --git a/AnkiDroid/src/main/java/com/ichi2/libanki/backend/DroidBackendFactory.java b/AnkiDroid/src/main/java/com/ichi2/libanki/backend/DroidBackendFactory.java new file mode 100644 index 000000000000..adfc4b58bb03 --- /dev/null +++ b/AnkiDroid/src/main/java/com/ichi2/libanki/backend/DroidBackendFactory.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2020 David Allison + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation; either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +package com.ichi2.libanki.backend; + +import com.ichi2.anki.AnkiDroidApp; + +import net.ankiweb.rsdroid.BackendFactory; +import net.ankiweb.rsdroid.RustBackendFailedException; + +import androidx.annotation.Nullable; +import timber.log.Timber; + +/** Responsible for selection of either the Rust or Java-based backend */ +public class DroidBackendFactory { + + /** Intentionally private - use {@link DroidBackendFactory#getInstance(boolean)}} */ + private DroidBackendFactory() { + + } + + /** + * Obtains an instance of a {@link DroidBackend}. + * Each call to this method will generate a separate instance which can handle a new Anki collection + */ + public static DroidBackend getInstance(boolean useBackend) { + + BackendFactory backendFactory = null; + if (useBackend) { + try { + backendFactory = BackendFactory.createInstance(); + } catch (RustBackendFailedException e) { + Timber.w(e, "Rust backend failed to load - falling back to Java"); + AnkiDroidApp.sendExceptionReport(e, "DroidBackendFactory::getInstance"); + } + } + return getInstance(backendFactory); + } + + private static DroidBackend getInstance(@Nullable BackendFactory backendFactory) { + if (backendFactory == null) { + return new JavaDroidBackend(); + } else { + return new RustDroidBackend(backendFactory); + } + } +} diff --git a/AnkiDroid/src/main/java/com/ichi2/libanki/backend/JavaDroidBackend.java b/AnkiDroid/src/main/java/com/ichi2/libanki/backend/JavaDroidBackend.java new file mode 100644 index 000000000000..89be743be896 --- /dev/null +++ b/AnkiDroid/src/main/java/com/ichi2/libanki/backend/JavaDroidBackend.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2020 David Allison + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation; either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +package com.ichi2.libanki.backend; + +import com.ichi2.libanki.DB; + +import net.ankiweb.rsdroid.RustCleanup; + +/** + * A class which implements the Rust backend functionality in Java - this is to allow moving our current Java code to + * the rust-based interface so we are able to perform regression testing against the converted interface + * + * This also allows an easy switch of functionality once we are happy that there are no regressions + */ +@RustCleanup("After the rust conversion is complete - this will be removed") +public class JavaDroidBackend implements DroidBackend { + @Override + public DB openCollectionDatabase(String path) { + return new DB(path); + } + + + @Override + public void closeCollection() { + // Nothing to do + } + + + @Override + public boolean databaseCreationCreatesSchema() { + return false; + } +} diff --git a/AnkiDroid/src/main/java/com/ichi2/libanki/backend/RustDroidBackend.java b/AnkiDroid/src/main/java/com/ichi2/libanki/backend/RustDroidBackend.java new file mode 100644 index 000000000000..50a1fdbaa5b8 --- /dev/null +++ b/AnkiDroid/src/main/java/com/ichi2/libanki/backend/RustDroidBackend.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2020 David Allison + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation; either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +package com.ichi2.libanki.backend; + +import com.ichi2.libanki.DB; + +import net.ankiweb.rsdroid.BackendFactory; +import net.ankiweb.rsdroid.BackendUtils; + +import timber.log.Timber; + +/** The Backend in Rust */ +public class RustDroidBackend implements DroidBackend { + // I think we can change this to BackendV1 once new DB() accepts it. + private final BackendFactory mBackend; + + + public RustDroidBackend(BackendFactory mBackend) { + this.mBackend = mBackend; + } + + + @Override + public DB openCollectionDatabase(String path) { + return new DB(path, mBackend); + } + + + @Override + public void closeCollection() { + mBackend.closeCollection(); + } + + + @Override + public boolean databaseCreationCreatesSchema() { + return true; + } +} diff --git a/AnkiDroid/src/test/java/com/ichi2/anki/RobolectricTest.java b/AnkiDroid/src/test/java/com/ichi2/anki/RobolectricTest.java index ed040ce9aa64..5e6e45939caf 100644 --- a/AnkiDroid/src/test/java/com/ichi2/anki/RobolectricTest.java +++ b/AnkiDroid/src/test/java/com/ichi2/anki/RobolectricTest.java @@ -44,6 +44,7 @@ import com.ichi2.utils.BooleanGetter; import com.ichi2.utils.JSONException; +import net.ankiweb.rsdroid.BackendException; import net.ankiweb.rsdroid.testing.RustBackendLoader; import org.hamcrest.Matcher; @@ -99,6 +100,7 @@ public void setUp() { DB.setSqliteOpenHelperFactory(getHelperFactory()); // But, don't use the helper unless useLegacyHelper is true Storage.setUseBackend(!useLegacyHelper()); + Storage.setUseInMemory(useInMemoryDatabase()); //Reset static variable for custom tabs failure. CustomTabActivityHelper.resetFailed(); @@ -145,6 +147,12 @@ public void tearDown() { try { // If you don't tear down the database you'll get unexpected IllegalStateExceptions related to connections CollectionHelper.getInstance().closeCollection(false, "RoboelectricTest: End"); + } catch (BackendException ex) { + if ("CollectionNotOpen".equals(ex.getMessage())) { + Timber.w(ex, "Collection was already disposed - may have been a problem"); + } else { + throw ex; + } } finally { // After every test make sure the CollectionHelper is no longer overridden (done for null testing) disableNullCollection(); From c03e51599db3a817a7818b694c63157f74082861 Mon Sep 17 00:00:00 2001 From: David Allison <62114487+david-allison-1@users.noreply.github.com> Date: Mon, 25 Jan 2021 01:39:46 +0000 Subject: [PATCH 3/4] Add Rust usage to Debug Info --- AnkiDroid/src/main/java/com/ichi2/anki/Info.java | 11 ++++++++++- .../src/main/java/com/ichi2/libanki/Collection.java | 4 ++++ .../java/com/ichi2/libanki/backend/DroidBackend.java | 2 ++ .../com/ichi2/libanki/backend/JavaDroidBackend.java | 6 ++++++ .../com/ichi2/libanki/backend/RustDroidBackend.java | 6 ++++++ 5 files changed, 28 insertions(+), 1 deletion(-) diff --git a/AnkiDroid/src/main/java/com/ichi2/anki/Info.java b/AnkiDroid/src/main/java/com/ichi2/anki/Info.java index 838032d4920d..d14fd18de818 100644 --- a/AnkiDroid/src/main/java/com/ichi2/anki/Info.java +++ b/AnkiDroid/src/main/java/com/ichi2/anki/Info.java @@ -189,11 +189,20 @@ public String copyDebugInfo() { Timber.e(e, "Sched name not found"); } + Boolean dbV2Enabled = null; + try { + dbV2Enabled = getCol().isUsingRustBackend(); + } catch (Throwable e) { + Timber.w(e, "Unable to detect Rust Backend"); + } + + String debugInfo = "AnkiDroid Version = " + VersionUtils.getPkgVersionName() + "\n\n" + "Android Version = " + Build.VERSION.RELEASE + "\n\n" + "ACRA UUID = " + Installation.id(this) + "\n\n" + "Scheduler = " + schedName + "\n\n" + - "Crash Reports Enabled = " + isSendingCrashReports() + "\n"; + "Crash Reports Enabled = " + isSendingCrashReports() + "\n\n" + + "DatabaseV2 Enabled = " + dbV2Enabled + "\n"; android.content.ClipboardManager clipboardManager = (android.content.ClipboardManager)getSystemService(Context.CLIPBOARD_SERVICE); if (clipboardManager != null) { diff --git a/AnkiDroid/src/main/java/com/ichi2/libanki/Collection.java b/AnkiDroid/src/main/java/com/ichi2/libanki/Collection.java index 86d21ac55b16..66c6bebcdd84 100644 --- a/AnkiDroid/src/main/java/com/ichi2/libanki/Collection.java +++ b/AnkiDroid/src/main/java/com/ichi2/libanki/Collection.java @@ -2166,6 +2166,10 @@ public AbstractSched createScheduler(int reportLimit) { return mSched; } + public boolean isUsingRustBackend() { + return mDroidBackend.isUsingRustBackend(); + } + /** Allows a mock db to be inserted for testing */ @VisibleForTesting public void setDb(DB database) { diff --git a/AnkiDroid/src/main/java/com/ichi2/libanki/backend/DroidBackend.java b/AnkiDroid/src/main/java/com/ichi2/libanki/backend/DroidBackend.java index bad75ad4c9a9..ddea26fdcaf9 100644 --- a/AnkiDroid/src/main/java/com/ichi2/libanki/backend/DroidBackend.java +++ b/AnkiDroid/src/main/java/com/ichi2/libanki/backend/DroidBackend.java @@ -27,4 +27,6 @@ public interface DroidBackend { /** Whether a call to {@link DroidBackend#openCollectionDatabase(String)} will generate a schema and indices for the database */ boolean databaseCreationCreatesSchema(); + + boolean isUsingRustBackend(); } diff --git a/AnkiDroid/src/main/java/com/ichi2/libanki/backend/JavaDroidBackend.java b/AnkiDroid/src/main/java/com/ichi2/libanki/backend/JavaDroidBackend.java index 89be743be896..1d6fba94f527 100644 --- a/AnkiDroid/src/main/java/com/ichi2/libanki/backend/JavaDroidBackend.java +++ b/AnkiDroid/src/main/java/com/ichi2/libanki/backend/JavaDroidBackend.java @@ -44,4 +44,10 @@ public void closeCollection() { public boolean databaseCreationCreatesSchema() { return false; } + + + @Override + public boolean isUsingRustBackend() { + return false; + } } diff --git a/AnkiDroid/src/main/java/com/ichi2/libanki/backend/RustDroidBackend.java b/AnkiDroid/src/main/java/com/ichi2/libanki/backend/RustDroidBackend.java index 50a1fdbaa5b8..48274445dc35 100644 --- a/AnkiDroid/src/main/java/com/ichi2/libanki/backend/RustDroidBackend.java +++ b/AnkiDroid/src/main/java/com/ichi2/libanki/backend/RustDroidBackend.java @@ -50,4 +50,10 @@ public void closeCollection() { public boolean databaseCreationCreatesSchema() { return true; } + + + @Override + public boolean isUsingRustBackend() { + return true; + } } From beb354eb3762cfd2d0402be688f083d71c62b75a Mon Sep 17 00:00:00 2001 From: David Allison <62114487+david-allison-1@users.noreply.github.com> Date: Tue, 26 Jan 2021 03:07:41 +0000 Subject: [PATCH 4/4] docs: Improve documentation for RobolectricTest --- AnkiDroid/build.gradle | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/AnkiDroid/build.gradle b/AnkiDroid/build.gradle index 712a4f005090..bcf9cd643485 100644 --- a/AnkiDroid/build.gradle +++ b/AnkiDroid/build.gradle @@ -242,15 +242,14 @@ dependencies { implementation 'androidx.webkit:webkit:1.4.0' // == Rust conversion (from Anki-Android-Backend on GitHub) == - // use a variable as we want both testing and implementation on the same version - String backendVersion = "0.0.7" + String backendVersion = "0.1.0" // We want both testing and implementation on the same version // build with ./gradlew rsdroid:assembleRelease // In my experience, using `files()` currently requires a reindex operation, which is slow. // implementation files("C:\\GitHub\\Rust-Test\\rsdroid\\build\\outputs\\aar\\rsdroid-release.aar") implementation "io.github.david-allison-1:anki-android-backend:$backendVersion" // build with ./gradlew rsdroid-testing:assembleRelease - // add a one-time call to Runtime.getRuntime().load(path); to test locally - // TODO: Add method in RustBackendLoader to load native library on disk + // RobolectricTest.java: replace RustBackendLoader.init(); with RustBackendLoader.loadRsdroid(path); + // A path for a testing library is typically under rsdroid-testing/assets testImplementation "io.github.david-allison-1:anki-android-backend-testing:$backendVersion" // May need a resolution strategy for support libs to our versions