Anvil App Server v1.2

Changes:

- Users service supports two-factor authentication
- URLMedia fetch fixes
- PDF rendering fix for apps with dependencies
- Doc updates
- OAuth fixes (MS and Google authentication)
- Stricter bytes/str enforcement in BlobMedia construction
- Downlink is more robust about cleaning up subprocesses

Based-on: anvil 96b13c0b30d2acdab28a6e9e1ac02a49de06bc4d

README.md

@@ -278,7 +278,7 @@ Port for outbound SMTP server (see `smtp-host`).
 
 #### smtp-encryption
 
-Enable TLS for connecting to outbound SMTP server (see `smtp-host`).
+Enable TLS for connecting to outbound SMTP server (see `smtp-host`). Takes "starttls" or "ssl" as a string value.
 
 #### smtp-username
 

client/js/modules/anvil.js

@@ -13,6 +13,16 @@ module.exports = function(appOrigin, uncaughtExceptions) {
 
     let ByteString = Sk.__future__.python3 ? Sk.builtin.bytes : Sk.builtin.str;
 
+    let arrayBufferToStr = arrayBuffer => {
+        let binary = "";
+        var bytes = new Uint8Array(arrayBuffer);
+        var length = bytes.byteLength;
+        for (var i = 0; i < length; i++) {
+            binary += String.fromCharCode(bytes[i]);
+        }
+        return binary;
+    }
+
     /**
     id: anvil_module
     docs_url: /docs/client/python#the-anvil-module
@@ -164,7 +174,13 @@ module.exports = function(appOrigin, uncaughtExceptions) {
                 let leafName = ps[ps.length-1];
                 // Yes, sysmodules is indexed with JS strings.
                 // No, this makes no sense.
-                let pyFormMod = Sk.sysmodules.mp$subscript(window.anvilAppMainPackage + "." + formName);
+                let pyFormMod;
+                try {
+                    pyFormMod = Sk.sysmodules.mp$subscript(window.anvilAppMainPackage + "." + formName);
+                } catch (e) {
+                    pyFormMod = Sk.sysmodules.mp$subscript(formName);
+                }
+
 
                 var formConstructor = pyFormMod.$d[leafName];
 
@@ -299,7 +315,7 @@ module.exports = function(appOrigin, uncaughtExceptions) {
                     $.ajax({
                       url: self._url,
                       type: "GET",
-                      dataType: "text",
+                      dataType: "binary",
                       processData: false,
                     }).then(function(r,ts,xhr) {
                         window.setLoading(false);
@@ -326,7 +342,11 @@ module.exports = function(appOrigin, uncaughtExceptions) {
 
         $loc["get_bytes"] = new Sk.builtin.func(function(self) {
             return new PyDefUtils.suspensionPromise(function(resolve, reject) {
-                doFetch(self).then(function(r) { resolve(new ByteString(r.data)); }, reject);
+                doFetch(self).then(function(r) {
+                    return r.data.arrayBuffer();
+                }, reject).then(function(arrayBuffer) {
+                    resolve(new ByteString(arrayBufferToStr(arrayBuffer)))
+                });
             });
         });
 
@@ -407,13 +427,7 @@ module.exports = function(appOrigin, uncaughtExceptions) {
                         fr.readAsBinaryString(self._data);
                     } else {
                         fr.onloadend = function() { 
-                            let binary = "";
-                            var bytes = new Uint8Array(fr.result);
-                            var length = bytes.byteLength;
-                            for (var i = 0; i < length; i++) {
-                                binary += String.fromCharCode(bytes[i]);
-                            }
-                            resolve(new ByteString(binary));
+                            resolve(new ByteString(arrayBufferToStr(fr.result)));
                         };
                         fr.readAsArrayBuffer(self._data);
                     }

client/js/runner.js

@@ -228,7 +228,7 @@ function loadApp(app, appId, appOrigin, preloadModules) {
 
     var accumulatingPrints = null;
 
-    var sendLog = function(details) { console.log(logDetails); };
+    var sendLog = function(details) { console.log(details); };
 
     var stdout = function(text, fromServer) {
         if (text != "\n") {
@@ -301,7 +301,7 @@ function loadApp(app, appId, appOrigin, preloadModules) {
                 }
             };
 
-            if (errorObj instanceof serverModuleAndLog.pyMod["SessionExpiredError"]) {
+            if (serverModuleAndLog && errorObj instanceof serverModuleAndLog.pyMod["SessionExpiredError"]) {
                 $("#session-expired-modal button.refresh").off("click").on("click", function() {
                     document.location.href = window.anvilAppOrigin + "/" + (window.anvilParams.accessKey || '');
                 });

client/js/webpack.config.js

@@ -6,7 +6,7 @@ module.exports = {
 
   // We want to generate two bundles. One for runner. one for its Service Worker
   entry: {
-    runner: ['babel-polyfill', './runner.js'],
+    runner: ['./runner.js'],
     sw: ['babel-polyfill',  './sw.js'],
   },
 

client/runner.html

@@ -254,11 +254,14 @@ <h4 class="modal-title">Kerberos Authentication</h4>
   <script src="{{cdn-origin}}/runtime/node_modules/moment/min/moment.min.js?buildTime=0" crossorigin></script>
   <script src="{{cdn-origin}}/runtime/node_modules/moment-timezone/builds/moment-timezone-with-data-2012-2022.min.js?buildTime=0" crossorigin></script>
   <script src="{{cdn-origin}}/runtime/js/lib/daterangepicker.js?buildTime=0" crossorigin></script>
+  <script src="{{cdn-origin}}/runtime/js/lib/b64.js?buildTime=0" crossorigin></script>
   <script src="{{cdn-origin}}/runtime/js/lib/bootstrap-notify.min.js?buildTime=0" crossorigin></script>
 
   <script src="{{cdn-origin}}/runtime/node_modules/js-yaml/dist/js-yaml.min.js?buildTime=0" crossorigin></script>
   <script src="{{cdn-origin}}/runtime/js/lib/mutationobserver.min.js?buildTime=0" crossorigin></script>
 
+  <script src="{{cdn-origin}}/runtime/node_modules/core-js/client/core.min.js?buildTime=0" crossorigin></script>
+
   <script src="https://www.youtube.com/iframe_api"></script>
   <script>
     function gm_authFailure(e) { window.googleMapsAuthFailure = true; };
@@ -267,6 +270,7 @@ <h4 class="modal-title">Kerberos Authentication</h4>
 
   <script src="https://checkout.stripe.com/checkout.js"></script>
 
+  <script src='{{cdn-origin}}/runtime/js/lib/unorm.js?buildTime=0' crossorigin></script>
 
   <script src='{{cdn-origin}}/runtime/js/lib/skulpt.min.js?buildTime=0' crossorigin></script>
   <script src='{{cdn-origin}}/runtime/js/lib/skulpt-stdlib.js?buildTime=0' crossorigin></script>

client/user_email_confirmed.html

@@ -6,14 +6,14 @@
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <title>Email confirmed</title>
 
-    <link rel="stylesheet" href="{{cdn-origin}}/runtime/core/css/bootstrap.css?buildTime=0">
-    <link rel="stylesheet" href="{{cdn-origin}}/runtime/core/css/bootstrap-theme.min.css?buildTime=0">
+    <link rel="stylesheet" href="{{cdn-origin}}/runtime/css/bootstrap.css?buildTime=0">
+    <link rel="stylesheet" href="{{cdn-origin}}/runtime/css/bootstrap-theme.min.css?buildTime=0">
 
-    <script src="{{cdn-origin}}/runtime/core/node_modules/html5-boilerplate/dist/js/vendor/modernizr-3.8.0.min.js?buildTime=0"></script>
+    <script src="{{cdn-origin}}/runtime/node_modules/html5-boilerplate/dist/js/vendor/modernizr-3.8.0.min.js?buildTime=0"></script>
 
 
-    <script src="{{cdn-origin}}/runtime/core/node_modules/jquery/dist/jquery.min.js?buildTime=0"></script>
-    <script src="{{cdn-origin}}/runtime/core/node_modules/bootstrap/dist/js/bootstrap.min.js?buildTime=0"></script>
+    <script src="{{cdn-origin}}/runtime/node_modules/jquery/dist/jquery.min.js?buildTime=0"></script>
+    <script src="{{cdn-origin}}/runtime/node_modules/bootstrap/dist/js/bootstrap.min.js?buildTime=0"></script>
 
 </head>
 <body>

client/user_email_password_reset.html

@@ -6,14 +6,14 @@
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <title>Reset your password</title>
 
-    <link rel="stylesheet" href="{{cdn-origin}}/runtime/core/css/bootstrap.css?buildTime=0">
-    <link rel="stylesheet" href="{{cdn-origin}}/runtime/core/css/bootstrap-theme.min.css?buildTime=0">
+    <link rel="stylesheet" href="{{cdn-origin}}/runtime/css/bootstrap.css?buildTime=0">
+    <link rel="stylesheet" href="{{cdn-origin}}/runtime/css/bootstrap-theme.min.css?buildTime=0">
 
-    <script src="{{cdn-origin}}/runtime/core/node_modules/html5-boilerplate/dist/js/vendor/modernizr-3.8.0.min.js?buildTime=0"></script>
+    <script src="{{cdn-origin}}/runtime/node_modules/html5-boilerplate/dist/js/vendor/modernizr-3.8.0.min.js?buildTime=0"></script>
 
 
-    <script src="{{cdn-origin}}/runtime/core/node_modules/jquery/dist/jquery.min.js?buildTime=0"></script>
-    <script src="{{cdn-origin}}/runtime/core/node_modules/bootstrap/dist/js/bootstrap.min.js?buildTime=0"></script>
+    <script src="{{cdn-origin}}/runtime/node_modules/jquery/dist/jquery.min.js?buildTime=0"></script>
+    <script src="{{cdn-origin}}/runtime/node_modules/bootstrap/dist/js/bootstrap.min.js?buildTime=0"></script>
 
     <style>
         .row { margin-top: 5px; }

client/user_email_password_reset_done.html

@@ -6,14 +6,14 @@
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <title>Email confirmed</title>
 
-    <link rel="stylesheet" href="{{cdn-origin}}/runtime/core/css/bootstrap.css?buildTime=0">
-    <link rel="stylesheet" href="{{cdn-origin}}/runtime/core/css/bootstrap-theme.min.css?buildTime=0">
+    <link rel="stylesheet" href="{{cdn-origin}}/runtime/css/bootstrap.css?buildTime=0">
+    <link rel="stylesheet" href="{{cdn-origin}}/runtime/css/bootstrap-theme.min.css?buildTime=0">
 
-    <script src="{{cdn-origin}}/runtime/core/node_modules/html5-boilerplate/dist/js/vendor/modernizr-3.8.0.min.js?buildTime=0"></script>
+    <script src="{{cdn-origin}}/runtime/node_modules/html5-boilerplate/dist/js/vendor/modernizr-3.8.0.min.js?buildTime=0"></script>
 
 
-    <script src="{{cdn-origin}}/runtime/core/node_modules/jquery/dist/jquery.min.js?buildTime=0"></script>
-    <script src="{{cdn-origin}}/runtime/core/node_modules/bootstrap/dist/js/bootstrap.min.js?buildTime=0"></script>
+    <script src="{{cdn-origin}}/runtime/node_modules/jquery/dist/jquery.min.js?buildTime=0"></script>
+    <script src="{{cdn-origin}}/runtime/node_modules/bootstrap/dist/js/bootstrap.min.js?buildTime=0"></script>
 
 </head>
 <body>

database/migrator-core/resources/migrations/base/schema.sql

@@ -49,7 +49,6 @@ GRANT SELECT ON db_version TO $ANVIL_USER;
 GRANT EXECUTE ON FUNCTION parse_anvil_timestamp TO $ANVIL_USER;
 GRANT EXECUTE ON FUNCTION to_anvil_timestamp TO $ANVIL_USER;
 
-GRANT ALL ON app_storage_tables, app_storage_access, app_storage_data, app_storage_media TO $ANVIL_USER;
-GRANT USAGE ON app_storage_data_id_seq TO $ANVIL_USER;
+GRANT ALL ON app_storage_tables, app_storage_access, app_storage_data, app_storage_media, app_storage_data_id_seq TO $ANVIL_USER;
 GRANT SELECT ON pg_largeobject TO $ANVIL_USER; -- To measure the size of Media objects
 --[/GRANTS]--

doc/HACKING.md

@@ -48,10 +48,15 @@ $ npm run build
 Next, build the server components and produce a Python package ready for distribution:
 
 ```bash
-$ cd ../packaging/standalone
+$ cd ../packaging/app-server
 $ ./build-all
 ```
 
+And now you can install the app-server:
+```bash
+pip install /anvil/runtime/packaging/app-server/python-package-build
+```
+
 ## Building and Running for Development
 
 If you are doing development, then building the entire system every time is not ideal. You'll want to set up each component so it is easy to rebuild and restart. First, perform all the build steps above. Then:

doc/app-structure.md

@@ -380,26 +380,62 @@ If the database is already configured with a different schema, the App Server wi
 
 Anvil's [Users Service](https://anvil.works/docs/users) handles authentication, including signup, login and user permissions, and provides a range of functionality to make user management easy and flexible.
 
-You can enable the Users Service in `anvil.yaml`, as well as configuring the service itself. If you’re using the Users Service, user accounts are stored in a Data Table, conventionally called "users" (see above for information on Data Tables).
+The best way to enable the Users Service is to enable it in the [online IDE](https://anvil.works/build), and then [clone your app onto your local machine](https://anvil.works/docs/version-control/git). You can also enable the Users Service in `anvil.yaml`, as well as configuring the service itself. 
 
-The Users service supports a number of [sign-in methods](https://anvil.works/docs/users/authentication_choices). These are configured using `client_config` in the `anvil.yaml` entry. The `server_config` tells the database which of your Data Tables should be used to store user accounts. By convention, this should be "users".
+If you're using the Users Service, user accounts are stored in a Data Table, conventionally called "users" (see above for information on Data Tables).
 
-Here's an example entry in `anvil.yaml` which adds and configures the User Service (the Data Tables service is required to use the Users service):
+The Users service supports a number of [sign-in methods](https://anvil.works/docs/users/authentication_choices). These are configured using `client_config` in the `anvil.yaml` entry. The `server_config` tells the database which of your Data Tables should be used to store user accounts. By convention, this should be "users".
 
-<!-- TODO: should this also include the db_schema for the users table? -->
+Here's an example entry in `anvil.yaml`, produced by enabling the Users Service in the online IDE. It adds and configures the User Service (the Data Tables service is required to use the Users service):
 
 ```
 services:
 - source: /runtime/services/tables.yml
   client_config: {}
   server_config: {auto_create_missing_columns: false}
 - source: /runtime/services/anvil/users.yml
-  client_config: {use_microsoft: true, require_secure_passwords: true, share_login_status: true,
+  client_config: {use_microsoft: false, require_secure_passwords: true, share_login_status: true,
     use_email: true, allow_remember_me: true, allow_signup: true, enable_automatically: true,
-    confirm_email: true, remember_me_days: 7, use_google: true, use_facebook: true}
+    confirm_email: true, remember_me_days: 7, use_google: false, use_facebook: false}
   server_config: {user_table: 'users'}
+db_schema:
+- name: Users
+  id: 3
+  python_name: users
+  columns:
+    Jiv3u_GvZ+M=:
+      name: email
+      type: string
+      admin_ui: {order: 0, width: 200}
+    e5qNZNN248Y=:
+      name: enabled
+      type: bool
+      admin_ui: {order: 1, width: 100}
+    haSy3ivjtXM=:
+      name: signed_up
+      type: datetime
+      admin_ui: {order: 2, width: 200}
+    aHRjjIgDub0=:
+      name: password_hash
+      type: string
+      admin_ui: {order: 3, width: 200}
+    uJDnnYdBrt8=:
+      name: confirmed_email
+      type: bool
+      admin_ui: {order: 4, width: 100}
+    jUfJHJ+557v=:
+      name: email_confirmation_key
+      type: string
+      admin_ui: {order: 5, width: 200}
+    jALyyGoERn0=:
+      name: last_login
+      type: datetime
+      admin_ui: {order: 6, width: 200}
+  access: {python_name: users, server: full, client: none, table_id: 3}
 ```
 
+For more information on the Users Table, and the columns in the `db_schema` above, see the [reference docs](https://anvil.works/docs/users/the_users_table)
+
 #### Email
 
 Your apps can send and receive email using the built-in [Email Service](https://anvil.works/docs/email).

doc/getting-started.md

@@ -142,7 +142,7 @@ $ sudo su
 \# exit
 $ virtualenv -p python3 venv
 $ . venv/bin/activate
-$ pip install anvil-standalone-runtime
+$ pip install anvil-app-server
 $ git clone [email protected]:2222/SOME_APP.git MyApp
 $ anvil-app-server --app MyApp --origin https://my-hostname.example.com
 ```

downlink/python/anvil/__init__.py

@@ -60,13 +60,16 @@ def get_name(self):
         return None
 
 
+_byte_string_type = bytes if sys.version_info >= (3,) else basestring
 _unicode_string_type = str if sys.version_info >= (3,) else unicode
 
 
 class BlobMedia(Media):
     def __init__(self, content_type, content, name=None):
         self._content_type = content_type
-        if isinstance(content, _unicode_string_type):
+        if not isinstance(content, _byte_string_type):
+            raise TypeError("BlobMedia content must be a byte string.")
+        elif isinstance(content, _unicode_string_type):
             content = content.encode("utf-8")
         self._bytes = content
         self._name = name

downlink/python/anvil/_threaded_server.py

@@ -138,9 +138,14 @@ def _setup(self, **kwargs):
 anvil.app = LocalAppInfo()
 
 
+class SendNoResponse(Exception):
+    pass
+
+
 class IncomingRequest(_serialise.IncomingReqResp):
-    def __init__(self, json, import_modules=None, dump_task_state=False):
+    def __init__(self, json, import_modules=None, run_fn=None, dump_task_state=False):
         self.import_modules = import_modules
+        self.run_fn = run_fn
         self.dump_task_state = dump_task_state
         _serialise.IncomingReqResp.__init__(self, json)
 
@@ -170,16 +175,18 @@ def make_call():
                 self.reconstruct_remaining_data()
                 call_info.session = _server._reconstruct_objects(sjson, None).get("session", {})
 
-                if 'liveObjectCall' in self.json:
+                if self.run_fn is not None:
+                    response = self.run_fn()
+                elif 'liveObjectCall' in self.json:
                     loc = self.json['liveObjectCall']
                     spec = dict(loc)
 
-                    if self.json["id"].startswith("server-"):
+                    if call_context.remote_caller is None:
+                        spec["source"] = "UNKNOWN"
+                    elif call_context.remote_caller.is_trusted:
                         spec["source"] = "server"
-                    elif self.json["id"].startswith("client-"):
-                        spec["source"] = "client"
                     else:
-                        spec["source"] = "UNKNOWN"
+                        spec["source"] = "client"
 
                     del spec["method"]
                     backend = loc['backend']
@@ -238,6 +245,8 @@ def err(*args):
                     send_reqresp(resp)
                 except _server.SerializationError as e:
                     raise _server.SerializationError("Cannot serialize return value from function. " + str(e))
+            except SendNoResponse:
+                pass
             except:
 
                 e = _server._report_exception(self.json["id"])

downlink/python/anvil/users

@@ -0,0 +1 @@
+../../../services/anvil/users