Move cpu backend out of tfjs-core (#3008)

BREAKING This moves the javascript cpu backend out of tfjs-core into its own package. It can be used as a peer dependency alongside tfjs-core and is part of our larger work to modularise tfjs.
tensorflow · Apr 8, 2020 · b72893a · b72893a
1 parent 51a8f4b
commit b72893a
Show file tree

Hide file tree

Showing 58 changed files with 4,693 additions and 552 deletions.
diff --git a/cloudbuild.yml b/cloudbuild.yml
@@ -24,6 +24,14 @@ steps:
   waitFor: ['find-affected-packages']
   env: ['NIGHTLY=$_NIGHTLY']
 
+# CPU backend.
+- name: 'gcr.io/cloud-builders/gcloud'
+  entrypoint: 'bash'
+  id: 'tfjs-backend-cpu'
+  args: ['./scripts/run-build.sh', 'tfjs-backend-cpu']
+  waitFor: ['find-affected-packages']
+  env: ['NIGHTLY=$_NIGHTLY']
+
 # Converter.
 - name: 'gcr.io/cloud-builders/gcloud'
   entrypoint: 'bash'

diff --git a/scripts/package_dependencies.json b/scripts/package_dependencies.json
@@ -4,5 +4,6 @@
   "tfjs-converter": ["tfjs-core"],
   "tfjs-layers": ["tfjs-core"],
   "tfjs-data": ["tfjs-core", "tfjs-layers"],
-  "tfjs-backend-wasm": ["tfjs-core"]
+  "tfjs-backend-wasm": ["tfjs-core"],
+  "tfjs-backend-cpu": ["tfjs-core"]
 }
diff --git a/tfjs-backend-cpu/.npmignore b/tfjs-backend-cpu/.npmignore
@@ -0,0 +1,27 @@
+.babelrc
+.DS_Store
+.idea/
+.rpt2_cache
+.travis.yml
+.vscode
+*.tgz
+*.txt
+**.yalc
+**yalc.lock
+cloudbuild.yml
+coverage/
+demo/
+dist/**/*_test.d.ts
+dist/**/*_test.js
+karma.conf.js
+node_modules/
+npm-debug.log
+package-lock.json
+package/
+rollup.config.js
+scripts/
+src/**/*_test.ts
+tsconfig.json
+tslint.json
+yarn-error.log
+yarn.lock
diff --git a/tfjs-backend-cpu/.npmrc b/tfjs-backend-cpu/.npmrc
@@ -0,0 +1 @@
+package-lock=false
diff --git a/tfjs-backend-cpu/cloudbuild.yml b/tfjs-backend-cpu/cloudbuild.yml
@@ -0,0 +1,45 @@
+steps:
+# Install common dependencies.
+- name: 'node:10'
+  id: 'yarn-common'
+  entrypoint: 'yarn'
+  args: ['install']
+
+# Install webgpu dependencies.
+- name: 'node:10'
+  dir: 'tfjs-backend-cpu'
+  id: 'yarn'
+  entrypoint: 'yarn'
+  args: ['install']
+  waitFor: ['yarn-common']
+
+# Build core from master.
+- name: 'node:10'
+  dir: 'tfjs-backend-cpu'
+  id: 'build-core'
+  entrypoint: 'yarn'
+  args: ['build-core-ci']
+  waitFor: ['yarn-common']
+
+# Run tests.
+- name: 'node:10'
+  dir: 'tfjs-backend-cpu'
+  entrypoint: 'yarn'
+  id: 'test-backend-cpu'
+  args: ['test-ci']
+  waitFor: ['build-core']
+  env: ['BROWSERSTACK_USERNAME=deeplearnjs1']
+  secretEnv: ['BROWSERSTACK_KEY']
+
+# General configuration
+secrets:
+- kmsKeyName: projects/learnjs-174218/locations/global/keyRings/tfjs/cryptoKeys/enc
+  secretEnv:
+    BROWSERSTACK_KEY: CiQAkwyoIW0LcnxymzotLwaH4udVTQFBEN4AEA5CA+a3+yflL2ASPQAD8BdZnGARf78MhH5T9rQqyz9HNODwVjVIj64CTkFlUCGrP1B2HX9LXHWHLmtKutEGTeFFX9XhuBzNExA=
+timeout: 1800s
+logsBucket: 'gs://tfjs-build-logs'
+substitutions:
+  _NIGHTLY: ''
+options:
+  logStreamingOption: 'STREAM_ON'
+  substitution_option: 'ALLOW_LOOSE'
diff --git a/tfjs-backend-cpu/karma.conf.js b/tfjs-backend-cpu/karma.conf.js
@@ -0,0 +1,118 @@
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+
+const karmaTypescriptConfig = {
+  tsconfig: 'tsconfig.json',
+  // Disable coverage reports and instrumentation by default for tests
+  coverageOptions: {instrumentation: false},
+  reports: {},
+  bundlerOptions: {
+    sourceMap: true,
+    // Ignore the import of the `worker_threads` package used in a core test
+    // meant to run in node.
+    exclude: ['worker_threads'],
+    // worker_node_test in tfjs-core contains a conditional require statement
+    // that confuses the bundler of karma-typescript.
+    ignore: ['./worker_node_test']
+  }
+};
+
+module.exports = function(config) {
+  const args = [];
+  if (config.grep) {
+    args.push('--grep', config.grep);
+  }
+  if (config.flags) {
+    args.push('--flags', config.flags);
+  }
+  let exclude = [];
+  if (config.excludeTest != null) {
+    exclude.push(config.excludeTest);
+  }
+
+  config.set({
+    frameworks: ['jasmine', 'karma-typescript'],
+    files: [
+      'src/setup_test.ts',
+      {pattern: 'src/**/*.ts'},
+    ],
+    preprocessors: {'**/*.ts': ['karma-typescript']},
+    karmaTypescriptConfig,
+    reporters: ['dots', 'karma-typescript'],
+    exclude,
+    colors: true,
+    autoWatch: false,
+    browsers: ['Chrome'],
+    singleRun: true,
+    client: {jasmine: {random: false}, args: args},
+    browserStack: {
+      username: process.env.BROWSERSTACK_USERNAME,
+      accessKey: process.env.BROWSERSTACK_KEY
+    },
+    captureTimeout: 120000,
+    reportSlowerThan: 500,
+    browserNoActivityTimeout: 240000,
+    customLaunchers: {
+      // For browserstack configs see:
+      // https://www.browserstack.com/automate/node
+      bs_chrome_mac: {
+        base: 'BrowserStack',
+        browser: 'chrome',
+        browser_version: 'latest',
+        os: 'OS X',
+        os_version: 'High Sierra'
+      },
+      bs_firefox_mac: {
+        base: 'BrowserStack',
+        browser: 'firefox',
+        browser_version: 'latest',
+        os: 'OS X',
+        os_version: 'High Sierra'
+      },
+      bs_safari_mac: {
+        base: 'BrowserStack',
+        browser: 'safari',
+        browser_version: 'latest',
+        os: 'OS X',
+        os_version: 'High Sierra'
+      },
+      bs_ios_11: {
+        base: 'BrowserStack',
+        device: 'iPhone X',
+        os: 'iOS',
+        os_version: '11.0',
+        real_mobile: true
+      },
+      bs_android_9: {
+        base: 'BrowserStack',
+        device: 'Google Pixel 3 XL',
+        os: 'android',
+        os_version: '9.0',
+        real_mobile: true
+      },
+      win_10_chrome: {
+        base: 'BrowserStack',
+        browser: 'chrome',
+        // Latest Chrome on Windows has WebGL problems:
+        // https://github.com/tensorflow/tfjs/issues/2272
+        browser_version: '77.0',
+        os: 'Windows',
+        os_version: '10'
+      },
+    },
+  })
+}
diff --git a/tfjs-backend-cpu/package.json b/tfjs-backend-cpu/package.json
@@ -0,0 +1,70 @@
+{
+  "name": "@tensorflow/tfjs-backend-cpu",
+  "version": "0.0.0",
+  "description": "Vanilla JavaScript backend for TensorFlow.js",
+  "private": false,
+  "main": "dist/index.js",
+  "jsdelivr": "dist/tf-backend-cpu.min.js",
+  "unpkg": "dist/tf-backend-cpu.min.js",
+  "types": "dist/index.d.ts",
+  "jsnext:main": "dist/tf-backend-cpu.esm.js",
+  "module": "dist/tf-backend-cpu.esm.js",
+  "engines": {
+    "yarn": ">= 1.3.2"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/tensorflow/tfjs.git"
+  },
+  "license": "Apache-2.0",
+  "devDependencies": {
+    "@rollup/plugin-commonjs": "^11.0.2",
+    "@rollup/plugin-node-resolve": "^7.1.1",
+    "@rollup/plugin-typescript": "^3.0.0",
+    "@tensorflow/tfjs-core": "link:../tfjs-core",
+    "@types/jasmine": "~3.0.0",
+    "clang-format": "~1.2.4",
+    "jasmine": "~3.1.0",
+    "jasmine-core": "~3.1.0",
+    "karma": "~4.4.1",
+    "karma-browserstack-launcher": "~1.4.0",
+    "karma-chrome-launcher": "~2.2.0",
+    "karma-jasmine": "~1.1.0",
+    "karma-typescript": "~4.1.1",
+    "npm-run-all": "~4.1.3",
+    "rimraf": "~2.6.2",
+    "rollup": "~2.3.2",
+    "rollup-plugin-terser": "~5.3.0",
+    "rollup-plugin-visualizer": "~3.3.2",
+    "ts-node": "~7.0.0",
+    "tslint": "~5.11.0",
+    "tslint-no-circular-imports": "~0.5.0",
+    "typescript": "3.5.3",
+    "yalc": "~1.0.0-pre.21"
+  },
+  "scripts": {
+    "build-ci": "tsc",
+    "build": "tsc",
+    "build-core": "cd ../tfjs-core && yarn && yarn build",
+    "build-core-ci": "cd ../tfjs-core && yarn && yarn build-ci",
+    "build-npm": "./scripts/build-npm.sh",
+    "link-local": "yalc link",
+    "publish-local": "rimraf dist/ && yarn build && rollup -c && yalc push",
+    "publish-npm": "npm publish",
+    "lint": "tslint -p . -t verbose",
+    "test": "yarn build-core && karma start",
+    "run-browserstack": "karma start --browserstack",
+    "test-ci": "./scripts/test-ci.sh"
+  },
+  "dependencies": {
+    "@types/seedrandom": "2.4.27",
+    "seedrandom": "2.4.3"
+  },
+  "peerDependencies": {
+    "@tensorflow/tfjs-core": "link:../tfjs-core"
+  },
+  "browser": {
+    "util": false,
+    "crypto": false
+  }
+}