Move core to its own folder and enable Monorepo Build (#1876)

Move core to its own folder `tfjs-core` and enable Monorepo build system where each package build runs only if the PR touches a file in that package folder.

DEV

Author: dsmilkov

.gitignore

@@ -13,6 +13,7 @@ bazel-out/
 yalc.lock
 .rpt2_cache/
 package/
+*/diff
 
 tfjs-backend-wasm/dist
 tfjs-backend-wasm/wasm-out

cloudbuild.yml

@@ -1,85 +1,34 @@
 steps:
-#
-# Core tests
-#
-# install dependencies
+# Install top-level deps.
 - name: 'node:10'
   entrypoint: 'yarn'
   id: 'yarn'
   args: ['install']
-# build
-- name: 'node:10'
-  entrypoint: 'yarn'
-  id: 'build'
-  args: ['build-ci']
-  waitFor: ['yarn']
-# run unit tests
-- name: 'node:10'
-  entrypoint: 'yarn'
-  id: 'test'
-  args: ['test-ci']
-  waitFor: ['build']
-  env: ['BROWSERSTACK_USERNAME=deeplearnjs1']
-  secretEnv: ['BROWSERSTACK_KEY']
-# Run integration tests of other packages against core
-- name: 'node:10'
-  entrypoint: 'yarn'
-  id: 'test-integration'
-  args: ['test-integration']
-  waitFor: ['build']
-  env: ['BROWSERSTACK_USERNAME=deeplearnjs1', 'NIGHTLY=$_NIGHTLY']
-  secretEnv: ['BROWSERSTACK_KEY']
-# bundle size check
-- name: 'node:10'
-  entrypoint: 'yarn'
-  id: 'test-bundle-size'
-  args: ['test-bundle-size']
-  waitFor: ['yarn']
-# test doc snippets
-- name: 'node:10'
-  entrypoint: 'yarn'
-  args: ['test-snippets']
-  id: 'test-snippets'
-  waitFor: ['yarn']
-# test Async backends
-- name: 'node:10'
-  entrypoint: 'yarn'
-  id: 'test-async-backends'
-  args: ['test-async-backends-ci']
-  waitFor: ['build']
-#
-# WebGPU tests
-#
-- name: 'node:10'
-  entrypoint: 'yarn'
-  id: 'test-webgpu'
-  dir: 'tfjs-webgpu/'
-  args: ['test-ci']
-  # Wait for parent because subfolders look at parent node_modules.
-  waitFor: ['yarn']
-#
-# React-Native tests
-#
+
+# Run diff to find modified files in each folder.
 - name: 'node:10'
   entrypoint: 'yarn'
-  id: 'test-react-native'
-  dir: 'tfjs-react-native/'
-  args: ['test-ci']
-  # Wait for parent because subfolders look at parent node_modules.
+  id: 'diff'
+  args: ['diff']
   waitFor: ['yarn']
-  env: ['BROWSERSTACK_USERNAME=deeplearnjs1']
-  secretEnv: ['BROWSERSTACK_KEY']
 
-# General configuration
-secrets:
-- kmsKeyName: projects/learnjs-174218/locations/global/keyRings/tfjs/cryptoKeys/enc
-  secretEnv:
-    BROWSERSTACK_KEY: CiQAkwyoIW0LcnxymzotLwaH4udVTQFBEN4AEA5CA+a3+yflL2ASPQAD8BdZnGARf78MhH5T9rQqyz9HNODwVjVIj64CTkFlUCGrP1B2HX9LXHWHLmtKutEGTeFFX9XhuBzNExA=
-timeout: 1800s
-logsBucket: 'gs://tfjs-build-logs'
-substitutions:
-  _NIGHTLY: ''
-options:
-  logStreamingOption: 'STREAM_ON'
-  machineType: 'N1_HIGHCPU_8'
-  substitution_option: 'ALLOW_LOOSE'
+# Core.
+- name: 'gcr.io/cloud-builders/gcloud'
+  entrypoint: 'bash'
+  id: 'core'
+  args: ['./scripts/run-build.sh', 'tfjs-core']
+  waitFor: ['diff']
+
+# WebGPU.
+- name: 'gcr.io/cloud-builders/gcloud'
+  entrypoint: 'bash'
+  id: 'webgpu'
+  args: ['./scripts/run-build.sh', 'tfjs-webgpu']
+  waitFor: ['diff']
+
+# React Native.
+- name: 'gcr.io/cloud-builders/gcloud'
+  entrypoint: 'bash'
+  id: 'react-native'
+  args: ['./scripts/run-build.sh', 'tfjs-react-native']
+  waitFor: ['diff']

package.json

@@ -1,90 +1,9 @@
 {
-  "name": "@tensorflow/tfjs-core",
-  "version": "1.2.7",
-  "description": "Hardware-accelerated JavaScript library for machine intelligence",
-  "private": false,
-  "main": "dist/index.js",
-  "jsdelivr": "dist/tf-core.min.js",
-  "unpkg": "dist/tf-core.min.js",
-  "types": "dist/index.d.ts",
-  "jsnext:main": "dist/tf-core.esm.js",
-  "module": "dist/tf-core.esm.js",
-  "miniprogram": "dist/miniprogram",
-  "engines": {
-    "yarn": ">= 1.3.2"
-  },
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/tensorflow/tfjs-core.git"
-  },
-  "license": "Apache-2.0",
   "devDependencies": {
-    "@bazel/bazel": "^0.24.0",
-    "@bazel/typescript": "^0.27.8",
-    "@types/jasmine": "~2.5.53",
-    "@types/node": "~9.6.0",
-    "@types/node-fetch": "~2.1.2",
-    "browserify": "~16.2.3",
     "clang-format": "~1.2.4",
-    "jasmine": "~3.1.0",
-    "jasmine-core": "~3.1.0",
-    "karma": "~4.2.0",
-    "karma-browserify": "~6.0.0",
-    "karma-browserstack-launcher": "~1.4.0",
-    "karma-chrome-launcher": "~2.2.0",
-    "karma-jasmine": "~1.1.0",
-    "karma-typescript": "~4.1.1",
-    "npm-run-all": "~4.1.3",
-    "rimraf": "~2.6.2",
-    "rollup": "~1.9.0",
-    "rollup-plugin-commonjs": "~9.3.4",
-    "rollup-plugin-node-resolve": "~3.4.0",
-    "rollup-plugin-terser": "~4.0.4",
-    "rollup-plugin-typescript2": "~0.20.1",
-    "rollup-plugin-visualizer": "~1.1.1",
-    "shelljs": "~0.8.3",
-    "ts-node": "~7.0.0",
-    "tslint": "~5.11.0",
-    "tslint-no-circular-imports": "~0.5.0",
-    "typescript": "3.3.3333",
-    "watchify": "~3.11.1",
-    "yalc": "~1.0.0-pre.21",
-    "yargs": "~13.2.2"
+    "shelljs": "~0.8.3"
   },
   "scripts": {
-    "build-ci": "./scripts/enumerate-tests.js --ci && tsc",
-    "build": "./scripts/enumerate-tests.js && tsc",
-    "build:bazel": "bazel build //...",
-    "build-npm": "./scripts/build-npm.sh",
-    "build-test-snippets": "yarn tsc --project ./scripts/test_snippets/tsconfig.json",
-    "format-all": "clang-format -i -style=Google --glob=src/**/*.ts",
-    "link-local": "yalc link",
-    "publish-local": "rimraf dist/ && yarn build && rollup -c && yalc push",
-    "lint": "tslint -p . -t verbose",
-    "coverage": "KARMA_COVERAGE=1 karma start --browsers='Chrome' --singleRun",
-    "test": "karma start",
-    "test-webworker": "karma start --worker",
-    "run-browserstack": "karma start --browserstack",
-    "test-bundle-size": "./scripts/test-bundle-size.js",
-    "test-node": "rimraf dist/ && tsc && node dist/test_node.js",
-    "test-node-ci": "node dist/test_node.js",
-    "test-async-backends": "rimraf dist/ && tsc && node dist/test_async_backends.js",
-    "test-async-backends-ci": "node dist/test_async_backends.js",
-    "test-integration": "./scripts/test-integration.js",
-    "test-ci": "./scripts/test-ci.sh",
-    "test-snippets": "ts-node ./scripts/test_snippets/test_snippets.ts"
-  },
-  "dependencies": {
-    "@types/offscreencanvas": "~2019.3.0",
-    "@types/seedrandom": "2.4.27",
-    "@types/webgl-ext": "0.0.30",
-    "@types/webgl2": "0.0.4",
-    "node-fetch": "~2.1.2",
-    "seedrandom": "2.4.3"
-  },
-  "browser": {
-    "node-fetch": false,
-    "util": false,
-    "crypto": false
+    "diff": "./scripts/diff.js"
   }
 }

scripts/diff.js

@@ -0,0 +1,42 @@
+#!/usr/bin/env node
+// Copyright 2019 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// =============================================================================
+
+const {exec} = require('./test-util');
+const {readdirSync, statSync, writeFileSync} = require('fs');
+const {join} = require('path');
+
+const CLONE_PATH = 'clone';
+
+const dirs = readdirSync('.').filter(f => {
+  return f !== 'node_modules' && f !== '.git' && statSync(f).isDirectory();
+});
+
+exec(
+    `git clone --depth=1 --single-branch ` +
+    `https://github.com/tensorflow/tfjs-core.git ${CLONE_PATH}`);
+
+
+dirs.forEach(dir => {
+  const diffCmd = `diff -rq ${CLONE_PATH}/${dir}/ ./${dir}/`;
+  const diffOutput = exec(diffCmd, {silent: true}, true).stdout.trim();
+
+  if (diffOutput !== '') {
+    console.log(`${dir} has modified files.`);
+    writeFileSync(join(dir, 'diff'), diffOutput);
+  } else {
+    console.log(`No modified files found in ${dir}`);
+  }
+});

scripts/run-build.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# Copyright 2019 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+
+set -e
+
+DIR=$1
+if test -f "$DIR/diff"; then
+  gcloud builds submit . --config=$DIR/cloudbuild.yml
+fi

.bazelignore renamed to tfjs-core/.bazelignore

@@ -0,0 +1,80 @@
+steps:
+# Install common dependencies.
+- name: 'node:10'
+  id: 'yarn-common'
+  entrypoint: 'yarn'
+  args: ['install']
+
+# Install tfjs-core dependencies.
+- name: 'node:10'
+  dir: 'tfjs-core'
+  id: 'yarn'
+  entrypoint: 'yarn'
+  args: ['install']
+  waitFor: ['yarn-common']
+
+# Build
+- name: 'node:10'
+  dir: 'tfjs-core'
+  id: 'build'
+  entrypoint: 'yarn'
+  args: ['build-ci']
+  waitFor: ['yarn']
+
+# Run unit tests.
+- name: 'node:10'
+  dir: 'tfjs-core'
+  id: 'test'
+  entrypoint: 'yarn'
+  args: ['test-ci']
+  waitFor: ['build']
+  env: ['BROWSERSTACK_USERNAME=deeplearnjs1']
+  secretEnv: ['BROWSERSTACK_KEY']
+
+# Run integration tests of other packages against core.
+- name: 'node:10'
+  dir: 'tfjs-core'
+  id: 'test-integration'
+  entrypoint: 'yarn'
+  args: ['test-integration']
+  waitFor: ['build']
+  env: ['BROWSERSTACK_USERNAME=deeplearnjs1', 'NIGHTLY=$_NIGHTLY']
+  secretEnv: ['BROWSERSTACK_KEY']
+
+# bundle size check
+- name: 'node:10'
+  dir: 'tfjs-core'
+  id: 'test-bundle-size'
+  entrypoint: 'yarn'
+  args: ['test-bundle-size']
+  waitFor: ['yarn']
+
+# test doc snippets
+- name: 'node:10'
+  dir: 'tfjs-core'
+  id: 'test-snippets'
+  entrypoint: 'yarn'
+  args: ['test-snippets']
+  waitFor: ['yarn']
+
+# test Async backends
+- name: 'node:10'
+  dir: 'tfjs-core'
+  id: 'test-async-backends'
+  entrypoint: 'yarn'
+  args: ['test-async-backends-ci']
+  waitFor: ['build']
+
+# General configuration
+secrets:
+- kmsKeyName: projects/learnjs-174218/locations/global/keyRings/tfjs/cryptoKeys/enc
+  secretEnv:
+    BROWSERSTACK_KEY: CiQAkwyoIW0LcnxymzotLwaH4udVTQFBEN4AEA5CA+a3+yflL2ASPQAD8BdZnGARf78MhH5T9rQqyz9HNODwVjVIj64CTkFlUCGrP1B2HX9LXHWHLmtKutEGTeFFX9XhuBzNExA=
+timeout: 1800s
+logsBucket: 'gs://tfjs-build-logs'
+substitutions:
+  _NIGHTLY: ''
+options:
+  logStreamingOption: 'STREAM_ON'
+  machineType: 'N1_HIGHCPU_8'
+  substitution_option: 'ALLOW_LOOSE'