Refine tar ball size (#42)

* publishing * refein * publishing * publishing * publishing * refein * refein * refein * add check * refein * refein * refein * refein * refien * refien * fix * fix * publishing * add * publishing
Oneflow-Inc · Sep 30, 2021 · 151a0a5 · 151a0a5
1 parent a0ab40d
commit 151a0a5
Show file tree

Hide file tree

Showing 51 changed files with 447 additions and 1,641 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -255,7 +255,7 @@ jobs:
             3.6
       - name: Upload bin
         if: ${{ !fromJson(matrix.cache-hit) && matrix.test-type != 'do-nothing' }}
-        uses: ./degist/upload
+        uses: ./digest/upload
         with:
           digest: ${{ steps.save-cache.outputs.build-digest }}
           entry: ${{ matrix.entry }}
@@ -265,7 +265,7 @@ jobs:
           dst-dir: bin
       - name: Upload whl
         if: ${{ !fromJson(matrix.cache-hit) && matrix.test-type != 'do-nothing' }}
-        uses: ./degist/upload
+        uses: ./digest/upload
         with:
           digest: ${{ steps.save-cache.outputs.build-digest }}
           entry: ${{ matrix.entry }}
@@ -387,7 +387,7 @@ jobs:
           exit 1
       - name: Download wheel and binary
         if: ${{ !fromJson(matrix.cache-hit) && matrix.test-type != 'do-nothing' }}
-        uses: ./degist/download
+        uses: ./digest/download
         id: download-digest
         with:
           digest: ${{ steps.save-cache.outputs.build-digest }}

diff --git a/.gitignore b/.gitignore
@@ -105,3 +105,4 @@ lib/**/*
 
 # oneflow
 /log
+oneflow-src
diff --git a/__tests__/action.test.ts b/__tests__/action.test.ts
@@ -0,0 +1,22 @@
+import * as process from 'process'
+import {test} from '@jest/globals'
+import os from 'os'
+import * as actionMeta from '../src/utils/actionMeta'
+import {setTestMatrix} from '../src/utils/matrix'
+
+process.env['RUNNER_TOOL_CACHE'] = '~/runner_tool_cache'.replace(
+  '~',
+  os.homedir
+)
+process.env['RUNNER_TEMP'] = '~/runner_temp'.replace('~', os.homedir)
+process.env['GITHUB_REPOSITORY'] = 'jest-test/jest-test'
+const MINUTES15 = 1000 * 60 * 15
+// shows how the runner will run a javascript action with env / stdout protocol
+
+test(
+  'cache test matrix',
+  async () => {
+    await actionMeta.checkAction()
+  },
+  MINUTES15
+)
diff --git a/__tests__/cache-complete.test.ts b/__tests__/cache-complete.test.ts
@@ -5,11 +5,7 @@ import os from 'os'
 import * as env from '../src/utils/env'
 import {ok} from 'assert'
 import * as cpExec from '../src/utils/cpExec'
-import {
-  checkComplete,
-  getOneFlowBuildCacheKeys,
-  removeComplete
-} from '../src/utils/cache'
+import * as cache from '../src/utils/cache'
 import * as matrix from '../src/utils/matrix'
 process.env['RUNNER_TOOL_CACHE'] = '~/runner_tool_cache'.replace(
   '~',
@@ -31,34 +27,25 @@ test(
     env.setInput('entry', ENTRY)
     // TODO: test multiple types of digest
     env.setInput('digest-type', 'single-client-test')
-    const keys = await getOneFlowBuildCacheKeys(ENTRY)
+    const keys = await cache.getOneFlowBuildCacheKeys(ENTRY)
     env.setBooleanInput('mark-as-completed', true)
     env.setBooleanInput('check-not-completed', true)
     env.setMultilineInput('runner-labels', [
       'self-hosted',
       'linux',
       'provision'
     ])
-    await removeComplete(keys)
-    ok(!(await checkComplete(keys)))
-    await cpExec.cpExec(
-      np,
-      path.join(__dirname, '..', 'lib', 'cacheComplete.js')
-    )
+    await cache.removeComplete(keys)
+    ok(!(await cache.checkComplete(keys)))
+    await cache.cacheRun()
     env.setTestState('keys', keys)
-    await cpExec.cpExec(
-      np,
-      path.join(__dirname, '..', 'lib', 'postCacheComplete.js')
-    )
+    await cache.postCacheRun()
     process.env['OSS_ACCESS_KEY_ID'] = ''
     process.env['OSS_ACCESS_KEY_SECRET'] = ''
     // await new Promise(resolve => setTimeout(resolve, 2000))
-    ok(await checkComplete(keys))
+    ok(await cache.checkComplete(keys))
     env.setBooleanInput('check-not-completed', false)
-    await cpExec.cpExec(
-      np,
-      path.join(__dirname, '..', 'lib', 'cacheComplete.js')
-    )
+    await cache.cacheRun()
   },
   MINUTES15
 )

diff --git a/action.yml b/action.yml
@@ -1,6 +1,10 @@
 name: 'Build/Get OneFlow'
 description: ''
 inputs:
+  action-type:
+    description: ''
+    required: false
+    default: 'build-oneflow'
   # OneFlow args
   oneflow-build-env:
     description: 'environment to build OneFlow: conda, manylinux'
@@ -100,4 +104,4 @@ outputs:
     description: 'pip index url could be use to install oneflow'
 runs:
   using: 'node12'
-  main: 'dist/buildOneFlow/index.js'
+  main: 'dist/index.js'
diff --git a/cache-complete/action.yml b/cache-complete/action.yml
@@ -1,11 +1,15 @@
 name: 'Build/Get OneFlow'
 description: ''
 inputs:
+  action-type:
+    description: ''
+    required: false
+    default: 'cache-complete'
   oneflow-src:
     description: 'path to OneFlow source code'
     required: true
   entry:
-    description: 'entry to cache completetion status'
+    description: 'entry to cache completion status'
     required: true
   runner-labels:
     required: false
@@ -31,8 +35,8 @@ outputs:
 
 runs:
   using: 'node12'
-  main: '../dist/cacheComplete/index.js'
-  post: '../dist/postCacheComplete/index.js'
+  main: '../dist/index.js'
+  post: '../dist/index.js'
   post-if: success()
 branding:
   icon: 'archive'

diff --git a/cache-complete/matrix/build/action.yml b/cache-complete/matrix/build/action.yml
@@ -1,6 +1,10 @@
 name: 'Build/Get OneFlow'
 description: ''
 inputs:
+  action-type:
+    description: ''
+    required: false
+    default: 'cache-complete/matrix/build'
   oneflow-src:
     description: 'path to OneFlow source code'
     required: true
@@ -16,7 +20,7 @@ outputs:
 
 runs:
   using: 'node12'
-  main: '../../../dist/genBuildMatrix/index.js'
+  main: '../../../dist/index.js'
 branding:
   icon: 'archive'
   color: 'gray-dark'
diff --git a/cache-complete/matrix/test/action.yml b/cache-complete/matrix/test/action.yml
@@ -1,6 +1,10 @@
 name: 'Build/Get OneFlow'
 description: ''
 inputs:
+  action-type:
+    description: ''
+    required: false
+    default: 'cache-complete/matrix/test'
   oneflow-src:
     description: 'path to OneFlow source code'
     required: true
@@ -13,7 +17,7 @@ outputs:
 
 runs:
   using: 'node12'
-  main: '../../../dist/genTestMatrix/index.js'
+  main: '../../../dist/index.js'
 branding:
   icon: 'archive'
   color: 'gray-dark'
diff --git a/degist/download/action.yml → digest/download/action.yml b/degist/download/action.yml → digest/download/action.yml
@@ -1,6 +1,10 @@
 name: 'Download by digest'
 description: ''
 inputs:
+  action-type:
+    description: ''
+    required: false
+    default: 'digest/download'
   entry:
     description: ''
     required: true
@@ -25,7 +29,7 @@ outputs:
 
 runs:
   using: 'node12'
-  main: '../../dist/downloadByDigest/index.js'
+  main: '../../dist/index.js'
 branding:
   icon: 'archive'
   color: 'gray-dark'
diff --git a/degist/upload/action.yml → digest/upload/action.yml b/degist/upload/action.yml → digest/upload/action.yml
@@ -1,6 +1,10 @@
 name: 'Upload by digest'
 description: ''
 inputs:
+  action-type:
+    description: ''
+    required: false
+    default: 'digest/upload'
   src-dir:
     description: 'path to OneFlow source code'
     required: true
@@ -23,7 +27,7 @@ inputs:
 
 runs:
   using: 'node12'
-  main: '../../dist/uploadByDigest/index.js'
+  main: '../../dist/index.js'
 branding:
   icon: 'archive'
   color: 'gray-dark'
diff --git a/...adByDigest/build/Release/cpufeatures.node → dist/build/Release/cpufeatures.node b/...adByDigest/build/Release/cpufeatures.node → dist/build/Release/cpufeatures.node
diff --git a/dist/buildImage/index.js b/dist/buildImage/index.js
diff --git a/dist/buildOneFlow/index.js b/dist/buildOneFlow/index.js
diff --git a/dist/cacheComplete/index.js b/dist/cacheComplete/index.js
diff --git a/dist/downloadByDigest/index.js b/dist/downloadByDigest/index.js
diff --git a/dist/downloadByDigest/pagent.exe b/dist/downloadByDigest/pagent.exe
diff --git a/dist/genBuildMatrix/index.js b/dist/genBuildMatrix/index.js
diff --git a/dist/genTestMatrix/index.js b/dist/genTestMatrix/index.js
diff --git a/dist/index.js b/dist/index.js
diff --git a/...tocol/crypto/build/Release/sshcrypto.node → ...tocol/crypto/build/Release/sshcrypto.node b/...tocol/crypto/build/Release/sshcrypto.node → ...tocol/crypto/build/Release/sshcrypto.node
diff --git a/...ocol/crypto/build/Release/sshcrypto1.node → ...ocol/crypto/build/Release/sshcrypto1.node b/...ocol/crypto/build/Release/sshcrypto1.node → ...ocol/crypto/build/Release/sshcrypto1.node
diff --git a/dist/mirrorTools/index.js b/dist/mirrorTools/index.js
diff --git a/dist/mirrorTools/pagent.exe b/dist/mirrorTools/pagent.exe
diff --git a/dist/buildImage/pagent.exe → dist/pagent.exe b/dist/buildImage/pagent.exe → dist/pagent.exe
diff --git a/dist/downloadByDigest/pagent1.exe → dist/pagent1.exe b/dist/downloadByDigest/pagent1.exe → dist/pagent1.exe
diff --git a/dist/buildOneFlow/pagent.exe → dist/pagent2.exe b/dist/buildOneFlow/pagent.exe → dist/pagent2.exe
diff --git a/dist/postCacheComplete/index.js b/dist/postCacheComplete/index.js
diff --git a/dist/uploadByDigest/build/Release/cpufeatures.node b/dist/uploadByDigest/build/Release/cpufeatures.node
diff --git a/dist/uploadByDigest/index.js b/dist/uploadByDigest/index.js
diff --git a/dist/uploadByDigest/lib/protocol/crypto/build/Release/sshcrypto.node b/dist/uploadByDigest/lib/protocol/crypto/build/Release/sshcrypto.node
diff --git a/dist/uploadByDigest/lib/protocol/crypto/build/Release/sshcrypto1.node b/dist/uploadByDigest/lib/protocol/crypto/build/Release/sshcrypto1.node
diff --git a/dist/uploadByDigest/pagent.exe b/dist/uploadByDigest/pagent.exe
diff --git a/dist/uploadByDigest/pagent1.exe b/dist/uploadByDigest/pagent1.exe
diff --git a/manylinux/action.yml b/manylinux/action.yml
@@ -1,6 +1,10 @@
 name: 'Build/Get OneFlow'
 description: ''
 inputs:
+  action-type:
+    description: ''
+    required: false
+    default: 'manylinux'
   manylinux-version:
     description: ''
     required: true
@@ -14,4 +18,4 @@ outputs:
     description: 'pip index url could be use to install oneflow'
 runs:
   using: 'node12'
-  main: '../dist/buildImage/index.js'
+  main: '../dist/index.js'
diff --git a/mirror/action.yml b/mirror/action.yml
@@ -1,5 +1,10 @@
 name: 'Mirror Tools'
 description: ''
+inputs:
+  action-type:
+    description: ''
+    required: false
+    default: 'mirror'
 runs:
   using: 'node12'
-  main: '../dist/mirrorTools/index.js'
+  main: '../dist/index.js'
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -9,15 +9,7 @@
         "format": "prettier --write **/*.ts",
         "format-check": "prettier --check **/*.ts",
         "lint": "eslint src/**/*.ts",
-        "package-buildImage": "ncc build --minify -o dist/buildImage src/buildImage.ts",
-        "package-buildOneFlow": "ncc build --minify -o dist/buildOneFlow src/buildOneFlow.ts",
-        "package-mirrorTools": "ncc build --minify -o dist/mirrorTools src/mirrorTools.ts",
-        "package-cacheComplete": "ncc build --minify -o dist/cacheComplete src/cacheComplete.ts",
-        "package-postCacheComplete": "ncc build --minify -o dist/postCacheComplete src/postCacheComplete.ts",
-        "package-genBuildMatrix": "ncc build --minify -o dist/genBuildMatrix src/genBuildMatrix.ts",
-        "package-genTestMatrix": "ncc build --minify -o dist/genTestMatrix src/genTestMatrix.ts",
-        "package-uploadByDigest": "ncc build --minify -o dist/uploadByDigest src/uploadByDigest.ts",
-        "package-downloadByDigest": "ncc build --minify -o dist/downloadByDigest src/downloadByDigest.ts",
+        "package-main": "ncc build --minify -o dist src/main.ts",
         "package": "concurrently 'npm:package-*'",
         "test": "jest",
         "all": "npm run build && npm run format && npm run lint && npm run package && npm test",
@@ -54,6 +46,7 @@
     "devDependencies": {
         "@types/ali-oss": "^6.0.10",
         "@types/dockerode": "^3.2.7",
+        "@types/js-yaml": "^4.0.3",
         "@types/node": "^16.3.3",
         "@types/semver": "^7.3.8",
         "@types/ssh2": "^0.5.48",

diff --git a/src/buildImage.ts b/src/buildImage.ts
diff --git a/src/buildOneFlow.ts b/src/buildOneFlow.ts
@@ -85,7 +85,7 @@ async function buildWithConda(): Promise<void> {
   }
 }
 
-async function run(): Promise<void> {
+export async function buildWithCondaOrManyLinux(): Promise<void> {
   try {
     const buildEnv: string = core.getInput('oneflow-build-env')
     if (['conda', 'manylinux'].includes(buildEnv) === false) {
@@ -106,5 +106,3 @@ async function run(): Promise<void> {
     core.setFailed(error as Error)
   }
 }
-
-run()
diff --git a/src/cacheComplete.ts b/src/cacheComplete.ts
diff --git a/src/downloadByDigest.ts b/src/downloadByDigest.ts
diff --git a/src/genBuildMatrix.ts b/src/genBuildMatrix.ts
diff --git a/src/genTestMatrix.ts b/src/genTestMatrix.ts