Rework generate-api CLI command to use .build directory (qmk#16441)

.github/workflows/api.yml

@@ -0,0 +1,38 @@
+name: Update API Data
+
+on:
+  push:
+    branches:
+    - master
+    paths:
+    - 'keyboards/**'
+    - 'layouts/community/**'
+  workflow_dispatch:
+
+jobs:
+  api_data:
+    runs-on: ubuntu-latest
+    container: qmkfm/qmk_cli
+
+    # protect against those who develop with their fork on master
+    if: github.repository == 'qmk/qmk_firmware'
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 1
+        persist-credentials: false
+
+    - name: Generate API Data
+      run: qmk generate-api
+
+    - name: Upload API Data
+      uses: jakejarvis/s3-sync-action@master
+      with:
+        args: --acl public-read --follow-symlinks --delete
+      env:
+        AWS_S3_BUCKET: ${{ secrets.API_SPACE_MASTER }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.SPACES_ACCESS_KEY }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_KEY }}
+        AWS_S3_ENDPOINT: https://nyc3.digitaloceanspaces.com
+        SOURCE_DIR: '.build/api_data'

.github/workflows/develop_api.yml

@@ -0,0 +1,38 @@
+name: Update Develop API Data
+
+on:
+  push:
+    branches:
+    - develop
+    paths:
+    - 'keyboards/**'
+    - 'layouts/community/**'
+  workflow_dispatch:
+
+jobs:
+  api_data:
+    runs-on: ubuntu-latest
+    container: qmkfm/qmk_cli
+
+    # protect against those who work in their fork on develop
+    if: github.repository == 'qmk/qmk_firmware'
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 1
+        persist-credentials: false
+
+    - name: Generate API Data
+      run: qmk generate-api
+
+    - name: Upload API Data
+      uses: jakejarvis/s3-sync-action@master
+      with:
+        args: --acl public-read --follow-symlinks --delete
+      env:
+        AWS_S3_BUCKET: ${{ secrets.API_SPACE_DEVELOP }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.SPACES_ACCESS_KEY }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_KEY }}
+        AWS_S3_ENDPOINT: https://nyc3.digitaloceanspaces.com
+        SOURCE_DIR: '.build/api_data'

lib/python/qmk/cli/generate/api.py

@@ -1,7 +1,7 @@
 """This script automates the generation of the QMK API data.
 """
 from pathlib import Path
-from shutil import copyfile
+import shutil
 import json
 
 from milc import cli
@@ -12,28 +12,42 @@
 from qmk.json_schema import json_load
 from qmk.keyboard import find_readme, list_keyboards
 
+TEMPLATE_PATH = Path('data/templates/api/')
+BUILD_API_PATH = Path('.build/api_data/')
+
 
 @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't write the data to disk.")
+@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the list of keyboards based on partial name matches the supplied value. May be passed multiple times.")
 @cli.subcommand('Creates a new keymap for the keyboard of your choosing', hidden=False if cli.config.user.developer else True)
 def generate_api(cli):
     """Generates the QMK API data.
     """
-    api_data_dir = Path('api_data')
-    v1_dir = api_data_dir / 'v1'
+    if BUILD_API_PATH.exists():
+        shutil.rmtree(BUILD_API_PATH)
+
+    shutil.copytree(TEMPLATE_PATH, BUILD_API_PATH)
+
+    v1_dir = BUILD_API_PATH / 'v1'
     keyboard_all_file = v1_dir / 'keyboards.json'  # A massive JSON containing everything
     keyboard_list_file = v1_dir / 'keyboard_list.json'  # A simple list of keyboard targets
     keyboard_aliases_file = v1_dir / 'keyboard_aliases.json'  # A list of historical keyboard names and their new name
     keyboard_metadata_file = v1_dir / 'keyboard_metadata.json'  # All the data configurator/via needs for initialization
     usb_file = v1_dir / 'usb.json'  # A mapping of USB VID/PID -> keyboard target
 
-    if not api_data_dir.exists():
-        api_data_dir.mkdir()
+    # Filter down when required
+    keyboard_list = list_keyboards()
+    if cli.args.filter:
+        kb_list = []
+        for keyboard_name in keyboard_list:
+            if any(i in keyboard_name for i in cli.args.filter):
+                kb_list.append(keyboard_name)
+        keyboard_list = kb_list
 
     kb_all = {}
     usb_list = {}
 
     # Generate and write keyboard specific JSON files
-    for keyboard_name in list_keyboards():
+    for keyboard_name in keyboard_list:
         kb_all[keyboard_name] = info_json(keyboard_name)
         keyboard_dir = v1_dir / 'keyboards' / keyboard_name
         keyboard_info = keyboard_dir / 'info.json'
@@ -47,7 +61,7 @@ def generate_api(cli):
             cli.log.debug('Wrote file %s', keyboard_info)
 
             if keyboard_readme_src:
-                copyfile(keyboard_readme_src, keyboard_readme)
+                shutil.copyfile(keyboard_readme_src, keyboard_readme)
                 cli.log.debug('Copied %s -> %s', keyboard_readme_src, keyboard_readme)
 
         if 'usb' in kb_all[keyboard_name]:

lib/python/qmk/tests/test_cli_commands.py

@@ -244,7 +244,7 @@ def test_clean():
 
 
 def test_generate_api():
-    result = check_subcommand('generate-api', '--dry-run')
+    result = check_subcommand('generate-api', '--dry-run', '--filter', 'handwired/pytest')
     check_returncode(result)